[mirror_ovs.git] / tests / ovsdb-cluster.at

OVS_START_SHELL_HELPERS
# ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION...
ovsdb_check_cluster () {
    local n=$1 schema_func=$2 output=$3
    shift; shift; shift

    $schema_func > schema
    schema=`ovsdb-tool schema-name schema`
    AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
    AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
    cid=`ovsdb-tool db-cid s1.db`
    for i in `seq 2 $n`; do
        AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
    done

    on_exit 'kill `cat *.pid`'
    for i in `seq $n`; do
        AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
    done
    for i in `seq $n`; do
        AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected])
    done

    for txn
    do
      AT_CHECK([ovsdb-client --timeout=30 -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
      cat stdout >> output
    done
    AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
    for i in `seq $n`; do
        OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
    done

    AT_CHECK([ovsdb-tool check-cluster s*.db])
}
OVS_END_SHELL_HELPERS

# Test a 1-server cluster.
AT_BANNER([OVSDB - clustered transactions (1 server)])
m4_define([OVSDB_CHECK_EXECUTION],
  [AT_SETUP([$1 - cluster of 1])
   AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
   ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
   AT_CLEANUP])
EXECUTION_EXAMPLES

# Test a 3-server cluster.
AT_BANNER([OVSDB - clustered transactions (3 servers)])
m4_define([OVSDB_CHECK_EXECUTION],
  [AT_SETUP([$1 - cluster of 3])
   AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
   ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
   AT_CLEANUP])
EXECUTION_EXAMPLES

# Test a 5-server cluster.
AT_BANNER([OVSDB - clustered transactions (5 servers)])
m4_define([OVSDB_CHECK_EXECUTION],
  [AT_SETUP([$1 - cluster of 5])
   AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
   ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
   AT_CLEANUP])
EXECUTION_EXAMPLES
\f

OVS_START_SHELL_HELPERS
# ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION...
ovsdb_cluster_failure_test () {
    # Initial state: s1 is leader, s2 and s3 are followers
    remote_1=$1
    remote_2=$2
    crash_node=$3
    crash_command=$4
    if test "$crash_node" == "1"; then
        new_leader=$5
    fi

    cp $top_srcdir/ovn/ovn-nb.ovsschema schema
    schema=`ovsdb-tool schema-name schema`
    AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
ovsdb|WARN|schema: changed 2 columns in 'OVN_Northbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns
])

    n=3
    join_cluster() {
        local i=$1
        others=
        for j in `seq 1 $n`; do
            if test $i != $j; then
                others="$others unix:s$j.raft"
            fi
        done
        AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
    }
    start_server() {
        local i=$1
        printf "\ns$i: starting\n"
        AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
    }
    connect_server() {
        local i=$1
        printf "\ns$i: waiting to connect to storage\n"
        AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
    }
    cid=`ovsdb-tool db-cid s1.db`
    for i in `seq 2 $n`; do join_cluster $i; done

    on_exit 'kill `cat *.pid`'
    for i in `seq $n`; do start_server $i; done
    for i in `seq $n`; do connect_server $i; done

    export OVN_NB_DB=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb

    # To ensure $new_leader node the new leader, we delay election timer for
    # the other follower.
    if test -n "$new_leader"; then
        if test "$new_leader" == "2"; then
            delay_election_node=3
        else
            delay_election_node=2
        fi
        AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore])
    fi
    AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore])
    AT_CHECK([ovn-nbctl -v --timeout=10 --no-leader-only --no-shuffle-remotes create logical_switch name=ls1], [0], [ignore], [ignore])

    # Make sure that the node really crashed.
    AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore])
    # XXX: Client will fail if remotes contains unix socket that doesn't exist (killed).
    if test "$remote_1" == "$crash_node"; then
        export OVN_NB_DB=unix:s$remote_2.ovsdb
    fi
    AT_CHECK([ovn-nbctl --no-leader-only ls-list | awk '{ print $2 }'], [0], [(ls1)
])
}
OVS_END_SHELL_HELPERS
AT_BANNER([OVSDB - cluster failure with pending transaction])

AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
# XXX: Detect and skip repeated transaction before enabling this test
AT_CHECK([exit 77])
ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
# XXX: Detect and skip repeated transaction before enabling this test
AT_CHECK([exit 77])
ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
# XXX: Detect and skip repeated transaction before enabling this test
AT_CHECK([exit 77])
ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
# XXX: Detect and skip repeated transaction before enabling this test
AT_CHECK([exit 77])
ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update
AT_CLEANUP

AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update])
AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update
AT_CLEANUP

\f
AT_BANNER([OVSDB - cluster tests])

# Torture test.
OVS_START_SHELL_HELPERS
ovsdb_torture_test () {
    local n=$1                  # Number of cluster members
    local victim=$2             # Cluster member to kill or remove
    local variant=$3            # 'kill' and restart or 'remove' and add
    cp $top_srcdir/ovn/ovn-sb.ovsschema schema
    schema=`ovsdb-tool schema-name schema`
    AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
ovsdb|WARN|schema: changed 2 columns in 'OVN_Southbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns
])

    join_cluster() {
        local i=$1
        others=
        for j in `seq 1 $n`; do
            if test $i != $j; then
                others="$others unix:s$j.raft"
            fi
        done
        AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
    }

    start_server() {
        local i=$1
        printf "\ns$i: starting\n"
        AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
    }
    stop_server() {
        local i=$1
        printf "\ns$i: stopping\n"
        OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
    }
    connect_server() {
        local i=$1
        printf "\ns$i: waiting to connect to storage\n"
        AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
    }
    remove_server() {
        local i=$1
        printf "\ns$i: removing from cluster\n"
        AT_CHECK([ovs-appctl --timeout=30 -t "`pwd`"/s$i cluster/leave OVN_Southbound])
        printf "\ns$i: waiting for removal to complete\n"
        AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed])
        stop_server $i
    }
    add_server() {
        local i=$1
        rm s$i.db
        join_cluster $i
        start_server $i
        connect_server $i
    }

    cid=`ovsdb-tool db-cid s1.db`
    for i in `seq 2 $n`; do join_cluster $i; done

    on_exit 'kill `cat *.pid`'
    for i in `seq $n`; do start_server $i; done
    for i in `seq $n`; do connect_server $i; done

    OVN_SB_DB=unix:s1.ovsdb
    for i in `seq 2 $n`; do
        OVN_SB_DB=$OVN_SB_DB,unix:s$i.ovsdb
    done
    export OVN_SB_DB

    n1=10 n2=5 n3=50
    echo "starting $n1*$n2 ovn-sbctl processes..."
    for i in $(seq 0 $(expr $n1 - 1) ); do
        (for j in $(seq $n2); do
             : > $i-$j.running
             txn="add SB_Global . external_ids $i-$j=$i-$j"
             for k in $(seq $n3); do
                 txn="$txn -- add SB_Global . external_ids $i-$j-$k=$i-$j-$k"
             done
             run_as "ovn-sbctl($i-$j)" ovn-sbctl "-vPATTERN:console:ovn-sbctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --no-leader-only $txn
             status=$?
             if test $status != 0; then
                 echo "$i-$j exited with status $status" > $i-$j:$status
             fi
             rm $i-$j.running
         done
         : > $i.done)&
    done
    echo "...done"

    echo "waiting for ovn-sbctl processes to exit..."
    # Use file instead of var because code inside "while" runs in a subshell.
    echo 0 > phase
    i=0
    (while :; do echo; sleep 0.1; done) | while read REPLY; do
        printf "t=%2d s:" $i
        done=0
        for j in $(seq 0 $(expr $n1 - 1)); do
            if test -f $j.done; then
                printf " $j"
                done=$(expr $done + 1)
            fi
        done
        printf '\n'
        if test $done = $n1; then
            break
        fi

        case $(cat phase) in # (
        0)
            if test $done -ge $(expr $n1 / 10); then
                if test $variant = kill; then
                    stop_server $victim
                else
                    remove_server $victim
                fi
                echo 1 > phase
                next=$(expr $i + 2)
            fi
            ;; # (
        1)
            if test $i -ge $next; then
                if test $variant = kill; then
                    start_server $victim
                    connect_server $victim
                else
                    add_server $victim
                fi
                echo 2 > phase
            fi
            ;;
        esac

        i=$(expr $i + 1)
    done
    echo "...done"
    AT_CHECK([if test $(cat phase) != 2; then exit 77; fi])

    for i in $(seq 0 $(expr $n1 - 1) ); do
        for j in `seq $n2`; do
            echo "$i-$j=$i-$j"
            for k in `seq $n3`; do
                echo "$i-$j-$k=$i-$j-$k"
            done
        done
    done | sort > expout
    AT_CHECK([ovn-sbctl --timeout=30 --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get SB_Global . external-ids | tr ',' '\n' | sed 's/[[{}"" ]]//g' | sort], [0], [expout])

    for i in `seq $n`; do
        if test $i != $victim || test $(cat phase) != 1; then
            stop_server $i
        fi
    done

    # We ignore stdout because non-fatal warnings get printed there.
    AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
}
OVS_END_SHELL_HELPERS

AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 1 kill
AT_CLEANUP
AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 2 kill
AT_CLEANUP
AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 3 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 1 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 2 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 3 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 4 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 5 kill
AT_CLEANUP

AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 1 remove
AT_CLEANUP
AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 2 remove
AT_CLEANUP
AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 3 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 1 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 2 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 3 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 4 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 5 remove
AT_CLEANUP
Commit	Line	Data
1b1d2e6d BP	1	OVS_START_SHELL_HELPERS
	2	# ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION...
	3	ovsdb_check_cluster () {
	4	local n=$1 schema_func=$2 output=$3
	5	shift; shift; shift
	6
	7	$schema_func > schema
	8	schema=`ovsdb-tool schema-name schema`
	9	AT_CHECK([ovsdb-tool '-vPATTERN:console:%c\|%p\|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
	10	AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
	11	cid=`ovsdb-tool db-cid s1.db`
	12	for i in `seq 2 $n`; do
	13	AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
	14	done
	15
	16	on_exit 'kill `cat *.pid`'
	17	for i in `seq $n`; do
5a0e4aec	18	AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
1b1d2e6d BP	19	done
1b1d2e6d BP	20	for i in `seq $n`; do
d97af428	21	AT_CHECK([ovsdb_client_wait unix:s$i.ovsdb $schema connected])
1b1d2e6d BP	22	done
	23
	24	for txn
	25	do
	26	AT_CHECK([ovsdb-client --timeout=30 -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
	27	cat stdout >> output
	28	done
	29	AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
	30	for i in `seq $n`; do
5a0e4aec	31	OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
1b1d2e6d BP	32	done
	33
	34	AT_CHECK([ovsdb-tool check-cluster s*.db])
	35	}
	36	OVS_END_SHELL_HELPERS
	37
	38	# Test a 1-server cluster.
	39	AT_BANNER([OVSDB - clustered transactions (1 server)])
	40	m4_define([OVSDB_CHECK_EXECUTION],
	41	[AT_SETUP([$1 - cluster of 1])
	42	AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
	43	ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
	44	AT_CLEANUP])
	45	EXECUTION_EXAMPLES
	46
	47	# Test a 3-server cluster.
	48	AT_BANNER([OVSDB - clustered transactions (3 servers)])
	49	m4_define([OVSDB_CHECK_EXECUTION],
	50	[AT_SETUP([$1 - cluster of 3])
	51	AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
	52	ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
	53	AT_CLEANUP])
	54	EXECUTION_EXAMPLES
	55
	56	# Test a 5-server cluster.
	57	AT_BANNER([OVSDB - clustered transactions (5 servers)])
	58	m4_define([OVSDB_CHECK_EXECUTION],
	59	[AT_SETUP([$1 - cluster of 5])
	60	AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
	61	ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
	62	AT_CLEANUP])
	63	EXECUTION_EXAMPLES
eb692258 HZ	64	\f
	65
	66	OVS_START_SHELL_HELPERS
	67	# ovsdb_cluster_failure_test SCHEMA_FUNC OUTPUT TRANSACTION...
	68	ovsdb_cluster_failure_test () {
	69	# Initial state: s1 is leader, s2 and s3 are followers
	70	remote_1=$1
	71	remote_2=$2
	72	crash_node=$3
	73	crash_command=$4
	74	if test "$crash_node" == "1"; then
	75	new_leader=$5
	76	fi
	77
	78	cp $top_srcdir/ovn/ovn-nb.ovsschema schema
	79	schema=`ovsdb-tool schema-name schema`
	80	AT_CHECK([ovsdb-tool '-vPATTERN:console:%c\|%p\|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
	81	ovsdb\|WARN\|schema: changed 2 columns in 'OVN_Northbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns
	82	])
	83
	84	n=3
	85	join_cluster() {
	86	local i=$1
	87	others=
	88	for j in `seq 1 $n`; do
	89	if test $i != $j; then
	90	others="$others unix:s$j.raft"
	91	fi
	92	done
	93	AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
	94	}
	95	start_server() {
	96	local i=$1
	97	printf "\ns$i: starting\n"
	98	AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
	99	}
	100	connect_server() {
	101	local i=$1
	102	printf "\ns$i: waiting to connect to storage\n"
	103	AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
	104	}
	105	cid=`ovsdb-tool db-cid s1.db`
	106	for i in `seq 2 $n`; do join_cluster $i; done
	107
	108	on_exit 'kill `cat *.pid`'
	109	for i in `seq $n`; do start_server $i; done
	110	for i in `seq $n`; do connect_server $i; done
	111
	112	export OVN_NB_DB=unix:s$remote_1.ovsdb,unix:s$remote_2.ovsdb
	113
	114	# To ensure $new_leader node the new leader, we delay election timer for
	115	# the other follower.
	116	if test -n "$new_leader"; then
	117	if test "$new_leader" == "2"; then
	118	delay_election_node=3
	119	else
	120	delay_election_node=2
	121	fi
	122	AT_CHECK([ovs-appctl -t "`pwd`"/s$delay_election_node cluster/failure-test delay-election], [0], [ignore])
	123	fi
	124	AT_CHECK([ovs-appctl -t "`pwd`"/s$crash_node cluster/failure-test $crash_command], [0], [ignore])
	125	AT_CHECK([ovn-nbctl -v --timeout=10 --no-leader-only --no-shuffle-remotes create logical_switch name=ls1], [0], [ignore], [ignore])
	126
	127	# Make sure that the node really crashed.
128	AT_CHECK([ls s$crash_node.ovsdb], [2], [ignore], [ignore])
129	# XXX: Client will fail if remotes contains unix socket that doesn't exist (killed).
130	if test "$remote_1" == "$crash_node"; then
131	export OVN_NB_DB=unix:s$remote_2.ovsdb
132	fi
133	AT_CHECK([ovn-nbctl --no-leader-only ls-list \| awk '{ print $2 }'], [0], [(ls1)
134	])
135	}
136	OVS_END_SHELL_HELPERS
137	AT_BANNER([OVSDB - cluster failure with pending transaction])
138
139	AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-2 becomes leader])
140	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
eb692258 HZ	141	ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 2
	142	AT_CLEANUP
	143
	144	AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending appendReq, follower-3 becomes leader])
	145	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	146	ovsdb_cluster_failure_test 2 3 1 crash-before-sending-append-request 3
	147	AT_CLEANUP
	148
	149	AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-2 becomes leader])
	150	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
eb692258 HZ	151	ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 2
	152	AT_CLEANUP
	153
	154	AT_SETUP([OVSDB cluster - txn on follower-2, leader crash before sending execRep, follower-3 becomes leader])
	155	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
eb692258 HZ	156	ovsdb_cluster_failure_test 2 3 1 crash-before-sending-execute-command-reply 3
	157	AT_CLEANUP
	158
	159	AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-2 becomes leader])
	160	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
eb692258 HZ	161	ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 2
	162	AT_CLEANUP
	163
	164	AT_SETUP([OVSDB cluster - txn on follower-2, leader crash after sending execRep, follower-3 becomes leader])
	165	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	166	ovsdb_cluster_failure_test 2 3 1 crash-after-sending-execute-command-reply 3
	167	AT_CLEANUP
	168
	169	AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-2 becomes leader])
	170	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
eb692258 HZ	171	ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 2
	172	AT_CLEANUP
	173
	174	AT_SETUP([OVSDB cluster - txn on leader, leader crash before sending appendReq, follower-3 becomes leader])
	175	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	176	ovsdb_cluster_failure_test 1 2 1 crash-before-sending-append-request 3
	177	AT_CLEANUP
	178
	179	AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-2 becomes leader])
	180	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	181	# XXX: Detect and skip repeated transaction before enabling this test
	182	AT_CHECK([exit 77])
	183	ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 2
	184	AT_CLEANUP
	185
	186	AT_SETUP([OVSDB cluster - txn on leader, leader crash after sending appendReq, follower-3 becomes leader])
	187	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	188	# XXX: Detect and skip repeated transaction before enabling this test
	189	AT_CHECK([exit 77])
	190	ovsdb_cluster_failure_test 1 2 1 crash-after-sending-append-request 3
	191	AT_CLEANUP
	192
	193	AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to follower-3])
	194	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	195	ovsdb_cluster_failure_test 2 3 2 crash-before-sending-execute-command-request
	196	AT_CLEANUP
	197
	198	AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash before sending execReq, reconnect to leader])
	199	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	200	ovsdb_cluster_failure_test 2 1 2 crash-before-sending-execute-command-request
	201	AT_CLEANUP
	202
	203	AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to follower-3])
	204	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	205	# XXX: Detect and skip repeated transaction before enabling this test
	206	AT_CHECK([exit 77])
	207	ovsdb_cluster_failure_test 2 3 2 crash-after-sending-execute-command-request
	208	AT_CLEANUP
	209
	210	AT_SETUP([OVSDB cluster - txn on follower-2, follower-2 crash after sending execReq, reconnect to leader])
	211	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	212	# XXX: Detect and skip repeated transaction before enabling this test
	213	AT_CHECK([exit 77])
	214	ovsdb_cluster_failure_test 2 1 2 crash-after-sending-execute-command-request
	215	AT_CLEANUP
	216
	217	AT_SETUP([OVSDB cluster - txn on leader, follower-2 crash after receiving appendReq for the update])
	218	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	219	ovsdb_cluster_failure_test 1 1 2 crash-after-receiving-append-request-update
	220	AT_CLEANUP
	221
	222	AT_SETUP([OVSDB cluster - txn on follower-2, follower-3 crash after receiving appendReq for the update])
	223	AT_KEYWORDS([ovsdb server negative unix cluster pending-txn])
	224	ovsdb_cluster_failure_test 2 2 3 crash-after-receiving-append-request-update
	225	AT_CLEANUP
	226
1b1d2e6d BP	227	\f
	228	AT_BANNER([OVSDB - cluster tests])
	229
	230	# Torture test.
	231	OVS_START_SHELL_HELPERS
	232	ovsdb_torture_test () {
5a0e4aec BP	233	local n=$1 # Number of cluster members
5a0e4aec BP	234	local victim=$2 # Cluster member to kill or remove
1b1d2e6d BP	235	local variant=$3 # 'kill' and restart or 'remove' and add
	236	cp $top_srcdir/ovn/ovn-sb.ovsschema schema
	237	schema=`ovsdb-tool schema-name schema`
	238	AT_CHECK([ovsdb-tool '-vPATTERN:console:%c\|%p\|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
	239	ovsdb\|WARN\|schema: changed 2 columns in 'OVN_Southbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns
	240	])
	241
	242	join_cluster() {
	243	local i=$1
5a0e4aec BP	244	others=
	245	for j in `seq 1 $n`; do
	246	if test $i != $j; then
	247	others="$others unix:s$j.raft"
1b1d2e6d	248	fi
5a0e4aec BP	249	done
5a0e4aec BP	250	AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
1b1d2e6d BP	251	}
	252
	253	start_server() {
	254	local i=$1
5a0e4aec BP	255	printf "\ns$i: starting\n"
5a0e4aec BP	256	AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
1b1d2e6d BP	257	}
	258	stop_server() {
	259	local i=$1
5a0e4aec	260	printf "\ns$i: stopping\n"
1b1d2e6d BP	261	OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
	262	}
	263	connect_server() {
	264	local i=$1
5a0e4aec	265	printf "\ns$i: waiting to connect to storage\n"
d97af428	266	AT_CHECK([ovsdb_client_wait --log-file=connect$i.log unix:s$i.ovsdb $schema connected])
1b1d2e6d BP	267	}
	268	remove_server() {
	269	local i=$1
5a0e4aec BP	270	printf "\ns$i: removing from cluster\n"
	271	AT_CHECK([ovs-appctl --timeout=30 -t "`pwd`"/s$i cluster/leave OVN_Southbound])
	272	printf "\ns$i: waiting for removal to complete\n"
d97af428	273	AT_CHECK([ovsdb_client_wait --log-file=remove$i.log unix:s$i.ovsdb $schema removed])
c7b5c534	274	stop_server $i
1b1d2e6d BP	275	}
	276	add_server() {
	277	local i=$1
5a0e4aec	278	rm s$i.db
1b1d2e6d	279	join_cluster $i
5a0e4aec BP	280	start_server $i
5a0e4aec BP	281	connect_server $i
1b1d2e6d BP	282	}
	283
	284	cid=`ovsdb-tool db-cid s1.db`
	285	for i in `seq 2 $n`; do join_cluster $i; done
	286
	287	on_exit 'kill `cat *.pid`'
	288	for i in `seq $n`; do start_server $i; done
	289	for i in `seq $n`; do connect_server $i; done
	290
	291	OVN_SB_DB=unix:s1.ovsdb
	292	for i in `seq 2 $n`; do
	293	OVN_SB_DB=$OVN_SB_DB,unix:s$i.ovsdb
	294	done
	295	export OVN_SB_DB
	296
0f954f32	297	n1=10 n2=5 n3=50
1b1d2e6d BP	298	echo "starting $n1*$n2 ovn-sbctl processes..."
1b1d2e6d BP	299	for i in $(seq 0 $(expr $n1 - 1) ); do
5a0e4aec	300	(for j in $(seq $n2); do
1b1d2e6d	301	: > $i-$j.running
0f954f32 HZ	302	txn="add SB_Global . external_ids $i-$j=$i-$j"
	303	for k in $(seq $n3); do
	304	txn="$txn -- add SB_Global . external_ids $i-$j-$k=$i-$j-$k"
	305	done
	306	run_as "ovn-sbctl($i-$j)" ovn-sbctl "-vPATTERN:console:ovn-sbctl($i-$j)\|%D{%H:%M:%S}\|%05N\|%c\|%p\|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --no-leader-only $txn
5a0e4aec BP	307	status=$?
	308	if test $status != 0; then
	309	echo "$i-$j exited with status $status" > $i-$j:$status
1b1d2e6d BP	310	fi
1b1d2e6d BP	311	rm $i-$j.running
5a0e4aec BP	312	done
5a0e4aec BP	313	: > $i.done)&
1b1d2e6d BP	314	done
1b1d2e6d BP	315	echo "...done"
1b1d2e6d BP	316
1b1d2e6d BP	317	echo "waiting for ovn-sbctl processes to exit..."
6c8dd8ca	318	# Use file instead of var because code inside "while" runs in a subshell.
7ee9c6e0	319	echo 0 > phase
0f03ae37	320	i=0
0f954f32	321	(while :; do echo; sleep 0.1; done) \| while read REPLY; do
1b1d2e6d	322	printf "t=%2d s:" $i
5a0e4aec	323	done=0
1b1d2e6d	324	for j in $(seq 0 $(expr $n1 - 1)); do
5a0e4aec BP	325	if test -f $j.done; then
	326	printf " $j"
	327	done=$(expr $done + 1)
1b1d2e6d	328	fi
5a0e4aec BP	329	done
	330	printf '\n'
	331	if test $done = $n1; then
	332	break
1b1d2e6d BP	333	fi
1b1d2e6d BP	334
7ee9c6e0	335	case $(cat phase) in # (
5a0e4aec	336	0)
0f954f32	337	if test $done -ge $(expr $n1 / 10); then
5a0e4aec BP	338	if test $variant = kill; then
	339	stop_server $victim
	340	else
	341	remove_server $victim
	342	fi
7ee9c6e0	343	echo 1 > phase
5a0e4aec	344	next=$(expr $i + 2)
1b1d2e6d	345	fi
5a0e4aec	346	;; # (
1b1d2e6d	347	1)
5a0e4aec BP	348	if test $i -ge $next; then
	349	if test $variant = kill; then
	350	start_server $victim
	351	connect_server $victim
	352	else
	353	add_server $victim
	354	fi
7ee9c6e0	355	echo 2 > phase
5a0e4aec BP	356	fi
	357	;;
	358	esac
	359
	360	i=$(expr $i + 1)
1b1d2e6d BP	361	done
1b1d2e6d BP	362	echo "...done"
7ee9c6e0	363	AT_CHECK([if test $(cat phase) != 2; then exit 77; fi])
1b1d2e6d	364
dff60a1e BP	365	for i in $(seq 0 $(expr $n1 - 1) ); do
	366	for j in `seq $n2`; do
	367	echo "$i-$j=$i-$j"
0f954f32 HZ	368	for k in `seq $n3`; do
	369	echo "$i-$j-$k=$i-$j-$k"
	370	done
dff60a1e BP	371	done
dff60a1e BP	372	done \| sort > expout
b6c9325b	373	AT_CHECK([ovn-sbctl --timeout=30 --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get SB_Global . external-ids \| tr ',' '\n' \| sed 's/[[{}"" ]]//g' \| sort], [0], [expout])
1b1d2e6d BP	374
1b1d2e6d BP	375	for i in `seq $n`; do
7ee9c6e0	376	if test $i != $victim \|\| test $(cat phase) != 1; then
5a0e4aec BP	377	stop_server $i
5a0e4aec BP	378	fi
1b1d2e6d BP	379	done
	380
	381	# We ignore stdout because non-fatal warnings get printed there.
	382	AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
	383	}
	384	OVS_END_SHELL_HELPERS
	385
	386	AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
	387	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
	388	ovsdb_torture_test 3 1 kill
	389	AT_CLEANUP
	390	AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
	391	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
	392	ovsdb_torture_test 3 2 kill
	393	AT_CLEANUP
	394	AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
	395	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
	396	ovsdb_torture_test 3 3 kill
	397	AT_CLEANUP
	398	AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
	399	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
	400	ovsdb_torture_test 5 1 kill
	401	AT_CLEANUP
	402	AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
	403	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
	404	ovsdb_torture_test 5 2 kill
	405	AT_CLEANUP
	406	AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
	407	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
	408	ovsdb_torture_test 5 3 kill
	409	AT_CLEANUP
	410	AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
	411	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
	412	ovsdb_torture_test 5 4 kill
	413	AT_CLEANUP
	414	AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
	415	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
	416	ovsdb_torture_test 5 5 kill
	417	AT_CLEANUP
	418
	419	AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
	420	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
	421	ovsdb_torture_test 3 1 remove
	422	AT_CLEANUP
	423	AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
	424	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
	425	ovsdb_torture_test 3 2 remove
	426	AT_CLEANUP
	427	AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
	428	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
	429	ovsdb_torture_test 3 3 remove
	430	AT_CLEANUP
	431	AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
	432	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
	433	ovsdb_torture_test 5 1 remove
	434	AT_CLEANUP
	435	AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
	436	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
	437	ovsdb_torture_test 5 2 remove
	438	AT_CLEANUP
	439	AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
	440	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
	441	ovsdb_torture_test 5 3 remove
	442	AT_CLEANUP
443	AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
444	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
445	ovsdb_torture_test 5 4 remove
446	AT_CLEANUP
447	AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
448	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
449	ovsdb_torture_test 5 5 remove
450	AT_CLEANUP