[ovs.git] / tests / ovsdb-cluster.at

OVS_START_SHELL_HELPERS
# ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION...
ovsdb_check_cluster () {
    local n=$1 schema_func=$2 output=$3
    shift; shift; shift

    $schema_func > schema
    schema=`ovsdb-tool schema-name schema`
    AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
    AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
    cid=`ovsdb-tool db-cid s1.db`
    for i in `seq 2 $n`; do
        AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
    done

    on_exit 'kill `cat *.pid`'
    for i in `seq $n`; do
	AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
    done
    for i in `seq $n`; do
	AT_CHECK([ovsdb-client --timeout=30 wait unix:s$i.ovsdb $schema connected])
    done

    for txn
    do
      AT_CHECK([ovsdb-client --timeout=30 -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
      cat stdout >> output
    done
    AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
    for i in `seq $n`; do
	OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
    done

    AT_CHECK([ovsdb-tool check-cluster s*.db])
}
OVS_END_SHELL_HELPERS

# Test a 1-server cluster.
AT_BANNER([OVSDB - clustered transactions (1 server)])
m4_define([OVSDB_CHECK_EXECUTION],
  [AT_SETUP([$1 - cluster of 1])
   AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
   ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
   AT_CLEANUP])
EXECUTION_EXAMPLES

# Test a 3-server cluster.
AT_BANNER([OVSDB - clustered transactions (3 servers)])
m4_define([OVSDB_CHECK_EXECUTION],
  [AT_SETUP([$1 - cluster of 3])
   AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
   ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
   AT_CLEANUP])
EXECUTION_EXAMPLES

# Test a 5-server cluster.
AT_BANNER([OVSDB - clustered transactions (5 servers)])
m4_define([OVSDB_CHECK_EXECUTION],
  [AT_SETUP([$1 - cluster of 5])
   AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
   ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
   AT_CLEANUP])
EXECUTION_EXAMPLES
\f
AT_BANNER([OVSDB - cluster tests])

# Torture test.
OVS_START_SHELL_HELPERS
ovsdb_torture_test () {
    local n=$1			# Number of cluster members
    local victim=$2		# Cluster member to kill or remove
    local variant=$3            # 'kill' and restart or 'remove' and add
    cp $top_srcdir/ovn/ovn-sb.ovsschema schema
    schema=`ovsdb-tool schema-name schema`
    AT_CHECK([ovsdb-tool '-vPATTERN:console:%c|%p|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
ovsdb|WARN|schema: changed 2 columns in 'OVN_Southbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns
])

    join_cluster() {
        local i=$1
	others=
	for j in `seq 1 $n`; do
	    if test $i != $j; then
	        others="$others unix:s$j.raft"
            fi
	done
	AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
    }

    start_server() {
        local i=$1
	printf "\ns$i: starting\n"
	AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
    }
    stop_server() {
        local i=$1
	printf "\ns$i: stopping\n"
        OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
    }
    connect_server() {
        local i=$1
	printf "\ns$i: waiting to connect to storage\n"
	AT_CHECK([ovsdb-client --timeout=30 -vfile -vsyslog:off -vvlog:off --log-file=connect$i.log wait unix:s$i.ovsdb $schema connected])
    }
    remove_server() {
        local i=$1
	printf "\ns$i: removing from cluster\n"
	AT_CHECK([ovs-appctl --timeout=30 -t "`pwd`"/s$i cluster/leave OVN_Southbound])
	printf "\ns$i: waiting for removal to complete\n"
	AT_CHECK([ovsdb-client --timeout=30 -vfile -vsyslog:off -vvlog:off --log-file=remove$i.log wait unix:s$i.ovsdb $schema removed])
	stop_server $victim
    }
    add_server() {
        local i=$1
	rm s$i.db
        join_cluster $i
	start_server $i
	connect_server $i
    }

    cid=`ovsdb-tool db-cid s1.db`
    for i in `seq 2 $n`; do join_cluster $i; done

    on_exit 'kill `cat *.pid`'
    for i in `seq $n`; do start_server $i; done
    for i in `seq $n`; do connect_server $i; done

    OVN_SB_DB=unix:s1.ovsdb
    for i in `seq 2 $n`; do
        OVN_SB_DB=$OVN_SB_DB,unix:s$i.ovsdb
    done
    export OVN_SB_DB

    n1=10 n2=5
    echo "starting $n1*$n2 ovn-sbctl processes..."
    for i in $(seq 0 $(expr $n1 - 1) ); do
	(for j in $(seq $n2); do
             : > $i-$j.running
	     run_as "ovn-sbctl($i-$j)" ovn-sbctl "-vPATTERN:console:ovn-sbctl($i-$j)|%D{%H:%M:%S}|%05N|%c|%p|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --no-leader-only add SB_Global . external_ids $i-$j=$i-$j
	     status=$?
	     if test $status != 0; then
	         echo "$i-$j exited with status $status" > $i-$j:$status
             fi
             rm $i-$j.running
	 done
	 : > $i.done)&
    done
    echo "...done"
    sleep 2

    echo "waiting for ovn-sbctl processes to exit..."
    i=0
    phase=0
    while :; do
        printf "t=%2d s:" $i
	done=0
        for j in $(seq 0 $(expr $n1 - 1)); do
	    if test -f $j.done; then
	        printf " $j"
		done=$(expr $done + 1)
            fi
	done
	printf '\n'
	if test $done = $n1; then
	    break
        fi

        case $phase in # (
	0)
            if test $done -ge $(expr $n1 / 4); then
	        if test $variant = kill; then
		    stop_server $victim
		else
		    remove_server $victim
		fi
		phase=1
		next=$(expr $i + 2)
            fi
	    ;; # (
        1)
	    if test $i -ge $next; then
	        if test $variant = kill; then
		    start_server $victim
		    connect_server $victim
		else
		    add_server $victim
		fi
		phase=2
	    fi
	    ;;
	esac

        sleep 1
	i=$(expr $i + 1)
    done
    echo "...done"
    AT_CHECK([if test $phase != 2; then exit 77; fi])

    for i in `seq 0 9`; do
	for j in `seq 5`; do
	    echo "$i-$j=$i-$j"
	done
    done > expout
    AT_CHECK([ovn-sbctl --timeout=30 --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get SB_Global . external-ids | sed 's/, /\n/g; s/[[{}""]]//g;'], [0], [expout])

    for i in `seq $n`; do
	if test $i != $victim || test $phase != 1; then
	    stop_server $i
	fi
    done

    # We ignore stdout because non-fatal warnings get printed there.
    AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
}
OVS_END_SHELL_HELPERS

AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 1 kill
AT_CLEANUP
AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 2 kill
AT_CLEANUP
AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 3 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 1 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 2 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 3 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 4 kill
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 5 kill
AT_CLEANUP

AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 1 remove
AT_CLEANUP
AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 2 remove
AT_CLEANUP
AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
ovsdb_torture_test 3 3 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 1 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 2 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 3 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 4 remove
AT_CLEANUP
AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
ovsdb_torture_test 5 5 remove
AT_CLEANUP
Commit	Line	Data
1b1d2e6d BP	1	OVS_START_SHELL_HELPERS
	2	# ovsdb_check_cluster N_SERVERS SCHEMA_FUNC OUTPUT TRANSACTION...
	3	ovsdb_check_cluster () {
	4	local n=$1 schema_func=$2 output=$3
	5	shift; shift; shift
	6
	7	$schema_func > schema
	8	schema=`ovsdb-tool schema-name schema`
	9	AT_CHECK([ovsdb-tool '-vPATTERN:console:%c\|%p\|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [stderr])
	10	AT_CHECK([grep -v 'from ephemeral to persistent' stderr], [1])
	11	cid=`ovsdb-tool db-cid s1.db`
	12	for i in `seq 2 $n`; do
	13	AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft unix:s1.raft])
	14	done
	15
	16	on_exit 'kill `cat *.pid`'
	17	for i in `seq $n`; do
	18	AT_CHECK([ovsdb-server -vraft -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
	19	done
	20	for i in `seq $n`; do
	21	AT_CHECK([ovsdb-client --timeout=30 wait unix:s$i.ovsdb $schema connected])
	22	done
	23
	24	for txn
	25	do
	26	AT_CHECK([ovsdb-client --timeout=30 -vjsonrpc -vconsole:off -vsyslog:off -vvlog:off --log-file transact unix:s1.ovsdb,unix:s2.ovsdb,unix:s3.ovsdb "$txn"], [0], [stdout])
	27	cat stdout >> output
	28	done
	29	AT_CHECK_UNQUOTED([uuidfilt output], [0], [$output])
	30	for i in `seq $n`; do
	31	OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
	32	done
	33
	34	AT_CHECK([ovsdb-tool check-cluster s*.db])
	35	}
	36	OVS_END_SHELL_HELPERS
	37
	38	# Test a 1-server cluster.
	39	AT_BANNER([OVSDB - clustered transactions (1 server)])
	40	m4_define([OVSDB_CHECK_EXECUTION],
	41	[AT_SETUP([$1 - cluster of 1])
	42	AT_KEYWORDS([ovsdb server positive unix cluster cluster1 $5])
	43	ovsdb_check_cluster 1 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
	44	AT_CLEANUP])
	45	EXECUTION_EXAMPLES
	46
	47	# Test a 3-server cluster.
	48	AT_BANNER([OVSDB - clustered transactions (3 servers)])
	49	m4_define([OVSDB_CHECK_EXECUTION],
	50	[AT_SETUP([$1 - cluster of 3])
	51	AT_KEYWORDS([ovsdb server positive unix cluster cluster3 $5])
	52	ovsdb_check_cluster 3 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
	53	AT_CLEANUP])
	54	EXECUTION_EXAMPLES
	55
	56	# Test a 5-server cluster.
	57	AT_BANNER([OVSDB - clustered transactions (5 servers)])
	58	m4_define([OVSDB_CHECK_EXECUTION],
	59	[AT_SETUP([$1 - cluster of 5])
	60	AT_KEYWORDS([ovsdb server positive unix cluster cluster5 $5])
	61	ovsdb_check_cluster 5 "$2" '$4' m4_foreach([txn], [$3], ['txn' ])
	62	AT_CLEANUP])
	63	EXECUTION_EXAMPLES
	64	\f
65	AT_BANNER([OVSDB - cluster tests])
66
67	# Torture test.
68	OVS_START_SHELL_HELPERS
69	ovsdb_torture_test () {
70	local n=$1 # Number of cluster members
71	local victim=$2 # Cluster member to kill or remove
72	local variant=$3 # 'kill' and restart or 'remove' and add
73	cp $top_srcdir/ovn/ovn-sb.ovsschema schema
74	schema=`ovsdb-tool schema-name schema`
75	AT_CHECK([ovsdb-tool '-vPATTERN:console:%c\|%p\|%m' create-cluster s1.db schema unix:s1.raft], [0], [], [dnl
76	ovsdb\|WARN\|schema: changed 2 columns in 'OVN_Southbound' database from ephemeral to persistent, including 'status' column in 'Connection' table, because clusters do not support ephemeral columns
77	])
78
79	join_cluster() {
80	local i=$1
81	others=
82	for j in `seq 1 $n`; do
83	if test $i != $j; then
84	others="$others unix:s$j.raft"
85	fi
86	done
87	AT_CHECK([ovsdb-tool join-cluster s$i.db $schema unix:s$i.raft $others])
88	}
89
90	start_server() {
91	local i=$1
92	printf "\ns$i: starting\n"
93	AT_CHECK([ovsdb-server -vjsonrpc -vconsole:off -vsyslog:off --detach --no-chdir --log-file=s$i.log --pidfile=s$i.pid --unixctl=s$i --remote=punix:s$i.ovsdb s$i.db])
94	}
95	stop_server() {
96	local i=$1
97	printf "\ns$i: stopping\n"
98	OVS_APP_EXIT_AND_WAIT_BY_TARGET([`pwd`/s$i], [s$i.pid])
99	}
100	connect_server() {
101	local i=$1
102	printf "\ns$i: waiting to connect to storage\n"
103	AT_CHECK([ovsdb-client --timeout=30 -vfile -vsyslog:off -vvlog:off --log-file=connect$i.log wait unix:s$i.ovsdb $schema connected])
104	}
105	remove_server() {
106	local i=$1
107	printf "\ns$i: removing from cluster\n"
108	AT_CHECK([ovs-appctl --timeout=30 -t "`pwd`"/s$i cluster/leave OVN_Southbound])
109	printf "\ns$i: waiting for removal to complete\n"
110	AT_CHECK([ovsdb-client --timeout=30 -vfile -vsyslog:off -vvlog:off --log-file=remove$i.log wait unix:s$i.ovsdb $schema removed])
111	stop_server $victim
112	}
113	add_server() {
114	local i=$1
115	rm s$i.db
116	join_cluster $i
117	start_server $i
118	connect_server $i
119	}
120
121	cid=`ovsdb-tool db-cid s1.db`
122	for i in `seq 2 $n`; do join_cluster $i; done
123
124	on_exit 'kill `cat *.pid`'
125	for i in `seq $n`; do start_server $i; done
126	for i in `seq $n`; do connect_server $i; done
127
128	OVN_SB_DB=unix:s1.ovsdb
129	for i in `seq 2 $n`; do
130	OVN_SB_DB=$OVN_SB_DB,unix:s$i.ovsdb
131	done
132	export OVN_SB_DB
133
134	n1=10 n2=5
135	echo "starting $n1*$n2 ovn-sbctl processes..."
136	for i in $(seq 0 $(expr $n1 - 1) ); do
137	(for j in $(seq $n2); do
138	: > $i-$j.running
139	run_as "ovn-sbctl($i-$j)" ovn-sbctl "-vPATTERN:console:ovn-sbctl($i-$j)\|%D{%H:%M:%S}\|%05N\|%c\|%p\|%m" --log-file=$i-$j.log -vfile -vsyslog:off -vtimeval:off --timeout=120 --no-leader-only add SB_Global . external_ids $i-$j=$i-$j
140	status=$?
141	if test $status != 0; then
142	echo "$i-$j exited with status $status" > $i-$j:$status
143	fi
144	rm $i-$j.running
145	done
146	: > $i.done)&
147	done
148	echo "...done"
149	sleep 2
150
151	echo "waiting for ovn-sbctl processes to exit..."
152	i=0
153	phase=0
154	while :; do
155	printf "t=%2d s:" $i
156	done=0
157	for j in $(seq 0 $(expr $n1 - 1)); do
158	if test -f $j.done; then
159	printf " $j"
160	done=$(expr $done + 1)
161	fi
162	done
163	printf '\n'
164	if test $done = $n1; then
165	break
166	fi
167
168	case $phase in # (
169	0)
170	if test $done -ge $(expr $n1 / 4); then
171	if test $variant = kill; then
172	stop_server $victim
173	else
174	remove_server $victim
175	fi
176	phase=1
177	next=$(expr $i + 2)
178	fi
179	;; # (
180	1)
181	if test $i -ge $next; then
182	if test $variant = kill; then
183	start_server $victim
184	connect_server $victim
185	else
186	add_server $victim
187	fi
188	phase=2
189	fi
190	;;
191	esac
192
193	sleep 1
194	i=$(expr $i + 1)
195	done
196	echo "...done"
197	AT_CHECK([if test $phase != 2; then exit 77; fi])
198
199	for i in `seq 0 9`; do
200	for j in `seq 5`; do
201	echo "$i-$j=$i-$j"
202	done
203	done > expout
204	AT_CHECK([ovn-sbctl --timeout=30 --log-file=finalize.log -vtimeval:off -vfile -vsyslog:off --bare get SB_Global . external-ids \| sed 's/, /\n/g; s/[[{}""]]//g;'], [0], [expout])
205
206	for i in `seq $n`; do
207	if test $i != $victim \|\| test $phase != 1; then
208	stop_server $i
209	fi
210	done
211
212	# We ignore stdout because non-fatal warnings get printed there.
213	AT_CHECK([ovsdb-tool check-cluster s*.db], [0], [ignore])
214	}
215	OVS_END_SHELL_HELPERS
216
217	AT_SETUP([OVSDB 3-server torture test - kill/restart leader])
218	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
219	ovsdb_torture_test 3 1 kill
220	AT_CLEANUP
221	AT_SETUP([OVSDB 3-server torture test - kill/restart follower 1])
222	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
223	ovsdb_torture_test 3 2 kill
224	AT_CLEANUP
225	AT_SETUP([OVSDB 3-server torture test - kill/restart follower 2])
226	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
227	ovsdb_torture_test 3 3 kill
228	AT_CLEANUP
229	AT_SETUP([OVSDB 5-server torture test - kill/restart leader])
230	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
231	ovsdb_torture_test 5 1 kill
232	AT_CLEANUP
233	AT_SETUP([OVSDB 5-server torture test - kill/restart follower 1])
234	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
235	ovsdb_torture_test 5 2 kill
236	AT_CLEANUP
237	AT_SETUP([OVSDB 5-server torture test - kill/restart follower 2])
238	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
239	ovsdb_torture_test 5 3 kill
240	AT_CLEANUP
241	AT_SETUP([OVSDB 5-server torture test - kill/restart follower 3])
242	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
243	ovsdb_torture_test 5 4 kill
244	AT_CLEANUP
245	AT_SETUP([OVSDB 5-server torture test - kill/restart follower 4])
246	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
247	ovsdb_torture_test 5 5 kill
248	AT_CLEANUP
249
250	AT_SETUP([OVSDB 3-server torture test - remove/re-add leader])
251	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
252	ovsdb_torture_test 3 1 remove
253	AT_CLEANUP
254	AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 1])
255	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
256	ovsdb_torture_test 3 2 remove
257	AT_CLEANUP
258	AT_SETUP([OVSDB 3-server torture test - remove/re-add follower 2])
259	AT_KEYWORDS([ovsdb server positive unix cluster cluster3])
260	ovsdb_torture_test 3 3 remove
261	AT_CLEANUP
262	AT_SETUP([OVSDB 5-server torture test - remove/re-add leader])
263	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
264	ovsdb_torture_test 5 1 remove
265	AT_CLEANUP
266	AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 1])
267	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
268	ovsdb_torture_test 5 2 remove
269	AT_CLEANUP
270	AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 2])
271	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
272	ovsdb_torture_test 5 3 remove
273	AT_CLEANUP
274	AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 3])
275	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
276	ovsdb_torture_test 5 4 remove
277	AT_CLEANUP
278	AT_SETUP([OVSDB 5-server torture test - remove/re-add follower 4])
279	AT_KEYWORDS([ovsdb server positive unix cluster cluster5])
280	ovsdb_torture_test 5 5 remove
281	AT_CLEANUP