4 # The contents of this file are subject to the terms of the
5 # Common Development and Distribution License (the "License").
6 # You may not use this file except in compliance with the License.
8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 # or http://www.opensolaris.org/os/licensing.
10 # See the License for the specific language governing permissions
11 # and limitations under the License.
13 # When distributing Covered Code, include this CDDL HEADER in each
14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 # If applicable, add the following below this CDDL HEADER, with the
16 # fields enclosed by brackets "[]" replaced with your own identifying
17 # information: Portions Copyright [yyyy] [name of copyright owner]
23 # Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
24 # Use is subject to license terms.
27 . $STF_SUITE/include/libtest.shlib
28 . $STF_SUITE/tests/functional/mmp/mmp.cfg
31 function check_pool_import # pool opts token keyword
33 typeset pool=${1:-$MMP_POOL}
38 zpool import $opts 2>&1 | \
39 nawk -v token="$token:" '($1==token) {print $0}' | \
40 grep -i "$keyword" > /dev/null 2>&1
45 function is_pool_imported # pool opts
47 typeset pool=${1:-$MMP_POOL}
50 check_pool_import "$pool" "$opts" "status" \
51 "The pool is currently imported"
55 function wait_pool_imported # pool opts
57 typeset pool=${1:-$MMP_POOL}
60 while is_pool_imported "$pool" "$opts"; do
67 function try_pool_import # pool opts message
69 typeset pool=${1:-$MMP_POOL}
73 zpool import $opts $pool 2>&1 | grep -i "$msg"
78 function mmp_set_hostid
84 if [ $(hostid) != "$hostid" ]; then
91 function mmp_clear_hostid
96 function mmp_pool_create_simple # pool dir
98 typeset pool=${1:-$MMP_POOL}
99 typeset dir=${2:-$MMP_DIR}
101 log_must mkdir -p $dir
102 log_must rm -f $dir/*
103 log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2
105 log_must mmp_clear_hostid
106 log_must mmp_set_hostid $HOSTID1
107 log_must zpool create -f -o cachefile=$MMP_CACHE $pool \
108 mirror $dir/vdev1 $dir/vdev2
109 log_must zpool set multihost=on $pool
112 function mmp_pool_create # pool dir
114 typeset pool=${1:-$MMP_POOL}
115 typeset dir=${2:-$MMP_DIR}
116 typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool"
118 mmp_pool_create_simple $pool $dir
120 log_must mv $MMP_CACHE ${MMP_CACHE}.stale
121 log_must zpool export $pool
122 log_must mmp_clear_hostid
123 log_must mmp_set_hostid $HOSTID2
125 log_note "Starting ztest in the background as hostid $HOSTID1"
126 log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &"
128 while ! is_pool_imported "$pool" "-d $dir"; do
134 function mmp_pool_destroy # pool dir
136 typeset pool=${1:-$MMP_POOL}
137 typeset dir=${2:-$MMP_DIR}
139 ZTESTPID=$(pgrep ztest)
140 if [ -n "$ZTESTPID" ]; then
141 log_must kill $ZTESTPID
145 if poolexists $pool; then
149 log_must rm -f $dir/*
153 function mmp_pool_set_hostid # pool hostid
158 log_must mmp_clear_hostid
159 log_must mmp_set_hostid $hostid
160 log_must zpool export $pool
161 log_must zpool import $pool
165 # Return the number of seconds the activity check portion of the import process
166 # will take. Does not include the time to find devices and assemble a config.
167 # Note that the activity check may be skipped, e.g. if the pool and host
168 # hostid's match, but this will return non-zero because mmp_* are populated.
169 function seconds_mmp_waits_for_activity
175 typeset devices=${#DISK[@]}
176 typeset import_intervals=$(get_tunable zfs_multihost_import_intervals)
177 typeset import_interval=$(get_tunable zfs_multihost_interval)
178 typeset tmpfile=$(mktemp)
183 log_must zdb -e -p $devpath $pool >$tmpfile 2>/dev/null
184 mmp_fail=$(awk '/mmp_fail/ {print $NF}' $tmpfile)
185 mmp_write=$(awk '/mmp_write/ {print $NF}' $tmpfile)
186 mmp_delay=$(awk '/mmp_delay/ {print $NF}' $tmpfile)
187 if [ -f $tmpfile ]; then
191 # In order of preference:
192 if [ -n $mmp_fail -a -n $mmp_write ]; then
193 seconds=$((2*mmp_fail*mmp_write/1000))
194 elif [ -n $mmp_delay ]; then
195 # MMP V0: Based on mmp_delay from the best Uberblock
196 seconds=$((import_intervals*devices*mmp_delay/1000000000))
198 # Non-MMP aware: Based on zfs_multihost_interval and import_intervals
199 seconds=$((import_intervals*import_interval/1000))
205 function import_no_activity_check # pool opts
210 typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1))
213 zpool import $opts $pool
216 if [[ $SECONDS -gt $max_duration ]]; then
217 log_fail "ERROR: import_no_activity_check unexpected activity \
218 check (${SECONDS}s gt $max_duration)"
224 function import_activity_check # pool opts act_test_duration
228 typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT}
231 zpool import $opts $pool
234 if [[ $SECONDS -le $min_duration ]]; then
235 log_fail "ERROR: import_activity_check expected activity check \
236 (${SECONDS}s le min_duration $min_duration)"
242 function clear_mmp_history
244 log_must set_tunable64 zfs_multihost_history $MMP_HISTORY_OFF
245 log_must set_tunable64 zfs_multihost_history $MMP_HISTORY
248 function count_skipped_mmp_writes # pool duration
251 typeset -i duration=$2
252 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost"
255 awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' "$hist_path"
258 function count_mmp_writes # pool duration
261 typeset -i duration=$2
262 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost"
265 awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' "$hist_path"
268 function summarize_uberblock_mmp # device
272 zdb -luuuu $device | awk '
273 BEGIN {write_fail_present=0; write_fail_missing=0; uber_invalid=0;}
274 /Uberblock\[[0-9][0-9]*\]/ {delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};};
275 /mmp_fail/ {fail=$3};
277 /mmp_write/ {write=$3};
278 /mmp_delay/ {delay=$3; if (delay==0) {delay_zero++};};
279 /mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++};
280 /mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++};
281 /mmp_valid/ && delay>0 && write<=0 {write_missing++};
282 /mmp_valid/ && delay>0 && fail<=0 {fail_missing++};
283 /mmp_valid/ && delay>0 && seq>0 {seq_nonzero++};
285 print "total_uberblocks " total;
286 print "delay_zero " delay_zero;
287 print "write_fail_present " write_fail_present;
288 print "write_fail_missing " write_fail_missing;
289 print "write_missing " write_missing;
290 print "fail_missing " fail_missing;
291 print "seq_nonzero " seq_nonzero;
292 print "uberblock_invalid " uber_invalid;
296 function count_mmp_write_fail_present # device
300 summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}'
303 function count_mmp_write_fail_missing # device
307 summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}'
310 function verify_mmp_write_fail_present # device
314 count=$(count_mmp_write_fail_present $device)
315 log_note "present count: $count"
316 if [ $count -eq 0 ]; then
317 summarize_uberblock_mmp $device
318 log_note "----- snip -----"
320 log_note "----- snip -----"
321 log_fail "No Uberblocks contain valid mmp_write and fail values"
324 count=$(count_mmp_write_fail_missing $device)
325 log_note "missing count: $count"
326 if [ $count -gt 0 ]; then
327 summarize_uberblock_mmp $device
328 log_note "----- snip -----"
330 log_note "----- snip -----"
331 log_fail "Uberblocks missing mmp_write or mmp_fail"