OpenZFS 9185 - Enable testing over NFS in ZFS performance tests

This change makes additions to the ZFS test suite that allows the
performance tests to run over NFS. The test is run and performance data
collected from the server side, while IO is generated on the NFS client.

This has been tested with Linux and illumos NFS clients.

Authored by: Ahmed Ghanem <ahmedg@delphix.com>
Reviewed by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Reviewed by: Kevin Greene <kevin.greene@delphix.com>
Reviewed-by: Richard Elling <Richard.Elling@RichardElling.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Ported-by: John Kennedy <john.kennedy@delphix.com>
Signed-off-by: John Kennedy <john.kennedy@delphix.com>
OpenZFS-issue: https://www.illumos.org/issues/9185
Closes #8367

diff --git a/scripts/zfs-tests.sh b/scripts/zfs-tests.sh
index f00a284847acf17b46ff0eb7ff01a58c9e694d6a..7c5286ba70ff3eb657066382c6b7a9386f662be2 100755 (executable)
--- a/scripts/zfs-tests.sh
+++ b/scripts/zfs-tests.sh
@@ -273,6 +273,7 @@ OPTIONS:
        -f          Use files only, disables block device tests
        -S          Enable stack tracer (negative performance impact)
        -c          Only create and populate constrained path
+       -n NFSFILE  Use the nfsfile to determine the NFS configuration
        -I NUM      Number of iterations
        -d DIR      Use DIR for files and loopback devices
        -s SIZE     Use vdevs of SIZE (default: 4G)
@@ -295,7 +296,7 @@ $0 -x
 EOF
 }
 
-while getopts 'hvqxkfScd:s:r:?t:T:u:I:' OPTION; do
+while getopts 'hvqxkfScn:d:s:r:?t:T:u:I:' OPTION; do
        case $OPTION in
        h)
                usage
@@ -324,6 +325,12 @@ while getopts 'hvqxkfScd:s:r:?t:T:u:I:' OPTION; do
                constrain_path
                exit
                ;;
+       n)
+               nfsfile=$OPTARG
+               [[ -f $nfsfile ]] || fail "Cannot read file: $nfsfile"
+               export NFS=1
+               . "$nfsfile"
+               ;;
        d)
                FILEDIR="$OPTARG"
                ;;

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index 78ba2488d460dc655da9a7adff65bbfbbda662fc..127a1477d4262726e6f01963567db0b390a91fb6 100644 (file)
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -100,6 +100,7 @@ export SYSTEM_FILES='arp
     readlink
     rm
     rmdir
+    scp
     sed
     seq
     setenforce
@@ -110,6 +111,7 @@ export SYSTEM_FILES='arp
     shuf
     sleep
     sort
+    ssh
     stat
     strings
     su

diff --git a/tests/zfs-tests/tests/perf/Makefile.am b/tests/zfs-tests/tests/perf/Makefile.am
index d31bba0e22514b83764f53d0895142f146782bde..68dd31ec12b1b027bb45b9424a3be5b684b6c466 100644 (file)
--- a/tests/zfs-tests/tests/perf/Makefile.am
+++ b/tests/zfs-tests/tests/perf/Makefile.am
@@ -1,5 +1,7 @@
 pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/perf
-dist_pkgdata_DATA = perf.shlib
+dist_pkgdata_SCRIPTS = \
+       nfs-sample.cfg \
+       perf.shlib
 
 SUBDIRS = \
        fio \

diff --git a/tests/zfs-tests/tests/perf/fio/random_reads.fio b/tests/zfs-tests/tests/perf/fio/random_reads.fio
index 79610f9b28aa8692cd89e9301a7f773dd5dfd083..e6e7034e0acb42dad98d228fdc7795635a2190f5 100644 (file)
--- a/tests/zfs-tests/tests/perf/fio/random_reads.fio
+++ b/tests/zfs-tests/tests/perf/fio/random_reads.fio
@@ -26,6 +26,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 
 [job]

diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite.fio
index 7d01c38ada964bf7dd6d26d33bafa154821b60d1..852d4bed69b804b6117e7c447c5bc452ec4f0907 100644 (file)
--- a/tests/zfs-tests/tests/perf/fio/random_readwrite.fio
+++ b/tests/zfs-tests/tests/perf/fio/random_readwrite.fio
@@ -28,6 +28,7 @@ runtime=${RUNTIME}
 bssplit=4k/50:8k/30:128k/10:1m/10
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 buffer_compress_percentage=66
 buffer_compress_chunk=4096

diff --git a/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio
index ed449555328c08d8134fb91fbfb6e1a56b702bd6..67b88c09d79f131aa76c91d96eb43a9c0a456c16 100644 (file)
--- a/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio
+++ b/tests/zfs-tests/tests/perf/fio/random_readwrite_fixed.fio
@@ -28,6 +28,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 buffer_compress_percentage=66
 buffer_compress_chunk=4096

diff --git a/tests/zfs-tests/tests/perf/fio/random_writes.fio b/tests/zfs-tests/tests/perf/fio/random_writes.fio
index 5e2cb30026c710610b1ae7e6e5c364489c7d9bc5..90db5ce3bfd17b9da873c04bc76cd98fd8aecb15 100644 (file)
--- a/tests/zfs-tests/tests/perf/fio/random_writes.fio
+++ b/tests/zfs-tests/tests/perf/fio/random_writes.fio
@@ -25,6 +25,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
 buffer_compress_percentage=66

diff --git a/tests/zfs-tests/tests/perf/fio/sequential_reads.fio b/tests/zfs-tests/tests/perf/fio/sequential_reads.fio
index 33a9a1d89396e8627fbe7becada44f26cff7b4a2..b4b45e08413504e553bae99905282cb8314b66a3 100644 (file)
--- a/tests/zfs-tests/tests/perf/fio/sequential_reads.fio
+++ b/tests/zfs-tests/tests/perf/fio/sequential_reads.fio
@@ -26,6 +26,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 
 [job]

diff --git a/tests/zfs-tests/tests/perf/fio/sequential_writes.fio b/tests/zfs-tests/tests/perf/fio/sequential_writes.fio
index 65a65910fd4faec6beaf0566008514f282e9b6ac..714993e92f163a58e864731740348a79af6631c8 100644 (file)
--- a/tests/zfs-tests/tests/perf/fio/sequential_writes.fio
+++ b/tests/zfs-tests/tests/perf/fio/sequential_writes.fio
@@ -25,6 +25,7 @@ runtime=${RUNTIME}
 bs=${BLOCKSIZE}
 ioengine=psync
 sync=${SYNC_TYPE}
+direct=${DIRECT}
 numjobs=${NUMJOBS}
 filesize=${FILESIZE}
 buffer_compress_percentage=66

diff --git a/tests/zfs-tests/tests/perf/nfs-sample.cfg b/tests/zfs-tests/tests/perf/nfs-sample.cfg
new file mode 100644 (file)
index 0000000..f7ac2da
--- /dev/null
+++ b/tests/zfs-tests/tests/perf/nfs-sample.cfg
@@ -0,0 +1,46 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2016 by Delphix. All rights reserved.
+#
+
+#
+# This file is a sample NFS configuration for the performance tests. To use the
+# performance tests over NFS you must have:
+#   - a client machine with fio and sudo installed
+#   - passwordless SSH set up from this host
+#     for delphix and root users to the client
+#   - passwordless sudo for the user on the client
+#
+
+
+# The IP address for the server
+export NFS_SERVER=127.0.0.1
+
+# The IP address for the client
+export NFS_CLIENT=127.0.0.1
+
+# The mountpoint to use inside the client
+export NFS_MOUNT=/var/tmp/nfs
+
+# The user to run the tests as on the client
+export NFS_USER=delphix
+
+# Common NFS client mount options
+export NFS_OPTIONS="-o rw,nosuid,bg,hard,rsize=1048576,wsize=1048576,"
+NFS_OPTIONS+="nointr,timeo=600,proto=tcp,actimeo=0,port=2049"
+
+# illumos NFS client mount options
+# export NFS_OPTIONS="$NFS_OPTIONS,vers=3"
+
+# Linux NFS client mount options
+export NFS_OPTIONS="-t nfs $NFS_OPTIONS,noacl,nfsvers=3"

diff --git a/tests/zfs-tests/tests/perf/perf.shlib b/tests/zfs-tests/tests/perf/perf.shlib
index 7165df759b1fa6276ffdbfe58407405420863159..69e61e9fd122259c8ac514fbf10eda68396f8df1 100644 (file)
--- a/tests/zfs-tests/tests/perf/perf.shlib
+++ b/tests/zfs-tests/tests/perf/perf.shlib
@@ -112,6 +112,16 @@ function do_fio_run_impl
        export BLOCKSIZE=$iosize
        sync
 
+       # When running locally, we want to keep the default behavior of
+       # DIRECT == 0, so only set it when we're running over NFS to
+       # disable client cache for reads.
+       if [[ $NFS -eq 1 ]]; then
+               export DIRECT=1
+               do_setup_nfs $script
+       else
+               export DIRECT=0
+       fi
+
        # This will be part of the output filename.
        typeset suffix=$(get_suffix $threads $sync $iosize)
 
@@ -124,7 +134,14 @@ function do_fio_run_impl
        typeset outfile="$logbase.fio.$suffix"
 
        # Start the load
-       log_must fio --output $outfile $FIO_SCRIPTS/$script
+       if [[ $NFS -eq 1 ]]; then
+               log_must ssh -t $NFS_USER@$NFS_CLIENT "
+                       fio --output /tmp/fio.out /tmp/test.fio
+               "
+               log_must scp $NFS_USER@$NFS_CLIENT:/tmp/fio.out $outfile
+       else
+               log_must fio --output $outfile $FIO_SCRIPTS/$script
+       fi
 }
 
 #
@@ -166,6 +183,34 @@ function do_fio_run
        done
 }
 
+# This function sets NFS mount on the client and make sure all correct
+# permissions are in place
+#
+function do_setup_nfs
+{
+       typeset script=$1
+       zfs  set sharenfs=on $TESTFS
+       log_must chmod  -R 777 /$TESTFS
+
+       ssh -t $NFS_USER@$NFS_CLIENT "mkdir -m 777 -p $NFS_MOUNT"
+       ssh -t $NFS_USER@$NFS_CLIENT "sudo -S umount $NFS_MOUNT"
+       log_must ssh -t $NFS_USER@$NFS_CLIENT "
+               sudo -S mount $NFS_OPTIONS $NFS_SERVER:/$TESTFS $NFS_MOUNT
+       "
+       #
+       # The variables in the fio script are only available in our current
+       # shell session, so we have to evaluate them here before copying
+       # the resulting script over to the target machine.
+       #
+       export jobnum='$jobnum'
+       while read line; do
+               eval echo "$line"
+       done < $FIO_SCRIPTS/$script > /tmp/test.fio
+       log_must sed -i -e "s%directory.*%directory=$NFS_MOUNT%" /tmp/test.fio
+       log_must scp /tmp/test.fio $NFS_USER@$NFS_CLIENT:/tmp
+       log_must rm /tmp/test.fio
+}
+
 #
 # This function iterates through the value pairs in $PERF_COLLECT_SCRIPTS.
 # The script at index N is launched in the background, with its output

diff --git a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
index 0f3256351c484b4801938ccc389c662c2f47c03e..e0b253200954c8983848e9f56c637eddc16eb232 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_writes_zil.ksh
@@ -60,6 +60,10 @@ elif [[ -n $PERF_REGRESSION_NIGHTLY ]]; then
        export PERF_IOSIZES=${PERF_IOSIZES:-'8k'}
 fi
 
+# Until the performance tests over NFS can deal with multiple file systems,
+# force the use of only one file system when testing over NFS.
+[[ $NFS -eq 1 ]] && PERF_NTHREADS_PER_FS='0'
+
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then