Profiling for perf tests

Stack profiling is quite useful and Linux ZFS test suite does not
current collect that data.

Linux perf is a common tool for this purpose though the perf record
data file can be quite large. With this change, Linux ZFS perf tests
capture perf record data if perf is installed on the system and
PERF_DO_PROFILING environment variable is set.

Reviewed by: John Wren Kennedy <jwk404@gmail.com>
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Signed-off-by: Tony Nguyen <tony.nguyen@delphix.com>
External-issue: LX-971
Closes #7549

diff --git a/tests/zfs-tests/include/commands.cfg b/tests/zfs-tests/include/commands.cfg
index 2a6c3b13d2faf969c93a203d259a4bf5abc79216..50eb6bd6bc1a94e60420d92deaa52f0697d4854a 100644 (file)
--- a/tests/zfs-tests/include/commands.cfg
+++ b/tests/zfs-tests/include/commands.cfg
@@ -84,6 +84,7 @@ export SYSTEM_FILES='arp
     openssl
     parted
     pax
+    perf
     pgrep
     ping
     pkill

diff --git a/tests/zfs-tests/tests/perf/regression/random_reads.ksh b/tests/zfs-tests/tests/perf/regression/random_reads.ksh
index 5e91fe0a7073b945532d90a25ddaa26dc62749d9..5bf269a859db25a75ca096611a84a29e34501dd6 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/random_reads.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_reads.ksh
@@ -85,9 +85,12 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
-        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
-            "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
-            "iostat")
+       typeset perf_record_cmd="perf record -F 99 -a -g -q \
+           -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
+       export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
+           "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
+           "iostat" "$perf_record_cmd" "perf")
 else
        export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
            "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")

diff --git a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
index 446a440ec13a8b0ec3993e77487d4e6c5cfd0f5b..e6d6e3a11d30ac3ed3ad091abe6e6b80388cdddd 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_readwrite.ksh
@@ -85,9 +85,12 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+       typeset perf_record_cmd="perf record -F 99 -a -g -q \
+           -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
            "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
-           "iostat")
+           "iostat" "$perf_record_cmd" "perf")
 else
        export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
            "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")

diff --git a/tests/zfs-tests/tests/perf/regression/random_writes.ksh b/tests/zfs-tests/tests/perf/regression/random_writes.ksh
index e724aefe2dff702edb339a101ca71e0b5f2ce44c..d85a3d98a1a8e5293f900c4f819f5b193e0c44dd 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/random_writes.ksh
+++ b/tests/zfs-tests/tests/perf/regression/random_writes.ksh
@@ -77,9 +77,12 @@ fi
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+       typeset perf_record_cmd="perf record -F 99 -a -g -q \
+           -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
            "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
-           "iostat")
+           "iostat" "$perf_record_cmd" "perf")
 else
        export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
            "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")

diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
index e60dbb18176af130ad5267c5f5aaa61fb7f79e5f..a9c62fe5a977e3791ef007d98e94c095ea637e6b 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads.ksh
@@ -85,9 +85,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+    typeset perf_record_cmd="perf record -F 99 -a -g -q \
+        -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
            "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1"
-           "vmstat" "mpstat  -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat")
+           "vmstat" "mpstat  -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat"
+           "$perf_record_cmd" "perf")
 else
        export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
            "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat"

diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
index 5445f814e82c9ecffd19e67328ca01099a093057..6622ac9737c496e4e51eae7e9a94af4d133ff0a3 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached.ksh
@@ -75,9 +75,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+       typeset perf_record_cmd="perf record -F 99 -a -g -q \
+           -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
            "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1"
-           "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat")
+           "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat"
+           "$perf_record_cmd" "perf")
 else
        export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
            "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat"

diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
index 5d7175b1ad078f9428d6488f16e048df9c3894ca..9ed0e4792dd371e3d9d01d46916d0c1b6fa792e8 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_arc_cached_clone.ksh
@@ -91,9 +91,13 @@ export TESTFS=$PERFPOOL/$TESTCLONE
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+       typeset perf_record_cmd="perf record -F 99 -a -g -q \
+           -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
            "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1"
-           "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat")
+           "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat"
+           "$perf_record_cmd" "perf")
 else
        export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
            "$PERF_SCRIPTS/prefetch_io.d $PERFPOOL 1" "prefetch" "vmstat 1" "vmstat"

diff --git a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
index 5d029280eb1c05b02bdd4160f93aa632536060a4..edb7a96c99f26fa16b5e57a70f4d60e7cf022a4d 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_reads_dbuf_cached.ksh
@@ -77,9 +77,13 @@ log_must fio $FIO_SCRIPTS/mkfiles.fio
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+       typeset perf_record_cmd="perf record -F 99 -a -g -q \
+           -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
            "$PERF_SCRIPTS/prefetch_io.sh $PERFPOOL 1" "prefetch" "vmstat 1"
-           "vmstat" "mpstat  -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat")
+           "vmstat" "mpstat  -P ALL 1" "mpstat" "iostat -dxyz 1" "iostat"
+           "$perf_record_cmd" "perf")
 else
        export collect_scripts=("kstat zfs:0 1" "kstat" "vmstat -T d 1" "vmstat"
            "mpstat -T d 1" "mpstat" "iostat -T d -xcnz 1" "iostat"

diff --git a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
index d5fb890201a99b2115381a4e5a053fc940ccd0a8..01ab80d4a81038c772f9eecd16afdd4348d885e3 100755 (executable)
--- a/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
+++ b/tests/zfs-tests/tests/perf/regression/sequential_writes.ksh
@@ -77,9 +77,12 @@ fi
 lun_list=$(pool_to_lun_list $PERFPOOL)
 log_note "Collecting backend IO stats with lun list $lun_list"
 if is_linux; then
+       typeset perf_record_cmd="perf record -F 99 -a -g -q \
+           -o /dev/stdout -- sleep ${PERF_RUNTIME}"
+
        export collect_scripts=("zpool iostat -lpvyL $PERFPOOL 1" "zpool.iostat"
            "vmstat 1" "vmstat" "mpstat -P ALL 1" "mpstat" "iostat -dxyz 1"
-           "iostat")
+           "iostat" "$perf_record_cmd" "perf")
 else
        export collect_scripts=("$PERF_SCRIPTS/io.d $PERFPOOL $lun_list 1" "io"
            "vmstat 1" "vmstat" "mpstat 1" "mpstat" "iostat -xcnz 1" "iostat")