Allow to limit zed's syslog chattiness

Some usage patterns like send/recv of replication streams can
produce a large number of events. In such a case, the current
all-syslog.sh zedlet will hold up to its name, and flood the
logs with mostly redundant information. Two mitigate this
situation, this changeset introduces to new variables
ZED_SYSLOG_SUBCLASS_INCLUDE and ZED_SYSLOG_SUBCLASS_EXCLUDE
to zed.rc that give more control over which event classes end
up in the syslog.

Reviewed-by: loli10K <ezomori.nozomu@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Daniel Kobras <d.kobras@science-computing.de>
Closes #6886
Closes #7260

diff --git a/cmd/zed/zed.d/all-debug.sh b/cmd/zed/zed.d/all-debug.sh
index 057e39b504b51660b1ffb28ac96a187e3f176c1b..14b39caacd9d814ef27600012ed9429bf7511957 100755 (executable)
--- a/cmd/zed/zed.d/all-debug.sh
+++ b/cmd/zed/zed.d/all-debug.sh
@@ -10,6 +10,8 @@
 
 : "${ZED_DEBUG_LOG:="${TMPDIR:="/tmp"}/zed.debug.log"}"
 
+zed_exit_if_ignoring_this_event
+
 lockfile="$(basename -- "${ZED_DEBUG_LOG}").lock"
 
 umask 077

diff --git a/cmd/zed/zed.d/all-syslog.sh b/cmd/zed/zed.d/all-syslog.sh
index 68d3cf3608ff561dfdb929edca61ae1087da4f96..cb9286500136577b97090733a07d637f0bd2343e 100755 (executable)
--- a/cmd/zed/zed.d/all-syslog.sh
+++ b/cmd/zed/zed.d/all-syslog.sh
@@ -5,6 +5,8 @@
 [ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
 . "${ZED_ZEDLET_DIR}/zed-functions.sh"
 
+zed_exit_if_ignoring_this_event
+
 zed_log_msg "eid=${ZEVENT_EID}" "class=${ZEVENT_SUBCLASS}" \
     "${ZEVENT_POOL_GUID:+"pool_guid=${ZEVENT_POOL_GUID}"}" \
     "${ZEVENT_VDEV_PATH:+"vdev_path=${ZEVENT_VDEV_PATH}"}" \

diff --git a/cmd/zed/zed.d/zed-functions.sh b/cmd/zed/zed.d/zed-functions.sh
index ed6a95914ee6232b0f02901d7f7b736396022185..fb16e9d36f3217be167dc7d1c6e71f49912b71b0 100644 (file)
--- a/cmd/zed/zed.d/zed-functions.sh
+++ b/cmd/zed/zed.d/zed-functions.sh
@@ -438,3 +438,23 @@ zed_guid_to_pool()
                $ZPOOL get -H -ovalue,name guid | awk '$1=='"$guid"' {print $2}'
        fi
 }
+
+# zed_exit_if_ignoring_this_event
+#
+# Exit the script if we should ignore this event, as determined by
+# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc.
+# This function assumes you've imported the normal zed variables.
+zed_exit_if_ignoring_this_event()
+{
+       if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then
+           eval "case ${ZEVENT_SUBCLASS} in
+           ${ZED_SYSLOG_SUBCLASS_INCLUDE});;
+           *) exit 0;;
+           esac"
+       elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then
+           eval "case ${ZEVENT_SUBCLASS} in
+           ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;;
+           *);;
+           esac"
+       fi
+}

diff --git a/cmd/zed/zed.d/zed.rc b/cmd/zed/zed.d/zed.rc
index 8b0e476d5a1af264d9ed9dcf78b542c7ef535d73..35a4d1275755859509608c5f993512b57c44458d 100644 (file)
--- a/cmd/zed/zed.d/zed.rc
+++ b/cmd/zed/zed.d/zed.rc
@@ -100,3 +100,14 @@ ZED_USE_ENCLOSURE_LEDS=1
 #
 #ZED_SYSLOG_TAG="zed"
 
+##
+# Which set of event subclasses to log
+# By default, events from all subclasses are logged.
+# If ZED_SYSLOG_SUBCLASS_INCLUDE is set, only subclasses
+# matching the pattern are logged. Use the pipe symbol (|)
+# or shell wildcards (*, ?) to match multiple subclasses.
+# Otherwise, if ZED_SYSLOG_SUBCLASS_EXCLUDE is set, the
+# matching subclasses are excluded from logging.
+#ZED_SYSLOG_SUBCLASS_INCLUDE="checksum|scrub_*|vdev.*"
+#ZED_SYSLOG_SUBCLASS_EXCLUDE="statechange|config_*|history_event"
+

diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index a0e561188260bfb0cfcb818516df13320c92103e..516c1c3b852d2c2364d02e548aeb851c0d5bd364 100644 (file)
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -413,7 +413,7 @@ tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
 tags = ['functional', 'devices']
 
 [tests/functional/events]
-tests = ['events_001_pos', 'events_002_pos']
+tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter']
 tags = ['functional', 'events']
 
 [tests/functional/exec]

diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib
index c1862c983cfa7ca47c8ff071339b13cc402cc0ee..a7344c1621df935abc8b4d42009a47a4807afe0d 100644 (file)
--- a/tests/zfs-tests/include/libtest.shlib
+++ b/tests/zfs-tests/include/libtest.shlib
@@ -3398,6 +3398,21 @@ function wait_scrubbed
        return 1
 }
 
+# Backup the zed.rc in our test directory so that we can edit it for our test.
+#
+# Returns: Backup file name.  You will need to pass this to zed_rc_restore().
+function zed_rc_backup
+{
+       zedrc_backup="$(mktemp)"
+       cp $ZEDLET_DIR/zed.rc $zedrc_backup
+       echo $zedrc_backup
+}
+
+function zed_rc_restore
+{
+       mv $1 $ZEDLET_DIR/zed.rc
+}
+
 #
 # Setup custom environment for the ZED.
 #
@@ -3536,6 +3551,23 @@ function zed_events_drain
        done
 }
 
+# Set a variable in zed.rc to something, un-commenting it in the process.
+#
+# $1 variable
+# $2 value
+function zed_rc_set
+{
+       var="$1"
+       val="$2"
+       # Remove the line
+       cmd="'/$var/d'"
+       eval sed -i $cmd $ZEDLET_DIR/zed.rc
+
+       # Add it at the end
+       echo "$var=$val" >> $ZEDLET_DIR/zed.rc
+}
+
+
 #
 # Check is provided device is being active used as a swap device.
 #

diff --git a/tests/zfs-tests/tests/functional/events/Makefile.am b/tests/zfs-tests/tests/functional/events/Makefile.am
index 7813c18be13588a3210130bc0f54710b3e94c401..415e44b6baffe7ff94ca4ac26fd8d95ff0352d66 100644 (file)
--- a/tests/zfs-tests/tests/functional/events/Makefile.am
+++ b/tests/zfs-tests/tests/functional/events/Makefile.am
@@ -5,4 +5,5 @@ dist_pkgdata_SCRIPTS = \
        events.cfg \
        events_common.kshlib \
        events_001_pos.ksh \
-       events_002_pos.ksh
+       events_002_pos.ksh \
+       zed_rc_filter.ksh

diff --git a/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh b/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh
new file mode 100755 (executable)
index 0000000..44652ee
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh
@@ -0,0 +1,104 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+# DESCRIPTION:
+# Verify zed.rc ZED_SYSLOG_SUBCLASS_INCLUDE/EXCLUDE event filtering works.
+#
+# STRATEGY:
+# 1. Execute zpool sub-commands on a pool.
+# 2. Test different combinations of ZED_SYSLOG_SUBCLASS_INCLUDE filtering.
+# 3. Execute zpool sub-commands on a pool.
+# 4. Test different combinations of ZED_SYSLOG_SUBCLASS_EXCLUDE filtering.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+       log_must zed_stop
+       zed_rc_restore $zedrc_backup
+}
+
+log_assert "Verify zpool sub-commands generate expected events"
+log_onexit cleanup
+
+log_must zpool events -c
+log_must zed_start
+
+# Backup our zed.rc
+zedrc_backup=$(zed_rc_backup)
+
+log_note "Include a single event type"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE history_event
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Include a single event type with wildcards"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE '*history_event*'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Test a filter of a non-match and a match"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'foobar|*history_event*'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.history_event" \
+    "zfs set compression=off $TESTPOOL/$TESTFS"
+
+log_note "Include multiple events"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'scrub_start|scrub_finish'
+run_and_verify -p "$TESTPOOL"  -e "sysevent.fs.zfs.scrub_start" \
+    -e "sysevent.fs.zfs.scrub_finish" \
+    "zpool scrub $TESTPOOL && wait_scrubbed $TESTPOOL"
+
+# We can't use run_and_verify() for exclusions, so run the rest of the tests
+# manually.
+log_note "Test one exclusion"
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE 'history_event'
+truncate -s 0 $ZED_DEBUG_LOG
+log_must zfs set compression=off $TESTPOOL/$TESTFS
+log_must file_wait $ZED_DEBUG_LOG 3
+log_mustnot grep -q history_event $ZED_DEBUG_LOG
+
+log_note "Test one exclusion with wildcards"
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE '*history_event*'
+truncate -s 0 $ZED_DEBUG_LOG
+log_must zfs set compression=off $TESTPOOL/$TESTFS
+log_must file_wait $ZED_DEBUG_LOG 3
+log_mustnot grep -q history_event $ZED_DEBUG_LOG
+
+log_note "Test one inclusion and one exclusion"
+zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'scrub_start'
+zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE 'scrub_finish'
+truncate -s 0 $ZED_DEBUG_LOG
+zpool scrub $TESTPOOL
+wait_scrubbed $TESTPOOL
+log_must file_wait $ZED_DEBUG_LOG 3
+log_must grep -q scrub_start $ZED_DEBUG_LOG
+log_mustnot grep -q scrub_finish $ZED_DEBUG_LOG
+
+log_pass "zed.rc event filtering works correctly."

diff --git a/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh b/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh
index 558cb065f7f903d2d2ad66fdfa990a1b0b1aab3a..3b124bdd598ed892c15d0acb2ac73bb24be056d6 100755 (executable)
--- a/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh
+++ b/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh
@@ -32,16 +32,15 @@
 log_assert "Testing the scrub after resilver zedlet"
 
 # Backup our zed.rc
-zedrc_backup="$(mktemp)"
-log_must cp $ZEDLET_DIR/zed.rc $zedrc_backup
+zedrc_backup=$(zed_rc_backup)
 
-# Enable ZED_SCRUB_AFTER_RESILVER
-eval "sed -i 's/\#ZED_SCRUB_AFTER_RESILVER/ZED_SCRUB_AFTER_RESILVER/g' $ZEDLET_DIR/zed.rc"
+# Enable ZED_SCRUB_AFTER_RESILVER in zed.rc
+zed_rc_set ZED_SCRUB_AFTER_RESILVER 1
 
 function cleanup
 {
        # Restore our zed.rc
-       log_must mv $zedrc_backup $ZEDLET_DIR/zed.rc
+       log_must zed_rc_restore $zedrc_backup
        default_cleanup_noexit
 }