From 6059ba27c44a5d63cf1ad9983aa17c5d49cdcde4 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Tue, 6 Mar 2018 15:41:52 -0800 Subject: [PATCH] Allow to limit zed's syslog chattiness Some usage patterns like send/recv of replication streams can produce a large number of events. In such a case, the current all-syslog.sh zedlet will hold up to its name, and flood the logs with mostly redundant information. Two mitigate this situation, this changeset introduces to new variables ZED_SYSLOG_SUBCLASS_INCLUDE and ZED_SYSLOG_SUBCLASS_EXCLUDE to zed.rc that give more control over which event classes end up in the syslog. Reviewed-by: loli10K Reviewed-by: Brian Behlendorf Reviewed-by: Giuseppe Di Natale Signed-off-by: Tony Hutter Signed-off-by: Daniel Kobras Closes #6886 Closes #7260 --- cmd/zed/zed.d/all-debug.sh | 2 + cmd/zed/zed.d/all-syslog.sh | 2 + cmd/zed/zed.d/zed-functions.sh | 20 ++++ cmd/zed/zed.d/zed.rc | 11 ++ tests/runfiles/linux.run | 2 +- tests/zfs-tests/include/libtest.shlib | 32 ++++++ .../tests/functional/events/Makefile.am | 3 +- .../tests/functional/events/zed_rc_filter.ksh | 104 ++++++++++++++++++ .../functional/fault/scrub_after_resilver.ksh | 9 +- 9 files changed, 178 insertions(+), 7 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh diff --git a/cmd/zed/zed.d/all-debug.sh b/cmd/zed/zed.d/all-debug.sh index 057e39b50..14b39caac 100755 --- a/cmd/zed/zed.d/all-debug.sh +++ b/cmd/zed/zed.d/all-debug.sh @@ -10,6 +10,8 @@ : "${ZED_DEBUG_LOG:="${TMPDIR:="/tmp"}/zed.debug.log"}" +zed_exit_if_ignoring_this_event + lockfile="$(basename -- "${ZED_DEBUG_LOG}").lock" umask 077 diff --git a/cmd/zed/zed.d/all-syslog.sh b/cmd/zed/zed.d/all-syslog.sh index 68d3cf360..cb9286500 100755 --- a/cmd/zed/zed.d/all-syslog.sh +++ b/cmd/zed/zed.d/all-syslog.sh @@ -5,6 +5,8 @@ [ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc" . "${ZED_ZEDLET_DIR}/zed-functions.sh" +zed_exit_if_ignoring_this_event + zed_log_msg "eid=${ZEVENT_EID}" "class=${ZEVENT_SUBCLASS}" \ "${ZEVENT_POOL_GUID:+"pool_guid=${ZEVENT_POOL_GUID}"}" \ "${ZEVENT_VDEV_PATH:+"vdev_path=${ZEVENT_VDEV_PATH}"}" \ diff --git a/cmd/zed/zed.d/zed-functions.sh b/cmd/zed/zed.d/zed-functions.sh index ed6a95914..fb16e9d36 100644 --- a/cmd/zed/zed.d/zed-functions.sh +++ b/cmd/zed/zed.d/zed-functions.sh @@ -438,3 +438,23 @@ zed_guid_to_pool() $ZPOOL get -H -ovalue,name guid | awk '$1=='"$guid"' {print $2}' fi } + +# zed_exit_if_ignoring_this_event +# +# Exit the script if we should ignore this event, as determined by +# $ZED_SYSLOG_SUBCLASS_INCLUDE and $ZED_SYSLOG_SUBCLASS_EXCLUDE in zed.rc. +# This function assumes you've imported the normal zed variables. +zed_exit_if_ignoring_this_event() +{ + if [ -n "${ZED_SYSLOG_SUBCLASS_INCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_INCLUDE});; + *) exit 0;; + esac" + elif [ -n "${ZED_SYSLOG_SUBCLASS_EXCLUDE}" ]; then + eval "case ${ZEVENT_SUBCLASS} in + ${ZED_SYSLOG_SUBCLASS_EXCLUDE}) exit 0;; + *);; + esac" + fi +} diff --git a/cmd/zed/zed.d/zed.rc b/cmd/zed/zed.d/zed.rc index 8b0e476d5..35a4d1275 100644 --- a/cmd/zed/zed.d/zed.rc +++ b/cmd/zed/zed.d/zed.rc @@ -100,3 +100,14 @@ ZED_USE_ENCLOSURE_LEDS=1 # #ZED_SYSLOG_TAG="zed" +## +# Which set of event subclasses to log +# By default, events from all subclasses are logged. +# If ZED_SYSLOG_SUBCLASS_INCLUDE is set, only subclasses +# matching the pattern are logged. Use the pipe symbol (|) +# or shell wildcards (*, ?) to match multiple subclasses. +# Otherwise, if ZED_SYSLOG_SUBCLASS_EXCLUDE is set, the +# matching subclasses are excluded from logging. +#ZED_SYSLOG_SUBCLASS_INCLUDE="checksum|scrub_*|vdev.*" +#ZED_SYSLOG_SUBCLASS_EXCLUDE="statechange|config_*|history_event" + diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index a0e561188..516c1c3b8 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -413,7 +413,7 @@ tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos'] tags = ['functional', 'devices'] [tests/functional/events] -tests = ['events_001_pos', 'events_002_pos'] +tests = ['events_001_pos', 'events_002_pos', 'zed_rc_filter'] tags = ['functional', 'events'] [tests/functional/exec] diff --git a/tests/zfs-tests/include/libtest.shlib b/tests/zfs-tests/include/libtest.shlib index c1862c983..a7344c162 100644 --- a/tests/zfs-tests/include/libtest.shlib +++ b/tests/zfs-tests/include/libtest.shlib @@ -3398,6 +3398,21 @@ function wait_scrubbed return 1 } +# Backup the zed.rc in our test directory so that we can edit it for our test. +# +# Returns: Backup file name. You will need to pass this to zed_rc_restore(). +function zed_rc_backup +{ + zedrc_backup="$(mktemp)" + cp $ZEDLET_DIR/zed.rc $zedrc_backup + echo $zedrc_backup +} + +function zed_rc_restore +{ + mv $1 $ZEDLET_DIR/zed.rc +} + # # Setup custom environment for the ZED. # @@ -3536,6 +3551,23 @@ function zed_events_drain done } +# Set a variable in zed.rc to something, un-commenting it in the process. +# +# $1 variable +# $2 value +function zed_rc_set +{ + var="$1" + val="$2" + # Remove the line + cmd="'/$var/d'" + eval sed -i $cmd $ZEDLET_DIR/zed.rc + + # Add it at the end + echo "$var=$val" >> $ZEDLET_DIR/zed.rc +} + + # # Check is provided device is being active used as a swap device. # diff --git a/tests/zfs-tests/tests/functional/events/Makefile.am b/tests/zfs-tests/tests/functional/events/Makefile.am index 7813c18be..415e44b6b 100644 --- a/tests/zfs-tests/tests/functional/events/Makefile.am +++ b/tests/zfs-tests/tests/functional/events/Makefile.am @@ -5,4 +5,5 @@ dist_pkgdata_SCRIPTS = \ events.cfg \ events_common.kshlib \ events_001_pos.ksh \ - events_002_pos.ksh + events_002_pos.ksh \ + zed_rc_filter.ksh diff --git a/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh b/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh new file mode 100755 index 000000000..44652ee4c --- /dev/null +++ b/tests/zfs-tests/tests/functional/events/zed_rc_filter.ksh @@ -0,0 +1,104 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2018 by Lawrence Livermore National Security, LLC. +# Use is subject to license terms. +# + +# DESCRIPTION: +# Verify zed.rc ZED_SYSLOG_SUBCLASS_INCLUDE/EXCLUDE event filtering works. +# +# STRATEGY: +# 1. Execute zpool sub-commands on a pool. +# 2. Test different combinations of ZED_SYSLOG_SUBCLASS_INCLUDE filtering. +# 3. Execute zpool sub-commands on a pool. +# 4. Test different combinations of ZED_SYSLOG_SUBCLASS_EXCLUDE filtering. + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/events/events_common.kshlib + +verify_runnable "both" + +function cleanup +{ + log_must zed_stop + zed_rc_restore $zedrc_backup +} + +log_assert "Verify zpool sub-commands generate expected events" +log_onexit cleanup + +log_must zpool events -c +log_must zed_start + +# Backup our zed.rc +zedrc_backup=$(zed_rc_backup) + +log_note "Include a single event type" +zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE history_event +run_and_verify -p "$TESTPOOL" -e "sysevent.fs.zfs.history_event" \ + "zfs set compression=off $TESTPOOL/$TESTFS" + +log_note "Include a single event type with wildcards" +zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE '*history_event*' +run_and_verify -p "$TESTPOOL" -e "sysevent.fs.zfs.history_event" \ + "zfs set compression=off $TESTPOOL/$TESTFS" + +log_note "Test a filter of a non-match and a match" +zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'foobar|*history_event*' +run_and_verify -p "$TESTPOOL" -e "sysevent.fs.zfs.history_event" \ + "zfs set compression=off $TESTPOOL/$TESTFS" + +log_note "Include multiple events" +zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'scrub_start|scrub_finish' +run_and_verify -p "$TESTPOOL" -e "sysevent.fs.zfs.scrub_start" \ + -e "sysevent.fs.zfs.scrub_finish" \ + "zpool scrub $TESTPOOL && wait_scrubbed $TESTPOOL" + +# We can't use run_and_verify() for exclusions, so run the rest of the tests +# manually. +log_note "Test one exclusion" +zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE 'history_event' +truncate -s 0 $ZED_DEBUG_LOG +log_must zfs set compression=off $TESTPOOL/$TESTFS +log_must file_wait $ZED_DEBUG_LOG 3 +log_mustnot grep -q history_event $ZED_DEBUG_LOG + +log_note "Test one exclusion with wildcards" +zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE '*history_event*' +truncate -s 0 $ZED_DEBUG_LOG +log_must zfs set compression=off $TESTPOOL/$TESTFS +log_must file_wait $ZED_DEBUG_LOG 3 +log_mustnot grep -q history_event $ZED_DEBUG_LOG + +log_note "Test one inclusion and one exclusion" +zed_rc_set ZED_SYSLOG_SUBCLASS_INCLUDE 'scrub_start' +zed_rc_set ZED_SYSLOG_SUBCLASS_EXCLUDE 'scrub_finish' +truncate -s 0 $ZED_DEBUG_LOG +zpool scrub $TESTPOOL +wait_scrubbed $TESTPOOL +log_must file_wait $ZED_DEBUG_LOG 3 +log_must grep -q scrub_start $ZED_DEBUG_LOG +log_mustnot grep -q scrub_finish $ZED_DEBUG_LOG + +log_pass "zed.rc event filtering works correctly." diff --git a/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh b/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh index 558cb065f..3b124bdd5 100755 --- a/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh +++ b/tests/zfs-tests/tests/functional/fault/scrub_after_resilver.ksh @@ -32,16 +32,15 @@ log_assert "Testing the scrub after resilver zedlet" # Backup our zed.rc -zedrc_backup="$(mktemp)" -log_must cp $ZEDLET_DIR/zed.rc $zedrc_backup +zedrc_backup=$(zed_rc_backup) -# Enable ZED_SCRUB_AFTER_RESILVER -eval "sed -i 's/\#ZED_SCRUB_AFTER_RESILVER/ZED_SCRUB_AFTER_RESILVER/g' $ZEDLET_DIR/zed.rc" +# Enable ZED_SCRUB_AFTER_RESILVER in zed.rc +zed_rc_set ZED_SCRUB_AFTER_RESILVER 1 function cleanup { # Restore our zed.rc - log_must mv $zedrc_backup $ZEDLET_DIR/zed.rc + log_must zed_rc_restore $zedrc_backup default_cleanup_noexit } -- 2.39.5