Add callback for zfs_multihost_interval

Add a callback to wake all running mmp threads when
zfs_multihost_interval is changed.

This is necessary when the interval is changed from a very large value
to a significantly lower one, while pools are imported that have the
multihost property enabled.

Without this commit, the mmp thread does not wake up and detect the new
interval until after it has waited the old multihost interval time.  A
user monitoring mmp writes via the provided kstat would be led to
believe that the changed setting did not work.

Added a test in the ZTS under mmp to verify the new functionality is
working.

Added a test to ztest which starts and stops mmp threads, and calls into
the code to signal sleeping mmp threads, to test for deadlocks or
similar locking issues.

Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Olaf Faaland <faaland1@llnl.gov>
Closes #6387

diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c
index b4cedbdbaf6649dd71dbc502a8fe57aa61f7354e..788e6aa6ac0b04ca0e4c1f8e858eb57918db404e 100644 (file)
--- a/cmd/ztest/ztest.c
+++ b/cmd/ztest/ztest.c
@@ -326,6 +326,7 @@ ztest_func_t ztest_spa_create_destroy;
 ztest_func_t ztest_fault_inject;
 ztest_func_t ztest_ddt_repair;
 ztest_func_t ztest_dmu_snapshot_hold;
+ztest_func_t ztest_mmp_enable_disable;
 ztest_func_t ztest_spa_rename;
 ztest_func_t ztest_scrub;
 ztest_func_t ztest_dsl_dataset_promote_busy;
@@ -375,6 +376,7 @@ ztest_info_t ztest_info[] = {
        ZTI_INIT(ztest_fault_inject, 1, &zopt_sometimes),
        ZTI_INIT(ztest_ddt_repair, 1, &zopt_sometimes),
        ZTI_INIT(ztest_dmu_snapshot_hold, 1, &zopt_sometimes),
+       ZTI_INIT(ztest_mmp_enable_disable, 1, &zopt_sometimes),
        ZTI_INIT(ztest_reguid, 1, &zopt_rarely),
        ZTI_INIT(ztest_spa_rename, 1, &zopt_rarely),
        ZTI_INIT(ztest_scrub, 1, &zopt_rarely),
@@ -2660,6 +2662,47 @@ ztest_spa_create_destroy(ztest_ds_t *zd, uint64_t id)
        (void) rw_unlock(&ztest_name_lock);
 }
 
+/*
+ * Start and then stop the MMP threads to ensure the startup and shutdown code
+ * works properly.  Actual protection and property-related code tested via ZTS.
+ */
+/* ARGSUSED */
+void
+ztest_mmp_enable_disable(ztest_ds_t *zd, uint64_t id)
+{
+       ztest_shared_opts_t *zo = &ztest_opts;
+       spa_t *spa = ztest_spa;
+
+       if (zo->zo_mmp_test)
+               return;
+
+       spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
+       mutex_enter(&spa->spa_props_lock);
+
+       if (!spa_multihost(spa)) {
+               spa->spa_multihost = B_TRUE;
+               mmp_thread_start(spa);
+       }
+
+       mutex_exit(&spa->spa_props_lock);
+       spa_config_exit(spa, SCL_CONFIG, FTAG);
+
+       txg_wait_synced(spa_get_dsl(spa), 0);
+       mmp_signal_all_threads();
+       txg_wait_synced(spa_get_dsl(spa), 0);
+
+       spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
+       mutex_enter(&spa->spa_props_lock);
+
+       if (spa_multihost(spa)) {
+               mmp_thread_stop(spa);
+               spa->spa_multihost = B_FALSE;
+       }
+
+       mutex_exit(&spa->spa_props_lock);
+       spa_config_exit(spa, SCL_CONFIG, FTAG);
+}
+
 /* ARGSUSED */
 void
 ztest_spa_upgrade(ztest_ds_t *zd, uint64_t id)

diff --git a/include/sys/mmp.h b/include/sys/mmp.h
index 4da612d6a5a205c52ac46da9cc4c2d25af1586c4..5b2fea1a66b1c8b2fba760be569a993b8b3daf4d 100644 (file)
--- a/include/sys/mmp.h
+++ b/include/sys/mmp.h
@@ -50,6 +50,7 @@ extern void mmp_fini(struct spa *spa);
 extern void mmp_thread_start(struct spa *spa);
 extern void mmp_thread_stop(struct spa *spa);
 extern void mmp_update_uberblock(struct spa *spa, struct uberblock *ub);
+extern void mmp_signal_all_threads(void);
 
 /* Global tuning */
 extern ulong_t zfs_multihost_interval;

diff --git a/module/zfs/mmp.c b/module/zfs/mmp.c
index 539b76826a865f256b56bef19d7c4abdcee602c3..00478a39f2b330dc9fb644da870c0d0bee2362a4 100644 (file)
--- a/module/zfs/mmp.c
+++ b/module/zfs/mmp.c
@@ -459,13 +459,61 @@ mmp_thread(spa_t *spa)
        mmp_thread_exit(mmp, &mmp->mmp_thread, &cpr);
 }
 
+/*
+ * Signal the MMP thread to wake it, when it is sleeping on
+ * its cv.  Used when some module parameter has changed and
+ * we want the thread to know about it.
+ * Only signal if the pool is active and mmp thread is
+ * running, otherwise there is no thread to wake.
+ */
+static void
+mmp_signal_thread(spa_t *spa)
+{
+       mmp_thread_t *mmp = &spa->spa_mmp;
+
+       mutex_enter(&mmp->mmp_thread_lock);
+       if (mmp->mmp_thread)
+               cv_broadcast(&mmp->mmp_thread_cv);
+       mutex_exit(&mmp->mmp_thread_lock);
+}
+
+void
+mmp_signal_all_threads(void)
+{
+       spa_t *spa = NULL;
+
+       mutex_enter(&spa_namespace_lock);
+       while ((spa = spa_next(spa))) {
+               if (spa->spa_state == POOL_STATE_ACTIVE)
+                       mmp_signal_thread(spa);
+       }
+       mutex_exit(&spa_namespace_lock);
+}
+
 #if defined(_KERNEL) && defined(HAVE_SPL)
+#include <linux/mod_compat.h>
+
+static int
+param_set_multihost_interval(const char *val, zfs_kernel_param_t *kp)
+{
+       int ret;
+
+       ret = param_set_ulong(val, kp);
+       if (ret < 0)
+               return (ret);
+
+       mmp_signal_all_threads();
+
+       return (ret);
+}
+
 /* BEGIN CSTYLED */
 module_param(zfs_multihost_fail_intervals, uint, 0644);
 MODULE_PARM_DESC(zfs_multihost_fail_intervals,
        "Max allowed period without a successful mmp write");
 
-module_param(zfs_multihost_interval, ulong, 0644);
+module_param_call(zfs_multihost_interval, param_set_multihost_interval,
+    param_get_ulong, &zfs_multihost_interval, 0644);
 MODULE_PARM_DESC(zfs_multihost_interval,
        "Milliseconds between mmp writes to each leaf");
 

diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index 842f6dd0c3b9976fb5c3d0a896f13ee108264e58..2017affa1b4d1964f1c67a4b1b3bd9e7f3e8cb03 100644 (file)
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -413,7 +413,7 @@ tests = ['mmap_write_001_pos', 'mmap_read_001_pos']
 [tests/functional/mmp]
 tests = ['mmp_on_thread', 'mmp_on_uberblocks', 'mmp_on_off', 'mmp_interval',
     'mmp_active_import', 'mmp_inactive_import', 'mmp_exported_import',
-    'mmp_write_uberblocks']
+    'mmp_write_uberblocks', 'mmp_reset_interval']
 
 [tests/functional/mount]
 tests = ['umount_001', 'umountall_001']

diff --git a/tests/zfs-tests/tests/functional/mmp/Makefile.am b/tests/zfs-tests/tests/functional/mmp/Makefile.am
index 75af0cdb71485b9b583614b4a4d3cba0c666f1d9..f81f07fef97ed6a587b844a04fc3437ff6f4c207 100644 (file)
--- a/tests/zfs-tests/tests/functional/mmp/Makefile.am
+++ b/tests/zfs-tests/tests/functional/mmp/Makefile.am
@@ -8,6 +8,7 @@ dist_pkgdata_SCRIPTS = \
        mmp_inactive_import.ksh \
        mmp_exported_import.ksh \
        mmp_write_uberblocks.ksh \
+       mmp_reset_interval.ksh \
        setup.ksh \
        cleanup.ksh \
        mmp.kshlib \

diff --git a/tests/zfs-tests/tests/functional/mmp/mmp.cfg b/tests/zfs-tests/tests/functional/mmp/mmp.cfg
index f17108a876307a7ce93d408f90c4cd293987190e..29e030a4301a81a36eba51a48f363e460f7926e6 100644 (file)
--- a/tests/zfs-tests/tests/functional/mmp/mmp.cfg
+++ b/tests/zfs-tests/tests/functional/mmp/mmp.cfg
@@ -31,7 +31,9 @@ export TXG_TIMEOUT_DEFAULT=5
 export MMP_POOL=mmppool
 export MMP_DIR=$TEST_BASE_DIR/mmp
 export MMP_HISTORY=100
+export MMP_HISTORY_OFF=0
 
+export MMP_INTERVAL_HOUR=$((60*60*1000))
 export MMP_INTERVAL_DEFAULT=1000
 export MMP_INTERVAL_MIN=100
 

diff --git a/tests/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh b/tests/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh
new file mode 100755 (executable)
index 0000000..ee408a8
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/mmp/mmp_reset_interval.ksh
@@ -0,0 +1,67 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+# DESCRIPTION:
+#      Ensure that the MMP thread is notified when zfs_multihost_interval is
+#      reduced.
+#
+# STRATEGY:
+#      1. Set zfs_multihost_interval to much longer than the test duration
+#      2. Create a zpool and enable multihost
+#      3. Verify no MMP writes occurred
+#      4. Set zfs_multihost_interval to 1 second
+#      5. Sleep briefly
+#      6. Verify MMP writes began
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/mmp/mmp.cfg
+. $STF_SUITE/tests/functional/mmp/mmp.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+       default_cleanup_noexit
+       log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT
+       log_must mmp_clear_hostid
+}
+
+log_assert "mmp threads notified when zfs_multihost_interval reduced"
+log_onexit cleanup
+
+log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_HOUR
+log_must mmp_set_hostid $HOSTID1
+
+default_setup_noexit $DISK
+log_must zpool set multihost=on $TESTPOOL
+
+prev_count=$(wc -l /proc/spl/kstat/zfs/$TESTPOOL/multihost | cut -f1 -d' ')
+log_must set_tunable64 zfs_multihost_interval $MMP_INTERVAL_DEFAULT
+
+# slight delay to allow time for the mmp write to complete
+sleep 1
+curr_count=$(wc -l /proc/spl/kstat/zfs/$TESTPOOL/multihost | cut -f1 -d' ')
+
+if [ $curr_count -eq $prev_count ]; then
+       log_fail "mmp writes did not start when zfs_multihost_interval reduced"
+fi
+
+log_pass "mmp threads notified when zfs_multihost_interval reduced"