Don't allow shrinking a PF_FSTRANS context

Avoid deadlocks when entering the shrinker from a PF_FSTRANS context.

This patch also reverts commit d0d5dd7 which added MUTEX_FSTRANS.  Its
use has been deprecated within ZFS as it was an ineffective mechanism
to eliminate deadlocks.  Among other things, it introduced the need for
strict ordering of mutex locking and unlocking in order that the
PF_FSTRANS flag wouldn't set incorrectly.

Signed-off-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #446

diff --git a/include/sys/mutex.h b/include/sys/mutex.h
index 6355782c0592e97d4f0c40c3f63d58328f2959bf..9b297e9faea38e63dcbe5dc39a9f6db69117c92c 100644 (file)
--- a/include/sys/mutex.h
+++ b/include/sys/mutex.h
@@ -32,19 +32,29 @@
 typedef enum {
        MUTEX_DEFAULT   = 0,
        MUTEX_SPIN      = 1,
-       MUTEX_ADAPTIVE  = 2,
-       MUTEX_FSTRANS   = 3,
+       MUTEX_ADAPTIVE  = 2
 } kmutex_type_t;
 
 typedef struct {
        struct mutex            m_mutex;
-       kmutex_type_t           m_type;
        spinlock_t              m_lock; /* used for serializing mutex_exit */
        kthread_t               *m_owner;
-       unsigned int            m_saved_flags;
 } kmutex_t;
 
 #define        MUTEX(mp)               (&((mp)->m_mutex))
+
+static inline void
+spl_mutex_set_owner(kmutex_t *mp)
+{
+       mp->m_owner = current;
+}
+
+static inline void
+spl_mutex_clear_owner(kmutex_t *mp)
+{
+       mp->m_owner = NULL;
+}
+
 #define        mutex_owner(mp)         (ACCESS_ONCE((mp)->m_owner))
 #define        mutex_owned(mp)         (mutex_owner(mp) == current)
 #define        MUTEX_HELD(mp)          mutex_owned(mp)
@@ -60,18 +70,11 @@ typedef struct {
 #define        mutex_init(mp, name, type, ibc)                         \
 {                                                              \
        static struct lock_class_key __key;                     \
-                                                               \
-       ASSERT3P(mp, !=, NULL);                                 \
-       ASSERT3P(ibc, ==, NULL);                                \
-       ASSERT((type == MUTEX_DEFAULT) ||                       \
-           (type == MUTEX_ADAPTIVE) ||                         \
-           (type == MUTEX_FSTRANS));                           \
+       ASSERT(type == MUTEX_DEFAULT);                          \
                                                                \
        __mutex_init(MUTEX(mp), (name) ? (#name) : (#mp), &__key); \
        spin_lock_init(&(mp)->m_lock);                          \
-       (mp)->m_type = type;                                    \
-       (mp)->m_owner = NULL;                                   \
-       (mp)->m_saved_flags = 0;                                \
+       spl_mutex_clear_owner(mp);                              \
 }
 
 #undef mutex_destroy
@@ -84,13 +87,8 @@ typedef struct {
 ({                                                             \
        int _rc_;                                               \
                                                                \
-       if ((_rc_ = mutex_trylock(MUTEX(mp))) == 1) {           \
-               (mp)->m_owner = current;                        \
-               if ((mp)->m_type == MUTEX_FSTRANS) {            \
-                       (mp)->m_saved_flags = current->flags;   \
-                       current->flags |= PF_FSTRANS;           \
-               }                                               \
-       }                                                       \
+       if ((_rc_ = mutex_trylock(MUTEX(mp))) == 1)             \
+               spl_mutex_set_owner(mp);                        \
                                                                \
        _rc_;                                                   \
 })
@@ -100,22 +98,14 @@ typedef struct {
 {                                                              \
        ASSERT3P(mutex_owner(mp), !=, current);                 \
        mutex_lock_nested(MUTEX(mp), (subclass));               \
-       (mp)->m_owner = current;                                \
-       if ((mp)->m_type == MUTEX_FSTRANS) {                    \
-               (mp)->m_saved_flags = current->flags;           \
-               current->flags |= PF_FSTRANS;                   \
-       }                                                       \
+       spl_mutex_set_owner(mp);                                \
 }
 #else /* CONFIG_DEBUG_LOCK_ALLOC */
 #define        mutex_enter_nested(mp, subclass)                        \
 {                                                              \
        ASSERT3P(mutex_owner(mp), !=, current);                 \
        mutex_lock(MUTEX(mp));                                  \
-       (mp)->m_owner = current;                                \
-       if ((mp)->m_type == MUTEX_FSTRANS) {                    \
-               (mp)->m_saved_flags = current->flags;           \
-               current->flags |= PF_FSTRANS;                   \
-       }                                                       \
+       spl_mutex_set_owner(mp);                                \
 }
 #endif /*  CONFIG_DEBUG_LOCK_ALLOC */
 
@@ -143,11 +133,7 @@ typedef struct {
 #define        mutex_exit(mp)                                          \
 {                                                              \
        spin_lock(&(mp)->m_lock);                               \
-       if ((mp)->m_type == MUTEX_FSTRANS) {                    \
-               current->flags &= ~(PF_FSTRANS);                \
-               current->flags |= (mp)->m_saved_flags;          \
-       }                                                       \
-       (mp)->m_owner = NULL;                                   \
+       spl_mutex_clear_owner(mp);                              \
        mutex_unlock(MUTEX(mp));                                \
        spin_unlock(&(mp)->m_lock);                             \
 }

diff --git a/module/spl/spl-kmem-cache.c b/module/spl/spl-kmem-cache.c
index 6fcc7c4e1b6179b4256fc15465bce1da49491359..cd3e543ba08ebed0e3d75c13f38d9d558b9fe8f3 100644 (file)
--- a/module/spl/spl-kmem-cache.c
+++ b/module/spl/spl-kmem-cache.c
@@ -1574,6 +1574,12 @@ __spl_kmem_cache_generic_shrinker(struct shrinker *shrink,
        spl_kmem_cache_t *skc;
        int alloc = 0;
 
+       /*
+        * No shrinking in a transaction context.  Can cause deadlocks.
+        */
+       if (sc->nr_to_scan && spl_fstrans_check())
+               return (SHRINK_STOP);
+
        down_read(&spl_kmem_cache_sem);
        list_for_each_entry(skc, &spl_kmem_cache_list, skc_list) {
                if (sc->nr_to_scan) {