inode: convert inode_stat.nr_unused to per-cpu counters

[mirror_ubuntu-bionic-kernel.git] / fs / inode.c

diff --git a/fs/inode.c b/fs/inode.c
index 990d284877a12472393a8d444b732ba9ec105c7c..9a03611217123b138f4010a94c1e2135d8d43709 100644 (file)
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -1,9 +1,7 @@
 /*
- * linux/fs/inode.c
- *
  * (C) 1997 Linus Torvalds
+ * (C) 1999 Andrea Arcangeli <andrea@suse.de> (dynamic inode allocation)
  */
-
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/dcache.h>
@@ -27,10 +25,11 @@
 #include <linux/prefetch.h>
 #include <linux/ima.h>
 #include <linux/cred.h>
+#include <linux/buffer_head.h> /* for inode_has_buffers */
 #include "internal.h"
 
 /*
- * inode locking rules.
+ * Inode locking rules:
  *
  * inode->i_lock protects:
  *   inode->i_state, inode->i_hash, __iget()
@@ -60,54 +59,11 @@
  *   inode_hash_lock
  */
 
-/*
- * This is needed for the following functions:
- *  - inode_has_buffers
- *  - invalidate_bdev
- *
- * FIXME: remove all knowledge of the buffer layer from this file
- */
-#include <linux/buffer_head.h>
-
-/*
- * New inode.c implementation.
- *
- * This implementation has the basic premise of trying
- * to be extremely low-overhead and SMP-safe, yet be
- * simple enough to be "obviously correct".
- *
- * Famous last words.
- */
-
-/* inode dynamic allocation 1999, Andrea Arcangeli <andrea@suse.de> */
-
-/* #define INODE_PARANOIA 1 */
-/* #define INODE_DEBUG 1 */
-
-/*
- * Inode lookup is no longer as critical as it used to be:
- * most of the lookups are going to be through the dcache.
- */
-#define I_HASHBITS     i_hash_shift
-#define I_HASHMASK     i_hash_mask
-
 static unsigned int i_hash_mask __read_mostly;
 static unsigned int i_hash_shift __read_mostly;
 static struct hlist_head *inode_hashtable __read_mostly;
 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
 
-/*
- * Each inode can be on two separate lists. One is
- * the hash list of the inode, used for lookups. The
- * other linked list is the "type" list:
- *  "in_use" - valid inode, i_count > 0, i_nlink > 0
- *  "dirty"  - as "in_use" but also dirty
- *  "unused" - valid inode, i_count = 0
- *
- * A "dirty" list is maintained for each super block,
- * allowing for low-overhead inode sync() operations.
- */
-
 static LIST_HEAD(inode_lru);
 static DEFINE_SPINLOCK(inode_lru_lock);
 
@@ -139,6 +95,7 @@ EXPORT_SYMBOL(empty_aops);
 struct inodes_stat_t inodes_stat;
 
 static DEFINE_PER_CPU(unsigned int, nr_inodes);
+static DEFINE_PER_CPU(unsigned int, nr_unused);
 
 static struct kmem_cache *inode_cachep __read_mostly;
 
@@ -153,7 +110,11 @@ static int get_nr_inodes(void)
 
 static inline int get_nr_inodes_unused(void)
 {
-       return inodes_stat.nr_unused;
+       int i;
+       int sum = 0;
+       for_each_possible_cpu(i)
+               sum += per_cpu(nr_unused, i);
+       return sum < 0 ? 0 : sum;
 }
 
 int get_nr_dirty_inodes(void)
@@ -171,6 +132,7 @@ int proc_nr_inodes(ctl_table *table, int write,
                   void __user *buffer, size_t *lenp, loff_t *ppos)
 {
        inodes_stat.nr_inodes = get_nr_inodes();
+       inodes_stat.nr_unused = get_nr_inodes_unused();
        return proc_dointvec(table, write, buffer, lenp, ppos);
 }
 #endif
@@ -384,7 +346,7 @@ static void inode_lru_list_add(struct inode *inode)
        spin_lock(&inode_lru_lock);
        if (list_empty(&inode->i_lru)) {
                list_add(&inode->i_lru, &inode_lru);
-               inodes_stat.nr_unused++;
+               this_cpu_inc(nr_unused);
        }
        spin_unlock(&inode_lru_lock);
 }
@@ -394,7 +356,7 @@ static void inode_lru_list_del(struct inode *inode)
        spin_lock(&inode_lru_lock);
        if (!list_empty(&inode->i_lru)) {
                list_del_init(&inode->i_lru);
-               inodes_stat.nr_unused--;
+               this_cpu_dec(nr_unused);
        }
        spin_unlock(&inode_lru_lock);
 }
@@ -424,8 +386,8 @@ static unsigned long hash(struct super_block *sb, unsigned long hashval)
 
        tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) /
                        L1_CACHE_BYTES;
-       tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> I_HASHBITS);
-       return tmp & I_HASHMASK;
+       tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift);
+       return tmp & i_hash_mask;
 }
 
 /**
@@ -467,7 +429,14 @@ EXPORT_SYMBOL(remove_inode_hash);
 void end_writeback(struct inode *inode)
 {
        might_sleep();
+       /*
+        * We have to cycle tree_lock here because reclaim can be still in the
+        * process of removing the last page (in __delete_from_page_cache())
+        * and we must not free mapping under it.
+        */
+       spin_lock_irq(&inode->i_data.tree_lock);
        BUG_ON(inode->i_data.nrpages);
+       spin_unlock_irq(&inode->i_data.tree_lock);
        BUG_ON(!list_empty(&inode->i_data.private_list));
        BUG_ON(!(inode->i_state & I_FREEING));
        BUG_ON(inode->i_state & I_CLEAR);
@@ -693,7 +662,7 @@ static void prune_icache(int nr_to_scan)
                    (inode->i_state & ~I_REFERENCED)) {
                        list_del_init(&inode->i_lru);
                        spin_unlock(&inode->i_lock);
-                       inodes_stat.nr_unused--;
+                       this_cpu_dec(nr_unused);
                        continue;
                }
 
@@ -730,7 +699,7 @@ static void prune_icache(int nr_to_scan)
                spin_unlock(&inode->i_lock);
 
                list_move(&inode->i_lru, &freeable);
-               inodes_stat.nr_unused--;
+               this_cpu_dec(nr_unused);
        }
        if (current_is_kswapd())
                __count_vm_events(KSWAPD_INODESTEAL, reap);
@@ -1368,7 +1337,7 @@ static void iput_final(struct inode *inode)
 
        WARN_ON(inode->i_state & I_NEW);
 
-       if (op && op->drop_inode)
+       if (op->drop_inode)
                drop = op->drop_inode(inode);
        else
                drop = generic_drop_inode(inode);