mm: memcontrol: account slab stats per lruvec

Josef's redesign of the balancing between slab caches and the page cache
requires slab cache statistics at the lruvec level.

Link: http://lkml.kernel.org/r/20170530181724.27197-7-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: Josef Bacik <josef@toxicpanda.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/slab.c b/mm/slab.c
index a38634ed478e3d5967fc1465004a2f1521c6c2db..04dec48c3ed7a12af3fef4958e928d0ddc31b64e 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -1425,11 +1425,9 @@ static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
 
        nr_pages = (1 << cachep->gfporder);
        if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
-               add_node_page_state(page_pgdat(page),
-                       NR_SLAB_RECLAIMABLE, nr_pages);
+               mod_lruvec_page_state(page, NR_SLAB_RECLAIMABLE, nr_pages);
        else
-               add_node_page_state(page_pgdat(page),
-                       NR_SLAB_UNRECLAIMABLE, nr_pages);
+               mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE, nr_pages);
 
        __SetPageSlab(page);
        /* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
@@ -1459,11 +1457,9 @@ static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
        kmemcheck_free_shadow(page, order);
 
        if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
-               sub_node_page_state(page_pgdat(page),
-                               NR_SLAB_RECLAIMABLE, nr_freed);
+               mod_lruvec_page_state(page, NR_SLAB_RECLAIMABLE, -nr_freed);
        else
-               sub_node_page_state(page_pgdat(page),
-                               NR_SLAB_UNRECLAIMABLE, nr_freed);
+               mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE, -nr_freed);
 
        BUG_ON(!PageSlab(page));
        __ClearPageSlabPfmemalloc(page);

diff --git a/mm/slab.h b/mm/slab.h
index 7b84e3839dfef16b3338900c2ddbc6dcc5b95a03..6885e1192ec5369d4edf7ee14aeeaf1f69c860bf 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -274,22 +274,11 @@ static __always_inline int memcg_charge_slab(struct page *page,
                                             gfp_t gfp, int order,
                                             struct kmem_cache *s)
 {
-       int ret;
-
        if (!memcg_kmem_enabled())
                return 0;
        if (is_root_cache(s))
                return 0;
-
-       ret = memcg_kmem_charge_memcg(page, gfp, order, s->memcg_params.memcg);
-       if (ret)
-               return ret;
-
-       mod_memcg_page_state(page,
-                            (s->flags & SLAB_RECLAIM_ACCOUNT) ?
-                            NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
-                            1 << order);
-       return 0;
+       return memcg_kmem_charge_memcg(page, gfp, order, s->memcg_params.memcg);
 }
 
 static __always_inline void memcg_uncharge_slab(struct page *page, int order,
@@ -297,11 +286,6 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order,
 {
        if (!memcg_kmem_enabled())
                return;
-
-       mod_memcg_page_state(page,
-                            (s->flags & SLAB_RECLAIM_ACCOUNT) ?
-                            NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
-                            -(1 << order));
        memcg_kmem_uncharge(page, order);
 }
 

diff --git a/mm/slub.c b/mm/slub.c
index aa5aa6bfb35e3f2d33caac3845cc55431e765df0..1d3f9835f4eabe91494f48d8ace08a5e42a895f9 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1615,7 +1615,7 @@ out:
        if (!page)
                return NULL;
 
-       mod_node_page_state(page_pgdat(page),
+       mod_lruvec_page_state(page,
                (s->flags & SLAB_RECLAIM_ACCOUNT) ?
                NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
                1 << oo_order(oo));
@@ -1655,7 +1655,7 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
 
        kmemcheck_free_shadow(page, compound_order(page));
 
-       mod_node_page_state(page_pgdat(page),
+       mod_lruvec_page_state(page,
                (s->flags & SLAB_RECLAIM_ACCOUNT) ?
                NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
                -pages);