mm: memcontrol: switch to native NR_FILE_PAGES and NR_SHMEM counters

Memcg maintains private MEMCG_CACHE and NR_SHMEM counters.  This
divergence from the generic VM accounting means unnecessary code overhead,
and creates a dependency for memcg that page->mapping is set up at the
time of charging, so that page types can be told apart.

Convert the generic accounting sites to mod_lruvec_page_state and friends
to maintain the per-cgroup vmstat counters of NR_FILE_PAGES and NR_SHMEM.
The page is already locked in these places, so page->mem_cgroup is stable;
we only need minimal tweaks of two mem_cgroup_migrate() calls to ensure
it's set up in time.

Then replace MEMCG_CACHE with NR_FILE_PAGES and delete the private
NR_SHMEM accounting sites.

Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reviewed-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Alex Shi <alex.shi@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Roman Gushchin <guro@fb.com>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Balbir Singh <bsingharora@gmail.com>
Link: http://lkml.kernel.org/r/20200508183105.225460-10-hannes@cmpxchg.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 8f00dd75581855ba700cab28db31616cf10b9b52..f6ea68ceed2c26ab08ba901eed47f0ef65fac013 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -29,8 +29,7 @@ struct kmem_cache;
 
 /* Cgroup-specific page state, on top of universal node page state */
 enum memcg_stat_item {
-       MEMCG_CACHE = NR_VM_NODE_STAT_ITEMS,
-       MEMCG_RSS,
+       MEMCG_RSS = NR_VM_NODE_STAT_ITEMS,
        MEMCG_RSS_HUGE,
        MEMCG_SWAP,
        MEMCG_SOCK,

diff --git a/mm/filemap.c b/mm/filemap.c
index 38e6a37336a65fe98e98b038e9085e53d3ec4384..a6565890cdf0fc585a215003d8cceaa5d76cef0b 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -199,9 +199,9 @@ static void unaccount_page_cache_page(struct address_space *mapping,
 
        nr = hpage_nr_pages(page);
 
-       __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
+       __mod_lruvec_page_state(page, NR_FILE_PAGES, -nr);
        if (PageSwapBacked(page)) {
-               __mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);
+               __mod_lruvec_page_state(page, NR_SHMEM, -nr);
                if (PageTransHuge(page))
                        __dec_node_page_state(page, NR_SHMEM_THPS);
        } else if (PageTransHuge(page)) {
@@ -802,21 +802,22 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
        new->mapping = mapping;
        new->index = offset;
 
+       mem_cgroup_migrate(old, new);
+
        xas_lock_irqsave(&xas, flags);
        xas_store(&xas, new);
 
        old->mapping = NULL;
        /* hugetlb pages do not participate in page cache accounting. */
        if (!PageHuge(old))
-               __dec_node_page_state(old, NR_FILE_PAGES);
+               __dec_lruvec_page_state(old, NR_FILE_PAGES);
        if (!PageHuge(new))
-               __inc_node_page_state(new, NR_FILE_PAGES);
+               __inc_lruvec_page_state(new, NR_FILE_PAGES);
        if (PageSwapBacked(old))
-               __dec_node_page_state(old, NR_SHMEM);
+               __dec_lruvec_page_state(old, NR_SHMEM);
        if (PageSwapBacked(new))
-               __inc_node_page_state(new, NR_SHMEM);
+               __inc_lruvec_page_state(new, NR_SHMEM);
        xas_unlock_irqrestore(&xas, flags);
-       mem_cgroup_migrate(old, new);
        if (freepage)
                freepage(old);
        put_page(old);
@@ -867,7 +868,7 @@ static int __add_to_page_cache_locked(struct page *page,
 
                /* hugetlb pages do not participate in page cache accounting */
                if (!huge)
-                       __inc_node_page_state(page, NR_FILE_PAGES);
+                       __inc_lruvec_page_state(page, NR_FILE_PAGES);
 unlock:
                xas_unlock_irq(&xas);
        } while (xas_nomem(&xas, gfp_mask & GFP_RECLAIM_MASK));

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 0a29b51494fdf735cdd32ebe2b3e0ced97603583..ddbdc1e3a694d200400311fb1ec51fe08d25ea6b 100644 (file)
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1844,12 +1844,18 @@ out_unlock:
        }
 
        if (nr_none) {
-               struct zone *zone = page_zone(new_page);
-
-               __mod_node_page_state(zone->zone_pgdat, NR_FILE_PAGES, nr_none);
+               struct lruvec *lruvec;
+               /*
+                * XXX: We have started try_charge and pinned the
+                * memcg, but the page isn't committed yet so we
+                * cannot use mod_lruvec_page_state(). This hackery
+                * will be cleaned up when remove the page->mapping
+                * dependency from memcg and fully charge above.
+                */
+               lruvec = mem_cgroup_lruvec(memcg, page_pgdat(new_page));
+               __mod_lruvec_state(lruvec, NR_FILE_PAGES, nr_none);
                if (is_shmem)
-                       __mod_node_page_state(zone->zone_pgdat,
-                                             NR_SHMEM, nr_none);
+                       __mod_lruvec_state(lruvec, NR_SHMEM, nr_none);
        }
 
 xa_locked:

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index a875a97067a307e2ce236fb833e5b1549c205020..ab3497ba0e3584419b5a676eaea1210d3c124708 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -842,11 +842,6 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
         */
        if (PageAnon(page))
                __mod_memcg_state(memcg, MEMCG_RSS, nr_pages);
-       else {
-               __mod_memcg_state(memcg, MEMCG_CACHE, nr_pages);
-               if (PageSwapBacked(page))
-                       __mod_memcg_state(memcg, NR_SHMEM, nr_pages);
-       }
 
        if (abs(nr_pages) > 1) {
                VM_BUG_ON_PAGE(!PageTransHuge(page), page);
@@ -1392,7 +1387,7 @@ static char *memory_stat_format(struct mem_cgroup *memcg)
                       (u64)memcg_page_state(memcg, MEMCG_RSS) *
                       PAGE_SIZE);
        seq_buf_printf(&s, "file %llu\n",
-                      (u64)memcg_page_state(memcg, MEMCG_CACHE) *
+                      (u64)memcg_page_state(memcg, NR_FILE_PAGES) *
                       PAGE_SIZE);
        seq_buf_printf(&s, "kernel_stack %llu\n",
                       (u64)memcg_page_state(memcg, MEMCG_KERNEL_STACK_KB) *
@@ -3357,7 +3352,7 @@ static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
        unsigned long val;
 
        if (mem_cgroup_is_root(memcg)) {
-               val = memcg_page_state(memcg, MEMCG_CACHE) +
+               val = memcg_page_state(memcg, NR_FILE_PAGES) +
                        memcg_page_state(memcg, MEMCG_RSS);
                if (swap)
                        val += memcg_page_state(memcg, MEMCG_SWAP);
@@ -3828,7 +3823,7 @@ static int memcg_numa_stat_show(struct seq_file *m, void *v)
 #endif /* CONFIG_NUMA */
 
 static const unsigned int memcg1_stats[] = {
-       MEMCG_CACHE,
+       NR_FILE_PAGES,
        MEMCG_RSS,
        MEMCG_RSS_HUGE,
        NR_SHMEM,
@@ -5461,6 +5456,14 @@ static int mem_cgroup_move_account(struct page *page,
        lock_page_memcg(page);
 
        if (!PageAnon(page)) {
+               __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
+               __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
+
+               if (PageSwapBacked(page)) {
+                       __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
+                       __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
+               }
+
                if (page_mapped(page)) {
                        __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
                        __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
@@ -6673,10 +6676,8 @@ struct uncharge_gather {
        unsigned long nr_pages;
        unsigned long pgpgout;
        unsigned long nr_anon;
-       unsigned long nr_file;
        unsigned long nr_kmem;
        unsigned long nr_huge;
-       unsigned long nr_shmem;
        struct page *dummy_page;
 };
 
@@ -6700,9 +6701,7 @@ static void uncharge_batch(const struct uncharge_gather *ug)
 
        local_irq_save(flags);
        __mod_memcg_state(ug->memcg, MEMCG_RSS, -ug->nr_anon);
-       __mod_memcg_state(ug->memcg, MEMCG_CACHE, -ug->nr_file);
        __mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);
-       __mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem);
        __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
        __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
        memcg_check_events(ug->memcg, ug->dummy_page);
@@ -6743,11 +6742,6 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
                        ug->nr_huge += nr_pages;
                if (PageAnon(page))
                        ug->nr_anon += nr_pages;
-               else {
-                       ug->nr_file += nr_pages;
-                       if (PageSwapBacked(page))
-                               ug->nr_shmem += nr_pages;
-               }
                ug->pgpgout++;
        } else {
                ug->nr_kmem += nr_pages;

diff --git a/mm/migrate.c b/mm/migrate.c
index 93c732213d124f43d315a084a3f8f38e81fcb6d7..0d1f796df75ff5345205696cfb46c2560b7d54c9 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -490,11 +490,18 @@ int migrate_page_move_mapping(struct address_space *mapping,
         * are mapped to swap space.
         */
        if (newzone != oldzone) {
-               __dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES);
-               __inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES);
+               struct lruvec *old_lruvec, *new_lruvec;
+               struct mem_cgroup *memcg;
+
+               memcg = page_memcg(page);
+               old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat);
+               new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat);
+
+               __dec_lruvec_state(old_lruvec, NR_FILE_PAGES);
+               __inc_lruvec_state(new_lruvec, NR_FILE_PAGES);
                if (PageSwapBacked(page) && !PageSwapCache(page)) {
-                       __dec_node_state(oldzone->zone_pgdat, NR_SHMEM);
-                       __inc_node_state(newzone->zone_pgdat, NR_SHMEM);
+                       __dec_lruvec_state(old_lruvec, NR_SHMEM);
+                       __inc_lruvec_state(new_lruvec, NR_SHMEM);
                }
                if (dirty && mapping_cap_account_dirty(mapping)) {
                        __dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);

diff --git a/mm/shmem.c b/mm/shmem.c
index 0d961572315291bbe4c2cb016fa56e6e151f806d..71842fd4a9d069bf3754317ce10a4691f9ead8eb 100644 (file)
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -653,8 +653,8 @@ next:
                        __inc_node_page_state(page, NR_SHMEM_THPS);
                }
                mapping->nrpages += nr;
-               __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
-               __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
+               __mod_lruvec_page_state(page, NR_FILE_PAGES, nr);
+               __mod_lruvec_page_state(page, NR_SHMEM, nr);
 unlock:
                xas_unlock_irq(&xas);
        } while (xas_nomem(&xas, gfp));
@@ -685,8 +685,8 @@ static void shmem_delete_from_page_cache(struct page *page, void *radswap)
        error = shmem_replace_entry(mapping, page->index, page, radswap);
        page->mapping = NULL;
        mapping->nrpages--;
-       __dec_node_page_state(page, NR_FILE_PAGES);
-       __dec_node_page_state(page, NR_SHMEM);
+       __dec_lruvec_page_state(page, NR_FILE_PAGES);
+       __dec_lruvec_page_state(page, NR_SHMEM);
        xa_unlock_irq(&mapping->i_pages);
        put_page(page);
        BUG_ON(error);
@@ -1593,8 +1593,9 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp,
        xa_lock_irq(&swap_mapping->i_pages);
        error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
        if (!error) {
-               __inc_node_page_state(newpage, NR_FILE_PAGES);
-               __dec_node_page_state(oldpage, NR_FILE_PAGES);
+               mem_cgroup_migrate(oldpage, newpage);
+               __inc_lruvec_page_state(newpage, NR_FILE_PAGES);
+               __dec_lruvec_page_state(oldpage, NR_FILE_PAGES);
        }
        xa_unlock_irq(&swap_mapping->i_pages);
 
@@ -1606,7 +1607,6 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp,
                 */
                oldpage = newpage;
        } else {
-               mem_cgroup_migrate(oldpage, newpage);
                lru_cache_add_anon(newpage);
                *pagep = newpage;
        }