mm: combine LRU and main union in struct page

This gives us five words of space in a single union in struct page.  The
compound_mapcount moves position (from offset 24 to offset 20) on 64-bit
systems, but that does not seem likely to cause any trouble.

Link: http://lkml.kernel.org/r/20180518194519.3820-11-willy@infradead.org
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Jérôme Glisse <jglisse@redhat.com>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 5b97bd445adab8063b5275e749f8ba7ff5cde7a5..dfe6d648e0557ce5fdfa664e7482c770a2b54f3b 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -73,59 +73,19 @@ struct page {
        unsigned long flags;            /* Atomic flags, some possibly
                                         * updated asynchronously */
        /*
-        * WARNING: bit 0 of the first word encode PageTail(). That means
-        * the rest users of the storage space MUST NOT use the bit to
+        * Five words (20/40 bytes) are available in this union.
+        * WARNING: bit 0 of the first word is used for PageTail(). That
+        * means the other users of this union MUST NOT use the bit to
         * avoid collision and false-positive PageTail().
         */
-       union {
-               struct list_head lru;   /* Pageout list, eg. active_list
-                                        * protected by zone_lru_lock !
-                                        * Can be used as a generic list
-                                        * by the page owner.
-                                        */
-               struct dev_pagemap *pgmap; /* ZONE_DEVICE pages are never on an
-                                           * lru or handled by a slab
-                                           * allocator, this points to the
-                                           * hosting device page map.
-                                           */
-               struct {                /* slub per cpu partial pages */
-                       struct page *next;      /* Next partial slab */
-#ifdef CONFIG_64BIT
-                       int pages;      /* Nr of partial slabs left */
-                       int pobjects;   /* Approximate # of objects */
-#else
-                       short int pages;
-                       short int pobjects;
-#endif
-               };
-
-               struct rcu_head rcu_head;       /* Used by SLAB
-                                                * when destroying via RCU
-                                                */
-               /* Tail pages of compound page */
-               struct {
-                       unsigned long compound_head; /* If bit zero is set */
-
-                       /* First tail page only */
-                       unsigned char compound_dtor;
-                       unsigned char compound_order;
-                       /* two/six bytes available here */
-               };
-
-#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
-               struct {
-                       unsigned long __pad;    /* do not overlay pmd_huge_pte
-                                                * with compound_head to avoid
-                                                * possible bit 0 collision.
-                                                */
-                       pgtable_t pmd_huge_pte; /* protected by page->ptl */
-               };
-#endif
-       };
-
-       /* Three words (12/24 bytes) are available in this union. */
        union {
                struct {        /* Page cache and anonymous pages */
+                       /**
+                        * @lru: Pageout list, eg. active_list protected by
+                        * zone_lru_lock.  Sometimes used as a generic list
+                        * by the page owner.
+                        */
+                       struct list_head lru;
                        /* See page-flags.h for PAGE_MAPPING_FLAGS */
                        struct address_space *mapping;
                        pgoff_t index;          /* Our offset within mapping. */
@@ -138,6 +98,19 @@ struct page {
                        unsigned long private;
                };
                struct {        /* slab, slob and slub */
+                       union {
+                               struct list_head slab_list;     /* uses lru */
+                               struct {        /* Partial pages */
+                                       struct page *next;
+#ifdef CONFIG_64BIT
+                                       int pages;      /* Nr of pages left */
+                                       int pobjects;   /* Approximate count */
+#else
+                                       short int pages;
+                                       short int pobjects;
+#endif
+                               };
+                       };
                        struct kmem_cache *slab_cache; /* not slob */
                        /* Double-word boundary */
                        void *freelist;         /* first free object */
@@ -151,9 +124,22 @@ struct page {
                                };
                        };
                };
-               atomic_t compound_mapcount;     /* first tail page */
-               struct list_head deferred_list; /* second tail page */
+               struct {        /* Tail pages of compound page */
+                       unsigned long compound_head;    /* Bit zero is set */
+
+                       /* First tail page only */
+                       unsigned char compound_dtor;
+                       unsigned char compound_order;
+                       atomic_t compound_mapcount;
+               };
+               struct {        /* Second tail page of compound page */
+                       unsigned long _compound_pad_1;  /* compound_head */
+                       unsigned long _compound_pad_2;
+                       struct list_head deferred_list;
+               };
                struct {        /* Page table pages */
+                       unsigned long _pt_pad_1;        /* compound_head */
+                       pgtable_t pmd_huge_pte; /* protected by page->ptl */
                        unsigned long _pt_pad_2;        /* mapping */
                        unsigned long _pt_pad_3;
 #if ALLOC_SPLIT_PTLOCKS
@@ -162,6 +148,15 @@ struct page {
                        spinlock_t ptl;
 #endif
                };
+
+               /** @rcu_head: You can use this to free a page by RCU. */
+               struct rcu_head rcu_head;
+
+               /**
+                * @pgmap: For ZONE_DEVICE pages, this points to the hosting
+                * device page map.
+                */
+               struct dev_pagemap *pgmap;
        };
 
        union {         /* This union is 4 bytes in size. */

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 628fd3e247313ea61530d326f99969c527dd7aed..ef181153199977235c98b6ae186272b7e32f85a8 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -943,7 +943,7 @@ static int free_tail_pages_check(struct page *head_page, struct page *page)
        }
        switch (page - head_page) {
        case 1:
-               /* the first tail page: ->mapping is compound_mapcount() */
+               /* the first tail page: ->mapping may be compound_mapcount() */
                if (unlikely(compound_mapcount(page))) {
                        bad_page(page, "nonzero compound_mapcount", 0);
                        goto out;