mm/page_alloc.c: memory hotplug: free pages as higher order

When freeing pages are done with higher order, time spent on coalescing
pages by buddy allocator can be reduced.  With section size of 256MB,
hot add latency of a single section shows improvement from 50-60 ms to
less than 1 ms, hence improving the hot add latency by 60 times.  Modify
external providers of online callback to align with the change.

[arunks@codeaurora.org: v11]
Link: http://lkml.kernel.org/r/1547792588-18032-1-git-send-email-arunks@codeaurora.org
[akpm@linux-foundation.org: remove unused local, per Arun]
[akpm@linux-foundation.org: avoid return of void-returning __free_pages_core(), per Oscar]
[akpm@linux-foundation.org: fix it for mm-convert-totalram_pages-and-totalhigh_pages-variables-to-atomic.patch]
[arunks@codeaurora.org: v8]
Link: http://lkml.kernel.org/r/1547032395-24582-1-git-send-email-arunks@codeaurora.org
[arunks@codeaurora.org: v9]
Link: http://lkml.kernel.org/r/1547098543-26452-1-git-send-email-arunks@codeaurora.org
Link: http://lkml.kernel.org/r/1538727006-5727-1-git-send-email-arunks@codeaurora.org
Signed-off-by: Arun KS <arunks@codeaurora.org>
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: Michal Hocko <mhocko@suse.com>
Reviewed-by: Oscar Salvador <osalvador@suse.de>
Reviewed-by: Alexander Duyck <alexander.h.duyck@linux.intel.com>
Cc: K. Y. Srinivasan <kys@microsoft.com>
Cc: Haiyang Zhang <haiyangz@microsoft.com>
Cc: Stephen Hemminger <sthemmin@microsoft.com>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Mathieu Malaterre <malat@debian.org>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Souptick Joarder <jrdr.linux@gmail.com>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Aaron Lu <aaron.lu@intel.com>
Cc: Srivatsa Vaddagiri <vatsa@codeaurora.org>
Cc: Vinayak Menon <vinmenon@codeaurora.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c
index 7c6349a50ef173421cdafea3b289ae9868da711e..a50b7624b2a3ab9b5cf5500dd1b040bb4007c5d4 100644 (file)
--- a/drivers/hv/hv_balloon.c
+++ b/drivers/hv/hv_balloon.c
@@ -771,7 +771,7 @@ static void hv_mem_hot_add(unsigned long start, unsigned long size,
        }
 }
 
-static void hv_online_page(struct page *pg)
+static void hv_online_page(struct page *pg, unsigned int order)
 {
        struct hv_hotadd_state *has;
        unsigned long flags;
@@ -780,10 +780,11 @@ static void hv_online_page(struct page *pg)
        spin_lock_irqsave(&dm_device.ha_lock, flags);
        list_for_each_entry(has, &dm_device.ha_region_list, list) {
                /* The page belongs to a different HAS. */
-               if ((pfn < has->start_pfn) || (pfn >= has->end_pfn))
+               if ((pfn < has->start_pfn) ||
+                               (pfn + (1UL << order) > has->end_pfn))
                        continue;
 
-               hv_page_online_one(has, pg);
+               hv_bring_pgs_online(has, pfn, 1UL << order);
                break;
        }
        spin_unlock_irqrestore(&dm_device.ha_lock, flags);

diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index ceb5048de9a7eb8f2e861380964ebf3c15f8a23a..d107447c47de40a1b22c0a3101fa8f0a45af3359 100644 (file)
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -369,14 +369,19 @@ static enum bp_state reserve_additional_memory(void)
        return BP_ECANCELED;
 }
 
-static void xen_online_page(struct page *page)
+static void xen_online_page(struct page *page, unsigned int order)
 {
-       __online_page_set_limits(page);
+       unsigned long i, size = (1 << order);
+       unsigned long start_pfn = page_to_pfn(page);
+       struct page *p;
 
+       pr_debug("Online %lu pages starting at pfn 0x%lx\n", size, start_pfn);
        mutex_lock(&balloon_mutex);
-
-       __balloon_append(page);
-
+       for (i = 0; i < size; i++) {
+               p = pfn_to_page(start_pfn + i);
+               __online_page_set_limits(p);
+               __balloon_append(p);
+       }
        mutex_unlock(&balloon_mutex);
 }
 

diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index 368267c1b71b101f3d1d9927197014e2a24efa85..52869d6d38b37307a36280654aa0d289eb773d10 100644 (file)
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -89,7 +89,7 @@ extern int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn,
        unsigned long *valid_start, unsigned long *valid_end);
 extern void __offline_isolated_pages(unsigned long, unsigned long);
 
-typedef void (*online_page_callback_t)(struct page *page);
+typedef void (*online_page_callback_t)(struct page *page, unsigned int order);
 
 extern int set_online_page_callback(online_page_callback_t callback);
 extern int restore_online_page_callback(online_page_callback_t callback);

diff --git a/mm/internal.h b/mm/internal.h
index f4a7bb02decfea1a5509c0a96cc8aa653ebc816c..536bc2a839b97010762d9347cf120ef25b2efb2a 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -163,6 +163,7 @@ static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
 extern int __isolate_free_page(struct page *page, unsigned int order);
 extern void memblock_free_pages(struct page *page, unsigned long pfn,
                                        unsigned int order);
+extern void __free_pages_core(struct page *page, unsigned int order);
 extern void prep_compound_page(struct page *page, unsigned int order);
 extern void post_alloc_hook(struct page *page, unsigned int order,
                                        gfp_t gfp_flags);

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 1ad28323fb9faceabb98cfc9f1b3834b365919bb..4f07c8ddfdd79e32a0f307579db9d8acdf899699 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -47,7 +47,7 @@
  * and restore_online_page_callback() for generic callback restore.
  */
 
-static void generic_online_page(struct page *page);
+static void generic_online_page(struct page *page, unsigned int order);
 
 static online_page_callback_t online_page_callback = generic_online_page;
 static DEFINE_MUTEX(online_page_callback_lock);
@@ -656,26 +656,39 @@ void __online_page_free(struct page *page)
 }
 EXPORT_SYMBOL_GPL(__online_page_free);
 
-static void generic_online_page(struct page *page)
+static void generic_online_page(struct page *page, unsigned int order)
 {
-       __online_page_set_limits(page);
-       __online_page_increment_counters(page);
-       __online_page_free(page);
+       __free_pages_core(page, order);
+       totalram_pages_add(1UL << order);
+#ifdef CONFIG_HIGHMEM
+       if (PageHighMem(page))
+               totalhigh_pages_add(1UL << order);
+#endif
+}
+
+static int online_pages_blocks(unsigned long start, unsigned long nr_pages)
+{
+       unsigned long end = start + nr_pages;
+       int order, onlined_pages = 0;
+
+       while (start < end) {
+               order = min(MAX_ORDER - 1,
+                       get_order(PFN_PHYS(end) - PFN_PHYS(start)));
+               (*online_page_callback)(pfn_to_page(start), order);
+
+               onlined_pages += (1UL << order);
+               start += (1UL << order);
+       }
+       return onlined_pages;
 }
 
 static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
                        void *arg)
 {
-       unsigned long i;
        unsigned long onlined_pages = *(unsigned long *)arg;
-       struct page *page;
 
        if (PageReserved(pfn_to_page(start_pfn)))
-               for (i = 0; i < nr_pages; i++) {
-                       page = pfn_to_page(start_pfn + i);
-                       (*online_page_callback)(page);
-                       onlined_pages++;
-               }
+               onlined_pages += online_pages_blocks(start_pfn, nr_pages);
 
        online_mem_sections(start_pfn, start_pfn + nr_pages);
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 10d0f2ed9f6988926d29b636923703b70470132a..5361bd078493029784be400bfc21fe937fa29cc0 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1303,7 +1303,7 @@ static void __free_pages_ok(struct page *page, unsigned int order)
        local_irq_restore(flags);
 }
 
-static void __init __free_pages_boot_core(struct page *page, unsigned int order)
+void __free_pages_core(struct page *page, unsigned int order)
 {
        unsigned int nr_pages = 1 << order;
        struct page *p = page;
@@ -1382,7 +1382,7 @@ void __init memblock_free_pages(struct page *page, unsigned long pfn,
 {
        if (early_page_uninitialised(pfn))
                return;
-       return __free_pages_boot_core(page, order);
+       __free_pages_core(page, order);
 }
 
 /*
@@ -1472,14 +1472,14 @@ static void __init deferred_free_range(unsigned long pfn,
        if (nr_pages == pageblock_nr_pages &&
            (pfn & (pageblock_nr_pages - 1)) == 0) {
                set_pageblock_migratetype(page, MIGRATE_MOVABLE);
-               __free_pages_boot_core(page, pageblock_order);
+               __free_pages_core(page, pageblock_order);
                return;
        }
 
        for (i = 0; i < nr_pages; i++, page++, pfn++) {
                if ((pfn & (pageblock_nr_pages - 1)) == 0)
                        set_pageblock_migratetype(page, MIGRATE_MOVABLE);
-               __free_pages_boot_core(page, 0);
+               __free_pages_core(page, 0);
        }
 }