]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - mm/page_alloc.c
media: rc: fix lots of documentation warnings
[mirror_ubuntu-bionic-kernel.git] / mm / page_alloc.c
1 /*
2 * linux/mm/page_alloc.c
3 *
4 * Manages the free list, the system allocates free pages here.
5 * Note that kmalloc() lives in slab.c
6 *
7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * Swap reorganised 29.12.95, Stephen Tweedie
9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
15 */
16
17 #include <linux/stddef.h>
18 #include <linux/mm.h>
19 #include <linux/swap.h>
20 #include <linux/interrupt.h>
21 #include <linux/pagemap.h>
22 #include <linux/jiffies.h>
23 #include <linux/bootmem.h>
24 #include <linux/memblock.h>
25 #include <linux/compiler.h>
26 #include <linux/kernel.h>
27 #include <linux/kasan.h>
28 #include <linux/module.h>
29 #include <linux/suspend.h>
30 #include <linux/pagevec.h>
31 #include <linux/blkdev.h>
32 #include <linux/slab.h>
33 #include <linux/ratelimit.h>
34 #include <linux/oom.h>
35 #include <linux/notifier.h>
36 #include <linux/topology.h>
37 #include <linux/sysctl.h>
38 #include <linux/cpu.h>
39 #include <linux/cpuset.h>
40 #include <linux/memory_hotplug.h>
41 #include <linux/nodemask.h>
42 #include <linux/vmalloc.h>
43 #include <linux/vmstat.h>
44 #include <linux/mempolicy.h>
45 #include <linux/memremap.h>
46 #include <linux/stop_machine.h>
47 #include <linux/sort.h>
48 #include <linux/pfn.h>
49 #include <linux/backing-dev.h>
50 #include <linux/fault-inject.h>
51 #include <linux/page-isolation.h>
52 #include <linux/page_ext.h>
53 #include <linux/debugobjects.h>
54 #include <linux/kmemleak.h>
55 #include <linux/compaction.h>
56 #include <trace/events/kmem.h>
57 #include <trace/events/oom.h>
58 #include <linux/prefetch.h>
59 #include <linux/mm_inline.h>
60 #include <linux/migrate.h>
61 #include <linux/hugetlb.h>
62 #include <linux/sched/rt.h>
63 #include <linux/sched/mm.h>
64 #include <linux/page_owner.h>
65 #include <linux/kthread.h>
66 #include <linux/memcontrol.h>
67 #include <linux/ftrace.h>
68 #include <linux/lockdep.h>
69 #include <linux/nmi.h>
70
71 #include <asm/sections.h>
72 #include <asm/tlbflush.h>
73 #include <asm/div64.h>
74 #include "internal.h"
75
76 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
77 static DEFINE_MUTEX(pcp_batch_high_lock);
78 #define MIN_PERCPU_PAGELIST_FRACTION (8)
79
80 #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
81 DEFINE_PER_CPU(int, numa_node);
82 EXPORT_PER_CPU_SYMBOL(numa_node);
83 #endif
84
85 DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
86
87 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
88 /*
89 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
90 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
91 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
92 * defined in <linux/topology.h>.
93 */
94 DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
95 EXPORT_PER_CPU_SYMBOL(_numa_mem_);
96 int _node_numa_mem_[MAX_NUMNODES];
97 #endif
98
99 /* work_structs for global per-cpu drains */
100 DEFINE_MUTEX(pcpu_drain_mutex);
101 DEFINE_PER_CPU(struct work_struct, pcpu_drain);
102
103 #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
104 volatile unsigned long latent_entropy __latent_entropy;
105 EXPORT_SYMBOL(latent_entropy);
106 #endif
107
108 /*
109 * Array of node states.
110 */
111 nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
112 [N_POSSIBLE] = NODE_MASK_ALL,
113 [N_ONLINE] = { { [0] = 1UL } },
114 #ifndef CONFIG_NUMA
115 [N_NORMAL_MEMORY] = { { [0] = 1UL } },
116 #ifdef CONFIG_HIGHMEM
117 [N_HIGH_MEMORY] = { { [0] = 1UL } },
118 #endif
119 [N_MEMORY] = { { [0] = 1UL } },
120 [N_CPU] = { { [0] = 1UL } },
121 #endif /* NUMA */
122 };
123 EXPORT_SYMBOL(node_states);
124
125 /* Protect totalram_pages and zone->managed_pages */
126 static DEFINE_SPINLOCK(managed_page_count_lock);
127
128 unsigned long totalram_pages __read_mostly;
129 unsigned long totalreserve_pages __read_mostly;
130 unsigned long totalcma_pages __read_mostly;
131
132 int percpu_pagelist_fraction;
133 gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
134
135 /*
136 * A cached value of the page's pageblock's migratetype, used when the page is
137 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
138 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
139 * Also the migratetype set in the page does not necessarily match the pcplist
140 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
141 * other index - this ensures that it will be put on the correct CMA freelist.
142 */
143 static inline int get_pcppage_migratetype(struct page *page)
144 {
145 return page->index;
146 }
147
148 static inline void set_pcppage_migratetype(struct page *page, int migratetype)
149 {
150 page->index = migratetype;
151 }
152
153 #ifdef CONFIG_PM_SLEEP
154 /*
155 * The following functions are used by the suspend/hibernate code to temporarily
156 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
157 * while devices are suspended. To avoid races with the suspend/hibernate code,
158 * they should always be called with pm_mutex held (gfp_allowed_mask also should
159 * only be modified with pm_mutex held, unless the suspend/hibernate code is
160 * guaranteed not to run in parallel with that modification).
161 */
162
163 static gfp_t saved_gfp_mask;
164
165 void pm_restore_gfp_mask(void)
166 {
167 WARN_ON(!mutex_is_locked(&pm_mutex));
168 if (saved_gfp_mask) {
169 gfp_allowed_mask = saved_gfp_mask;
170 saved_gfp_mask = 0;
171 }
172 }
173
174 void pm_restrict_gfp_mask(void)
175 {
176 WARN_ON(!mutex_is_locked(&pm_mutex));
177 WARN_ON(saved_gfp_mask);
178 saved_gfp_mask = gfp_allowed_mask;
179 gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
180 }
181
182 bool pm_suspended_storage(void)
183 {
184 if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
185 return false;
186 return true;
187 }
188 #endif /* CONFIG_PM_SLEEP */
189
190 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
191 unsigned int pageblock_order __read_mostly;
192 #endif
193
194 static void __free_pages_ok(struct page *page, unsigned int order);
195
196 /*
197 * results with 256, 32 in the lowmem_reserve sysctl:
198 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
199 * 1G machine -> (16M dma, 784M normal, 224M high)
200 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
201 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
202 * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
203 *
204 * TBD: should special case ZONE_DMA32 machines here - in those we normally
205 * don't need any ZONE_NORMAL reservation
206 */
207 int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
208 #ifdef CONFIG_ZONE_DMA
209 256,
210 #endif
211 #ifdef CONFIG_ZONE_DMA32
212 256,
213 #endif
214 #ifdef CONFIG_HIGHMEM
215 32,
216 #endif
217 32,
218 };
219
220 EXPORT_SYMBOL(totalram_pages);
221
222 static char * const zone_names[MAX_NR_ZONES] = {
223 #ifdef CONFIG_ZONE_DMA
224 "DMA",
225 #endif
226 #ifdef CONFIG_ZONE_DMA32
227 "DMA32",
228 #endif
229 "Normal",
230 #ifdef CONFIG_HIGHMEM
231 "HighMem",
232 #endif
233 "Movable",
234 #ifdef CONFIG_ZONE_DEVICE
235 "Device",
236 #endif
237 };
238
239 char * const migratetype_names[MIGRATE_TYPES] = {
240 "Unmovable",
241 "Movable",
242 "Reclaimable",
243 "HighAtomic",
244 #ifdef CONFIG_CMA
245 "CMA",
246 #endif
247 #ifdef CONFIG_MEMORY_ISOLATION
248 "Isolate",
249 #endif
250 };
251
252 compound_page_dtor * const compound_page_dtors[] = {
253 NULL,
254 free_compound_page,
255 #ifdef CONFIG_HUGETLB_PAGE
256 free_huge_page,
257 #endif
258 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
259 free_transhuge_page,
260 #endif
261 };
262
263 int min_free_kbytes = 1024;
264 int user_min_free_kbytes = -1;
265 int watermark_scale_factor = 10;
266
267 static unsigned long __meminitdata nr_kernel_pages;
268 static unsigned long __meminitdata nr_all_pages;
269 static unsigned long __meminitdata dma_reserve;
270
271 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
272 static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
273 static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
274 static unsigned long __initdata required_kernelcore;
275 static unsigned long __initdata required_movablecore;
276 static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
277 static bool mirrored_kernelcore;
278
279 /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
280 int movable_zone;
281 EXPORT_SYMBOL(movable_zone);
282 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
283
284 #if MAX_NUMNODES > 1
285 int nr_node_ids __read_mostly = MAX_NUMNODES;
286 int nr_online_nodes __read_mostly = 1;
287 EXPORT_SYMBOL(nr_node_ids);
288 EXPORT_SYMBOL(nr_online_nodes);
289 #endif
290
291 int page_group_by_mobility_disabled __read_mostly;
292
293 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
294
295 /*
296 * Determine how many pages need to be initialized durig early boot
297 * (non-deferred initialization).
298 * The value of first_deferred_pfn will be set later, once non-deferred pages
299 * are initialized, but for now set it ULONG_MAX.
300 */
301 static inline void reset_deferred_meminit(pg_data_t *pgdat)
302 {
303 phys_addr_t start_addr, end_addr;
304 unsigned long max_pgcnt;
305 unsigned long reserved;
306
307 /*
308 * Initialise at least 2G of a node but also take into account that
309 * two large system hashes that can take up 1GB for 0.25TB/node.
310 */
311 max_pgcnt = max(2UL << (30 - PAGE_SHIFT),
312 (pgdat->node_spanned_pages >> 8));
313
314 /*
315 * Compensate the all the memblock reservations (e.g. crash kernel)
316 * from the initial estimation to make sure we will initialize enough
317 * memory to boot.
318 */
319 start_addr = PFN_PHYS(pgdat->node_start_pfn);
320 end_addr = PFN_PHYS(pgdat->node_start_pfn + max_pgcnt);
321 reserved = memblock_reserved_memory_within(start_addr, end_addr);
322 max_pgcnt += PHYS_PFN(reserved);
323
324 pgdat->static_init_pgcnt = min(max_pgcnt, pgdat->node_spanned_pages);
325 pgdat->first_deferred_pfn = ULONG_MAX;
326 }
327
328 /* Returns true if the struct page for the pfn is uninitialised */
329 static inline bool __meminit early_page_uninitialised(unsigned long pfn)
330 {
331 int nid = early_pfn_to_nid(pfn);
332
333 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
334 return true;
335
336 return false;
337 }
338
339 /*
340 * Returns false when the remaining initialisation should be deferred until
341 * later in the boot cycle when it can be parallelised.
342 */
343 static inline bool update_defer_init(pg_data_t *pgdat,
344 unsigned long pfn, unsigned long zone_end,
345 unsigned long *nr_initialised)
346 {
347 /* Always populate low zones for address-contrained allocations */
348 if (zone_end < pgdat_end_pfn(pgdat))
349 return true;
350 (*nr_initialised)++;
351 if ((*nr_initialised > pgdat->static_init_pgcnt) &&
352 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
353 pgdat->first_deferred_pfn = pfn;
354 return false;
355 }
356
357 return true;
358 }
359 #else
360 static inline void reset_deferred_meminit(pg_data_t *pgdat)
361 {
362 }
363
364 static inline bool early_page_uninitialised(unsigned long pfn)
365 {
366 return false;
367 }
368
369 static inline bool update_defer_init(pg_data_t *pgdat,
370 unsigned long pfn, unsigned long zone_end,
371 unsigned long *nr_initialised)
372 {
373 return true;
374 }
375 #endif
376
377 /* Return a pointer to the bitmap storing bits affecting a block of pages */
378 static inline unsigned long *get_pageblock_bitmap(struct page *page,
379 unsigned long pfn)
380 {
381 #ifdef CONFIG_SPARSEMEM
382 return __pfn_to_section(pfn)->pageblock_flags;
383 #else
384 return page_zone(page)->pageblock_flags;
385 #endif /* CONFIG_SPARSEMEM */
386 }
387
388 static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
389 {
390 #ifdef CONFIG_SPARSEMEM
391 pfn &= (PAGES_PER_SECTION-1);
392 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
393 #else
394 pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
395 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
396 #endif /* CONFIG_SPARSEMEM */
397 }
398
399 /**
400 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
401 * @page: The page within the block of interest
402 * @pfn: The target page frame number
403 * @end_bitidx: The last bit of interest to retrieve
404 * @mask: mask of bits that the caller is interested in
405 *
406 * Return: pageblock_bits flags
407 */
408 static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
409 unsigned long pfn,
410 unsigned long end_bitidx,
411 unsigned long mask)
412 {
413 unsigned long *bitmap;
414 unsigned long bitidx, word_bitidx;
415 unsigned long word;
416
417 bitmap = get_pageblock_bitmap(page, pfn);
418 bitidx = pfn_to_bitidx(page, pfn);
419 word_bitidx = bitidx / BITS_PER_LONG;
420 bitidx &= (BITS_PER_LONG-1);
421
422 word = bitmap[word_bitidx];
423 bitidx += end_bitidx;
424 return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
425 }
426
427 unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
428 unsigned long end_bitidx,
429 unsigned long mask)
430 {
431 return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
432 }
433
434 static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
435 {
436 return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
437 }
438
439 /**
440 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
441 * @page: The page within the block of interest
442 * @flags: The flags to set
443 * @pfn: The target page frame number
444 * @end_bitidx: The last bit of interest
445 * @mask: mask of bits that the caller is interested in
446 */
447 void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
448 unsigned long pfn,
449 unsigned long end_bitidx,
450 unsigned long mask)
451 {
452 unsigned long *bitmap;
453 unsigned long bitidx, word_bitidx;
454 unsigned long old_word, word;
455
456 BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
457
458 bitmap = get_pageblock_bitmap(page, pfn);
459 bitidx = pfn_to_bitidx(page, pfn);
460 word_bitidx = bitidx / BITS_PER_LONG;
461 bitidx &= (BITS_PER_LONG-1);
462
463 VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
464
465 bitidx += end_bitidx;
466 mask <<= (BITS_PER_LONG - bitidx - 1);
467 flags <<= (BITS_PER_LONG - bitidx - 1);
468
469 word = READ_ONCE(bitmap[word_bitidx]);
470 for (;;) {
471 old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
472 if (word == old_word)
473 break;
474 word = old_word;
475 }
476 }
477
478 void set_pageblock_migratetype(struct page *page, int migratetype)
479 {
480 if (unlikely(page_group_by_mobility_disabled &&
481 migratetype < MIGRATE_PCPTYPES))
482 migratetype = MIGRATE_UNMOVABLE;
483
484 set_pageblock_flags_group(page, (unsigned long)migratetype,
485 PB_migrate, PB_migrate_end);
486 }
487
488 #ifdef CONFIG_DEBUG_VM
489 static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
490 {
491 int ret = 0;
492 unsigned seq;
493 unsigned long pfn = page_to_pfn(page);
494 unsigned long sp, start_pfn;
495
496 do {
497 seq = zone_span_seqbegin(zone);
498 start_pfn = zone->zone_start_pfn;
499 sp = zone->spanned_pages;
500 if (!zone_spans_pfn(zone, pfn))
501 ret = 1;
502 } while (zone_span_seqretry(zone, seq));
503
504 if (ret)
505 pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
506 pfn, zone_to_nid(zone), zone->name,
507 start_pfn, start_pfn + sp);
508
509 return ret;
510 }
511
512 static int page_is_consistent(struct zone *zone, struct page *page)
513 {
514 if (!pfn_valid_within(page_to_pfn(page)))
515 return 0;
516 if (zone != page_zone(page))
517 return 0;
518
519 return 1;
520 }
521 /*
522 * Temporary debugging check for pages not lying within a given zone.
523 */
524 static int __maybe_unused bad_range(struct zone *zone, struct page *page)
525 {
526 if (page_outside_zone_boundaries(zone, page))
527 return 1;
528 if (!page_is_consistent(zone, page))
529 return 1;
530
531 return 0;
532 }
533 #else
534 static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
535 {
536 return 0;
537 }
538 #endif
539
540 static void bad_page(struct page *page, const char *reason,
541 unsigned long bad_flags)
542 {
543 static unsigned long resume;
544 static unsigned long nr_shown;
545 static unsigned long nr_unshown;
546
547 /*
548 * Allow a burst of 60 reports, then keep quiet for that minute;
549 * or allow a steady drip of one report per second.
550 */
551 if (nr_shown == 60) {
552 if (time_before(jiffies, resume)) {
553 nr_unshown++;
554 goto out;
555 }
556 if (nr_unshown) {
557 pr_alert(
558 "BUG: Bad page state: %lu messages suppressed\n",
559 nr_unshown);
560 nr_unshown = 0;
561 }
562 nr_shown = 0;
563 }
564 if (nr_shown++ == 0)
565 resume = jiffies + 60 * HZ;
566
567 pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
568 current->comm, page_to_pfn(page));
569 __dump_page(page, reason);
570 bad_flags &= page->flags;
571 if (bad_flags)
572 pr_alert("bad because of flags: %#lx(%pGp)\n",
573 bad_flags, &bad_flags);
574 dump_page_owner(page);
575
576 print_modules();
577 dump_stack();
578 out:
579 /* Leave bad fields for debug, except PageBuddy could make trouble */
580 page_mapcount_reset(page); /* remove PageBuddy */
581 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
582 }
583
584 /*
585 * Higher-order pages are called "compound pages". They are structured thusly:
586 *
587 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
588 *
589 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
590 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
591 *
592 * The first tail page's ->compound_dtor holds the offset in array of compound
593 * page destructors. See compound_page_dtors.
594 *
595 * The first tail page's ->compound_order holds the order of allocation.
596 * This usage means that zero-order pages may not be compound.
597 */
598
599 void free_compound_page(struct page *page)
600 {
601 __free_pages_ok(page, compound_order(page));
602 }
603
604 void prep_compound_page(struct page *page, unsigned int order)
605 {
606 int i;
607 int nr_pages = 1 << order;
608
609 set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
610 set_compound_order(page, order);
611 __SetPageHead(page);
612 for (i = 1; i < nr_pages; i++) {
613 struct page *p = page + i;
614 set_page_count(p, 0);
615 p->mapping = TAIL_MAPPING;
616 set_compound_head(p, page);
617 }
618 atomic_set(compound_mapcount_ptr(page), -1);
619 }
620
621 #ifdef CONFIG_DEBUG_PAGEALLOC
622 unsigned int _debug_guardpage_minorder;
623 bool _debug_pagealloc_enabled __read_mostly
624 = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
625 EXPORT_SYMBOL(_debug_pagealloc_enabled);
626 bool _debug_guardpage_enabled __read_mostly;
627
628 static int __init early_debug_pagealloc(char *buf)
629 {
630 if (!buf)
631 return -EINVAL;
632 return kstrtobool(buf, &_debug_pagealloc_enabled);
633 }
634 early_param("debug_pagealloc", early_debug_pagealloc);
635
636 static bool need_debug_guardpage(void)
637 {
638 /* If we don't use debug_pagealloc, we don't need guard page */
639 if (!debug_pagealloc_enabled())
640 return false;
641
642 if (!debug_guardpage_minorder())
643 return false;
644
645 return true;
646 }
647
648 static void init_debug_guardpage(void)
649 {
650 if (!debug_pagealloc_enabled())
651 return;
652
653 if (!debug_guardpage_minorder())
654 return;
655
656 _debug_guardpage_enabled = true;
657 }
658
659 struct page_ext_operations debug_guardpage_ops = {
660 .need = need_debug_guardpage,
661 .init = init_debug_guardpage,
662 };
663
664 static int __init debug_guardpage_minorder_setup(char *buf)
665 {
666 unsigned long res;
667
668 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
669 pr_err("Bad debug_guardpage_minorder value\n");
670 return 0;
671 }
672 _debug_guardpage_minorder = res;
673 pr_info("Setting debug_guardpage_minorder to %lu\n", res);
674 return 0;
675 }
676 early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
677
678 static inline bool set_page_guard(struct zone *zone, struct page *page,
679 unsigned int order, int migratetype)
680 {
681 struct page_ext *page_ext;
682
683 if (!debug_guardpage_enabled())
684 return false;
685
686 if (order >= debug_guardpage_minorder())
687 return false;
688
689 page_ext = lookup_page_ext(page);
690 if (unlikely(!page_ext))
691 return false;
692
693 __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
694
695 INIT_LIST_HEAD(&page->lru);
696 set_page_private(page, order);
697 /* Guard pages are not available for any usage */
698 __mod_zone_freepage_state(zone, -(1 << order), migratetype);
699
700 return true;
701 }
702
703 static inline void clear_page_guard(struct zone *zone, struct page *page,
704 unsigned int order, int migratetype)
705 {
706 struct page_ext *page_ext;
707
708 if (!debug_guardpage_enabled())
709 return;
710
711 page_ext = lookup_page_ext(page);
712 if (unlikely(!page_ext))
713 return;
714
715 __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
716
717 set_page_private(page, 0);
718 if (!is_migrate_isolate(migratetype))
719 __mod_zone_freepage_state(zone, (1 << order), migratetype);
720 }
721 #else
722 struct page_ext_operations debug_guardpage_ops;
723 static inline bool set_page_guard(struct zone *zone, struct page *page,
724 unsigned int order, int migratetype) { return false; }
725 static inline void clear_page_guard(struct zone *zone, struct page *page,
726 unsigned int order, int migratetype) {}
727 #endif
728
729 static inline void set_page_order(struct page *page, unsigned int order)
730 {
731 set_page_private(page, order);
732 __SetPageBuddy(page);
733 }
734
735 static inline void rmv_page_order(struct page *page)
736 {
737 __ClearPageBuddy(page);
738 set_page_private(page, 0);
739 }
740
741 /*
742 * This function checks whether a page is free && is the buddy
743 * we can do coalesce a page and its buddy if
744 * (a) the buddy is not in a hole (check before calling!) &&
745 * (b) the buddy is in the buddy system &&
746 * (c) a page and its buddy have the same order &&
747 * (d) a page and its buddy are in the same zone.
748 *
749 * For recording whether a page is in the buddy system, we set ->_mapcount
750 * PAGE_BUDDY_MAPCOUNT_VALUE.
751 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
752 * serialized by zone->lock.
753 *
754 * For recording page's order, we use page_private(page).
755 */
756 static inline int page_is_buddy(struct page *page, struct page *buddy,
757 unsigned int order)
758 {
759 if (page_is_guard(buddy) && page_order(buddy) == order) {
760 if (page_zone_id(page) != page_zone_id(buddy))
761 return 0;
762
763 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
764
765 return 1;
766 }
767
768 if (PageBuddy(buddy) && page_order(buddy) == order) {
769 /*
770 * zone check is done late to avoid uselessly
771 * calculating zone/node ids for pages that could
772 * never merge.
773 */
774 if (page_zone_id(page) != page_zone_id(buddy))
775 return 0;
776
777 VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
778
779 return 1;
780 }
781 return 0;
782 }
783
784 /*
785 * Freeing function for a buddy system allocator.
786 *
787 * The concept of a buddy system is to maintain direct-mapped table
788 * (containing bit values) for memory blocks of various "orders".
789 * The bottom level table contains the map for the smallest allocatable
790 * units of memory (here, pages), and each level above it describes
791 * pairs of units from the levels below, hence, "buddies".
792 * At a high level, all that happens here is marking the table entry
793 * at the bottom level available, and propagating the changes upward
794 * as necessary, plus some accounting needed to play nicely with other
795 * parts of the VM system.
796 * At each level, we keep a list of pages, which are heads of continuous
797 * free pages of length of (1 << order) and marked with _mapcount
798 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
799 * field.
800 * So when we are allocating or freeing one, we can derive the state of the
801 * other. That is, if we allocate a small block, and both were
802 * free, the remainder of the region must be split into blocks.
803 * If a block is freed, and its buddy is also free, then this
804 * triggers coalescing into a block of larger size.
805 *
806 * -- nyc
807 */
808
809 static inline void __free_one_page(struct page *page,
810 unsigned long pfn,
811 struct zone *zone, unsigned int order,
812 int migratetype)
813 {
814 unsigned long combined_pfn;
815 unsigned long uninitialized_var(buddy_pfn);
816 struct page *buddy;
817 unsigned int max_order;
818
819 max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);
820
821 VM_BUG_ON(!zone_is_initialized(zone));
822 VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
823
824 VM_BUG_ON(migratetype == -1);
825 if (likely(!is_migrate_isolate(migratetype)))
826 __mod_zone_freepage_state(zone, 1 << order, migratetype);
827
828 VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
829 VM_BUG_ON_PAGE(bad_range(zone, page), page);
830
831 continue_merging:
832 while (order < max_order - 1) {
833 buddy_pfn = __find_buddy_pfn(pfn, order);
834 buddy = page + (buddy_pfn - pfn);
835
836 if (!pfn_valid_within(buddy_pfn))
837 goto done_merging;
838 if (!page_is_buddy(page, buddy, order))
839 goto done_merging;
840 /*
841 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
842 * merge with it and move up one order.
843 */
844 if (page_is_guard(buddy)) {
845 clear_page_guard(zone, buddy, order, migratetype);
846 } else {
847 list_del(&buddy->lru);
848 zone->free_area[order].nr_free--;
849 rmv_page_order(buddy);
850 }
851 combined_pfn = buddy_pfn & pfn;
852 page = page + (combined_pfn - pfn);
853 pfn = combined_pfn;
854 order++;
855 }
856 if (max_order < MAX_ORDER) {
857 /* If we are here, it means order is >= pageblock_order.
858 * We want to prevent merge between freepages on isolate
859 * pageblock and normal pageblock. Without this, pageblock
860 * isolation could cause incorrect freepage or CMA accounting.
861 *
862 * We don't want to hit this code for the more frequent
863 * low-order merging.
864 */
865 if (unlikely(has_isolate_pageblock(zone))) {
866 int buddy_mt;
867
868 buddy_pfn = __find_buddy_pfn(pfn, order);
869 buddy = page + (buddy_pfn - pfn);
870 buddy_mt = get_pageblock_migratetype(buddy);
871
872 if (migratetype != buddy_mt
873 && (is_migrate_isolate(migratetype) ||
874 is_migrate_isolate(buddy_mt)))
875 goto done_merging;
876 }
877 max_order++;
878 goto continue_merging;
879 }
880
881 done_merging:
882 set_page_order(page, order);
883
884 /*
885 * If this is not the largest possible page, check if the buddy
886 * of the next-highest order is free. If it is, it's possible
887 * that pages are being freed that will coalesce soon. In case,
888 * that is happening, add the free page to the tail of the list
889 * so it's less likely to be used soon and more likely to be merged
890 * as a higher order page
891 */
892 if ((order < MAX_ORDER-2) && pfn_valid_within(buddy_pfn)) {
893 struct page *higher_page, *higher_buddy;
894 combined_pfn = buddy_pfn & pfn;
895 higher_page = page + (combined_pfn - pfn);
896 buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
897 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
898 if (pfn_valid_within(buddy_pfn) &&
899 page_is_buddy(higher_page, higher_buddy, order + 1)) {
900 list_add_tail(&page->lru,
901 &zone->free_area[order].free_list[migratetype]);
902 goto out;
903 }
904 }
905
906 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
907 out:
908 zone->free_area[order].nr_free++;
909 }
910
911 /*
912 * A bad page could be due to a number of fields. Instead of multiple branches,
913 * try and check multiple fields with one check. The caller must do a detailed
914 * check if necessary.
915 */
916 static inline bool page_expected_state(struct page *page,
917 unsigned long check_flags)
918 {
919 if (unlikely(atomic_read(&page->_mapcount) != -1))
920 return false;
921
922 if (unlikely((unsigned long)page->mapping |
923 page_ref_count(page) |
924 #ifdef CONFIG_MEMCG
925 (unsigned long)page->mem_cgroup |
926 #endif
927 (page->flags & check_flags)))
928 return false;
929
930 return true;
931 }
932
933 static void free_pages_check_bad(struct page *page)
934 {
935 const char *bad_reason;
936 unsigned long bad_flags;
937
938 bad_reason = NULL;
939 bad_flags = 0;
940
941 if (unlikely(atomic_read(&page->_mapcount) != -1))
942 bad_reason = "nonzero mapcount";
943 if (unlikely(page->mapping != NULL))
944 bad_reason = "non-NULL mapping";
945 if (unlikely(page_ref_count(page) != 0))
946 bad_reason = "nonzero _refcount";
947 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
948 bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
949 bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
950 }
951 #ifdef CONFIG_MEMCG
952 if (unlikely(page->mem_cgroup))
953 bad_reason = "page still charged to cgroup";
954 #endif
955 bad_page(page, bad_reason, bad_flags);
956 }
957
958 static inline int free_pages_check(struct page *page)
959 {
960 if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
961 return 0;
962
963 /* Something has gone sideways, find it */
964 free_pages_check_bad(page);
965 return 1;
966 }
967
968 static int free_tail_pages_check(struct page *head_page, struct page *page)
969 {
970 int ret = 1;
971
972 /*
973 * We rely page->lru.next never has bit 0 set, unless the page
974 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
975 */
976 BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
977
978 if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
979 ret = 0;
980 goto out;
981 }
982 switch (page - head_page) {
983 case 1:
984 /* the first tail page: ->mapping is compound_mapcount() */
985 if (unlikely(compound_mapcount(page))) {
986 bad_page(page, "nonzero compound_mapcount", 0);
987 goto out;
988 }
989 break;
990 case 2:
991 /*
992 * the second tail page: ->mapping is
993 * page_deferred_list().next -- ignore value.
994 */
995 break;
996 default:
997 if (page->mapping != TAIL_MAPPING) {
998 bad_page(page, "corrupted mapping in tail page", 0);
999 goto out;
1000 }
1001 break;
1002 }
1003 if (unlikely(!PageTail(page))) {
1004 bad_page(page, "PageTail not set", 0);
1005 goto out;
1006 }
1007 if (unlikely(compound_head(page) != head_page)) {
1008 bad_page(page, "compound_head not consistent", 0);
1009 goto out;
1010 }
1011 ret = 0;
1012 out:
1013 page->mapping = NULL;
1014 clear_compound_head(page);
1015 return ret;
1016 }
1017
1018 static __always_inline bool free_pages_prepare(struct page *page,
1019 unsigned int order, bool check_free)
1020 {
1021 int bad = 0;
1022
1023 VM_BUG_ON_PAGE(PageTail(page), page);
1024
1025 trace_mm_page_free(page, order);
1026
1027 /*
1028 * Check tail pages before head page information is cleared to
1029 * avoid checking PageCompound for order-0 pages.
1030 */
1031 if (unlikely(order)) {
1032 bool compound = PageCompound(page);
1033 int i;
1034
1035 VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1036
1037 if (compound)
1038 ClearPageDoubleMap(page);
1039 for (i = 1; i < (1 << order); i++) {
1040 if (compound)
1041 bad += free_tail_pages_check(page, page + i);
1042 if (unlikely(free_pages_check(page + i))) {
1043 bad++;
1044 continue;
1045 }
1046 (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1047 }
1048 }
1049 if (PageMappingFlags(page))
1050 page->mapping = NULL;
1051 if (memcg_kmem_enabled() && PageKmemcg(page))
1052 memcg_kmem_uncharge(page, order);
1053 if (check_free)
1054 bad += free_pages_check(page);
1055 if (bad)
1056 return false;
1057
1058 page_cpupid_reset_last(page);
1059 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1060 reset_page_owner(page, order);
1061
1062 if (!PageHighMem(page)) {
1063 debug_check_no_locks_freed(page_address(page),
1064 PAGE_SIZE << order);
1065 debug_check_no_obj_freed(page_address(page),
1066 PAGE_SIZE << order);
1067 }
1068 arch_free_page(page, order);
1069 kernel_poison_pages(page, 1 << order, 0);
1070 kernel_map_pages(page, 1 << order, 0);
1071 kasan_free_pages(page, order);
1072
1073 return true;
1074 }
1075
1076 #ifdef CONFIG_DEBUG_VM
1077 static inline bool free_pcp_prepare(struct page *page)
1078 {
1079 return free_pages_prepare(page, 0, true);
1080 }
1081
1082 static inline bool bulkfree_pcp_prepare(struct page *page)
1083 {
1084 return false;
1085 }
1086 #else
1087 static bool free_pcp_prepare(struct page *page)
1088 {
1089 return free_pages_prepare(page, 0, false);
1090 }
1091
1092 static bool bulkfree_pcp_prepare(struct page *page)
1093 {
1094 return free_pages_check(page);
1095 }
1096 #endif /* CONFIG_DEBUG_VM */
1097
1098 /*
1099 * Frees a number of pages from the PCP lists
1100 * Assumes all pages on list are in same zone, and of same order.
1101 * count is the number of pages to free.
1102 *
1103 * If the zone was previously in an "all pages pinned" state then look to
1104 * see if this freeing clears that state.
1105 *
1106 * And clear the zone's pages_scanned counter, to hold off the "all pages are
1107 * pinned" detection logic.
1108 */
1109 static void free_pcppages_bulk(struct zone *zone, int count,
1110 struct per_cpu_pages *pcp)
1111 {
1112 int migratetype = 0;
1113 int batch_free = 0;
1114 bool isolated_pageblocks;
1115
1116 spin_lock(&zone->lock);
1117 isolated_pageblocks = has_isolate_pageblock(zone);
1118
1119 while (count) {
1120 struct page *page;
1121 struct list_head *list;
1122
1123 /*
1124 * Remove pages from lists in a round-robin fashion. A
1125 * batch_free count is maintained that is incremented when an
1126 * empty list is encountered. This is so more pages are freed
1127 * off fuller lists instead of spinning excessively around empty
1128 * lists
1129 */
1130 do {
1131 batch_free++;
1132 if (++migratetype == MIGRATE_PCPTYPES)
1133 migratetype = 0;
1134 list = &pcp->lists[migratetype];
1135 } while (list_empty(list));
1136
1137 /* This is the only non-empty list. Free them all. */
1138 if (batch_free == MIGRATE_PCPTYPES)
1139 batch_free = count;
1140
1141 do {
1142 int mt; /* migratetype of the to-be-freed page */
1143
1144 page = list_last_entry(list, struct page, lru);
1145 /* must delete as __free_one_page list manipulates */
1146 list_del(&page->lru);
1147
1148 mt = get_pcppage_migratetype(page);
1149 /* MIGRATE_ISOLATE page should not go to pcplists */
1150 VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
1151 /* Pageblock could have been isolated meanwhile */
1152 if (unlikely(isolated_pageblocks))
1153 mt = get_pageblock_migratetype(page);
1154
1155 if (bulkfree_pcp_prepare(page))
1156 continue;
1157
1158 __free_one_page(page, page_to_pfn(page), zone, 0, mt);
1159 trace_mm_page_pcpu_drain(page, 0, mt);
1160 } while (--count && --batch_free && !list_empty(list));
1161 }
1162 spin_unlock(&zone->lock);
1163 }
1164
1165 static void free_one_page(struct zone *zone,
1166 struct page *page, unsigned long pfn,
1167 unsigned int order,
1168 int migratetype)
1169 {
1170 spin_lock(&zone->lock);
1171 if (unlikely(has_isolate_pageblock(zone) ||
1172 is_migrate_isolate(migratetype))) {
1173 migratetype = get_pfnblock_migratetype(page, pfn);
1174 }
1175 __free_one_page(page, pfn, zone, order, migratetype);
1176 spin_unlock(&zone->lock);
1177 }
1178
1179 static void __meminit __init_single_page(struct page *page, unsigned long pfn,
1180 unsigned long zone, int nid)
1181 {
1182 mm_zero_struct_page(page);
1183 set_page_links(page, zone, nid, pfn);
1184 init_page_count(page);
1185 page_mapcount_reset(page);
1186 page_cpupid_reset_last(page);
1187
1188 INIT_LIST_HEAD(&page->lru);
1189 #ifdef WANT_PAGE_VIRTUAL
1190 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
1191 if (!is_highmem_idx(zone))
1192 set_page_address(page, __va(pfn << PAGE_SHIFT));
1193 #endif
1194 }
1195
1196 static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone,
1197 int nid)
1198 {
1199 return __init_single_page(pfn_to_page(pfn), pfn, zone, nid);
1200 }
1201
1202 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1203 static void __meminit init_reserved_page(unsigned long pfn)
1204 {
1205 pg_data_t *pgdat;
1206 int nid, zid;
1207
1208 if (!early_page_uninitialised(pfn))
1209 return;
1210
1211 nid = early_pfn_to_nid(pfn);
1212 pgdat = NODE_DATA(nid);
1213
1214 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1215 struct zone *zone = &pgdat->node_zones[zid];
1216
1217 if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
1218 break;
1219 }
1220 __init_single_pfn(pfn, zid, nid);
1221 }
1222 #else
1223 static inline void init_reserved_page(unsigned long pfn)
1224 {
1225 }
1226 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1227
1228 /*
1229 * Initialised pages do not have PageReserved set. This function is
1230 * called for each range allocated by the bootmem allocator and
1231 * marks the pages PageReserved. The remaining valid pages are later
1232 * sent to the buddy page allocator.
1233 */
1234 void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
1235 {
1236 unsigned long start_pfn = PFN_DOWN(start);
1237 unsigned long end_pfn = PFN_UP(end);
1238
1239 for (; start_pfn < end_pfn; start_pfn++) {
1240 if (pfn_valid(start_pfn)) {
1241 struct page *page = pfn_to_page(start_pfn);
1242
1243 init_reserved_page(start_pfn);
1244
1245 /* Avoid false-positive PageTail() */
1246 INIT_LIST_HEAD(&page->lru);
1247
1248 SetPageReserved(page);
1249 }
1250 }
1251 }
1252
1253 static void __free_pages_ok(struct page *page, unsigned int order)
1254 {
1255 unsigned long flags;
1256 int migratetype;
1257 unsigned long pfn = page_to_pfn(page);
1258
1259 if (!free_pages_prepare(page, order, true))
1260 return;
1261
1262 migratetype = get_pfnblock_migratetype(page, pfn);
1263 local_irq_save(flags);
1264 __count_vm_events(PGFREE, 1 << order);
1265 free_one_page(page_zone(page), page, pfn, order, migratetype);
1266 local_irq_restore(flags);
1267 }
1268
1269 static void __init __free_pages_boot_core(struct page *page, unsigned int order)
1270 {
1271 unsigned int nr_pages = 1 << order;
1272 struct page *p = page;
1273 unsigned int loop;
1274
1275 prefetchw(p);
1276 for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
1277 prefetchw(p + 1);
1278 __ClearPageReserved(p);
1279 set_page_count(p, 0);
1280 }
1281 __ClearPageReserved(p);
1282 set_page_count(p, 0);
1283
1284 page_zone(page)->managed_pages += nr_pages;
1285 set_page_refcounted(page);
1286 __free_pages(page, order);
1287 }
1288
1289 #if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
1290 defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1291
1292 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
1293
1294 int __meminit early_pfn_to_nid(unsigned long pfn)
1295 {
1296 static DEFINE_SPINLOCK(early_pfn_lock);
1297 int nid;
1298
1299 spin_lock(&early_pfn_lock);
1300 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
1301 if (nid < 0)
1302 nid = first_online_node;
1303 spin_unlock(&early_pfn_lock);
1304
1305 return nid;
1306 }
1307 #endif
1308
1309 #ifdef CONFIG_NODES_SPAN_OTHER_NODES
1310 static inline bool __meminit __maybe_unused
1311 meminit_pfn_in_nid(unsigned long pfn, int node,
1312 struct mminit_pfnnid_cache *state)
1313 {
1314 int nid;
1315
1316 nid = __early_pfn_to_nid(pfn, state);
1317 if (nid >= 0 && nid != node)
1318 return false;
1319 return true;
1320 }
1321
1322 /* Only safe to use early in boot when initialisation is single-threaded */
1323 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1324 {
1325 return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache);
1326 }
1327
1328 #else
1329
1330 static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
1331 {
1332 return true;
1333 }
1334 static inline bool __meminit __maybe_unused
1335 meminit_pfn_in_nid(unsigned long pfn, int node,
1336 struct mminit_pfnnid_cache *state)
1337 {
1338 return true;
1339 }
1340 #endif
1341
1342
1343 void __init __free_pages_bootmem(struct page *page, unsigned long pfn,
1344 unsigned int order)
1345 {
1346 if (early_page_uninitialised(pfn))
1347 return;
1348 return __free_pages_boot_core(page, order);
1349 }
1350
1351 /*
1352 * Check that the whole (or subset of) a pageblock given by the interval of
1353 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1354 * with the migration of free compaction scanner. The scanners then need to
1355 * use only pfn_valid_within() check for arches that allow holes within
1356 * pageblocks.
1357 *
1358 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1359 *
1360 * It's possible on some configurations to have a setup like node0 node1 node0
1361 * i.e. it's possible that all pages within a zones range of pages do not
1362 * belong to a single zone. We assume that a border between node0 and node1
1363 * can occur within a single pageblock, but not a node0 node1 node0
1364 * interleaving within a single pageblock. It is therefore sufficient to check
1365 * the first and last page of a pageblock and avoid checking each individual
1366 * page in a pageblock.
1367 */
1368 struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1369 unsigned long end_pfn, struct zone *zone)
1370 {
1371 struct page *start_page;
1372 struct page *end_page;
1373
1374 /* end_pfn is one past the range we are checking */
1375 end_pfn--;
1376
1377 if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
1378 return NULL;
1379
1380 start_page = pfn_to_online_page(start_pfn);
1381 if (!start_page)
1382 return NULL;
1383
1384 if (page_zone(start_page) != zone)
1385 return NULL;
1386
1387 end_page = pfn_to_page(end_pfn);
1388
1389 /* This gives a shorter code than deriving page_zone(end_page) */
1390 if (page_zone_id(start_page) != page_zone_id(end_page))
1391 return NULL;
1392
1393 return start_page;
1394 }
1395
1396 void set_zone_contiguous(struct zone *zone)
1397 {
1398 unsigned long block_start_pfn = zone->zone_start_pfn;
1399 unsigned long block_end_pfn;
1400
1401 block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
1402 for (; block_start_pfn < zone_end_pfn(zone);
1403 block_start_pfn = block_end_pfn,
1404 block_end_pfn += pageblock_nr_pages) {
1405
1406 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
1407
1408 if (!__pageblock_pfn_to_page(block_start_pfn,
1409 block_end_pfn, zone))
1410 return;
1411 }
1412
1413 /* We confirm that there is no hole */
1414 zone->contiguous = true;
1415 }
1416
1417 void clear_zone_contiguous(struct zone *zone)
1418 {
1419 zone->contiguous = false;
1420 }
1421
1422 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1423 static void __init deferred_free_range(unsigned long pfn,
1424 unsigned long nr_pages)
1425 {
1426 struct page *page;
1427 unsigned long i;
1428
1429 if (!nr_pages)
1430 return;
1431
1432 page = pfn_to_page(pfn);
1433
1434 /* Free a large naturally-aligned chunk if possible */
1435 if (nr_pages == pageblock_nr_pages &&
1436 (pfn & (pageblock_nr_pages - 1)) == 0) {
1437 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1438 __free_pages_boot_core(page, pageblock_order);
1439 return;
1440 }
1441
1442 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1443 if ((pfn & (pageblock_nr_pages - 1)) == 0)
1444 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1445 __free_pages_boot_core(page, 0);
1446 }
1447 }
1448
1449 /* Completion tracking for deferred_init_memmap() threads */
1450 static atomic_t pgdat_init_n_undone __initdata;
1451 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1452
1453 static inline void __init pgdat_init_report_one_done(void)
1454 {
1455 if (atomic_dec_and_test(&pgdat_init_n_undone))
1456 complete(&pgdat_init_all_done_comp);
1457 }
1458
1459 /*
1460 * Helper for deferred_init_range, free the given range, reset the counters, and
1461 * return number of pages freed.
1462 */
1463 static inline unsigned long __init __def_free(unsigned long *nr_free,
1464 unsigned long *free_base_pfn,
1465 struct page **page)
1466 {
1467 unsigned long nr = *nr_free;
1468
1469 deferred_free_range(*free_base_pfn, nr);
1470 *free_base_pfn = 0;
1471 *nr_free = 0;
1472 *page = NULL;
1473
1474 return nr;
1475 }
1476
1477 static unsigned long __init deferred_init_range(int nid, int zid,
1478 unsigned long start_pfn,
1479 unsigned long end_pfn)
1480 {
1481 struct mminit_pfnnid_cache nid_init_state = { };
1482 unsigned long nr_pgmask = pageblock_nr_pages - 1;
1483 unsigned long free_base_pfn = 0;
1484 unsigned long nr_pages = 0;
1485 unsigned long nr_free = 0;
1486 struct page *page = NULL;
1487 unsigned long pfn;
1488
1489 /*
1490 * First we check if pfn is valid on architectures where it is possible
1491 * to have holes within pageblock_nr_pages. On systems where it is not
1492 * possible, this function is optimized out.
1493 *
1494 * Then, we check if a current large page is valid by only checking the
1495 * validity of the head pfn.
1496 *
1497 * meminit_pfn_in_nid is checked on systems where pfns can interleave
1498 * within a node: a pfn is between start and end of a node, but does not
1499 * belong to this memory node.
1500 *
1501 * Finally, we minimize pfn page lookups and scheduler checks by
1502 * performing it only once every pageblock_nr_pages.
1503 *
1504 * We do it in two loops: first we initialize struct page, than free to
1505 * buddy allocator, becuse while we are freeing pages we can access
1506 * pages that are ahead (computing buddy page in __free_one_page()).
1507 */
1508 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1509 if (!pfn_valid_within(pfn))
1510 continue;
1511 if ((pfn & nr_pgmask) || pfn_valid(pfn)) {
1512 if (meminit_pfn_in_nid(pfn, nid, &nid_init_state)) {
1513 if (page && (pfn & nr_pgmask))
1514 page++;
1515 else
1516 page = pfn_to_page(pfn);
1517 __init_single_page(page, pfn, zid, nid);
1518 cond_resched();
1519 }
1520 }
1521 }
1522
1523 page = NULL;
1524 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1525 if (!pfn_valid_within(pfn)) {
1526 nr_pages += __def_free(&nr_free, &free_base_pfn, &page);
1527 } else if (!(pfn & nr_pgmask) && !pfn_valid(pfn)) {
1528 nr_pages += __def_free(&nr_free, &free_base_pfn, &page);
1529 } else if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) {
1530 nr_pages += __def_free(&nr_free, &free_base_pfn, &page);
1531 } else if (page && (pfn & nr_pgmask)) {
1532 page++;
1533 nr_free++;
1534 } else {
1535 nr_pages += __def_free(&nr_free, &free_base_pfn, &page);
1536 page = pfn_to_page(pfn);
1537 free_base_pfn = pfn;
1538 nr_free = 1;
1539 cond_resched();
1540 }
1541 }
1542 /* Free the last block of pages to allocator */
1543 nr_pages += __def_free(&nr_free, &free_base_pfn, &page);
1544
1545 return nr_pages;
1546 }
1547
1548 /* Initialise remaining memory on a node */
1549 static int __init deferred_init_memmap(void *data)
1550 {
1551 pg_data_t *pgdat = data;
1552 int nid = pgdat->node_id;
1553 unsigned long start = jiffies;
1554 unsigned long nr_pages = 0;
1555 unsigned long spfn, epfn;
1556 phys_addr_t spa, epa;
1557 int zid;
1558 struct zone *zone;
1559 unsigned long first_init_pfn = pgdat->first_deferred_pfn;
1560 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
1561 u64 i;
1562
1563 if (first_init_pfn == ULONG_MAX) {
1564 pgdat_init_report_one_done();
1565 return 0;
1566 }
1567
1568 /* Bind memory initialisation thread to a local node if possible */
1569 if (!cpumask_empty(cpumask))
1570 set_cpus_allowed_ptr(current, cpumask);
1571
1572 /* Sanity check boundaries */
1573 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
1574 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
1575 pgdat->first_deferred_pfn = ULONG_MAX;
1576
1577 /* Only the highest zone is deferred so find it */
1578 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1579 zone = pgdat->node_zones + zid;
1580 if (first_init_pfn < zone_end_pfn(zone))
1581 break;
1582 }
1583 first_init_pfn = max(zone->zone_start_pfn, first_init_pfn);
1584
1585 for_each_free_mem_range(i, nid, MEMBLOCK_NONE, &spa, &epa, NULL) {
1586 spfn = max_t(unsigned long, first_init_pfn, PFN_UP(spa));
1587 epfn = min_t(unsigned long, zone_end_pfn(zone), PFN_DOWN(epa));
1588 nr_pages += deferred_init_range(nid, zid, spfn, epfn);
1589 }
1590
1591 /* Sanity check that the next zone really is unpopulated */
1592 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
1593
1594 pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
1595 jiffies_to_msecs(jiffies - start));
1596
1597 pgdat_init_report_one_done();
1598 return 0;
1599 }
1600 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1601
1602 void __init page_alloc_init_late(void)
1603 {
1604 struct zone *zone;
1605
1606 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1607 int nid;
1608
1609 /* There will be num_node_state(N_MEMORY) threads */
1610 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
1611 for_each_node_state(nid, N_MEMORY) {
1612 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
1613 }
1614
1615 /* Block until all are initialised */
1616 wait_for_completion(&pgdat_init_all_done_comp);
1617
1618 /* Reinit limits that are based on free pages after the kernel is up */
1619 files_maxfiles_init();
1620 #endif
1621 #ifdef CONFIG_ARCH_DISCARD_MEMBLOCK
1622 /* Discard memblock private memory */
1623 memblock_discard();
1624 #endif
1625
1626 for_each_populated_zone(zone)
1627 set_zone_contiguous(zone);
1628 }
1629
1630 #ifdef CONFIG_CMA
1631 /* Free whole pageblock and set its migration type to MIGRATE_CMA. */
1632 void __init init_cma_reserved_pageblock(struct page *page)
1633 {
1634 unsigned i = pageblock_nr_pages;
1635 struct page *p = page;
1636
1637 do {
1638 __ClearPageReserved(p);
1639 set_page_count(p, 0);
1640 } while (++p, --i);
1641
1642 set_pageblock_migratetype(page, MIGRATE_CMA);
1643
1644 if (pageblock_order >= MAX_ORDER) {
1645 i = pageblock_nr_pages;
1646 p = page;
1647 do {
1648 set_page_refcounted(p);
1649 __free_pages(p, MAX_ORDER - 1);
1650 p += MAX_ORDER_NR_PAGES;
1651 } while (i -= MAX_ORDER_NR_PAGES);
1652 } else {
1653 set_page_refcounted(page);
1654 __free_pages(page, pageblock_order);
1655 }
1656
1657 adjust_managed_page_count(page, pageblock_nr_pages);
1658 }
1659 #endif
1660
1661 /*
1662 * The order of subdivision here is critical for the IO subsystem.
1663 * Please do not alter this order without good reasons and regression
1664 * testing. Specifically, as large blocks of memory are subdivided,
1665 * the order in which smaller blocks are delivered depends on the order
1666 * they're subdivided in this function. This is the primary factor
1667 * influencing the order in which pages are delivered to the IO
1668 * subsystem according to empirical testing, and this is also justified
1669 * by considering the behavior of a buddy system containing a single
1670 * large block of memory acted on by a series of small allocations.
1671 * This behavior is a critical factor in sglist merging's success.
1672 *
1673 * -- nyc
1674 */
1675 static inline void expand(struct zone *zone, struct page *page,
1676 int low, int high, struct free_area *area,
1677 int migratetype)
1678 {
1679 unsigned long size = 1 << high;
1680
1681 while (high > low) {
1682 area--;
1683 high--;
1684 size >>= 1;
1685 VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
1686
1687 /*
1688 * Mark as guard pages (or page), that will allow to
1689 * merge back to allocator when buddy will be freed.
1690 * Corresponding page table entries will not be touched,
1691 * pages will stay not present in virtual address space
1692 */
1693 if (set_page_guard(zone, &page[size], high, migratetype))
1694 continue;
1695
1696 list_add(&page[size].lru, &area->free_list[migratetype]);
1697 area->nr_free++;
1698 set_page_order(&page[size], high);
1699 }
1700 }
1701
1702 static void check_new_page_bad(struct page *page)
1703 {
1704 const char *bad_reason = NULL;
1705 unsigned long bad_flags = 0;
1706
1707 if (unlikely(atomic_read(&page->_mapcount) != -1))
1708 bad_reason = "nonzero mapcount";
1709 if (unlikely(page->mapping != NULL))
1710 bad_reason = "non-NULL mapping";
1711 if (unlikely(page_ref_count(page) != 0))
1712 bad_reason = "nonzero _count";
1713 if (unlikely(page->flags & __PG_HWPOISON)) {
1714 bad_reason = "HWPoisoned (hardware-corrupted)";
1715 bad_flags = __PG_HWPOISON;
1716 /* Don't complain about hwpoisoned pages */
1717 page_mapcount_reset(page); /* remove PageBuddy */
1718 return;
1719 }
1720 if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
1721 bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
1722 bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
1723 }
1724 #ifdef CONFIG_MEMCG
1725 if (unlikely(page->mem_cgroup))
1726 bad_reason = "page still charged to cgroup";
1727 #endif
1728 bad_page(page, bad_reason, bad_flags);
1729 }
1730
1731 /*
1732 * This page is about to be returned from the page allocator
1733 */
1734 static inline int check_new_page(struct page *page)
1735 {
1736 if (likely(page_expected_state(page,
1737 PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
1738 return 0;
1739
1740 check_new_page_bad(page);
1741 return 1;
1742 }
1743
1744 static inline bool free_pages_prezeroed(void)
1745 {
1746 return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
1747 page_poisoning_enabled();
1748 }
1749
1750 #ifdef CONFIG_DEBUG_VM
1751 static bool check_pcp_refill(struct page *page)
1752 {
1753 return false;
1754 }
1755
1756 static bool check_new_pcp(struct page *page)
1757 {
1758 return check_new_page(page);
1759 }
1760 #else
1761 static bool check_pcp_refill(struct page *page)
1762 {
1763 return check_new_page(page);
1764 }
1765 static bool check_new_pcp(struct page *page)
1766 {
1767 return false;
1768 }
1769 #endif /* CONFIG_DEBUG_VM */
1770
1771 static bool check_new_pages(struct page *page, unsigned int order)
1772 {
1773 int i;
1774 for (i = 0; i < (1 << order); i++) {
1775 struct page *p = page + i;
1776
1777 if (unlikely(check_new_page(p)))
1778 return true;
1779 }
1780
1781 return false;
1782 }
1783
1784 inline void post_alloc_hook(struct page *page, unsigned int order,
1785 gfp_t gfp_flags)
1786 {
1787 set_page_private(page, 0);
1788 set_page_refcounted(page);
1789
1790 arch_alloc_page(page, order);
1791 kernel_map_pages(page, 1 << order, 1);
1792 kernel_poison_pages(page, 1 << order, 1);
1793 kasan_alloc_pages(page, order);
1794 set_page_owner(page, order, gfp_flags);
1795 }
1796
1797 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
1798 unsigned int alloc_flags)
1799 {
1800 int i;
1801
1802 post_alloc_hook(page, order, gfp_flags);
1803
1804 if (!free_pages_prezeroed() && (gfp_flags & __GFP_ZERO))
1805 for (i = 0; i < (1 << order); i++)
1806 clear_highpage(page + i);
1807
1808 if (order && (gfp_flags & __GFP_COMP))
1809 prep_compound_page(page, order);
1810
1811 /*
1812 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
1813 * allocate the page. The expectation is that the caller is taking
1814 * steps that will free more memory. The caller should avoid the page
1815 * being used for !PFMEMALLOC purposes.
1816 */
1817 if (alloc_flags & ALLOC_NO_WATERMARKS)
1818 set_page_pfmemalloc(page);
1819 else
1820 clear_page_pfmemalloc(page);
1821 }
1822
1823 /*
1824 * Go through the free lists for the given migratetype and remove
1825 * the smallest available page from the freelists
1826 */
1827 static __always_inline
1828 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
1829 int migratetype)
1830 {
1831 unsigned int current_order;
1832 struct free_area *area;
1833 struct page *page;
1834
1835 /* Find a page of the appropriate size in the preferred list */
1836 for (current_order = order; current_order < MAX_ORDER; ++current_order) {
1837 area = &(zone->free_area[current_order]);
1838 page = list_first_entry_or_null(&area->free_list[migratetype],
1839 struct page, lru);
1840 if (!page)
1841 continue;
1842 list_del(&page->lru);
1843 rmv_page_order(page);
1844 area->nr_free--;
1845 expand(zone, page, order, current_order, area, migratetype);
1846 set_pcppage_migratetype(page, migratetype);
1847 return page;
1848 }
1849
1850 return NULL;
1851 }
1852
1853
1854 /*
1855 * This array describes the order lists are fallen back to when
1856 * the free lists for the desirable migrate type are depleted
1857 */
1858 static int fallbacks[MIGRATE_TYPES][4] = {
1859 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
1860 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
1861 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
1862 #ifdef CONFIG_CMA
1863 [MIGRATE_CMA] = { MIGRATE_TYPES }, /* Never used */
1864 #endif
1865 #ifdef CONFIG_MEMORY_ISOLATION
1866 [MIGRATE_ISOLATE] = { MIGRATE_TYPES }, /* Never used */
1867 #endif
1868 };
1869
1870 #ifdef CONFIG_CMA
1871 static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
1872 unsigned int order)
1873 {
1874 return __rmqueue_smallest(zone, order, MIGRATE_CMA);
1875 }
1876 #else
1877 static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
1878 unsigned int order) { return NULL; }
1879 #endif
1880
1881 /*
1882 * Move the free pages in a range to the free lists of the requested type.
1883 * Note that start_page and end_pages are not aligned on a pageblock
1884 * boundary. If alignment is required, use move_freepages_block()
1885 */
1886 static int move_freepages(struct zone *zone,
1887 struct page *start_page, struct page *end_page,
1888 int migratetype, int *num_movable)
1889 {
1890 struct page *page;
1891 unsigned int order;
1892 int pages_moved = 0;
1893
1894 #ifndef CONFIG_HOLES_IN_ZONE
1895 /*
1896 * page_zone is not safe to call in this context when
1897 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
1898 * anyway as we check zone boundaries in move_freepages_block().
1899 * Remove at a later date when no bug reports exist related to
1900 * grouping pages by mobility
1901 */
1902 VM_BUG_ON(page_zone(start_page) != page_zone(end_page));
1903 #endif
1904
1905 if (num_movable)
1906 *num_movable = 0;
1907
1908 for (page = start_page; page <= end_page;) {
1909 if (!pfn_valid_within(page_to_pfn(page))) {
1910 page++;
1911 continue;
1912 }
1913
1914 /* Make sure we are not inadvertently changing nodes */
1915 VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
1916
1917 if (!PageBuddy(page)) {
1918 /*
1919 * We assume that pages that could be isolated for
1920 * migration are movable. But we don't actually try
1921 * isolating, as that would be expensive.
1922 */
1923 if (num_movable &&
1924 (PageLRU(page) || __PageMovable(page)))
1925 (*num_movable)++;
1926
1927 page++;
1928 continue;
1929 }
1930
1931 order = page_order(page);
1932 list_move(&page->lru,
1933 &zone->free_area[order].free_list[migratetype]);
1934 page += 1 << order;
1935 pages_moved += 1 << order;
1936 }
1937
1938 return pages_moved;
1939 }
1940
1941 int move_freepages_block(struct zone *zone, struct page *page,
1942 int migratetype, int *num_movable)
1943 {
1944 unsigned long start_pfn, end_pfn;
1945 struct page *start_page, *end_page;
1946
1947 start_pfn = page_to_pfn(page);
1948 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
1949 start_page = pfn_to_page(start_pfn);
1950 end_page = start_page + pageblock_nr_pages - 1;
1951 end_pfn = start_pfn + pageblock_nr_pages - 1;
1952
1953 /* Do not cross zone boundaries */
1954 if (!zone_spans_pfn(zone, start_pfn))
1955 start_page = page;
1956 if (!zone_spans_pfn(zone, end_pfn))
1957 return 0;
1958
1959 return move_freepages(zone, start_page, end_page, migratetype,
1960 num_movable);
1961 }
1962
1963 static void change_pageblock_range(struct page *pageblock_page,
1964 int start_order, int migratetype)
1965 {
1966 int nr_pageblocks = 1 << (start_order - pageblock_order);
1967
1968 while (nr_pageblocks--) {
1969 set_pageblock_migratetype(pageblock_page, migratetype);
1970 pageblock_page += pageblock_nr_pages;
1971 }
1972 }
1973
1974 /*
1975 * When we are falling back to another migratetype during allocation, try to
1976 * steal extra free pages from the same pageblocks to satisfy further
1977 * allocations, instead of polluting multiple pageblocks.
1978 *
1979 * If we are stealing a relatively large buddy page, it is likely there will
1980 * be more free pages in the pageblock, so try to steal them all. For
1981 * reclaimable and unmovable allocations, we steal regardless of page size,
1982 * as fragmentation caused by those allocations polluting movable pageblocks
1983 * is worse than movable allocations stealing from unmovable and reclaimable
1984 * pageblocks.
1985 */
1986 static bool can_steal_fallback(unsigned int order, int start_mt)
1987 {
1988 /*
1989 * Leaving this order check is intended, although there is
1990 * relaxed order check in next check. The reason is that
1991 * we can actually steal whole pageblock if this condition met,
1992 * but, below check doesn't guarantee it and that is just heuristic
1993 * so could be changed anytime.
1994 */
1995 if (order >= pageblock_order)
1996 return true;
1997
1998 if (order >= pageblock_order / 2 ||
1999 start_mt == MIGRATE_RECLAIMABLE ||
2000 start_mt == MIGRATE_UNMOVABLE ||
2001 page_group_by_mobility_disabled)
2002 return true;
2003
2004 return false;
2005 }
2006
2007 /*
2008 * This function implements actual steal behaviour. If order is large enough,
2009 * we can steal whole pageblock. If not, we first move freepages in this
2010 * pageblock to our migratetype and determine how many already-allocated pages
2011 * are there in the pageblock with a compatible migratetype. If at least half
2012 * of pages are free or compatible, we can change migratetype of the pageblock
2013 * itself, so pages freed in the future will be put on the correct free list.
2014 */
2015 static void steal_suitable_fallback(struct zone *zone, struct page *page,
2016 int start_type, bool whole_block)
2017 {
2018 unsigned int current_order = page_order(page);
2019 struct free_area *area;
2020 int free_pages, movable_pages, alike_pages;
2021 int old_block_type;
2022
2023 old_block_type = get_pageblock_migratetype(page);
2024
2025 /*
2026 * This can happen due to races and we want to prevent broken
2027 * highatomic accounting.
2028 */
2029 if (is_migrate_highatomic(old_block_type))
2030 goto single_page;
2031
2032 /* Take ownership for orders >= pageblock_order */
2033 if (current_order >= pageblock_order) {
2034 change_pageblock_range(page, current_order, start_type);
2035 goto single_page;
2036 }
2037
2038 /* We are not allowed to try stealing from the whole block */
2039 if (!whole_block)
2040 goto single_page;
2041
2042 free_pages = move_freepages_block(zone, page, start_type,
2043 &movable_pages);
2044 /*
2045 * Determine how many pages are compatible with our allocation.
2046 * For movable allocation, it's the number of movable pages which
2047 * we just obtained. For other types it's a bit more tricky.
2048 */
2049 if (start_type == MIGRATE_MOVABLE) {
2050 alike_pages = movable_pages;
2051 } else {
2052 /*
2053 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2054 * to MOVABLE pageblock, consider all non-movable pages as
2055 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2056 * vice versa, be conservative since we can't distinguish the
2057 * exact migratetype of non-movable pages.
2058 */
2059 if (old_block_type == MIGRATE_MOVABLE)
2060 alike_pages = pageblock_nr_pages
2061 - (free_pages + movable_pages);
2062 else
2063 alike_pages = 0;
2064 }
2065
2066 /* moving whole block can fail due to zone boundary conditions */
2067 if (!free_pages)
2068 goto single_page;
2069
2070 /*
2071 * If a sufficient number of pages in the block are either free or of
2072 * comparable migratability as our allocation, claim the whole block.
2073 */
2074 if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2075 page_group_by_mobility_disabled)
2076 set_pageblock_migratetype(page, start_type);
2077
2078 return;
2079
2080 single_page:
2081 area = &zone->free_area[current_order];
2082 list_move(&page->lru, &area->free_list[start_type]);
2083 }
2084
2085 /*
2086 * Check whether there is a suitable fallback freepage with requested order.
2087 * If only_stealable is true, this function returns fallback_mt only if
2088 * we can steal other freepages all together. This would help to reduce
2089 * fragmentation due to mixed migratetype pages in one pageblock.
2090 */
2091 int find_suitable_fallback(struct free_area *area, unsigned int order,
2092 int migratetype, bool only_stealable, bool *can_steal)
2093 {
2094 int i;
2095 int fallback_mt;
2096
2097 if (area->nr_free == 0)
2098 return -1;
2099
2100 *can_steal = false;
2101 for (i = 0;; i++) {
2102 fallback_mt = fallbacks[migratetype][i];
2103 if (fallback_mt == MIGRATE_TYPES)
2104 break;
2105
2106 if (list_empty(&area->free_list[fallback_mt]))
2107 continue;
2108
2109 if (can_steal_fallback(order, migratetype))
2110 *can_steal = true;
2111
2112 if (!only_stealable)
2113 return fallback_mt;
2114
2115 if (*can_steal)
2116 return fallback_mt;
2117 }
2118
2119 return -1;
2120 }
2121
2122 /*
2123 * Reserve a pageblock for exclusive use of high-order atomic allocations if
2124 * there are no empty page blocks that contain a page with a suitable order
2125 */
2126 static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
2127 unsigned int alloc_order)
2128 {
2129 int mt;
2130 unsigned long max_managed, flags;
2131
2132 /*
2133 * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
2134 * Check is race-prone but harmless.
2135 */
2136 max_managed = (zone->managed_pages / 100) + pageblock_nr_pages;
2137 if (zone->nr_reserved_highatomic >= max_managed)
2138 return;
2139
2140 spin_lock_irqsave(&zone->lock, flags);
2141
2142 /* Recheck the nr_reserved_highatomic limit under the lock */
2143 if (zone->nr_reserved_highatomic >= max_managed)
2144 goto out_unlock;
2145
2146 /* Yoink! */
2147 mt = get_pageblock_migratetype(page);
2148 if (!is_migrate_highatomic(mt) && !is_migrate_isolate(mt)
2149 && !is_migrate_cma(mt)) {
2150 zone->nr_reserved_highatomic += pageblock_nr_pages;
2151 set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
2152 move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
2153 }
2154
2155 out_unlock:
2156 spin_unlock_irqrestore(&zone->lock, flags);
2157 }
2158
2159 /*
2160 * Used when an allocation is about to fail under memory pressure. This
2161 * potentially hurts the reliability of high-order allocations when under
2162 * intense memory pressure but failed atomic allocations should be easier
2163 * to recover from than an OOM.
2164 *
2165 * If @force is true, try to unreserve a pageblock even though highatomic
2166 * pageblock is exhausted.
2167 */
2168 static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
2169 bool force)
2170 {
2171 struct zonelist *zonelist = ac->zonelist;
2172 unsigned long flags;
2173 struct zoneref *z;
2174 struct zone *zone;
2175 struct page *page;
2176 int order;
2177 bool ret;
2178
2179 for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
2180 ac->nodemask) {
2181 /*
2182 * Preserve at least one pageblock unless memory pressure
2183 * is really high.
2184 */
2185 if (!force && zone->nr_reserved_highatomic <=
2186 pageblock_nr_pages)
2187 continue;
2188
2189 spin_lock_irqsave(&zone->lock, flags);
2190 for (order = 0; order < MAX_ORDER; order++) {
2191 struct free_area *area = &(zone->free_area[order]);
2192
2193 page = list_first_entry_or_null(
2194 &area->free_list[MIGRATE_HIGHATOMIC],
2195 struct page, lru);
2196 if (!page)
2197 continue;
2198
2199 /*
2200 * In page freeing path, migratetype change is racy so
2201 * we can counter several free pages in a pageblock
2202 * in this loop althoug we changed the pageblock type
2203 * from highatomic to ac->migratetype. So we should
2204 * adjust the count once.
2205 */
2206 if (is_migrate_highatomic_page(page)) {
2207 /*
2208 * It should never happen but changes to
2209 * locking could inadvertently allow a per-cpu
2210 * drain to add pages to MIGRATE_HIGHATOMIC
2211 * while unreserving so be safe and watch for
2212 * underflows.
2213 */
2214 zone->nr_reserved_highatomic -= min(
2215 pageblock_nr_pages,
2216 zone->nr_reserved_highatomic);
2217 }
2218
2219 /*
2220 * Convert to ac->migratetype and avoid the normal
2221 * pageblock stealing heuristics. Minimally, the caller
2222 * is doing the work and needs the pages. More
2223 * importantly, if the block was always converted to
2224 * MIGRATE_UNMOVABLE or another type then the number
2225 * of pageblocks that cannot be completely freed
2226 * may increase.
2227 */
2228 set_pageblock_migratetype(page, ac->migratetype);
2229 ret = move_freepages_block(zone, page, ac->migratetype,
2230 NULL);
2231 if (ret) {
2232 spin_unlock_irqrestore(&zone->lock, flags);
2233 return ret;
2234 }
2235 }
2236 spin_unlock_irqrestore(&zone->lock, flags);
2237 }
2238
2239 return false;
2240 }
2241
2242 /*
2243 * Try finding a free buddy page on the fallback list and put it on the free
2244 * list of requested migratetype, possibly along with other pages from the same
2245 * block, depending on fragmentation avoidance heuristics. Returns true if
2246 * fallback was found so that __rmqueue_smallest() can grab it.
2247 *
2248 * The use of signed ints for order and current_order is a deliberate
2249 * deviation from the rest of this file, to make the for loop
2250 * condition simpler.
2251 */
2252 static __always_inline bool
2253 __rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
2254 {
2255 struct free_area *area;
2256 int current_order;
2257 struct page *page;
2258 int fallback_mt;
2259 bool can_steal;
2260
2261 /*
2262 * Find the largest available free page in the other list. This roughly
2263 * approximates finding the pageblock with the most free pages, which
2264 * would be too costly to do exactly.
2265 */
2266 for (current_order = MAX_ORDER - 1; current_order >= order;
2267 --current_order) {
2268 area = &(zone->free_area[current_order]);
2269 fallback_mt = find_suitable_fallback(area, current_order,
2270 start_migratetype, false, &can_steal);
2271 if (fallback_mt == -1)
2272 continue;
2273
2274 /*
2275 * We cannot steal all free pages from the pageblock and the
2276 * requested migratetype is movable. In that case it's better to
2277 * steal and split the smallest available page instead of the
2278 * largest available page, because even if the next movable
2279 * allocation falls back into a different pageblock than this
2280 * one, it won't cause permanent fragmentation.
2281 */
2282 if (!can_steal && start_migratetype == MIGRATE_MOVABLE
2283 && current_order > order)
2284 goto find_smallest;
2285
2286 goto do_steal;
2287 }
2288
2289 return false;
2290
2291 find_smallest:
2292 for (current_order = order; current_order < MAX_ORDER;
2293 current_order++) {
2294 area = &(zone->free_area[current_order]);
2295 fallback_mt = find_suitable_fallback(area, current_order,
2296 start_migratetype, false, &can_steal);
2297 if (fallback_mt != -1)
2298 break;
2299 }
2300
2301 /*
2302 * This should not happen - we already found a suitable fallback
2303 * when looking for the largest page.
2304 */
2305 VM_BUG_ON(current_order == MAX_ORDER);
2306
2307 do_steal:
2308 page = list_first_entry(&area->free_list[fallback_mt],
2309 struct page, lru);
2310
2311 steal_suitable_fallback(zone, page, start_migratetype, can_steal);
2312
2313 trace_mm_page_alloc_extfrag(page, order, current_order,
2314 start_migratetype, fallback_mt);
2315
2316 return true;
2317
2318 }
2319
2320 /*
2321 * Do the hard work of removing an element from the buddy allocator.
2322 * Call me with the zone->lock already held.
2323 */
2324 static __always_inline struct page *
2325 __rmqueue(struct zone *zone, unsigned int order, int migratetype)
2326 {
2327 struct page *page;
2328
2329 retry:
2330 page = __rmqueue_smallest(zone, order, migratetype);
2331 if (unlikely(!page)) {
2332 if (migratetype == MIGRATE_MOVABLE)
2333 page = __rmqueue_cma_fallback(zone, order);
2334
2335 if (!page && __rmqueue_fallback(zone, order, migratetype))
2336 goto retry;
2337 }
2338
2339 trace_mm_page_alloc_zone_locked(page, order, migratetype);
2340 return page;
2341 }
2342
2343 /*
2344 * Obtain a specified number of elements from the buddy allocator, all under
2345 * a single hold of the lock, for efficiency. Add them to the supplied list.
2346 * Returns the number of new pages which were placed at *list.
2347 */
2348 static int rmqueue_bulk(struct zone *zone, unsigned int order,
2349 unsigned long count, struct list_head *list,
2350 int migratetype)
2351 {
2352 int i, alloced = 0;
2353
2354 spin_lock(&zone->lock);
2355 for (i = 0; i < count; ++i) {
2356 struct page *page = __rmqueue(zone, order, migratetype);
2357 if (unlikely(page == NULL))
2358 break;
2359
2360 if (unlikely(check_pcp_refill(page)))
2361 continue;
2362
2363 /*
2364 * Split buddy pages returned by expand() are received here in
2365 * physical page order. The page is added to the tail of
2366 * caller's list. From the callers perspective, the linked list
2367 * is ordered by page number under some conditions. This is
2368 * useful for IO devices that can forward direction from the
2369 * head, thus also in the physical page order. This is useful
2370 * for IO devices that can merge IO requests if the physical
2371 * pages are ordered properly.
2372 */
2373 list_add_tail(&page->lru, list);
2374 alloced++;
2375 if (is_migrate_cma(get_pcppage_migratetype(page)))
2376 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
2377 -(1 << order));
2378 }
2379
2380 /*
2381 * i pages were removed from the buddy list even if some leak due
2382 * to check_pcp_refill failing so adjust NR_FREE_PAGES based
2383 * on i. Do not confuse with 'alloced' which is the number of
2384 * pages added to the pcp list.
2385 */
2386 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
2387 spin_unlock(&zone->lock);
2388 return alloced;
2389 }
2390
2391 #ifdef CONFIG_NUMA
2392 /*
2393 * Called from the vmstat counter updater to drain pagesets of this
2394 * currently executing processor on remote nodes after they have
2395 * expired.
2396 *
2397 * Note that this function must be called with the thread pinned to
2398 * a single processor.
2399 */
2400 void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
2401 {
2402 unsigned long flags;
2403 int to_drain, batch;
2404
2405 local_irq_save(flags);
2406 batch = READ_ONCE(pcp->batch);
2407 to_drain = min(pcp->count, batch);
2408 if (to_drain > 0) {
2409 free_pcppages_bulk(zone, to_drain, pcp);
2410 pcp->count -= to_drain;
2411 }
2412 local_irq_restore(flags);
2413 }
2414 #endif
2415
2416 /*
2417 * Drain pcplists of the indicated processor and zone.
2418 *
2419 * The processor must either be the current processor and the
2420 * thread pinned to the current processor or a processor that
2421 * is not online.
2422 */
2423 static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2424 {
2425 unsigned long flags;
2426 struct per_cpu_pageset *pset;
2427 struct per_cpu_pages *pcp;
2428
2429 local_irq_save(flags);
2430 pset = per_cpu_ptr(zone->pageset, cpu);
2431
2432 pcp = &pset->pcp;
2433 if (pcp->count) {
2434 free_pcppages_bulk(zone, pcp->count, pcp);
2435 pcp->count = 0;
2436 }
2437 local_irq_restore(flags);
2438 }
2439
2440 /*
2441 * Drain pcplists of all zones on the indicated processor.
2442 *
2443 * The processor must either be the current processor and the
2444 * thread pinned to the current processor or a processor that
2445 * is not online.
2446 */
2447 static void drain_pages(unsigned int cpu)
2448 {
2449 struct zone *zone;
2450
2451 for_each_populated_zone(zone) {
2452 drain_pages_zone(cpu, zone);
2453 }
2454 }
2455
2456 /*
2457 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
2458 *
2459 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
2460 * the single zone's pages.
2461 */
2462 void drain_local_pages(struct zone *zone)
2463 {
2464 int cpu = smp_processor_id();
2465
2466 if (zone)
2467 drain_pages_zone(cpu, zone);
2468 else
2469 drain_pages(cpu);
2470 }
2471
2472 static void drain_local_pages_wq(struct work_struct *work)
2473 {
2474 /*
2475 * drain_all_pages doesn't use proper cpu hotplug protection so
2476 * we can race with cpu offline when the WQ can move this from
2477 * a cpu pinned worker to an unbound one. We can operate on a different
2478 * cpu which is allright but we also have to make sure to not move to
2479 * a different one.
2480 */
2481 preempt_disable();
2482 drain_local_pages(NULL);
2483 preempt_enable();
2484 }
2485
2486 /*
2487 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
2488 *
2489 * When zone parameter is non-NULL, spill just the single zone's pages.
2490 *
2491 * Note that this can be extremely slow as the draining happens in a workqueue.
2492 */
2493 void drain_all_pages(struct zone *zone)
2494 {
2495 int cpu;
2496
2497 /*
2498 * Allocate in the BSS so we wont require allocation in
2499 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
2500 */
2501 static cpumask_t cpus_with_pcps;
2502
2503 /*
2504 * Make sure nobody triggers this path before mm_percpu_wq is fully
2505 * initialized.
2506 */
2507 if (WARN_ON_ONCE(!mm_percpu_wq))
2508 return;
2509
2510 /* Workqueues cannot recurse */
2511 if (current->flags & PF_WQ_WORKER)
2512 return;
2513
2514 /*
2515 * Do not drain if one is already in progress unless it's specific to
2516 * a zone. Such callers are primarily CMA and memory hotplug and need
2517 * the drain to be complete when the call returns.
2518 */
2519 if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
2520 if (!zone)
2521 return;
2522 mutex_lock(&pcpu_drain_mutex);
2523 }
2524
2525 /*
2526 * We don't care about racing with CPU hotplug event
2527 * as offline notification will cause the notified
2528 * cpu to drain that CPU pcps and on_each_cpu_mask
2529 * disables preemption as part of its processing
2530 */
2531 for_each_online_cpu(cpu) {
2532 struct per_cpu_pageset *pcp;
2533 struct zone *z;
2534 bool has_pcps = false;
2535
2536 if (zone) {
2537 pcp = per_cpu_ptr(zone->pageset, cpu);
2538 if (pcp->pcp.count)
2539 has_pcps = true;
2540 } else {
2541 for_each_populated_zone(z) {
2542 pcp = per_cpu_ptr(z->pageset, cpu);
2543 if (pcp->pcp.count) {
2544 has_pcps = true;
2545 break;
2546 }
2547 }
2548 }
2549
2550 if (has_pcps)
2551 cpumask_set_cpu(cpu, &cpus_with_pcps);
2552 else
2553 cpumask_clear_cpu(cpu, &cpus_with_pcps);
2554 }
2555
2556 for_each_cpu(cpu, &cpus_with_pcps) {
2557 struct work_struct *work = per_cpu_ptr(&pcpu_drain, cpu);
2558 INIT_WORK(work, drain_local_pages_wq);
2559 queue_work_on(cpu, mm_percpu_wq, work);
2560 }
2561 for_each_cpu(cpu, &cpus_with_pcps)
2562 flush_work(per_cpu_ptr(&pcpu_drain, cpu));
2563
2564 mutex_unlock(&pcpu_drain_mutex);
2565 }
2566
2567 #ifdef CONFIG_HIBERNATION
2568
2569 /*
2570 * Touch the watchdog for every WD_PAGE_COUNT pages.
2571 */
2572 #define WD_PAGE_COUNT (128*1024)
2573
2574 void mark_free_pages(struct zone *zone)
2575 {
2576 unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
2577 unsigned long flags;
2578 unsigned int order, t;
2579 struct page *page;
2580
2581 if (zone_is_empty(zone))
2582 return;
2583
2584 spin_lock_irqsave(&zone->lock, flags);
2585
2586 max_zone_pfn = zone_end_pfn(zone);
2587 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
2588 if (pfn_valid(pfn)) {
2589 page = pfn_to_page(pfn);
2590
2591 if (!--page_count) {
2592 touch_nmi_watchdog();
2593 page_count = WD_PAGE_COUNT;
2594 }
2595
2596 if (page_zone(page) != zone)
2597 continue;
2598
2599 if (!swsusp_page_is_forbidden(page))
2600 swsusp_unset_page_free(page);
2601 }
2602
2603 for_each_migratetype_order(order, t) {
2604 list_for_each_entry(page,
2605 &zone->free_area[order].free_list[t], lru) {
2606 unsigned long i;
2607
2608 pfn = page_to_pfn(page);
2609 for (i = 0; i < (1UL << order); i++) {
2610 if (!--page_count) {
2611 touch_nmi_watchdog();
2612 page_count = WD_PAGE_COUNT;
2613 }
2614 swsusp_set_page_free(pfn_to_page(pfn + i));
2615 }
2616 }
2617 }
2618 spin_unlock_irqrestore(&zone->lock, flags);
2619 }
2620 #endif /* CONFIG_PM */
2621
2622 static bool free_unref_page_prepare(struct page *page, unsigned long pfn)
2623 {
2624 int migratetype;
2625
2626 if (!free_pcp_prepare(page))
2627 return false;
2628
2629 migratetype = get_pfnblock_migratetype(page, pfn);
2630 set_pcppage_migratetype(page, migratetype);
2631 return true;
2632 }
2633
2634 static void free_unref_page_commit(struct page *page, unsigned long pfn)
2635 {
2636 struct zone *zone = page_zone(page);
2637 struct per_cpu_pages *pcp;
2638 int migratetype;
2639
2640 migratetype = get_pcppage_migratetype(page);
2641 __count_vm_event(PGFREE);
2642
2643 /*
2644 * We only track unmovable, reclaimable and movable on pcp lists.
2645 * Free ISOLATE pages back to the allocator because they are being
2646 * offlined but treat HIGHATOMIC as movable pages so we can get those
2647 * areas back if necessary. Otherwise, we may have to free
2648 * excessively into the page allocator
2649 */
2650 if (migratetype >= MIGRATE_PCPTYPES) {
2651 if (unlikely(is_migrate_isolate(migratetype))) {
2652 free_one_page(zone, page, pfn, 0, migratetype);
2653 return;
2654 }
2655 migratetype = MIGRATE_MOVABLE;
2656 }
2657
2658 pcp = &this_cpu_ptr(zone->pageset)->pcp;
2659 list_add(&page->lru, &pcp->lists[migratetype]);
2660 pcp->count++;
2661 if (pcp->count >= pcp->high) {
2662 unsigned long batch = READ_ONCE(pcp->batch);
2663 free_pcppages_bulk(zone, batch, pcp);
2664 pcp->count -= batch;
2665 }
2666 }
2667
2668 /*
2669 * Free a 0-order page
2670 */
2671 void free_unref_page(struct page *page)
2672 {
2673 unsigned long flags;
2674 unsigned long pfn = page_to_pfn(page);
2675
2676 if (!free_unref_page_prepare(page, pfn))
2677 return;
2678
2679 local_irq_save(flags);
2680 free_unref_page_commit(page, pfn);
2681 local_irq_restore(flags);
2682 }
2683
2684 /*
2685 * Free a list of 0-order pages
2686 */
2687 void free_unref_page_list(struct list_head *list)
2688 {
2689 struct page *page, *next;
2690 unsigned long flags, pfn;
2691
2692 /* Prepare pages for freeing */
2693 list_for_each_entry_safe(page, next, list, lru) {
2694 pfn = page_to_pfn(page);
2695 if (!free_unref_page_prepare(page, pfn))
2696 list_del(&page->lru);
2697 set_page_private(page, pfn);
2698 }
2699
2700 local_irq_save(flags);
2701 list_for_each_entry_safe(page, next, list, lru) {
2702 unsigned long pfn = page_private(page);
2703
2704 set_page_private(page, 0);
2705 trace_mm_page_free_batched(page);
2706 free_unref_page_commit(page, pfn);
2707 }
2708 local_irq_restore(flags);
2709 }
2710
2711 /*
2712 * split_page takes a non-compound higher-order page, and splits it into
2713 * n (1<<order) sub-pages: page[0..n]
2714 * Each sub-page must be freed individually.
2715 *
2716 * Note: this is probably too low level an operation for use in drivers.
2717 * Please consult with lkml before using this in your driver.
2718 */
2719 void split_page(struct page *page, unsigned int order)
2720 {
2721 int i;
2722
2723 VM_BUG_ON_PAGE(PageCompound(page), page);
2724 VM_BUG_ON_PAGE(!page_count(page), page);
2725
2726 for (i = 1; i < (1 << order); i++)
2727 set_page_refcounted(page + i);
2728 split_page_owner(page, order);
2729 }
2730 EXPORT_SYMBOL_GPL(split_page);
2731
2732 int __isolate_free_page(struct page *page, unsigned int order)
2733 {
2734 unsigned long watermark;
2735 struct zone *zone;
2736 int mt;
2737
2738 BUG_ON(!PageBuddy(page));
2739
2740 zone = page_zone(page);
2741 mt = get_pageblock_migratetype(page);
2742
2743 if (!is_migrate_isolate(mt)) {
2744 /*
2745 * Obey watermarks as if the page was being allocated. We can
2746 * emulate a high-order watermark check with a raised order-0
2747 * watermark, because we already know our high-order page
2748 * exists.
2749 */
2750 watermark = min_wmark_pages(zone) + (1UL << order);
2751 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
2752 return 0;
2753
2754 __mod_zone_freepage_state(zone, -(1UL << order), mt);
2755 }
2756
2757 /* Remove page from free list */
2758 list_del(&page->lru);
2759 zone->free_area[order].nr_free--;
2760 rmv_page_order(page);
2761
2762 /*
2763 * Set the pageblock if the isolated page is at least half of a
2764 * pageblock
2765 */
2766 if (order >= pageblock_order - 1) {
2767 struct page *endpage = page + (1 << order) - 1;
2768 for (; page < endpage; page += pageblock_nr_pages) {
2769 int mt = get_pageblock_migratetype(page);
2770 if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)
2771 && !is_migrate_highatomic(mt))
2772 set_pageblock_migratetype(page,
2773 MIGRATE_MOVABLE);
2774 }
2775 }
2776
2777
2778 return 1UL << order;
2779 }
2780
2781 /*
2782 * Update NUMA hit/miss statistics
2783 *
2784 * Must be called with interrupts disabled.
2785 */
2786 static inline void zone_statistics(struct zone *preferred_zone, struct zone *z)
2787 {
2788 #ifdef CONFIG_NUMA
2789 enum numa_stat_item local_stat = NUMA_LOCAL;
2790
2791 /* skip numa counters update if numa stats is disabled */
2792 if (!static_branch_likely(&vm_numa_stat_key))
2793 return;
2794
2795 if (z->node != numa_node_id())
2796 local_stat = NUMA_OTHER;
2797
2798 if (z->node == preferred_zone->node)
2799 __inc_numa_state(z, NUMA_HIT);
2800 else {
2801 __inc_numa_state(z, NUMA_MISS);
2802 __inc_numa_state(preferred_zone, NUMA_FOREIGN);
2803 }
2804 __inc_numa_state(z, local_stat);
2805 #endif
2806 }
2807
2808 /* Remove page from the per-cpu list, caller must protect the list */
2809 static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
2810 struct per_cpu_pages *pcp,
2811 struct list_head *list)
2812 {
2813 struct page *page;
2814
2815 do {
2816 if (list_empty(list)) {
2817 pcp->count += rmqueue_bulk(zone, 0,
2818 pcp->batch, list,
2819 migratetype);
2820 if (unlikely(list_empty(list)))
2821 return NULL;
2822 }
2823
2824 page = list_first_entry(list, struct page, lru);
2825 list_del(&page->lru);
2826 pcp->count--;
2827 } while (check_new_pcp(page));
2828
2829 return page;
2830 }
2831
2832 /* Lock and remove page from the per-cpu list */
2833 static struct page *rmqueue_pcplist(struct zone *preferred_zone,
2834 struct zone *zone, unsigned int order,
2835 gfp_t gfp_flags, int migratetype)
2836 {
2837 struct per_cpu_pages *pcp;
2838 struct list_head *list;
2839 struct page *page;
2840 unsigned long flags;
2841
2842 local_irq_save(flags);
2843 pcp = &this_cpu_ptr(zone->pageset)->pcp;
2844 list = &pcp->lists[migratetype];
2845 page = __rmqueue_pcplist(zone, migratetype, pcp, list);
2846 if (page) {
2847 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
2848 zone_statistics(preferred_zone, zone);
2849 }
2850 local_irq_restore(flags);
2851 return page;
2852 }
2853
2854 /*
2855 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
2856 */
2857 static inline
2858 struct page *rmqueue(struct zone *preferred_zone,
2859 struct zone *zone, unsigned int order,
2860 gfp_t gfp_flags, unsigned int alloc_flags,
2861 int migratetype)
2862 {
2863 unsigned long flags;
2864 struct page *page;
2865
2866 if (likely(order == 0)) {
2867 page = rmqueue_pcplist(preferred_zone, zone, order,
2868 gfp_flags, migratetype);
2869 goto out;
2870 }
2871
2872 /*
2873 * We most definitely don't want callers attempting to
2874 * allocate greater than order-1 page units with __GFP_NOFAIL.
2875 */
2876 WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
2877 spin_lock_irqsave(&zone->lock, flags);
2878
2879 do {
2880 page = NULL;
2881 if (alloc_flags & ALLOC_HARDER) {
2882 page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
2883 if (page)
2884 trace_mm_page_alloc_zone_locked(page, order, migratetype);
2885 }
2886 if (!page)
2887 page = __rmqueue(zone, order, migratetype);
2888 } while (page && check_new_pages(page, order));
2889 spin_unlock(&zone->lock);
2890 if (!page)
2891 goto failed;
2892 __mod_zone_freepage_state(zone, -(1 << order),
2893 get_pcppage_migratetype(page));
2894
2895 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
2896 zone_statistics(preferred_zone, zone);
2897 local_irq_restore(flags);
2898
2899 out:
2900 VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
2901 return page;
2902
2903 failed:
2904 local_irq_restore(flags);
2905 return NULL;
2906 }
2907
2908 #ifdef CONFIG_FAIL_PAGE_ALLOC
2909
2910 static struct {
2911 struct fault_attr attr;
2912
2913 bool ignore_gfp_highmem;
2914 bool ignore_gfp_reclaim;
2915 u32 min_order;
2916 } fail_page_alloc = {
2917 .attr = FAULT_ATTR_INITIALIZER,
2918 .ignore_gfp_reclaim = true,
2919 .ignore_gfp_highmem = true,
2920 .min_order = 1,
2921 };
2922
2923 static int __init setup_fail_page_alloc(char *str)
2924 {
2925 return setup_fault_attr(&fail_page_alloc.attr, str);
2926 }
2927 __setup("fail_page_alloc=", setup_fail_page_alloc);
2928
2929 static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
2930 {
2931 if (order < fail_page_alloc.min_order)
2932 return false;
2933 if (gfp_mask & __GFP_NOFAIL)
2934 return false;
2935 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
2936 return false;
2937 if (fail_page_alloc.ignore_gfp_reclaim &&
2938 (gfp_mask & __GFP_DIRECT_RECLAIM))
2939 return false;
2940
2941 return should_fail(&fail_page_alloc.attr, 1 << order);
2942 }
2943
2944 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
2945
2946 static int __init fail_page_alloc_debugfs(void)
2947 {
2948 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
2949 struct dentry *dir;
2950
2951 dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
2952 &fail_page_alloc.attr);
2953 if (IS_ERR(dir))
2954 return PTR_ERR(dir);
2955
2956 if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
2957 &fail_page_alloc.ignore_gfp_reclaim))
2958 goto fail;
2959 if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
2960 &fail_page_alloc.ignore_gfp_highmem))
2961 goto fail;
2962 if (!debugfs_create_u32("min-order", mode, dir,
2963 &fail_page_alloc.min_order))
2964 goto fail;
2965
2966 return 0;
2967 fail:
2968 debugfs_remove_recursive(dir);
2969
2970 return -ENOMEM;
2971 }
2972
2973 late_initcall(fail_page_alloc_debugfs);
2974
2975 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
2976
2977 #else /* CONFIG_FAIL_PAGE_ALLOC */
2978
2979 static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
2980 {
2981 return false;
2982 }
2983
2984 #endif /* CONFIG_FAIL_PAGE_ALLOC */
2985
2986 /*
2987 * Return true if free base pages are above 'mark'. For high-order checks it
2988 * will return true of the order-0 watermark is reached and there is at least
2989 * one free page of a suitable size. Checking now avoids taking the zone lock
2990 * to check in the allocation paths if no pages are free.
2991 */
2992 bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
2993 int classzone_idx, unsigned int alloc_flags,
2994 long free_pages)
2995 {
2996 long min = mark;
2997 int o;
2998 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
2999
3000 /* free_pages may go negative - that's OK */
3001 free_pages -= (1 << order) - 1;
3002
3003 if (alloc_flags & ALLOC_HIGH)
3004 min -= min / 2;
3005
3006 /*
3007 * If the caller does not have rights to ALLOC_HARDER then subtract
3008 * the high-atomic reserves. This will over-estimate the size of the
3009 * atomic reserve but it avoids a search.
3010 */
3011 if (likely(!alloc_harder)) {
3012 free_pages -= z->nr_reserved_highatomic;
3013 } else {
3014 /*
3015 * OOM victims can try even harder than normal ALLOC_HARDER
3016 * users on the grounds that it's definitely going to be in
3017 * the exit path shortly and free memory. Any allocation it
3018 * makes during the free path will be small and short-lived.
3019 */
3020 if (alloc_flags & ALLOC_OOM)
3021 min -= min / 2;
3022 else
3023 min -= min / 4;
3024 }
3025
3026
3027 #ifdef CONFIG_CMA
3028 /* If allocation can't use CMA areas don't use free CMA pages */
3029 if (!(alloc_flags & ALLOC_CMA))
3030 free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
3031 #endif
3032
3033 /*
3034 * Check watermarks for an order-0 allocation request. If these
3035 * are not met, then a high-order request also cannot go ahead
3036 * even if a suitable page happened to be free.
3037 */
3038 if (free_pages <= min + z->lowmem_reserve[classzone_idx])
3039 return false;
3040
3041 /* If this is an order-0 request then the watermark is fine */
3042 if (!order)
3043 return true;
3044
3045 /* For a high-order request, check at least one suitable page is free */
3046 for (o = order; o < MAX_ORDER; o++) {
3047 struct free_area *area = &z->free_area[o];
3048 int mt;
3049
3050 if (!area->nr_free)
3051 continue;
3052
3053 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3054 if (!list_empty(&area->free_list[mt]))
3055 return true;
3056 }
3057
3058 #ifdef CONFIG_CMA
3059 if ((alloc_flags & ALLOC_CMA) &&
3060 !list_empty(&area->free_list[MIGRATE_CMA])) {
3061 return true;
3062 }
3063 #endif
3064 if (alloc_harder &&
3065 !list_empty(&area->free_list[MIGRATE_HIGHATOMIC]))
3066 return true;
3067 }
3068 return false;
3069 }
3070
3071 bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3072 int classzone_idx, unsigned int alloc_flags)
3073 {
3074 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3075 zone_page_state(z, NR_FREE_PAGES));
3076 }
3077
3078 static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
3079 unsigned long mark, int classzone_idx, unsigned int alloc_flags)
3080 {
3081 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3082 long cma_pages = 0;
3083
3084 #ifdef CONFIG_CMA
3085 /* If allocation can't use CMA areas don't use free CMA pages */
3086 if (!(alloc_flags & ALLOC_CMA))
3087 cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
3088 #endif
3089
3090 /*
3091 * Fast check for order-0 only. If this fails then the reserves
3092 * need to be calculated. There is a corner case where the check
3093 * passes but only the high-order atomic reserve are free. If
3094 * the caller is !atomic then it'll uselessly search the free
3095 * list. That corner case is then slower but it is harmless.
3096 */
3097 if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
3098 return true;
3099
3100 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
3101 free_pages);
3102 }
3103
3104 bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
3105 unsigned long mark, int classzone_idx)
3106 {
3107 long free_pages = zone_page_state(z, NR_FREE_PAGES);
3108
3109 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
3110 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
3111
3112 return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
3113 free_pages);
3114 }
3115
3116 #ifdef CONFIG_NUMA
3117 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3118 {
3119 return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3120 RECLAIM_DISTANCE;
3121 }
3122 #else /* CONFIG_NUMA */
3123 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3124 {
3125 return true;
3126 }
3127 #endif /* CONFIG_NUMA */
3128
3129 /*
3130 * get_page_from_freelist goes through the zonelist trying to allocate
3131 * a page.
3132 */
3133 static struct page *
3134 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3135 const struct alloc_context *ac)
3136 {
3137 struct zoneref *z = ac->preferred_zoneref;
3138 struct zone *zone;
3139 struct pglist_data *last_pgdat_dirty_limit = NULL;
3140
3141 /*
3142 * Scan zonelist, looking for a zone with enough free.
3143 * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
3144 */
3145 for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3146 ac->nodemask) {
3147 struct page *page;
3148 unsigned long mark;
3149
3150 if (cpusets_enabled() &&
3151 (alloc_flags & ALLOC_CPUSET) &&
3152 !__cpuset_zone_allowed(zone, gfp_mask))
3153 continue;
3154 /*
3155 * When allocating a page cache page for writing, we
3156 * want to get it from a node that is within its dirty
3157 * limit, such that no single node holds more than its
3158 * proportional share of globally allowed dirty pages.
3159 * The dirty limits take into account the node's
3160 * lowmem reserves and high watermark so that kswapd
3161 * should be able to balance it without having to
3162 * write pages from its LRU list.
3163 *
3164 * XXX: For now, allow allocations to potentially
3165 * exceed the per-node dirty limit in the slowpath
3166 * (spread_dirty_pages unset) before going into reclaim,
3167 * which is important when on a NUMA setup the allowed
3168 * nodes are together not big enough to reach the
3169 * global limit. The proper fix for these situations
3170 * will require awareness of nodes in the
3171 * dirty-throttling and the flusher threads.
3172 */
3173 if (ac->spread_dirty_pages) {
3174 if (last_pgdat_dirty_limit == zone->zone_pgdat)
3175 continue;
3176
3177 if (!node_dirty_ok(zone->zone_pgdat)) {
3178 last_pgdat_dirty_limit = zone->zone_pgdat;
3179 continue;
3180 }
3181 }
3182
3183 mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
3184 if (!zone_watermark_fast(zone, order, mark,
3185 ac_classzone_idx(ac), alloc_flags)) {
3186 int ret;
3187
3188 /* Checked here to keep the fast path fast */
3189 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3190 if (alloc_flags & ALLOC_NO_WATERMARKS)
3191 goto try_this_zone;
3192
3193 if (node_reclaim_mode == 0 ||
3194 !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
3195 continue;
3196
3197 ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3198 switch (ret) {
3199 case NODE_RECLAIM_NOSCAN:
3200 /* did not scan */
3201 continue;
3202 case NODE_RECLAIM_FULL:
3203 /* scanned but unreclaimable */
3204 continue;
3205 default:
3206 /* did we reclaim enough */
3207 if (zone_watermark_ok(zone, order, mark,
3208 ac_classzone_idx(ac), alloc_flags))
3209 goto try_this_zone;
3210
3211 continue;
3212 }
3213 }
3214
3215 try_this_zone:
3216 page = rmqueue(ac->preferred_zoneref->zone, zone, order,
3217 gfp_mask, alloc_flags, ac->migratetype);
3218 if (page) {
3219 prep_new_page(page, order, gfp_mask, alloc_flags);
3220
3221 /*
3222 * If this is a high-order atomic allocation then check
3223 * if the pageblock should be reserved for the future
3224 */
3225 if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
3226 reserve_highatomic_pageblock(page, zone, order);
3227
3228 return page;
3229 }
3230 }
3231
3232 return NULL;
3233 }
3234
3235 /*
3236 * Large machines with many possible nodes should not always dump per-node
3237 * meminfo in irq context.
3238 */
3239 static inline bool should_suppress_show_mem(void)
3240 {
3241 bool ret = false;
3242
3243 #if NODES_SHIFT > 8
3244 ret = in_interrupt();
3245 #endif
3246 return ret;
3247 }
3248
3249 static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
3250 {
3251 unsigned int filter = SHOW_MEM_FILTER_NODES;
3252 static DEFINE_RATELIMIT_STATE(show_mem_rs, HZ, 1);
3253
3254 if (should_suppress_show_mem() || !__ratelimit(&show_mem_rs))
3255 return;
3256
3257 /*
3258 * This documents exceptions given to allocations in certain
3259 * contexts that are allowed to allocate outside current's set
3260 * of allowed nodes.
3261 */
3262 if (!(gfp_mask & __GFP_NOMEMALLOC))
3263 if (tsk_is_oom_victim(current) ||
3264 (current->flags & (PF_MEMALLOC | PF_EXITING)))
3265 filter &= ~SHOW_MEM_FILTER_NODES;
3266 if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
3267 filter &= ~SHOW_MEM_FILTER_NODES;
3268
3269 show_mem(filter, nodemask);
3270 }
3271
3272 void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
3273 {
3274 struct va_format vaf;
3275 va_list args;
3276 static DEFINE_RATELIMIT_STATE(nopage_rs, DEFAULT_RATELIMIT_INTERVAL,
3277 DEFAULT_RATELIMIT_BURST);
3278
3279 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs))
3280 return;
3281
3282 va_start(args, fmt);
3283 vaf.fmt = fmt;
3284 vaf.va = &args;
3285 pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl\n",
3286 current->comm, &vaf, gfp_mask, &gfp_mask,
3287 nodemask_pr_args(nodemask));
3288 va_end(args);
3289
3290 cpuset_print_current_mems_allowed();
3291
3292 dump_stack();
3293 warn_alloc_show_mem(gfp_mask, nodemask);
3294 }
3295
3296 static inline struct page *
3297 __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
3298 unsigned int alloc_flags,
3299 const struct alloc_context *ac)
3300 {
3301 struct page *page;
3302
3303 page = get_page_from_freelist(gfp_mask, order,
3304 alloc_flags|ALLOC_CPUSET, ac);
3305 /*
3306 * fallback to ignore cpuset restriction if our nodes
3307 * are depleted
3308 */
3309 if (!page)
3310 page = get_page_from_freelist(gfp_mask, order,
3311 alloc_flags, ac);
3312
3313 return page;
3314 }
3315
3316 static inline struct page *
3317 __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
3318 const struct alloc_context *ac, unsigned long *did_some_progress)
3319 {
3320 struct oom_control oc = {
3321 .zonelist = ac->zonelist,
3322 .nodemask = ac->nodemask,
3323 .memcg = NULL,
3324 .gfp_mask = gfp_mask,
3325 .order = order,
3326 };
3327 struct page *page;
3328
3329 *did_some_progress = 0;
3330
3331 /*
3332 * Acquire the oom lock. If that fails, somebody else is
3333 * making progress for us.
3334 */
3335 if (!mutex_trylock(&oom_lock)) {
3336 *did_some_progress = 1;
3337 schedule_timeout_uninterruptible(1);
3338 return NULL;
3339 }
3340
3341 /*
3342 * Go through the zonelist yet one more time, keep very high watermark
3343 * here, this is only to catch a parallel oom killing, we must fail if
3344 * we're still under heavy pressure. But make sure that this reclaim
3345 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
3346 * allocation which will never fail due to oom_lock already held.
3347 */
3348 page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
3349 ~__GFP_DIRECT_RECLAIM, order,
3350 ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
3351 if (page)
3352 goto out;
3353
3354 /* Coredumps can quickly deplete all memory reserves */
3355 if (current->flags & PF_DUMPCORE)
3356 goto out;
3357 /* The OOM killer will not help higher order allocs */
3358 if (order > PAGE_ALLOC_COSTLY_ORDER)
3359 goto out;
3360 /*
3361 * We have already exhausted all our reclaim opportunities without any
3362 * success so it is time to admit defeat. We will skip the OOM killer
3363 * because it is very likely that the caller has a more reasonable
3364 * fallback than shooting a random task.
3365 */
3366 if (gfp_mask & __GFP_RETRY_MAYFAIL)
3367 goto out;
3368 /* The OOM killer does not needlessly kill tasks for lowmem */
3369 if (ac->high_zoneidx < ZONE_NORMAL)
3370 goto out;
3371 if (pm_suspended_storage())
3372 goto out;
3373 /*
3374 * XXX: GFP_NOFS allocations should rather fail than rely on
3375 * other request to make a forward progress.
3376 * We are in an unfortunate situation where out_of_memory cannot
3377 * do much for this context but let's try it to at least get
3378 * access to memory reserved if the current task is killed (see
3379 * out_of_memory). Once filesystems are ready to handle allocation
3380 * failures more gracefully we should just bail out here.
3381 */
3382
3383 /* The OOM killer may not free memory on a specific node */
3384 if (gfp_mask & __GFP_THISNODE)
3385 goto out;
3386
3387 /* Exhausted what can be done so it's blamo time */
3388 if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
3389 *did_some_progress = 1;
3390
3391 /*
3392 * Help non-failing allocations by giving them access to memory
3393 * reserves
3394 */
3395 if (gfp_mask & __GFP_NOFAIL)
3396 page = __alloc_pages_cpuset_fallback(gfp_mask, order,
3397 ALLOC_NO_WATERMARKS, ac);
3398 }
3399 out:
3400 mutex_unlock(&oom_lock);
3401 return page;
3402 }
3403
3404 /*
3405 * Maximum number of compaction retries wit a progress before OOM
3406 * killer is consider as the only way to move forward.
3407 */
3408 #define MAX_COMPACT_RETRIES 16
3409
3410 #ifdef CONFIG_COMPACTION
3411 /* Try memory compaction for high-order allocations before reclaim */
3412 static struct page *
3413 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3414 unsigned int alloc_flags, const struct alloc_context *ac,
3415 enum compact_priority prio, enum compact_result *compact_result)
3416 {
3417 struct page *page;
3418 unsigned int noreclaim_flag;
3419
3420 if (!order)
3421 return NULL;
3422
3423 noreclaim_flag = memalloc_noreclaim_save();
3424 *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
3425 prio);
3426 memalloc_noreclaim_restore(noreclaim_flag);
3427
3428 if (*compact_result <= COMPACT_INACTIVE)
3429 return NULL;
3430
3431 /*
3432 * At least in one zone compaction wasn't deferred or skipped, so let's
3433 * count a compaction stall
3434 */
3435 count_vm_event(COMPACTSTALL);
3436
3437 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3438
3439 if (page) {
3440 struct zone *zone = page_zone(page);
3441
3442 zone->compact_blockskip_flush = false;
3443 compaction_defer_reset(zone, order, true);
3444 count_vm_event(COMPACTSUCCESS);
3445 return page;
3446 }
3447
3448 /*
3449 * It's bad if compaction run occurs and fails. The most likely reason
3450 * is that pages exist, but not enough to satisfy watermarks.
3451 */
3452 count_vm_event(COMPACTFAIL);
3453
3454 cond_resched();
3455
3456 return NULL;
3457 }
3458
3459 static inline bool
3460 should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
3461 enum compact_result compact_result,
3462 enum compact_priority *compact_priority,
3463 int *compaction_retries)
3464 {
3465 int max_retries = MAX_COMPACT_RETRIES;
3466 int min_priority;
3467 bool ret = false;
3468 int retries = *compaction_retries;
3469 enum compact_priority priority = *compact_priority;
3470
3471 if (!order)
3472 return false;
3473
3474 if (compaction_made_progress(compact_result))
3475 (*compaction_retries)++;
3476
3477 /*
3478 * compaction considers all the zone as desperately out of memory
3479 * so it doesn't really make much sense to retry except when the
3480 * failure could be caused by insufficient priority
3481 */
3482 if (compaction_failed(compact_result))
3483 goto check_priority;
3484
3485 /*
3486 * make sure the compaction wasn't deferred or didn't bail out early
3487 * due to locks contention before we declare that we should give up.
3488 * But do not retry if the given zonelist is not suitable for
3489 * compaction.
3490 */
3491 if (compaction_withdrawn(compact_result)) {
3492 ret = compaction_zonelist_suitable(ac, order, alloc_flags);
3493 goto out;
3494 }
3495
3496 /*
3497 * !costly requests are much more important than __GFP_RETRY_MAYFAIL
3498 * costly ones because they are de facto nofail and invoke OOM
3499 * killer to move on while costly can fail and users are ready
3500 * to cope with that. 1/4 retries is rather arbitrary but we
3501 * would need much more detailed feedback from compaction to
3502 * make a better decision.
3503 */
3504 if (order > PAGE_ALLOC_COSTLY_ORDER)
3505 max_retries /= 4;
3506 if (*compaction_retries <= max_retries) {
3507 ret = true;
3508 goto out;
3509 }
3510
3511 /*
3512 * Make sure there are attempts at the highest priority if we exhausted
3513 * all retries or failed at the lower priorities.
3514 */
3515 check_priority:
3516 min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
3517 MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
3518
3519 if (*compact_priority > min_priority) {
3520 (*compact_priority)--;
3521 *compaction_retries = 0;
3522 ret = true;
3523 }
3524 out:
3525 trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
3526 return ret;
3527 }
3528 #else
3529 static inline struct page *
3530 __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
3531 unsigned int alloc_flags, const struct alloc_context *ac,
3532 enum compact_priority prio, enum compact_result *compact_result)
3533 {
3534 *compact_result = COMPACT_SKIPPED;
3535 return NULL;
3536 }
3537
3538 static inline bool
3539 should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
3540 enum compact_result compact_result,
3541 enum compact_priority *compact_priority,
3542 int *compaction_retries)
3543 {
3544 struct zone *zone;
3545 struct zoneref *z;
3546
3547 if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
3548 return false;
3549
3550 /*
3551 * There are setups with compaction disabled which would prefer to loop
3552 * inside the allocator rather than hit the oom killer prematurely.
3553 * Let's give them a good hope and keep retrying while the order-0
3554 * watermarks are OK.
3555 */
3556 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3557 ac->nodemask) {
3558 if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
3559 ac_classzone_idx(ac), alloc_flags))
3560 return true;
3561 }
3562 return false;
3563 }
3564 #endif /* CONFIG_COMPACTION */
3565
3566 #ifdef CONFIG_LOCKDEP
3567 struct lockdep_map __fs_reclaim_map =
3568 STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
3569
3570 static bool __need_fs_reclaim(gfp_t gfp_mask)
3571 {
3572 gfp_mask = current_gfp_context(gfp_mask);
3573
3574 /* no reclaim without waiting on it */
3575 if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
3576 return false;
3577
3578 /* this guy won't enter reclaim */
3579 if ((current->flags & PF_MEMALLOC) && !(gfp_mask & __GFP_NOMEMALLOC))
3580 return false;
3581
3582 /* We're only interested __GFP_FS allocations for now */
3583 if (!(gfp_mask & __GFP_FS))
3584 return false;
3585
3586 if (gfp_mask & __GFP_NOLOCKDEP)
3587 return false;
3588
3589 return true;
3590 }
3591
3592 void fs_reclaim_acquire(gfp_t gfp_mask)
3593 {
3594 if (__need_fs_reclaim(gfp_mask))
3595 lock_map_acquire(&__fs_reclaim_map);
3596 }
3597 EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
3598
3599 void fs_reclaim_release(gfp_t gfp_mask)
3600 {
3601 if (__need_fs_reclaim(gfp_mask))
3602 lock_map_release(&__fs_reclaim_map);
3603 }
3604 EXPORT_SYMBOL_GPL(fs_reclaim_release);
3605 #endif
3606
3607 /* Perform direct synchronous page reclaim */
3608 static int
3609 __perform_reclaim(gfp_t gfp_mask, unsigned int order,
3610 const struct alloc_context *ac)
3611 {
3612 struct reclaim_state reclaim_state;
3613 int progress;
3614 unsigned int noreclaim_flag;
3615
3616 cond_resched();
3617
3618 /* We now go into synchronous reclaim */
3619 cpuset_memory_pressure_bump();
3620 noreclaim_flag = memalloc_noreclaim_save();
3621 fs_reclaim_acquire(gfp_mask);
3622 reclaim_state.reclaimed_slab = 0;
3623 current->reclaim_state = &reclaim_state;
3624
3625 progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
3626 ac->nodemask);
3627
3628 current->reclaim_state = NULL;
3629 fs_reclaim_release(gfp_mask);
3630 memalloc_noreclaim_restore(noreclaim_flag);
3631
3632 cond_resched();
3633
3634 return progress;
3635 }
3636
3637 /* The really slow allocator path where we enter direct reclaim */
3638 static inline struct page *
3639 __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
3640 unsigned int alloc_flags, const struct alloc_context *ac,
3641 unsigned long *did_some_progress)
3642 {
3643 struct page *page = NULL;
3644 bool drained = false;
3645
3646 *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
3647 if (unlikely(!(*did_some_progress)))
3648 return NULL;
3649
3650 retry:
3651 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3652
3653 /*
3654 * If an allocation failed after direct reclaim, it could be because
3655 * pages are pinned on the per-cpu lists or in high alloc reserves.
3656 * Shrink them them and try again
3657 */
3658 if (!page && !drained) {
3659 unreserve_highatomic_pageblock(ac, false);
3660 drain_all_pages(NULL);
3661 drained = true;
3662 goto retry;
3663 }
3664
3665 return page;
3666 }
3667
3668 static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
3669 {
3670 struct zoneref *z;
3671 struct zone *zone;
3672 pg_data_t *last_pgdat = NULL;
3673
3674 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
3675 ac->high_zoneidx, ac->nodemask) {
3676 if (last_pgdat != zone->zone_pgdat)
3677 wakeup_kswapd(zone, order, ac->high_zoneidx);
3678 last_pgdat = zone->zone_pgdat;
3679 }
3680 }
3681
3682 static inline unsigned int
3683 gfp_to_alloc_flags(gfp_t gfp_mask)
3684 {
3685 unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
3686
3687 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
3688 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
3689
3690 /*
3691 * The caller may dip into page reserves a bit more if the caller
3692 * cannot run direct reclaim, or if the caller has realtime scheduling
3693 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
3694 * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
3695 */
3696 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
3697
3698 if (gfp_mask & __GFP_ATOMIC) {
3699 /*
3700 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
3701 * if it can't schedule.
3702 */
3703 if (!(gfp_mask & __GFP_NOMEMALLOC))
3704 alloc_flags |= ALLOC_HARDER;
3705 /*
3706 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
3707 * comment for __cpuset_node_allowed().
3708 */
3709 alloc_flags &= ~ALLOC_CPUSET;
3710 } else if (unlikely(rt_task(current)) && !in_interrupt())
3711 alloc_flags |= ALLOC_HARDER;
3712
3713 #ifdef CONFIG_CMA
3714 if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
3715 alloc_flags |= ALLOC_CMA;
3716 #endif
3717 return alloc_flags;
3718 }
3719
3720 static bool oom_reserves_allowed(struct task_struct *tsk)
3721 {
3722 if (!tsk_is_oom_victim(tsk))
3723 return false;
3724
3725 /*
3726 * !MMU doesn't have oom reaper so give access to memory reserves
3727 * only to the thread with TIF_MEMDIE set
3728 */
3729 if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
3730 return false;
3731
3732 return true;
3733 }
3734
3735 /*
3736 * Distinguish requests which really need access to full memory
3737 * reserves from oom victims which can live with a portion of it
3738 */
3739 static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
3740 {
3741 if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
3742 return 0;
3743 if (gfp_mask & __GFP_MEMALLOC)
3744 return ALLOC_NO_WATERMARKS;
3745 if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
3746 return ALLOC_NO_WATERMARKS;
3747 if (!in_interrupt()) {
3748 if (current->flags & PF_MEMALLOC)
3749 return ALLOC_NO_WATERMARKS;
3750 else if (oom_reserves_allowed(current))
3751 return ALLOC_OOM;
3752 }
3753
3754 return 0;
3755 }
3756
3757 bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
3758 {
3759 return !!__gfp_pfmemalloc_flags(gfp_mask);
3760 }
3761
3762 /*
3763 * Checks whether it makes sense to retry the reclaim to make a forward progress
3764 * for the given allocation request.
3765 *
3766 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
3767 * without success, or when we couldn't even meet the watermark if we
3768 * reclaimed all remaining pages on the LRU lists.
3769 *
3770 * Returns true if a retry is viable or false to enter the oom path.
3771 */
3772 static inline bool
3773 should_reclaim_retry(gfp_t gfp_mask, unsigned order,
3774 struct alloc_context *ac, int alloc_flags,
3775 bool did_some_progress, int *no_progress_loops)
3776 {
3777 struct zone *zone;
3778 struct zoneref *z;
3779
3780 /*
3781 * Costly allocations might have made a progress but this doesn't mean
3782 * their order will become available due to high fragmentation so
3783 * always increment the no progress counter for them
3784 */
3785 if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
3786 *no_progress_loops = 0;
3787 else
3788 (*no_progress_loops)++;
3789
3790 /*
3791 * Make sure we converge to OOM if we cannot make any progress
3792 * several times in the row.
3793 */
3794 if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
3795 /* Before OOM, exhaust highatomic_reserve */
3796 return unreserve_highatomic_pageblock(ac, true);
3797 }
3798
3799 /*
3800 * Keep reclaiming pages while there is a chance this will lead
3801 * somewhere. If none of the target zones can satisfy our allocation
3802 * request even if all reclaimable pages are considered then we are
3803 * screwed and have to go OOM.
3804 */
3805 for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
3806 ac->nodemask) {
3807 unsigned long available;
3808 unsigned long reclaimable;
3809 unsigned long min_wmark = min_wmark_pages(zone);
3810 bool wmark;
3811
3812 available = reclaimable = zone_reclaimable_pages(zone);
3813 available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
3814
3815 /*
3816 * Would the allocation succeed if we reclaimed all
3817 * reclaimable pages?
3818 */
3819 wmark = __zone_watermark_ok(zone, order, min_wmark,
3820 ac_classzone_idx(ac), alloc_flags, available);
3821 trace_reclaim_retry_zone(z, order, reclaimable,
3822 available, min_wmark, *no_progress_loops, wmark);
3823 if (wmark) {
3824 /*
3825 * If we didn't make any progress and have a lot of
3826 * dirty + writeback pages then we should wait for
3827 * an IO to complete to slow down the reclaim and
3828 * prevent from pre mature OOM
3829 */
3830 if (!did_some_progress) {
3831 unsigned long write_pending;
3832
3833 write_pending = zone_page_state_snapshot(zone,
3834 NR_ZONE_WRITE_PENDING);
3835
3836 if (2 * write_pending > reclaimable) {
3837 congestion_wait(BLK_RW_ASYNC, HZ/10);
3838 return true;
3839 }
3840 }
3841
3842 /*
3843 * Memory allocation/reclaim might be called from a WQ
3844 * context and the current implementation of the WQ
3845 * concurrency control doesn't recognize that
3846 * a particular WQ is congested if the worker thread is
3847 * looping without ever sleeping. Therefore we have to
3848 * do a short sleep here rather than calling
3849 * cond_resched().
3850 */
3851 if (current->flags & PF_WQ_WORKER)
3852 schedule_timeout_uninterruptible(1);
3853 else
3854 cond_resched();
3855
3856 return true;
3857 }
3858 }
3859
3860 return false;
3861 }
3862
3863 static inline bool
3864 check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
3865 {
3866 /*
3867 * It's possible that cpuset's mems_allowed and the nodemask from
3868 * mempolicy don't intersect. This should be normally dealt with by
3869 * policy_nodemask(), but it's possible to race with cpuset update in
3870 * such a way the check therein was true, and then it became false
3871 * before we got our cpuset_mems_cookie here.
3872 * This assumes that for all allocations, ac->nodemask can come only
3873 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
3874 * when it does not intersect with the cpuset restrictions) or the
3875 * caller can deal with a violated nodemask.
3876 */
3877 if (cpusets_enabled() && ac->nodemask &&
3878 !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
3879 ac->nodemask = NULL;
3880 return true;
3881 }
3882
3883 /*
3884 * When updating a task's mems_allowed or mempolicy nodemask, it is
3885 * possible to race with parallel threads in such a way that our
3886 * allocation can fail while the mask is being updated. If we are about
3887 * to fail, check if the cpuset changed during allocation and if so,
3888 * retry.
3889 */
3890 if (read_mems_allowed_retry(cpuset_mems_cookie))
3891 return true;
3892
3893 return false;
3894 }
3895
3896 static inline struct page *
3897 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
3898 struct alloc_context *ac)
3899 {
3900 bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
3901 const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
3902 struct page *page = NULL;
3903 unsigned int alloc_flags;
3904 unsigned long did_some_progress;
3905 enum compact_priority compact_priority;
3906 enum compact_result compact_result;
3907 int compaction_retries;
3908 int no_progress_loops;
3909 unsigned int cpuset_mems_cookie;
3910 int reserve_flags;
3911
3912 /*
3913 * In the slowpath, we sanity check order to avoid ever trying to
3914 * reclaim >= MAX_ORDER areas which will never succeed. Callers may
3915 * be using allocators in order of preference for an area that is
3916 * too large.
3917 */
3918 if (order >= MAX_ORDER) {
3919 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
3920 return NULL;
3921 }
3922
3923 /*
3924 * We also sanity check to catch abuse of atomic reserves being used by
3925 * callers that are not in atomic context.
3926 */
3927 if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
3928 (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
3929 gfp_mask &= ~__GFP_ATOMIC;
3930
3931 retry_cpuset:
3932 compaction_retries = 0;
3933 no_progress_loops = 0;
3934 compact_priority = DEF_COMPACT_PRIORITY;
3935 cpuset_mems_cookie = read_mems_allowed_begin();
3936
3937 /*
3938 * The fast path uses conservative alloc_flags to succeed only until
3939 * kswapd needs to be woken up, and to avoid the cost of setting up
3940 * alloc_flags precisely. So we do that now.
3941 */
3942 alloc_flags = gfp_to_alloc_flags(gfp_mask);
3943
3944 /*
3945 * We need to recalculate the starting point for the zonelist iterator
3946 * because we might have used different nodemask in the fast path, or
3947 * there was a cpuset modification and we are retrying - otherwise we
3948 * could end up iterating over non-eligible zones endlessly.
3949 */
3950 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
3951 ac->high_zoneidx, ac->nodemask);
3952 if (!ac->preferred_zoneref->zone)
3953 goto nopage;
3954
3955 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
3956 wake_all_kswapds(order, ac);
3957
3958 /*
3959 * The adjusted alloc_flags might result in immediate success, so try
3960 * that first
3961 */
3962 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
3963 if (page)
3964 goto got_pg;
3965
3966 /*
3967 * For costly allocations, try direct compaction first, as it's likely
3968 * that we have enough base pages and don't need to reclaim. For non-
3969 * movable high-order allocations, do that as well, as compaction will
3970 * try prevent permanent fragmentation by migrating from blocks of the
3971 * same migratetype.
3972 * Don't try this for allocations that are allowed to ignore
3973 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
3974 */
3975 if (can_direct_reclaim &&
3976 (costly_order ||
3977 (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
3978 && !gfp_pfmemalloc_allowed(gfp_mask)) {
3979 page = __alloc_pages_direct_compact(gfp_mask, order,
3980 alloc_flags, ac,
3981 INIT_COMPACT_PRIORITY,
3982 &compact_result);
3983 if (page)
3984 goto got_pg;
3985
3986 /*
3987 * Checks for costly allocations with __GFP_NORETRY, which
3988 * includes THP page fault allocations
3989 */
3990 if (costly_order && (gfp_mask & __GFP_NORETRY)) {
3991 /*
3992 * If compaction is deferred for high-order allocations,
3993 * it is because sync compaction recently failed. If
3994 * this is the case and the caller requested a THP
3995 * allocation, we do not want to heavily disrupt the
3996 * system, so we fail the allocation instead of entering
3997 * direct reclaim.
3998 */
3999 if (compact_result == COMPACT_DEFERRED)
4000 goto nopage;
4001
4002 /*
4003 * Looks like reclaim/compaction is worth trying, but
4004 * sync compaction could be very expensive, so keep
4005 * using async compaction.
4006 */
4007 compact_priority = INIT_COMPACT_PRIORITY;
4008 }
4009 }
4010
4011 retry:
4012 /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
4013 if (gfp_mask & __GFP_KSWAPD_RECLAIM)
4014 wake_all_kswapds(order, ac);
4015
4016 reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
4017 if (reserve_flags)
4018 alloc_flags = reserve_flags;
4019
4020 /*
4021 * Reset the zonelist iterators if memory policies can be ignored.
4022 * These allocations are high priority and system rather than user
4023 * orientated.
4024 */
4025 if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
4026 ac->zonelist = node_zonelist(numa_node_id(), gfp_mask);
4027 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4028 ac->high_zoneidx, ac->nodemask);
4029 }
4030
4031 /* Attempt with potentially adjusted zonelist and alloc_flags */
4032 page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4033 if (page)
4034 goto got_pg;
4035
4036 /* Caller is not willing to reclaim, we can't balance anything */
4037 if (!can_direct_reclaim)
4038 goto nopage;
4039
4040 /* Avoid recursion of direct reclaim */
4041 if (current->flags & PF_MEMALLOC)
4042 goto nopage;
4043
4044 /* Try direct reclaim and then allocating */
4045 page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
4046 &did_some_progress);
4047 if (page)
4048 goto got_pg;
4049
4050 /* Try direct compaction and then allocating */
4051 page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
4052 compact_priority, &compact_result);
4053 if (page)
4054 goto got_pg;
4055
4056 /* Do not loop if specifically requested */
4057 if (gfp_mask & __GFP_NORETRY)
4058 goto nopage;
4059
4060 /*
4061 * Do not retry costly high order allocations unless they are
4062 * __GFP_RETRY_MAYFAIL
4063 */
4064 if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
4065 goto nopage;
4066
4067 if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
4068 did_some_progress > 0, &no_progress_loops))
4069 goto retry;
4070
4071 /*
4072 * It doesn't make any sense to retry for the compaction if the order-0
4073 * reclaim is not able to make any progress because the current
4074 * implementation of the compaction depends on the sufficient amount
4075 * of free memory (see __compaction_suitable)
4076 */
4077 if (did_some_progress > 0 &&
4078 should_compact_retry(ac, order, alloc_flags,
4079 compact_result, &compact_priority,
4080 &compaction_retries))
4081 goto retry;
4082
4083
4084 /* Deal with possible cpuset update races before we start OOM killing */
4085 if (check_retry_cpuset(cpuset_mems_cookie, ac))
4086 goto retry_cpuset;
4087
4088 /* Reclaim has failed us, start killing things */
4089 page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
4090 if (page)
4091 goto got_pg;
4092
4093 /* Avoid allocations with no watermarks from looping endlessly */
4094 if (tsk_is_oom_victim(current) &&
4095 (alloc_flags == ALLOC_OOM ||
4096 (gfp_mask & __GFP_NOMEMALLOC)))
4097 goto nopage;
4098
4099 /* Retry as long as the OOM killer is making progress */
4100 if (did_some_progress) {
4101 no_progress_loops = 0;
4102 goto retry;
4103 }
4104
4105 nopage:
4106 /* Deal with possible cpuset update races before we fail */
4107 if (check_retry_cpuset(cpuset_mems_cookie, ac))
4108 goto retry_cpuset;
4109
4110 /*
4111 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
4112 * we always retry
4113 */
4114 if (gfp_mask & __GFP_NOFAIL) {
4115 /*
4116 * All existing users of the __GFP_NOFAIL are blockable, so warn
4117 * of any new users that actually require GFP_NOWAIT
4118 */
4119 if (WARN_ON_ONCE(!can_direct_reclaim))
4120 goto fail;
4121
4122 /*
4123 * PF_MEMALLOC request from this context is rather bizarre
4124 * because we cannot reclaim anything and only can loop waiting
4125 * for somebody to do a work for us
4126 */
4127 WARN_ON_ONCE(current->flags & PF_MEMALLOC);
4128
4129 /*
4130 * non failing costly orders are a hard requirement which we
4131 * are not prepared for much so let's warn about these users
4132 * so that we can identify them and convert them to something
4133 * else.
4134 */
4135 WARN_ON_ONCE(order > PAGE_ALLOC_COSTLY_ORDER);
4136
4137 /*
4138 * Help non-failing allocations by giving them access to memory
4139 * reserves but do not use ALLOC_NO_WATERMARKS because this
4140 * could deplete whole memory reserves which would just make
4141 * the situation worse
4142 */
4143 page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
4144 if (page)
4145 goto got_pg;
4146
4147 cond_resched();
4148 goto retry;
4149 }
4150 fail:
4151 warn_alloc(gfp_mask, ac->nodemask,
4152 "page allocation failure: order:%u", order);
4153 got_pg:
4154 return page;
4155 }
4156
4157 static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
4158 int preferred_nid, nodemask_t *nodemask,
4159 struct alloc_context *ac, gfp_t *alloc_mask,
4160 unsigned int *alloc_flags)
4161 {
4162 ac->high_zoneidx = gfp_zone(gfp_mask);
4163 ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
4164 ac->nodemask = nodemask;
4165 ac->migratetype = gfpflags_to_migratetype(gfp_mask);
4166
4167 if (cpusets_enabled()) {
4168 *alloc_mask |= __GFP_HARDWALL;
4169 if (!ac->nodemask)
4170 ac->nodemask = &cpuset_current_mems_allowed;
4171 else
4172 *alloc_flags |= ALLOC_CPUSET;
4173 }
4174
4175 fs_reclaim_acquire(gfp_mask);
4176 fs_reclaim_release(gfp_mask);
4177
4178 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
4179
4180 if (should_fail_alloc_page(gfp_mask, order))
4181 return false;
4182
4183 if (IS_ENABLED(CONFIG_CMA) && ac->migratetype == MIGRATE_MOVABLE)
4184 *alloc_flags |= ALLOC_CMA;
4185
4186 return true;
4187 }
4188
4189 /* Determine whether to spread dirty pages and what the first usable zone */
4190 static inline void finalise_ac(gfp_t gfp_mask,
4191 unsigned int order, struct alloc_context *ac)
4192 {
4193 /* Dirty zone balancing only done in the fast path */
4194 ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
4195
4196 /*
4197 * The preferred zone is used for statistics but crucially it is
4198 * also used as the starting point for the zonelist iterator. It
4199 * may get reset for allocations that ignore memory policies.
4200 */
4201 ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4202 ac->high_zoneidx, ac->nodemask);
4203 }
4204
4205 /*
4206 * This is the 'heart' of the zoned buddy allocator.
4207 */
4208 struct page *
4209 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
4210 nodemask_t *nodemask)
4211 {
4212 struct page *page;
4213 unsigned int alloc_flags = ALLOC_WMARK_LOW;
4214 gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
4215 struct alloc_context ac = { };
4216
4217 gfp_mask &= gfp_allowed_mask;
4218 alloc_mask = gfp_mask;
4219 if (!prepare_alloc_pages(gfp_mask, order, preferred_nid, nodemask, &ac, &alloc_mask, &alloc_flags))
4220 return NULL;
4221
4222 finalise_ac(gfp_mask, order, &ac);
4223
4224 /* First allocation attempt */
4225 page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
4226 if (likely(page))
4227 goto out;
4228
4229 /*
4230 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
4231 * resp. GFP_NOIO which has to be inherited for all allocation requests
4232 * from a particular context which has been marked by
4233 * memalloc_no{fs,io}_{save,restore}.
4234 */
4235 alloc_mask = current_gfp_context(gfp_mask);
4236 ac.spread_dirty_pages = false;
4237
4238 /*
4239 * Restore the original nodemask if it was potentially replaced with
4240 * &cpuset_current_mems_allowed to optimize the fast-path attempt.
4241 */
4242 if (unlikely(ac.nodemask != nodemask))
4243 ac.nodemask = nodemask;
4244
4245 page = __alloc_pages_slowpath(alloc_mask, order, &ac);
4246
4247 out:
4248 if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
4249 unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) {
4250 __free_pages(page, order);
4251 page = NULL;
4252 }
4253
4254 trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
4255
4256 return page;
4257 }
4258 EXPORT_SYMBOL(__alloc_pages_nodemask);
4259
4260 /*
4261 * Common helper functions.
4262 */
4263 unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
4264 {
4265 struct page *page;
4266
4267 /*
4268 * __get_free_pages() returns a 32-bit address, which cannot represent
4269 * a highmem page
4270 */
4271 VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
4272
4273 page = alloc_pages(gfp_mask, order);
4274 if (!page)
4275 return 0;
4276 return (unsigned long) page_address(page);
4277 }
4278 EXPORT_SYMBOL(__get_free_pages);
4279
4280 unsigned long get_zeroed_page(gfp_t gfp_mask)
4281 {
4282 return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
4283 }
4284 EXPORT_SYMBOL(get_zeroed_page);
4285
4286 void __free_pages(struct page *page, unsigned int order)
4287 {
4288 if (put_page_testzero(page)) {
4289 if (order == 0)
4290 free_unref_page(page);
4291 else
4292 __free_pages_ok(page, order);
4293 }
4294 }
4295
4296 EXPORT_SYMBOL(__free_pages);
4297
4298 void free_pages(unsigned long addr, unsigned int order)
4299 {
4300 if (addr != 0) {
4301 VM_BUG_ON(!virt_addr_valid((void *)addr));
4302 __free_pages(virt_to_page((void *)addr), order);
4303 }
4304 }
4305
4306 EXPORT_SYMBOL(free_pages);
4307
4308 /*
4309 * Page Fragment:
4310 * An arbitrary-length arbitrary-offset area of memory which resides
4311 * within a 0 or higher order page. Multiple fragments within that page
4312 * are individually refcounted, in the page's reference counter.
4313 *
4314 * The page_frag functions below provide a simple allocation framework for
4315 * page fragments. This is used by the network stack and network device
4316 * drivers to provide a backing region of memory for use as either an
4317 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
4318 */
4319 static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
4320 gfp_t gfp_mask)
4321 {
4322 struct page *page = NULL;
4323 gfp_t gfp = gfp_mask;
4324
4325 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4326 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
4327 __GFP_NOMEMALLOC;
4328 page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
4329 PAGE_FRAG_CACHE_MAX_ORDER);
4330 nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
4331 #endif
4332 if (unlikely(!page))
4333 page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
4334
4335 nc->va = page ? page_address(page) : NULL;
4336
4337 return page;
4338 }
4339
4340 void __page_frag_cache_drain(struct page *page, unsigned int count)
4341 {
4342 VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
4343
4344 if (page_ref_sub_and_test(page, count)) {
4345 unsigned int order = compound_order(page);
4346
4347 if (order == 0)
4348 free_unref_page(page);
4349 else
4350 __free_pages_ok(page, order);
4351 }
4352 }
4353 EXPORT_SYMBOL(__page_frag_cache_drain);
4354
4355 void *page_frag_alloc(struct page_frag_cache *nc,
4356 unsigned int fragsz, gfp_t gfp_mask)
4357 {
4358 unsigned int size = PAGE_SIZE;
4359 struct page *page;
4360 int offset;
4361
4362 if (unlikely(!nc->va)) {
4363 refill:
4364 page = __page_frag_cache_refill(nc, gfp_mask);
4365 if (!page)
4366 return NULL;
4367
4368 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4369 /* if size can vary use size else just use PAGE_SIZE */
4370 size = nc->size;
4371 #endif
4372 /* Even if we own the page, we do not use atomic_set().
4373 * This would break get_page_unless_zero() users.
4374 */
4375 page_ref_add(page, size - 1);
4376
4377 /* reset page count bias and offset to start of new frag */
4378 nc->pfmemalloc = page_is_pfmemalloc(page);
4379 nc->pagecnt_bias = size;
4380 nc->offset = size;
4381 }
4382
4383 offset = nc->offset - fragsz;
4384 if (unlikely(offset < 0)) {
4385 page = virt_to_page(nc->va);
4386
4387 if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
4388 goto refill;
4389
4390 #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
4391 /* if size can vary use size else just use PAGE_SIZE */
4392 size = nc->size;
4393 #endif
4394 /* OK, page count is 0, we can safely set it */
4395 set_page_count(page, size);
4396
4397 /* reset page count bias and offset to start of new frag */
4398 nc->pagecnt_bias = size;
4399 offset = size - fragsz;
4400 }
4401
4402 nc->pagecnt_bias--;
4403 nc->offset = offset;
4404
4405 return nc->va + offset;
4406 }
4407 EXPORT_SYMBOL(page_frag_alloc);
4408
4409 /*
4410 * Frees a page fragment allocated out of either a compound or order 0 page.
4411 */
4412 void page_frag_free(void *addr)
4413 {
4414 struct page *page = virt_to_head_page(addr);
4415
4416 if (unlikely(put_page_testzero(page)))
4417 __free_pages_ok(page, compound_order(page));
4418 }
4419 EXPORT_SYMBOL(page_frag_free);
4420
4421 static void *make_alloc_exact(unsigned long addr, unsigned int order,
4422 size_t size)
4423 {
4424 if (addr) {
4425 unsigned long alloc_end = addr + (PAGE_SIZE << order);
4426 unsigned long used = addr + PAGE_ALIGN(size);
4427
4428 split_page(virt_to_page((void *)addr), order);
4429 while (used < alloc_end) {
4430 free_page(used);
4431 used += PAGE_SIZE;
4432 }
4433 }
4434 return (void *)addr;
4435 }
4436
4437 /**
4438 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
4439 * @size: the number of bytes to allocate
4440 * @gfp_mask: GFP flags for the allocation
4441 *
4442 * This function is similar to alloc_pages(), except that it allocates the
4443 * minimum number of pages to satisfy the request. alloc_pages() can only
4444 * allocate memory in power-of-two pages.
4445 *
4446 * This function is also limited by MAX_ORDER.
4447 *
4448 * Memory allocated by this function must be released by free_pages_exact().
4449 */
4450 void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
4451 {
4452 unsigned int order = get_order(size);
4453 unsigned long addr;
4454
4455 addr = __get_free_pages(gfp_mask, order);
4456 return make_alloc_exact(addr, order, size);
4457 }
4458 EXPORT_SYMBOL(alloc_pages_exact);
4459
4460 /**
4461 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
4462 * pages on a node.
4463 * @nid: the preferred node ID where memory should be allocated
4464 * @size: the number of bytes to allocate
4465 * @gfp_mask: GFP flags for the allocation
4466 *
4467 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
4468 * back.
4469 */
4470 void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
4471 {
4472 unsigned int order = get_order(size);
4473 struct page *p = alloc_pages_node(nid, gfp_mask, order);
4474 if (!p)
4475 return NULL;
4476 return make_alloc_exact((unsigned long)page_address(p), order, size);
4477 }
4478
4479 /**
4480 * free_pages_exact - release memory allocated via alloc_pages_exact()
4481 * @virt: the value returned by alloc_pages_exact.
4482 * @size: size of allocation, same value as passed to alloc_pages_exact().
4483 *
4484 * Release the memory allocated by a previous call to alloc_pages_exact.
4485 */
4486 void free_pages_exact(void *virt, size_t size)
4487 {
4488 unsigned long addr = (unsigned long)virt;
4489 unsigned long end = addr + PAGE_ALIGN(size);
4490
4491 while (addr < end) {
4492 free_page(addr);
4493 addr += PAGE_SIZE;
4494 }
4495 }
4496 EXPORT_SYMBOL(free_pages_exact);
4497
4498 /**
4499 * nr_free_zone_pages - count number of pages beyond high watermark
4500 * @offset: The zone index of the highest zone
4501 *
4502 * nr_free_zone_pages() counts the number of counts pages which are beyond the
4503 * high watermark within all zones at or below a given zone index. For each
4504 * zone, the number of pages is calculated as:
4505 *
4506 * nr_free_zone_pages = managed_pages - high_pages
4507 */
4508 static unsigned long nr_free_zone_pages(int offset)
4509 {
4510 struct zoneref *z;
4511 struct zone *zone;
4512
4513 /* Just pick one node, since fallback list is circular */
4514 unsigned long sum = 0;
4515
4516 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
4517
4518 for_each_zone_zonelist(zone, z, zonelist, offset) {
4519 unsigned long size = zone->managed_pages;
4520 unsigned long high = high_wmark_pages(zone);
4521 if (size > high)
4522 sum += size - high;
4523 }
4524
4525 return sum;
4526 }
4527
4528 /**
4529 * nr_free_buffer_pages - count number of pages beyond high watermark
4530 *
4531 * nr_free_buffer_pages() counts the number of pages which are beyond the high
4532 * watermark within ZONE_DMA and ZONE_NORMAL.
4533 */
4534 unsigned long nr_free_buffer_pages(void)
4535 {
4536 return nr_free_zone_pages(gfp_zone(GFP_USER));
4537 }
4538 EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
4539
4540 /**
4541 * nr_free_pagecache_pages - count number of pages beyond high watermark
4542 *
4543 * nr_free_pagecache_pages() counts the number of pages which are beyond the
4544 * high watermark within all zones.
4545 */
4546 unsigned long nr_free_pagecache_pages(void)
4547 {
4548 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
4549 }
4550
4551 static inline void show_node(struct zone *zone)
4552 {
4553 if (IS_ENABLED(CONFIG_NUMA))
4554 printk("Node %d ", zone_to_nid(zone));
4555 }
4556
4557 long si_mem_available(void)
4558 {
4559 long available;
4560 unsigned long pagecache;
4561 unsigned long wmark_low = 0;
4562 unsigned long pages[NR_LRU_LISTS];
4563 struct zone *zone;
4564 int lru;
4565
4566 for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
4567 pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
4568
4569 for_each_zone(zone)
4570 wmark_low += zone->watermark[WMARK_LOW];
4571
4572 /*
4573 * Estimate the amount of memory available for userspace allocations,
4574 * without causing swapping.
4575 */
4576 available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
4577
4578 /*
4579 * Not all the page cache can be freed, otherwise the system will
4580 * start swapping. Assume at least half of the page cache, or the
4581 * low watermark worth of cache, needs to stay.
4582 */
4583 pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
4584 pagecache -= min(pagecache / 2, wmark_low);
4585 available += pagecache;
4586
4587 /*
4588 * Part of the reclaimable slab consists of items that are in use,
4589 * and cannot be freed. Cap this estimate at the low watermark.
4590 */
4591 available += global_node_page_state(NR_SLAB_RECLAIMABLE) -
4592 min(global_node_page_state(NR_SLAB_RECLAIMABLE) / 2,
4593 wmark_low);
4594
4595 if (available < 0)
4596 available = 0;
4597 return available;
4598 }
4599 EXPORT_SYMBOL_GPL(si_mem_available);
4600
4601 void si_meminfo(struct sysinfo *val)
4602 {
4603 val->totalram = totalram_pages;
4604 val->sharedram = global_node_page_state(NR_SHMEM);
4605 val->freeram = global_zone_page_state(NR_FREE_PAGES);
4606 val->bufferram = nr_blockdev_pages();
4607 val->totalhigh = totalhigh_pages;
4608 val->freehigh = nr_free_highpages();
4609 val->mem_unit = PAGE_SIZE;
4610 }
4611
4612 EXPORT_SYMBOL(si_meminfo);
4613
4614 #ifdef CONFIG_NUMA
4615 void si_meminfo_node(struct sysinfo *val, int nid)
4616 {
4617 int zone_type; /* needs to be signed */
4618 unsigned long managed_pages = 0;
4619 unsigned long managed_highpages = 0;
4620 unsigned long free_highpages = 0;
4621 pg_data_t *pgdat = NODE_DATA(nid);
4622
4623 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
4624 managed_pages += pgdat->node_zones[zone_type].managed_pages;
4625 val->totalram = managed_pages;
4626 val->sharedram = node_page_state(pgdat, NR_SHMEM);
4627 val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
4628 #ifdef CONFIG_HIGHMEM
4629 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
4630 struct zone *zone = &pgdat->node_zones[zone_type];
4631
4632 if (is_highmem(zone)) {
4633 managed_highpages += zone->managed_pages;
4634 free_highpages += zone_page_state(zone, NR_FREE_PAGES);
4635 }
4636 }
4637 val->totalhigh = managed_highpages;
4638 val->freehigh = free_highpages;
4639 #else
4640 val->totalhigh = managed_highpages;
4641 val->freehigh = free_highpages;
4642 #endif
4643 val->mem_unit = PAGE_SIZE;
4644 }
4645 #endif
4646
4647 /*
4648 * Determine whether the node should be displayed or not, depending on whether
4649 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
4650 */
4651 static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
4652 {
4653 if (!(flags & SHOW_MEM_FILTER_NODES))
4654 return false;
4655
4656 /*
4657 * no node mask - aka implicit memory numa policy. Do not bother with
4658 * the synchronization - read_mems_allowed_begin - because we do not
4659 * have to be precise here.
4660 */
4661 if (!nodemask)
4662 nodemask = &cpuset_current_mems_allowed;
4663
4664 return !node_isset(nid, *nodemask);
4665 }
4666
4667 #define K(x) ((x) << (PAGE_SHIFT-10))
4668
4669 static void show_migration_types(unsigned char type)
4670 {
4671 static const char types[MIGRATE_TYPES] = {
4672 [MIGRATE_UNMOVABLE] = 'U',
4673 [MIGRATE_MOVABLE] = 'M',
4674 [MIGRATE_RECLAIMABLE] = 'E',
4675 [MIGRATE_HIGHATOMIC] = 'H',
4676 #ifdef CONFIG_CMA
4677 [MIGRATE_CMA] = 'C',
4678 #endif
4679 #ifdef CONFIG_MEMORY_ISOLATION
4680 [MIGRATE_ISOLATE] = 'I',
4681 #endif
4682 };
4683 char tmp[MIGRATE_TYPES + 1];
4684 char *p = tmp;
4685 int i;
4686
4687 for (i = 0; i < MIGRATE_TYPES; i++) {
4688 if (type & (1 << i))
4689 *p++ = types[i];
4690 }
4691
4692 *p = '\0';
4693 printk(KERN_CONT "(%s) ", tmp);
4694 }
4695
4696 /*
4697 * Show free area list (used inside shift_scroll-lock stuff)
4698 * We also calculate the percentage fragmentation. We do this by counting the
4699 * memory on each free list with the exception of the first item on the list.
4700 *
4701 * Bits in @filter:
4702 * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
4703 * cpuset.
4704 */
4705 void show_free_areas(unsigned int filter, nodemask_t *nodemask)
4706 {
4707 unsigned long free_pcp = 0;
4708 int cpu;
4709 struct zone *zone;
4710 pg_data_t *pgdat;
4711
4712 for_each_populated_zone(zone) {
4713 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
4714 continue;
4715
4716 for_each_online_cpu(cpu)
4717 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
4718 }
4719
4720 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
4721 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
4722 " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
4723 " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
4724 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
4725 " free:%lu free_pcp:%lu free_cma:%lu\n",
4726 global_node_page_state(NR_ACTIVE_ANON),
4727 global_node_page_state(NR_INACTIVE_ANON),
4728 global_node_page_state(NR_ISOLATED_ANON),
4729 global_node_page_state(NR_ACTIVE_FILE),
4730 global_node_page_state(NR_INACTIVE_FILE),
4731 global_node_page_state(NR_ISOLATED_FILE),
4732 global_node_page_state(NR_UNEVICTABLE),
4733 global_node_page_state(NR_FILE_DIRTY),
4734 global_node_page_state(NR_WRITEBACK),
4735 global_node_page_state(NR_UNSTABLE_NFS),
4736 global_node_page_state(NR_SLAB_RECLAIMABLE),
4737 global_node_page_state(NR_SLAB_UNRECLAIMABLE),
4738 global_node_page_state(NR_FILE_MAPPED),
4739 global_node_page_state(NR_SHMEM),
4740 global_zone_page_state(NR_PAGETABLE),
4741 global_zone_page_state(NR_BOUNCE),
4742 global_zone_page_state(NR_FREE_PAGES),
4743 free_pcp,
4744 global_zone_page_state(NR_FREE_CMA_PAGES));
4745
4746 for_each_online_pgdat(pgdat) {
4747 if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
4748 continue;
4749
4750 printk("Node %d"
4751 " active_anon:%lukB"
4752 " inactive_anon:%lukB"
4753 " active_file:%lukB"
4754 " inactive_file:%lukB"
4755 " unevictable:%lukB"
4756 " isolated(anon):%lukB"
4757 " isolated(file):%lukB"
4758 " mapped:%lukB"
4759 " dirty:%lukB"
4760 " writeback:%lukB"
4761 " shmem:%lukB"
4762 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4763 " shmem_thp: %lukB"
4764 " shmem_pmdmapped: %lukB"
4765 " anon_thp: %lukB"
4766 #endif
4767 " writeback_tmp:%lukB"
4768 " unstable:%lukB"
4769 " all_unreclaimable? %s"
4770 "\n",
4771 pgdat->node_id,
4772 K(node_page_state(pgdat, NR_ACTIVE_ANON)),
4773 K(node_page_state(pgdat, NR_INACTIVE_ANON)),
4774 K(node_page_state(pgdat, NR_ACTIVE_FILE)),
4775 K(node_page_state(pgdat, NR_INACTIVE_FILE)),
4776 K(node_page_state(pgdat, NR_UNEVICTABLE)),
4777 K(node_page_state(pgdat, NR_ISOLATED_ANON)),
4778 K(node_page_state(pgdat, NR_ISOLATED_FILE)),
4779 K(node_page_state(pgdat, NR_FILE_MAPPED)),
4780 K(node_page_state(pgdat, NR_FILE_DIRTY)),
4781 K(node_page_state(pgdat, NR_WRITEBACK)),
4782 K(node_page_state(pgdat, NR_SHMEM)),
4783 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4784 K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
4785 K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
4786 * HPAGE_PMD_NR),
4787 K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
4788 #endif
4789 K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
4790 K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
4791 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
4792 "yes" : "no");
4793 }
4794
4795 for_each_populated_zone(zone) {
4796 int i;
4797
4798 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
4799 continue;
4800
4801 free_pcp = 0;
4802 for_each_online_cpu(cpu)
4803 free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count;
4804
4805 show_node(zone);
4806 printk(KERN_CONT
4807 "%s"
4808 " free:%lukB"
4809 " min:%lukB"
4810 " low:%lukB"
4811 " high:%lukB"
4812 " active_anon:%lukB"
4813 " inactive_anon:%lukB"
4814 " active_file:%lukB"
4815 " inactive_file:%lukB"
4816 " unevictable:%lukB"
4817 " writepending:%lukB"
4818 " present:%lukB"
4819 " managed:%lukB"
4820 " mlocked:%lukB"
4821 " kernel_stack:%lukB"
4822 " pagetables:%lukB"
4823 " bounce:%lukB"
4824 " free_pcp:%lukB"
4825 " local_pcp:%ukB"
4826 " free_cma:%lukB"
4827 "\n",
4828 zone->name,
4829 K(zone_page_state(zone, NR_FREE_PAGES)),
4830 K(min_wmark_pages(zone)),
4831 K(low_wmark_pages(zone)),
4832 K(high_wmark_pages(zone)),
4833 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
4834 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
4835 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
4836 K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
4837 K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
4838 K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
4839 K(zone->present_pages),
4840 K(zone->managed_pages),
4841 K(zone_page_state(zone, NR_MLOCK)),
4842 zone_page_state(zone, NR_KERNEL_STACK_KB),
4843 K(zone_page_state(zone, NR_PAGETABLE)),
4844 K(zone_page_state(zone, NR_BOUNCE)),
4845 K(free_pcp),
4846 K(this_cpu_read(zone->pageset->pcp.count)),
4847 K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
4848 printk("lowmem_reserve[]:");
4849 for (i = 0; i < MAX_NR_ZONES; i++)
4850 printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
4851 printk(KERN_CONT "\n");
4852 }
4853
4854 for_each_populated_zone(zone) {
4855 unsigned int order;
4856 unsigned long nr[MAX_ORDER], flags, total = 0;
4857 unsigned char types[MAX_ORDER];
4858
4859 if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
4860 continue;
4861 show_node(zone);
4862 printk(KERN_CONT "%s: ", zone->name);
4863
4864 spin_lock_irqsave(&zone->lock, flags);
4865 for (order = 0; order < MAX_ORDER; order++) {
4866 struct free_area *area = &zone->free_area[order];
4867 int type;
4868
4869 nr[order] = area->nr_free;
4870 total += nr[order] << order;
4871
4872 types[order] = 0;
4873 for (type = 0; type < MIGRATE_TYPES; type++) {
4874 if (!list_empty(&area->free_list[type]))
4875 types[order] |= 1 << type;
4876 }
4877 }
4878 spin_unlock_irqrestore(&zone->lock, flags);
4879 for (order = 0; order < MAX_ORDER; order++) {
4880 printk(KERN_CONT "%lu*%lukB ",
4881 nr[order], K(1UL) << order);
4882 if (nr[order])
4883 show_migration_types(types[order]);
4884 }
4885 printk(KERN_CONT "= %lukB\n", K(total));
4886 }
4887
4888 hugetlb_show_meminfo();
4889
4890 printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
4891
4892 show_swap_cache_info();
4893 }
4894
4895 static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
4896 {
4897 zoneref->zone = zone;
4898 zoneref->zone_idx = zone_idx(zone);
4899 }
4900
4901 /*
4902 * Builds allocation fallback zone lists.
4903 *
4904 * Add all populated zones of a node to the zonelist.
4905 */
4906 static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
4907 {
4908 struct zone *zone;
4909 enum zone_type zone_type = MAX_NR_ZONES;
4910 int nr_zones = 0;
4911
4912 do {
4913 zone_type--;
4914 zone = pgdat->node_zones + zone_type;
4915 if (managed_zone(zone)) {
4916 zoneref_set_zone(zone, &zonerefs[nr_zones++]);
4917 check_highest_zone(zone_type);
4918 }
4919 } while (zone_type);
4920
4921 return nr_zones;
4922 }
4923
4924 #ifdef CONFIG_NUMA
4925
4926 static int __parse_numa_zonelist_order(char *s)
4927 {
4928 /*
4929 * We used to support different zonlists modes but they turned
4930 * out to be just not useful. Let's keep the warning in place
4931 * if somebody still use the cmd line parameter so that we do
4932 * not fail it silently
4933 */
4934 if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
4935 pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s);
4936 return -EINVAL;
4937 }
4938 return 0;
4939 }
4940
4941 static __init int setup_numa_zonelist_order(char *s)
4942 {
4943 if (!s)
4944 return 0;
4945
4946 return __parse_numa_zonelist_order(s);
4947 }
4948 early_param("numa_zonelist_order", setup_numa_zonelist_order);
4949
4950 char numa_zonelist_order[] = "Node";
4951
4952 /*
4953 * sysctl handler for numa_zonelist_order
4954 */
4955 int numa_zonelist_order_handler(struct ctl_table *table, int write,
4956 void __user *buffer, size_t *length,
4957 loff_t *ppos)
4958 {
4959 char *str;
4960 int ret;
4961
4962 if (!write)
4963 return proc_dostring(table, write, buffer, length, ppos);
4964 str = memdup_user_nul(buffer, 16);
4965 if (IS_ERR(str))
4966 return PTR_ERR(str);
4967
4968 ret = __parse_numa_zonelist_order(str);
4969 kfree(str);
4970 return ret;
4971 }
4972
4973
4974 #define MAX_NODE_LOAD (nr_online_nodes)
4975 static int node_load[MAX_NUMNODES];
4976
4977 /**
4978 * find_next_best_node - find the next node that should appear in a given node's fallback list
4979 * @node: node whose fallback list we're appending
4980 * @used_node_mask: nodemask_t of already used nodes
4981 *
4982 * We use a number of factors to determine which is the next node that should
4983 * appear on a given node's fallback list. The node should not have appeared
4984 * already in @node's fallback list, and it should be the next closest node
4985 * according to the distance array (which contains arbitrary distance values
4986 * from each node to each node in the system), and should also prefer nodes
4987 * with no CPUs, since presumably they'll have very little allocation pressure
4988 * on them otherwise.
4989 * It returns -1 if no node is found.
4990 */
4991 static int find_next_best_node(int node, nodemask_t *used_node_mask)
4992 {
4993 int n, val;
4994 int min_val = INT_MAX;
4995 int best_node = NUMA_NO_NODE;
4996 const struct cpumask *tmp = cpumask_of_node(0);
4997
4998 /* Use the local node if we haven't already */
4999 if (!node_isset(node, *used_node_mask)) {
5000 node_set(node, *used_node_mask);
5001 return node;
5002 }
5003
5004 for_each_node_state(n, N_MEMORY) {
5005
5006 /* Don't want a node to appear more than once */
5007 if (node_isset(n, *used_node_mask))
5008 continue;
5009
5010 /* Use the distance array to find the distance */
5011 val = node_distance(node, n);
5012
5013 /* Penalize nodes under us ("prefer the next node") */
5014 val += (n < node);
5015
5016 /* Give preference to headless and unused nodes */
5017 tmp = cpumask_of_node(n);
5018 if (!cpumask_empty(tmp))
5019 val += PENALTY_FOR_NODE_WITH_CPUS;
5020
5021 /* Slight preference for less loaded node */
5022 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
5023 val += node_load[n];
5024
5025 if (val < min_val) {
5026 min_val = val;
5027 best_node = n;
5028 }
5029 }
5030
5031 if (best_node >= 0)
5032 node_set(best_node, *used_node_mask);
5033
5034 return best_node;
5035 }
5036
5037
5038 /*
5039 * Build zonelists ordered by node and zones within node.
5040 * This results in maximum locality--normal zone overflows into local
5041 * DMA zone, if any--but risks exhausting DMA zone.
5042 */
5043 static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
5044 unsigned nr_nodes)
5045 {
5046 struct zoneref *zonerefs;
5047 int i;
5048
5049 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5050
5051 for (i = 0; i < nr_nodes; i++) {
5052 int nr_zones;
5053
5054 pg_data_t *node = NODE_DATA(node_order[i]);
5055
5056 nr_zones = build_zonerefs_node(node, zonerefs);
5057 zonerefs += nr_zones;
5058 }
5059 zonerefs->zone = NULL;
5060 zonerefs->zone_idx = 0;
5061 }
5062
5063 /*
5064 * Build gfp_thisnode zonelists
5065 */
5066 static void build_thisnode_zonelists(pg_data_t *pgdat)
5067 {
5068 struct zoneref *zonerefs;
5069 int nr_zones;
5070
5071 zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
5072 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5073 zonerefs += nr_zones;
5074 zonerefs->zone = NULL;
5075 zonerefs->zone_idx = 0;
5076 }
5077
5078 /*
5079 * Build zonelists ordered by zone and nodes within zones.
5080 * This results in conserving DMA zone[s] until all Normal memory is
5081 * exhausted, but results in overflowing to remote node while memory
5082 * may still exist in local DMA zone.
5083 */
5084
5085 static void build_zonelists(pg_data_t *pgdat)
5086 {
5087 static int node_order[MAX_NUMNODES];
5088 int node, load, nr_nodes = 0;
5089 nodemask_t used_mask;
5090 int local_node, prev_node;
5091
5092 /* NUMA-aware ordering of nodes */
5093 local_node = pgdat->node_id;
5094 load = nr_online_nodes;
5095 prev_node = local_node;
5096 nodes_clear(used_mask);
5097
5098 memset(node_order, 0, sizeof(node_order));
5099 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
5100 /*
5101 * We don't want to pressure a particular node.
5102 * So adding penalty to the first node in same
5103 * distance group to make it round-robin.
5104 */
5105 if (node_distance(local_node, node) !=
5106 node_distance(local_node, prev_node))
5107 node_load[node] = load;
5108
5109 node_order[nr_nodes++] = node;
5110 prev_node = node;
5111 load--;
5112 }
5113
5114 build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
5115 build_thisnode_zonelists(pgdat);
5116 }
5117
5118 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
5119 /*
5120 * Return node id of node used for "local" allocations.
5121 * I.e., first node id of first zone in arg node's generic zonelist.
5122 * Used for initializing percpu 'numa_mem', which is used primarily
5123 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
5124 */
5125 int local_memory_node(int node)
5126 {
5127 struct zoneref *z;
5128
5129 z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
5130 gfp_zone(GFP_KERNEL),
5131 NULL);
5132 return z->zone->node;
5133 }
5134 #endif
5135
5136 static void setup_min_unmapped_ratio(void);
5137 static void setup_min_slab_ratio(void);
5138 #else /* CONFIG_NUMA */
5139
5140 static void build_zonelists(pg_data_t *pgdat)
5141 {
5142 int node, local_node;
5143 struct zoneref *zonerefs;
5144 int nr_zones;
5145
5146 local_node = pgdat->node_id;
5147
5148 zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5149 nr_zones = build_zonerefs_node(pgdat, zonerefs);
5150 zonerefs += nr_zones;
5151
5152 /*
5153 * Now we build the zonelist so that it contains the zones
5154 * of all the other nodes.
5155 * We don't want to pressure a particular node, so when
5156 * building the zones for node N, we make sure that the
5157 * zones coming right after the local ones are those from
5158 * node N+1 (modulo N)
5159 */
5160 for (node = local_node + 1; node < MAX_NUMNODES; node++) {
5161 if (!node_online(node))
5162 continue;
5163 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5164 zonerefs += nr_zones;
5165 }
5166 for (node = 0; node < local_node; node++) {
5167 if (!node_online(node))
5168 continue;
5169 nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
5170 zonerefs += nr_zones;
5171 }
5172
5173 zonerefs->zone = NULL;
5174 zonerefs->zone_idx = 0;
5175 }
5176
5177 #endif /* CONFIG_NUMA */
5178
5179 /*
5180 * Boot pageset table. One per cpu which is going to be used for all
5181 * zones and all nodes. The parameters will be set in such a way
5182 * that an item put on a list will immediately be handed over to
5183 * the buddy list. This is safe since pageset manipulation is done
5184 * with interrupts disabled.
5185 *
5186 * The boot_pagesets must be kept even after bootup is complete for
5187 * unused processors and/or zones. They do play a role for bootstrapping
5188 * hotplugged processors.
5189 *
5190 * zoneinfo_show() and maybe other functions do
5191 * not check if the processor is online before following the pageset pointer.
5192 * Other parts of the kernel may not check if the zone is available.
5193 */
5194 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
5195 static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
5196 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
5197
5198 static void __build_all_zonelists(void *data)
5199 {
5200 int nid;
5201 int __maybe_unused cpu;
5202 pg_data_t *self = data;
5203 static DEFINE_SPINLOCK(lock);
5204
5205 spin_lock(&lock);
5206
5207 #ifdef CONFIG_NUMA
5208 memset(node_load, 0, sizeof(node_load));
5209 #endif
5210
5211 /*
5212 * This node is hotadded and no memory is yet present. So just
5213 * building zonelists is fine - no need to touch other nodes.
5214 */
5215 if (self && !node_online(self->node_id)) {
5216 build_zonelists(self);
5217 } else {
5218 for_each_online_node(nid) {
5219 pg_data_t *pgdat = NODE_DATA(nid);
5220
5221 build_zonelists(pgdat);
5222 }
5223
5224 #ifdef CONFIG_HAVE_MEMORYLESS_NODES
5225 /*
5226 * We now know the "local memory node" for each node--
5227 * i.e., the node of the first zone in the generic zonelist.
5228 * Set up numa_mem percpu variable for on-line cpus. During
5229 * boot, only the boot cpu should be on-line; we'll init the
5230 * secondary cpus' numa_mem as they come on-line. During
5231 * node/memory hotplug, we'll fixup all on-line cpus.
5232 */
5233 for_each_online_cpu(cpu)
5234 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
5235 #endif
5236 }
5237
5238 spin_unlock(&lock);
5239 }
5240
5241 static noinline void __init
5242 build_all_zonelists_init(void)
5243 {
5244 int cpu;
5245
5246 __build_all_zonelists(NULL);
5247
5248 /*
5249 * Initialize the boot_pagesets that are going to be used
5250 * for bootstrapping processors. The real pagesets for
5251 * each zone will be allocated later when the per cpu
5252 * allocator is available.
5253 *
5254 * boot_pagesets are used also for bootstrapping offline
5255 * cpus if the system is already booted because the pagesets
5256 * are needed to initialize allocators on a specific cpu too.
5257 * F.e. the percpu allocator needs the page allocator which
5258 * needs the percpu allocator in order to allocate its pagesets
5259 * (a chicken-egg dilemma).
5260 */
5261 for_each_possible_cpu(cpu)
5262 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
5263
5264 mminit_verify_zonelist();
5265 cpuset_init_current_mems_allowed();
5266 }
5267
5268 /*
5269 * unless system_state == SYSTEM_BOOTING.
5270 *
5271 * __ref due to call of __init annotated helper build_all_zonelists_init
5272 * [protected by SYSTEM_BOOTING].
5273 */
5274 void __ref build_all_zonelists(pg_data_t *pgdat)
5275 {
5276 if (system_state == SYSTEM_BOOTING) {
5277 build_all_zonelists_init();
5278 } else {
5279 __build_all_zonelists(pgdat);
5280 /* cpuset refresh routine should be here */
5281 }
5282 vm_total_pages = nr_free_pagecache_pages();
5283 /*
5284 * Disable grouping by mobility if the number of pages in the
5285 * system is too low to allow the mechanism to work. It would be
5286 * more accurate, but expensive to check per-zone. This check is
5287 * made on memory-hotadd so a system can start with mobility
5288 * disabled and enable it later
5289 */
5290 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
5291 page_group_by_mobility_disabled = 1;
5292 else
5293 page_group_by_mobility_disabled = 0;
5294
5295 pr_info("Built %i zonelists, mobility grouping %s. Total pages: %ld\n",
5296 nr_online_nodes,
5297 page_group_by_mobility_disabled ? "off" : "on",
5298 vm_total_pages);
5299 #ifdef CONFIG_NUMA
5300 pr_info("Policy zone: %s\n", zone_names[policy_zone]);
5301 #endif
5302 }
5303
5304 /*
5305 * Initially all pages are reserved - free ones are freed
5306 * up by free_all_bootmem() once the early boot process is
5307 * done. Non-atomic initialization, single-pass.
5308 */
5309 void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
5310 unsigned long start_pfn, enum memmap_context context)
5311 {
5312 struct vmem_altmap *altmap = to_vmem_altmap(__pfn_to_phys(start_pfn));
5313 unsigned long end_pfn = start_pfn + size;
5314 pg_data_t *pgdat = NODE_DATA(nid);
5315 unsigned long pfn;
5316 unsigned long nr_initialised = 0;
5317 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5318 struct memblock_region *r = NULL, *tmp;
5319 #endif
5320
5321 if (highest_memmap_pfn < end_pfn - 1)
5322 highest_memmap_pfn = end_pfn - 1;
5323
5324 /*
5325 * Honor reservation requested by the driver for this ZONE_DEVICE
5326 * memory
5327 */
5328 if (altmap && start_pfn == altmap->base_pfn)
5329 start_pfn += altmap->reserve;
5330
5331 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
5332 /*
5333 * There can be holes in boot-time mem_map[]s handed to this
5334 * function. They do not exist on hotplugged memory.
5335 */
5336 if (context != MEMMAP_EARLY)
5337 goto not_early;
5338
5339 if (!early_pfn_valid(pfn)) {
5340 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5341 /*
5342 * Skip to the pfn preceding the next valid one (or
5343 * end_pfn), such that we hit a valid pfn (or end_pfn)
5344 * on our next iteration of the loop.
5345 */
5346 pfn = memblock_next_valid_pfn(pfn, end_pfn) - 1;
5347 #endif
5348 continue;
5349 }
5350 if (!early_pfn_in_nid(pfn, nid))
5351 continue;
5352 if (!update_defer_init(pgdat, pfn, end_pfn, &nr_initialised))
5353 break;
5354
5355 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5356 /*
5357 * Check given memblock attribute by firmware which can affect
5358 * kernel memory layout. If zone==ZONE_MOVABLE but memory is
5359 * mirrored, it's an overlapped memmap init. skip it.
5360 */
5361 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
5362 if (!r || pfn >= memblock_region_memory_end_pfn(r)) {
5363 for_each_memblock(memory, tmp)
5364 if (pfn < memblock_region_memory_end_pfn(tmp))
5365 break;
5366 r = tmp;
5367 }
5368 if (pfn >= memblock_region_memory_base_pfn(r) &&
5369 memblock_is_mirror(r)) {
5370 /* already initialized as NORMAL */
5371 pfn = memblock_region_memory_end_pfn(r);
5372 continue;
5373 }
5374 }
5375 #endif
5376
5377 not_early:
5378 /*
5379 * Mark the block movable so that blocks are reserved for
5380 * movable at startup. This will force kernel allocations
5381 * to reserve their blocks rather than leaking throughout
5382 * the address space during boot when many long-lived
5383 * kernel allocations are made.
5384 *
5385 * bitmap is created for zone's valid pfn range. but memmap
5386 * can be created for invalid pages (for alignment)
5387 * check here not to call set_pageblock_migratetype() against
5388 * pfn out of zone.
5389 */
5390 if (!(pfn & (pageblock_nr_pages - 1))) {
5391 struct page *page = pfn_to_page(pfn);
5392
5393 __init_single_page(page, pfn, zone, nid);
5394 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
5395 cond_resched();
5396 } else {
5397 __init_single_pfn(pfn, zone, nid);
5398 }
5399 }
5400 }
5401
5402 static void __meminit zone_init_free_lists(struct zone *zone)
5403 {
5404 unsigned int order, t;
5405 for_each_migratetype_order(order, t) {
5406 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
5407 zone->free_area[order].nr_free = 0;
5408 }
5409 }
5410
5411 #ifndef __HAVE_ARCH_MEMMAP_INIT
5412 #define memmap_init(size, nid, zone, start_pfn) \
5413 memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
5414 #endif
5415
5416 static int zone_batchsize(struct zone *zone)
5417 {
5418 #ifdef CONFIG_MMU
5419 int batch;
5420
5421 /*
5422 * The per-cpu-pages pools are set to around 1000th of the
5423 * size of the zone. But no more than 1/2 of a meg.
5424 *
5425 * OK, so we don't know how big the cache is. So guess.
5426 */
5427 batch = zone->managed_pages / 1024;
5428 if (batch * PAGE_SIZE > 512 * 1024)
5429 batch = (512 * 1024) / PAGE_SIZE;
5430 batch /= 4; /* We effectively *= 4 below */
5431 if (batch < 1)
5432 batch = 1;
5433
5434 /*
5435 * Clamp the batch to a 2^n - 1 value. Having a power
5436 * of 2 value was found to be more likely to have
5437 * suboptimal cache aliasing properties in some cases.
5438 *
5439 * For example if 2 tasks are alternately allocating
5440 * batches of pages, one task can end up with a lot
5441 * of pages of one half of the possible page colors
5442 * and the other with pages of the other colors.
5443 */
5444 batch = rounddown_pow_of_two(batch + batch/2) - 1;
5445
5446 return batch;
5447
5448 #else
5449 /* The deferral and batching of frees should be suppressed under NOMMU
5450 * conditions.
5451 *
5452 * The problem is that NOMMU needs to be able to allocate large chunks
5453 * of contiguous memory as there's no hardware page translation to
5454 * assemble apparent contiguous memory from discontiguous pages.
5455 *
5456 * Queueing large contiguous runs of pages for batching, however,
5457 * causes the pages to actually be freed in smaller chunks. As there
5458 * can be a significant delay between the individual batches being
5459 * recycled, this leads to the once large chunks of space being
5460 * fragmented and becoming unavailable for high-order allocations.
5461 */
5462 return 0;
5463 #endif
5464 }
5465
5466 /*
5467 * pcp->high and pcp->batch values are related and dependent on one another:
5468 * ->batch must never be higher then ->high.
5469 * The following function updates them in a safe manner without read side
5470 * locking.
5471 *
5472 * Any new users of pcp->batch and pcp->high should ensure they can cope with
5473 * those fields changing asynchronously (acording the the above rule).
5474 *
5475 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
5476 * outside of boot time (or some other assurance that no concurrent updaters
5477 * exist).
5478 */
5479 static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
5480 unsigned long batch)
5481 {
5482 /* start with a fail safe value for batch */
5483 pcp->batch = 1;
5484 smp_wmb();
5485
5486 /* Update high, then batch, in order */
5487 pcp->high = high;
5488 smp_wmb();
5489
5490 pcp->batch = batch;
5491 }
5492
5493 /* a companion to pageset_set_high() */
5494 static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch)
5495 {
5496 pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch));
5497 }
5498
5499 static void pageset_init(struct per_cpu_pageset *p)
5500 {
5501 struct per_cpu_pages *pcp;
5502 int migratetype;
5503
5504 memset(p, 0, sizeof(*p));
5505
5506 pcp = &p->pcp;
5507 pcp->count = 0;
5508 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
5509 INIT_LIST_HEAD(&pcp->lists[migratetype]);
5510 }
5511
5512 static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
5513 {
5514 pageset_init(p);
5515 pageset_set_batch(p, batch);
5516 }
5517
5518 /*
5519 * pageset_set_high() sets the high water mark for hot per_cpu_pagelist
5520 * to the value high for the pageset p.
5521 */
5522 static void pageset_set_high(struct per_cpu_pageset *p,
5523 unsigned long high)
5524 {
5525 unsigned long batch = max(1UL, high / 4);
5526 if ((high / 4) > (PAGE_SHIFT * 8))
5527 batch = PAGE_SHIFT * 8;
5528
5529 pageset_update(&p->pcp, high, batch);
5530 }
5531
5532 static void pageset_set_high_and_batch(struct zone *zone,
5533 struct per_cpu_pageset *pcp)
5534 {
5535 if (percpu_pagelist_fraction)
5536 pageset_set_high(pcp,
5537 (zone->managed_pages /
5538 percpu_pagelist_fraction));
5539 else
5540 pageset_set_batch(pcp, zone_batchsize(zone));
5541 }
5542
5543 static void __meminit zone_pageset_init(struct zone *zone, int cpu)
5544 {
5545 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
5546
5547 pageset_init(pcp);
5548 pageset_set_high_and_batch(zone, pcp);
5549 }
5550
5551 void __meminit setup_zone_pageset(struct zone *zone)
5552 {
5553 int cpu;
5554 zone->pageset = alloc_percpu(struct per_cpu_pageset);
5555 for_each_possible_cpu(cpu)
5556 zone_pageset_init(zone, cpu);
5557 }
5558
5559 /*
5560 * Allocate per cpu pagesets and initialize them.
5561 * Before this call only boot pagesets were available.
5562 */
5563 void __init setup_per_cpu_pageset(void)
5564 {
5565 struct pglist_data *pgdat;
5566 struct zone *zone;
5567
5568 for_each_populated_zone(zone)
5569 setup_zone_pageset(zone);
5570
5571 for_each_online_pgdat(pgdat)
5572 pgdat->per_cpu_nodestats =
5573 alloc_percpu(struct per_cpu_nodestat);
5574 }
5575
5576 static __meminit void zone_pcp_init(struct zone *zone)
5577 {
5578 /*
5579 * per cpu subsystem is not up at this point. The following code
5580 * relies on the ability of the linker to provide the
5581 * offset of a (static) per cpu variable into the per cpu area.
5582 */
5583 zone->pageset = &boot_pageset;
5584
5585 if (populated_zone(zone))
5586 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
5587 zone->name, zone->present_pages,
5588 zone_batchsize(zone));
5589 }
5590
5591 void __meminit init_currently_empty_zone(struct zone *zone,
5592 unsigned long zone_start_pfn,
5593 unsigned long size)
5594 {
5595 struct pglist_data *pgdat = zone->zone_pgdat;
5596
5597 pgdat->nr_zones = zone_idx(zone) + 1;
5598
5599 zone->zone_start_pfn = zone_start_pfn;
5600
5601 mminit_dprintk(MMINIT_TRACE, "memmap_init",
5602 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
5603 pgdat->node_id,
5604 (unsigned long)zone_idx(zone),
5605 zone_start_pfn, (zone_start_pfn + size));
5606
5607 zone_init_free_lists(zone);
5608 zone->initialized = 1;
5609 }
5610
5611 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
5612 #ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
5613
5614 /*
5615 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
5616 */
5617 int __meminit __early_pfn_to_nid(unsigned long pfn,
5618 struct mminit_pfnnid_cache *state)
5619 {
5620 unsigned long start_pfn, end_pfn;
5621 int nid;
5622
5623 if (state->last_start <= pfn && pfn < state->last_end)
5624 return state->last_nid;
5625
5626 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
5627 if (nid != -1) {
5628 state->last_start = start_pfn;
5629 state->last_end = end_pfn;
5630 state->last_nid = nid;
5631 }
5632
5633 return nid;
5634 }
5635 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
5636
5637 /**
5638 * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range
5639 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
5640 * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid
5641 *
5642 * If an architecture guarantees that all ranges registered contain no holes
5643 * and may be freed, this this function may be used instead of calling
5644 * memblock_free_early_nid() manually.
5645 */
5646 void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
5647 {
5648 unsigned long start_pfn, end_pfn;
5649 int i, this_nid;
5650
5651 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
5652 start_pfn = min(start_pfn, max_low_pfn);
5653 end_pfn = min(end_pfn, max_low_pfn);
5654
5655 if (start_pfn < end_pfn)
5656 memblock_free_early_nid(PFN_PHYS(start_pfn),
5657 (end_pfn - start_pfn) << PAGE_SHIFT,
5658 this_nid);
5659 }
5660 }
5661
5662 /**
5663 * sparse_memory_present_with_active_regions - Call memory_present for each active range
5664 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
5665 *
5666 * If an architecture guarantees that all ranges registered contain no holes and may
5667 * be freed, this function may be used instead of calling memory_present() manually.
5668 */
5669 void __init sparse_memory_present_with_active_regions(int nid)
5670 {
5671 unsigned long start_pfn, end_pfn;
5672 int i, this_nid;
5673
5674 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
5675 memory_present(this_nid, start_pfn, end_pfn);
5676 }
5677
5678 /**
5679 * get_pfn_range_for_nid - Return the start and end page frames for a node
5680 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
5681 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
5682 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
5683 *
5684 * It returns the start and end page frame of a node based on information
5685 * provided by memblock_set_node(). If called for a node
5686 * with no available memory, a warning is printed and the start and end
5687 * PFNs will be 0.
5688 */
5689 void __meminit get_pfn_range_for_nid(unsigned int nid,
5690 unsigned long *start_pfn, unsigned long *end_pfn)
5691 {
5692 unsigned long this_start_pfn, this_end_pfn;
5693 int i;
5694
5695 *start_pfn = -1UL;
5696 *end_pfn = 0;
5697
5698 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
5699 *start_pfn = min(*start_pfn, this_start_pfn);
5700 *end_pfn = max(*end_pfn, this_end_pfn);
5701 }
5702
5703 if (*start_pfn == -1UL)
5704 *start_pfn = 0;
5705 }
5706
5707 /*
5708 * This finds a zone that can be used for ZONE_MOVABLE pages. The
5709 * assumption is made that zones within a node are ordered in monotonic
5710 * increasing memory addresses so that the "highest" populated zone is used
5711 */
5712 static void __init find_usable_zone_for_movable(void)
5713 {
5714 int zone_index;
5715 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
5716 if (zone_index == ZONE_MOVABLE)
5717 continue;
5718
5719 if (arch_zone_highest_possible_pfn[zone_index] >
5720 arch_zone_lowest_possible_pfn[zone_index])
5721 break;
5722 }
5723
5724 VM_BUG_ON(zone_index == -1);
5725 movable_zone = zone_index;
5726 }
5727
5728 /*
5729 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
5730 * because it is sized independent of architecture. Unlike the other zones,
5731 * the starting point for ZONE_MOVABLE is not fixed. It may be different
5732 * in each node depending on the size of each node and how evenly kernelcore
5733 * is distributed. This helper function adjusts the zone ranges
5734 * provided by the architecture for a given node by using the end of the
5735 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
5736 * zones within a node are in order of monotonic increases memory addresses
5737 */
5738 static void __meminit adjust_zone_range_for_zone_movable(int nid,
5739 unsigned long zone_type,
5740 unsigned long node_start_pfn,
5741 unsigned long node_end_pfn,
5742 unsigned long *zone_start_pfn,
5743 unsigned long *zone_end_pfn)
5744 {
5745 /* Only adjust if ZONE_MOVABLE is on this node */
5746 if (zone_movable_pfn[nid]) {
5747 /* Size ZONE_MOVABLE */
5748 if (zone_type == ZONE_MOVABLE) {
5749 *zone_start_pfn = zone_movable_pfn[nid];
5750 *zone_end_pfn = min(node_end_pfn,
5751 arch_zone_highest_possible_pfn[movable_zone]);
5752
5753 /* Adjust for ZONE_MOVABLE starting within this range */
5754 } else if (!mirrored_kernelcore &&
5755 *zone_start_pfn < zone_movable_pfn[nid] &&
5756 *zone_end_pfn > zone_movable_pfn[nid]) {
5757 *zone_end_pfn = zone_movable_pfn[nid];
5758
5759 /* Check if this whole range is within ZONE_MOVABLE */
5760 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
5761 *zone_start_pfn = *zone_end_pfn;
5762 }
5763 }
5764
5765 /*
5766 * Return the number of pages a zone spans in a node, including holes
5767 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
5768 */
5769 static unsigned long __meminit zone_spanned_pages_in_node(int nid,
5770 unsigned long zone_type,
5771 unsigned long node_start_pfn,
5772 unsigned long node_end_pfn,
5773 unsigned long *zone_start_pfn,
5774 unsigned long *zone_end_pfn,
5775 unsigned long *ignored)
5776 {
5777 /* When hotadd a new node from cpu_up(), the node should be empty */
5778 if (!node_start_pfn && !node_end_pfn)
5779 return 0;
5780
5781 /* Get the start and end of the zone */
5782 *zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
5783 *zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
5784 adjust_zone_range_for_zone_movable(nid, zone_type,
5785 node_start_pfn, node_end_pfn,
5786 zone_start_pfn, zone_end_pfn);
5787
5788 /* Check that this node has pages within the zone's required range */
5789 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
5790 return 0;
5791
5792 /* Move the zone boundaries inside the node if necessary */
5793 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
5794 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
5795
5796 /* Return the spanned pages */
5797 return *zone_end_pfn - *zone_start_pfn;
5798 }
5799
5800 /*
5801 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
5802 * then all holes in the requested range will be accounted for.
5803 */
5804 unsigned long __meminit __absent_pages_in_range(int nid,
5805 unsigned long range_start_pfn,
5806 unsigned long range_end_pfn)
5807 {
5808 unsigned long nr_absent = range_end_pfn - range_start_pfn;
5809 unsigned long start_pfn, end_pfn;
5810 int i;
5811
5812 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
5813 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
5814 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
5815 nr_absent -= end_pfn - start_pfn;
5816 }
5817 return nr_absent;
5818 }
5819
5820 /**
5821 * absent_pages_in_range - Return number of page frames in holes within a range
5822 * @start_pfn: The start PFN to start searching for holes
5823 * @end_pfn: The end PFN to stop searching for holes
5824 *
5825 * It returns the number of pages frames in memory holes within a range.
5826 */
5827 unsigned long __init absent_pages_in_range(unsigned long start_pfn,
5828 unsigned long end_pfn)
5829 {
5830 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
5831 }
5832
5833 /* Return the number of page frames in holes in a zone on a node */
5834 static unsigned long __meminit zone_absent_pages_in_node(int nid,
5835 unsigned long zone_type,
5836 unsigned long node_start_pfn,
5837 unsigned long node_end_pfn,
5838 unsigned long *ignored)
5839 {
5840 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
5841 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
5842 unsigned long zone_start_pfn, zone_end_pfn;
5843 unsigned long nr_absent;
5844
5845 /* When hotadd a new node from cpu_up(), the node should be empty */
5846 if (!node_start_pfn && !node_end_pfn)
5847 return 0;
5848
5849 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
5850 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
5851
5852 adjust_zone_range_for_zone_movable(nid, zone_type,
5853 node_start_pfn, node_end_pfn,
5854 &zone_start_pfn, &zone_end_pfn);
5855 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
5856
5857 /*
5858 * ZONE_MOVABLE handling.
5859 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
5860 * and vice versa.
5861 */
5862 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
5863 unsigned long start_pfn, end_pfn;
5864 struct memblock_region *r;
5865
5866 for_each_memblock(memory, r) {
5867 start_pfn = clamp(memblock_region_memory_base_pfn(r),
5868 zone_start_pfn, zone_end_pfn);
5869 end_pfn = clamp(memblock_region_memory_end_pfn(r),
5870 zone_start_pfn, zone_end_pfn);
5871
5872 if (zone_type == ZONE_MOVABLE &&
5873 memblock_is_mirror(r))
5874 nr_absent += end_pfn - start_pfn;
5875
5876 if (zone_type == ZONE_NORMAL &&
5877 !memblock_is_mirror(r))
5878 nr_absent += end_pfn - start_pfn;
5879 }
5880 }
5881
5882 return nr_absent;
5883 }
5884
5885 #else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
5886 static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
5887 unsigned long zone_type,
5888 unsigned long node_start_pfn,
5889 unsigned long node_end_pfn,
5890 unsigned long *zone_start_pfn,
5891 unsigned long *zone_end_pfn,
5892 unsigned long *zones_size)
5893 {
5894 unsigned int zone;
5895
5896 *zone_start_pfn = node_start_pfn;
5897 for (zone = 0; zone < zone_type; zone++)
5898 *zone_start_pfn += zones_size[zone];
5899
5900 *zone_end_pfn = *zone_start_pfn + zones_size[zone_type];
5901
5902 return zones_size[zone_type];
5903 }
5904
5905 static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
5906 unsigned long zone_type,
5907 unsigned long node_start_pfn,
5908 unsigned long node_end_pfn,
5909 unsigned long *zholes_size)
5910 {
5911 if (!zholes_size)
5912 return 0;
5913
5914 return zholes_size[zone_type];
5915 }
5916
5917 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
5918
5919 static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
5920 unsigned long node_start_pfn,
5921 unsigned long node_end_pfn,
5922 unsigned long *zones_size,
5923 unsigned long *zholes_size)
5924 {
5925 unsigned long realtotalpages = 0, totalpages = 0;
5926 enum zone_type i;
5927
5928 for (i = 0; i < MAX_NR_ZONES; i++) {
5929 struct zone *zone = pgdat->node_zones + i;
5930 unsigned long zone_start_pfn, zone_end_pfn;
5931 unsigned long size, real_size;
5932
5933 size = zone_spanned_pages_in_node(pgdat->node_id, i,
5934 node_start_pfn,
5935 node_end_pfn,
5936 &zone_start_pfn,
5937 &zone_end_pfn,
5938 zones_size);
5939 real_size = size - zone_absent_pages_in_node(pgdat->node_id, i,
5940 node_start_pfn, node_end_pfn,
5941 zholes_size);
5942 if (size)
5943 zone->zone_start_pfn = zone_start_pfn;
5944 else
5945 zone->zone_start_pfn = 0;
5946 zone->spanned_pages = size;
5947 zone->present_pages = real_size;
5948
5949 totalpages += size;
5950 realtotalpages += real_size;
5951 }
5952
5953 pgdat->node_spanned_pages = totalpages;
5954 pgdat->node_present_pages = realtotalpages;
5955 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
5956 realtotalpages);
5957 }
5958
5959 #ifndef CONFIG_SPARSEMEM
5960 /*
5961 * Calculate the size of the zone->blockflags rounded to an unsigned long
5962 * Start by making sure zonesize is a multiple of pageblock_order by rounding
5963 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
5964 * round what is now in bits to nearest long in bits, then return it in
5965 * bytes.
5966 */
5967 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
5968 {
5969 unsigned long usemapsize;
5970
5971 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
5972 usemapsize = roundup(zonesize, pageblock_nr_pages);
5973 usemapsize = usemapsize >> pageblock_order;
5974 usemapsize *= NR_PAGEBLOCK_BITS;
5975 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
5976
5977 return usemapsize / 8;
5978 }
5979
5980 static void __init setup_usemap(struct pglist_data *pgdat,
5981 struct zone *zone,
5982 unsigned long zone_start_pfn,
5983 unsigned long zonesize)
5984 {
5985 unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize);
5986 zone->pageblock_flags = NULL;
5987 if (usemapsize)
5988 zone->pageblock_flags =
5989 memblock_virt_alloc_node_nopanic(usemapsize,
5990 pgdat->node_id);
5991 }
5992 #else
5993 static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone,
5994 unsigned long zone_start_pfn, unsigned long zonesize) {}
5995 #endif /* CONFIG_SPARSEMEM */
5996
5997 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
5998
5999 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
6000 void __paginginit set_pageblock_order(void)
6001 {
6002 unsigned int order;
6003
6004 /* Check that pageblock_nr_pages has not already been setup */
6005 if (pageblock_order)
6006 return;
6007
6008 if (HPAGE_SHIFT > PAGE_SHIFT)
6009 order = HUGETLB_PAGE_ORDER;
6010 else
6011 order = MAX_ORDER - 1;
6012
6013 /*
6014 * Assume the largest contiguous order of interest is a huge page.
6015 * This value may be variable depending on boot parameters on IA64 and
6016 * powerpc.
6017 */
6018 pageblock_order = order;
6019 }
6020 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
6021
6022 /*
6023 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
6024 * is unused as pageblock_order is set at compile-time. See
6025 * include/linux/pageblock-flags.h for the values of pageblock_order based on
6026 * the kernel config
6027 */
6028 void __paginginit set_pageblock_order(void)
6029 {
6030 }
6031
6032 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
6033
6034 static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages,
6035 unsigned long present_pages)
6036 {
6037 unsigned long pages = spanned_pages;
6038
6039 /*
6040 * Provide a more accurate estimation if there are holes within
6041 * the zone and SPARSEMEM is in use. If there are holes within the
6042 * zone, each populated memory region may cost us one or two extra
6043 * memmap pages due to alignment because memmap pages for each
6044 * populated regions may not be naturally aligned on page boundary.
6045 * So the (present_pages >> 4) heuristic is a tradeoff for that.
6046 */
6047 if (spanned_pages > present_pages + (present_pages >> 4) &&
6048 IS_ENABLED(CONFIG_SPARSEMEM))
6049 pages = present_pages;
6050
6051 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
6052 }
6053
6054 /*
6055 * Set up the zone data structures:
6056 * - mark all pages reserved
6057 * - mark all memory queues empty
6058 * - clear the memory bitmaps
6059 *
6060 * NOTE: pgdat should get zeroed by caller.
6061 */
6062 static void __paginginit free_area_init_core(struct pglist_data *pgdat)
6063 {
6064 enum zone_type j;
6065 int nid = pgdat->node_id;
6066
6067 pgdat_resize_init(pgdat);
6068 #ifdef CONFIG_NUMA_BALANCING
6069 spin_lock_init(&pgdat->numabalancing_migrate_lock);
6070 pgdat->numabalancing_migrate_nr_pages = 0;
6071 pgdat->numabalancing_migrate_next_window = jiffies;
6072 #endif
6073 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
6074 spin_lock_init(&pgdat->split_queue_lock);
6075 INIT_LIST_HEAD(&pgdat->split_queue);
6076 pgdat->split_queue_len = 0;
6077 #endif
6078 init_waitqueue_head(&pgdat->kswapd_wait);
6079 init_waitqueue_head(&pgdat->pfmemalloc_wait);
6080 #ifdef CONFIG_COMPACTION
6081 init_waitqueue_head(&pgdat->kcompactd_wait);
6082 #endif
6083 pgdat_page_ext_init(pgdat);
6084 spin_lock_init(&pgdat->lru_lock);
6085 lruvec_init(node_lruvec(pgdat));
6086
6087 pgdat->per_cpu_nodestats = &boot_nodestats;
6088
6089 for (j = 0; j < MAX_NR_ZONES; j++) {
6090 struct zone *zone = pgdat->node_zones + j;
6091 unsigned long size, realsize, freesize, memmap_pages;
6092 unsigned long zone_start_pfn = zone->zone_start_pfn;
6093
6094 size = zone->spanned_pages;
6095 realsize = freesize = zone->present_pages;
6096
6097 /*
6098 * Adjust freesize so that it accounts for how much memory
6099 * is used by this zone for memmap. This affects the watermark
6100 * and per-cpu initialisations
6101 */
6102 memmap_pages = calc_memmap_size(size, realsize);
6103 if (!is_highmem_idx(j)) {
6104 if (freesize >= memmap_pages) {
6105 freesize -= memmap_pages;
6106 if (memmap_pages)
6107 printk(KERN_DEBUG
6108 " %s zone: %lu pages used for memmap\n",
6109 zone_names[j], memmap_pages);
6110 } else
6111 pr_warn(" %s zone: %lu pages exceeds freesize %lu\n",
6112 zone_names[j], memmap_pages, freesize);
6113 }
6114
6115 /* Account for reserved pages */
6116 if (j == 0 && freesize > dma_reserve) {
6117 freesize -= dma_reserve;
6118 printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
6119 zone_names[0], dma_reserve);
6120 }
6121
6122 if (!is_highmem_idx(j))
6123 nr_kernel_pages += freesize;
6124 /* Charge for highmem memmap if there are enough kernel pages */
6125 else if (nr_kernel_pages > memmap_pages * 2)
6126 nr_kernel_pages -= memmap_pages;
6127 nr_all_pages += freesize;
6128
6129 /*
6130 * Set an approximate value for lowmem here, it will be adjusted
6131 * when the bootmem allocator frees pages into the buddy system.
6132 * And all highmem pages will be managed by the buddy system.
6133 */
6134 zone->managed_pages = is_highmem_idx(j) ? realsize : freesize;
6135 #ifdef CONFIG_NUMA
6136 zone->node = nid;
6137 #endif
6138 zone->name = zone_names[j];
6139 zone->zone_pgdat = pgdat;
6140 spin_lock_init(&zone->lock);
6141 zone_seqlock_init(zone);
6142 zone_pcp_init(zone);
6143
6144 if (!size)
6145 continue;
6146
6147 set_pageblock_order();
6148 setup_usemap(pgdat, zone, zone_start_pfn, size);
6149 init_currently_empty_zone(zone, zone_start_pfn, size);
6150 memmap_init(size, nid, j, zone_start_pfn);
6151 }
6152 }
6153
6154 #ifdef CONFIG_FLAT_NODE_MEM_MAP
6155 static void __ref alloc_node_mem_map(struct pglist_data *pgdat)
6156 {
6157 unsigned long __maybe_unused start = 0;
6158 unsigned long __maybe_unused offset = 0;
6159
6160 /* Skip empty nodes */
6161 if (!pgdat->node_spanned_pages)
6162 return;
6163
6164 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
6165 offset = pgdat->node_start_pfn - start;
6166 /* ia64 gets its own node_mem_map, before this, without bootmem */
6167 if (!pgdat->node_mem_map) {
6168 unsigned long size, end;
6169 struct page *map;
6170
6171 /*
6172 * The zone's endpoints aren't required to be MAX_ORDER
6173 * aligned but the node_mem_map endpoints must be in order
6174 * for the buddy allocator to function correctly.
6175 */
6176 end = pgdat_end_pfn(pgdat);
6177 end = ALIGN(end, MAX_ORDER_NR_PAGES);
6178 size = (end - start) * sizeof(struct page);
6179 map = alloc_remap(pgdat->node_id, size);
6180 if (!map)
6181 map = memblock_virt_alloc_node_nopanic(size,
6182 pgdat->node_id);
6183 pgdat->node_mem_map = map + offset;
6184 }
6185 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
6186 __func__, pgdat->node_id, (unsigned long)pgdat,
6187 (unsigned long)pgdat->node_mem_map);
6188 #ifndef CONFIG_NEED_MULTIPLE_NODES
6189 /*
6190 * With no DISCONTIG, the global mem_map is just set as node 0's
6191 */
6192 if (pgdat == NODE_DATA(0)) {
6193 mem_map = NODE_DATA(0)->node_mem_map;
6194 #if defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) || defined(CONFIG_FLATMEM)
6195 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
6196 mem_map -= offset;
6197 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6198 }
6199 #endif
6200 }
6201 #else
6202 static void __ref alloc_node_mem_map(struct pglist_data *pgdat) { }
6203 #endif /* CONFIG_FLAT_NODE_MEM_MAP */
6204
6205 void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
6206 unsigned long node_start_pfn, unsigned long *zholes_size)
6207 {
6208 pg_data_t *pgdat = NODE_DATA(nid);
6209 unsigned long start_pfn = 0;
6210 unsigned long end_pfn = 0;
6211
6212 /* pg_data_t should be reset to zero when it's allocated */
6213 WARN_ON(pgdat->nr_zones || pgdat->kswapd_classzone_idx);
6214
6215 pgdat->node_id = nid;
6216 pgdat->node_start_pfn = node_start_pfn;
6217 pgdat->per_cpu_nodestats = NULL;
6218 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6219 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
6220 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
6221 (u64)start_pfn << PAGE_SHIFT,
6222 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
6223 #else
6224 start_pfn = node_start_pfn;
6225 #endif
6226 calculate_node_totalpages(pgdat, start_pfn, end_pfn,
6227 zones_size, zholes_size);
6228
6229 alloc_node_mem_map(pgdat);
6230
6231 reset_deferred_meminit(pgdat);
6232 free_area_init_core(pgdat);
6233 }
6234
6235 #ifdef CONFIG_HAVE_MEMBLOCK
6236 /*
6237 * Only struct pages that are backed by physical memory are zeroed and
6238 * initialized by going through __init_single_page(). But, there are some
6239 * struct pages which are reserved in memblock allocator and their fields
6240 * may be accessed (for example page_to_pfn() on some configuration accesses
6241 * flags). We must explicitly zero those struct pages.
6242 */
6243 void __paginginit zero_resv_unavail(void)
6244 {
6245 phys_addr_t start, end;
6246 unsigned long pfn;
6247 u64 i, pgcnt;
6248
6249 /*
6250 * Loop through ranges that are reserved, but do not have reported
6251 * physical memory backing.
6252 */
6253 pgcnt = 0;
6254 for_each_resv_unavail_range(i, &start, &end) {
6255 for (pfn = PFN_DOWN(start); pfn < PFN_UP(end); pfn++) {
6256 mm_zero_struct_page(pfn_to_page(pfn));
6257 pgcnt++;
6258 }
6259 }
6260
6261 /*
6262 * Struct pages that do not have backing memory. This could be because
6263 * firmware is using some of this memory, or for some other reasons.
6264 * Once memblock is changed so such behaviour is not allowed: i.e.
6265 * list of "reserved" memory must be a subset of list of "memory", then
6266 * this code can be removed.
6267 */
6268 if (pgcnt)
6269 pr_info("Reserved but unavailable: %lld pages", pgcnt);
6270 }
6271 #endif /* CONFIG_HAVE_MEMBLOCK */
6272
6273 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
6274
6275 #if MAX_NUMNODES > 1
6276 /*
6277 * Figure out the number of possible node ids.
6278 */
6279 void __init setup_nr_node_ids(void)
6280 {
6281 unsigned int highest;
6282
6283 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
6284 nr_node_ids = highest + 1;
6285 }
6286 #endif
6287
6288 /**
6289 * node_map_pfn_alignment - determine the maximum internode alignment
6290 *
6291 * This function should be called after node map is populated and sorted.
6292 * It calculates the maximum power of two alignment which can distinguish
6293 * all the nodes.
6294 *
6295 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
6296 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
6297 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
6298 * shifted, 1GiB is enough and this function will indicate so.
6299 *
6300 * This is used to test whether pfn -> nid mapping of the chosen memory
6301 * model has fine enough granularity to avoid incorrect mapping for the
6302 * populated node map.
6303 *
6304 * Returns the determined alignment in pfn's. 0 if there is no alignment
6305 * requirement (single node).
6306 */
6307 unsigned long __init node_map_pfn_alignment(void)
6308 {
6309 unsigned long accl_mask = 0, last_end = 0;
6310 unsigned long start, end, mask;
6311 int last_nid = -1;
6312 int i, nid;
6313
6314 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
6315 if (!start || last_nid < 0 || last_nid == nid) {
6316 last_nid = nid;
6317 last_end = end;
6318 continue;
6319 }
6320
6321 /*
6322 * Start with a mask granular enough to pin-point to the
6323 * start pfn and tick off bits one-by-one until it becomes
6324 * too coarse to separate the current node from the last.
6325 */
6326 mask = ~((1 << __ffs(start)) - 1);
6327 while (mask && last_end <= (start & (mask << 1)))
6328 mask <<= 1;
6329
6330 /* accumulate all internode masks */
6331 accl_mask |= mask;
6332 }
6333
6334 /* convert mask to number of pages */
6335 return ~accl_mask + 1;
6336 }
6337
6338 /* Find the lowest pfn for a node */
6339 static unsigned long __init find_min_pfn_for_node(int nid)
6340 {
6341 unsigned long min_pfn = ULONG_MAX;
6342 unsigned long start_pfn;
6343 int i;
6344
6345 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
6346 min_pfn = min(min_pfn, start_pfn);
6347
6348 if (min_pfn == ULONG_MAX) {
6349 pr_warn("Could not find start_pfn for node %d\n", nid);
6350 return 0;
6351 }
6352
6353 return min_pfn;
6354 }
6355
6356 /**
6357 * find_min_pfn_with_active_regions - Find the minimum PFN registered
6358 *
6359 * It returns the minimum PFN based on information provided via
6360 * memblock_set_node().
6361 */
6362 unsigned long __init find_min_pfn_with_active_regions(void)
6363 {
6364 return find_min_pfn_for_node(MAX_NUMNODES);
6365 }
6366
6367 /*
6368 * early_calculate_totalpages()
6369 * Sum pages in active regions for movable zone.
6370 * Populate N_MEMORY for calculating usable_nodes.
6371 */
6372 static unsigned long __init early_calculate_totalpages(void)
6373 {
6374 unsigned long totalpages = 0;
6375 unsigned long start_pfn, end_pfn;
6376 int i, nid;
6377
6378 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
6379 unsigned long pages = end_pfn - start_pfn;
6380
6381 totalpages += pages;
6382 if (pages)
6383 node_set_state(nid, N_MEMORY);
6384 }
6385 return totalpages;
6386 }
6387
6388 /*
6389 * Find the PFN the Movable zone begins in each node. Kernel memory
6390 * is spread evenly between nodes as long as the nodes have enough
6391 * memory. When they don't, some nodes will have more kernelcore than
6392 * others
6393 */
6394 static void __init find_zone_movable_pfns_for_nodes(void)
6395 {
6396 int i, nid;
6397 unsigned long usable_startpfn;
6398 unsigned long kernelcore_node, kernelcore_remaining;
6399 /* save the state before borrow the nodemask */
6400 nodemask_t saved_node_state = node_states[N_MEMORY];
6401 unsigned long totalpages = early_calculate_totalpages();
6402 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
6403 struct memblock_region *r;
6404
6405 /* Need to find movable_zone earlier when movable_node is specified. */
6406 find_usable_zone_for_movable();
6407
6408 /*
6409 * If movable_node is specified, ignore kernelcore and movablecore
6410 * options.
6411 */
6412 if (movable_node_is_enabled()) {
6413 for_each_memblock(memory, r) {
6414 if (!memblock_is_hotpluggable(r))
6415 continue;
6416
6417 nid = r->nid;
6418
6419 usable_startpfn = PFN_DOWN(r->base);
6420 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
6421 min(usable_startpfn, zone_movable_pfn[nid]) :
6422 usable_startpfn;
6423 }
6424
6425 goto out2;
6426 }
6427
6428 /*
6429 * If kernelcore=mirror is specified, ignore movablecore option
6430 */
6431 if (mirrored_kernelcore) {
6432 bool mem_below_4gb_not_mirrored = false;
6433
6434 for_each_memblock(memory, r) {
6435 if (memblock_is_mirror(r))
6436 continue;
6437
6438 nid = r->nid;
6439
6440 usable_startpfn = memblock_region_memory_base_pfn(r);
6441
6442 if (usable_startpfn < 0x100000) {
6443 mem_below_4gb_not_mirrored = true;
6444 continue;
6445 }
6446
6447 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
6448 min(usable_startpfn, zone_movable_pfn[nid]) :
6449 usable_startpfn;
6450 }
6451
6452 if (mem_below_4gb_not_mirrored)
6453 pr_warn("This configuration results in unmirrored kernel memory.");
6454
6455 goto out2;
6456 }
6457
6458 /*
6459 * If movablecore=nn[KMG] was specified, calculate what size of
6460 * kernelcore that corresponds so that memory usable for
6461 * any allocation type is evenly spread. If both kernelcore
6462 * and movablecore are specified, then the value of kernelcore
6463 * will be used for required_kernelcore if it's greater than
6464 * what movablecore would have allowed.
6465 */
6466 if (required_movablecore) {
6467 unsigned long corepages;
6468
6469 /*
6470 * Round-up so that ZONE_MOVABLE is at least as large as what
6471 * was requested by the user
6472 */
6473 required_movablecore =
6474 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
6475 required_movablecore = min(totalpages, required_movablecore);
6476 corepages = totalpages - required_movablecore;
6477
6478 required_kernelcore = max(required_kernelcore, corepages);
6479 }
6480
6481 /*
6482 * If kernelcore was not specified or kernelcore size is larger
6483 * than totalpages, there is no ZONE_MOVABLE.
6484 */
6485 if (!required_kernelcore || required_kernelcore >= totalpages)
6486 goto out;
6487
6488 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
6489 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
6490
6491 restart:
6492 /* Spread kernelcore memory as evenly as possible throughout nodes */
6493 kernelcore_node = required_kernelcore / usable_nodes;
6494 for_each_node_state(nid, N_MEMORY) {
6495 unsigned long start_pfn, end_pfn;
6496
6497 /*
6498 * Recalculate kernelcore_node if the division per node
6499 * now exceeds what is necessary to satisfy the requested
6500 * amount of memory for the kernel
6501 */
6502 if (required_kernelcore < kernelcore_node)
6503 kernelcore_node = required_kernelcore / usable_nodes;
6504
6505 /*
6506 * As the map is walked, we track how much memory is usable
6507 * by the kernel using kernelcore_remaining. When it is
6508 * 0, the rest of the node is usable by ZONE_MOVABLE
6509 */
6510 kernelcore_remaining = kernelcore_node;
6511
6512 /* Go through each range of PFNs within this node */
6513 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
6514 unsigned long size_pages;
6515
6516 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
6517 if (start_pfn >= end_pfn)
6518 continue;
6519
6520 /* Account for what is only usable for kernelcore */
6521 if (start_pfn < usable_startpfn) {
6522 unsigned long kernel_pages;
6523 kernel_pages = min(end_pfn, usable_startpfn)
6524 - start_pfn;
6525
6526 kernelcore_remaining -= min(kernel_pages,
6527 kernelcore_remaining);
6528 required_kernelcore -= min(kernel_pages,
6529 required_kernelcore);
6530
6531 /* Continue if range is now fully accounted */
6532 if (end_pfn <= usable_startpfn) {
6533
6534 /*
6535 * Push zone_movable_pfn to the end so
6536 * that if we have to rebalance
6537 * kernelcore across nodes, we will
6538 * not double account here
6539 */
6540 zone_movable_pfn[nid] = end_pfn;
6541 continue;
6542 }
6543 start_pfn = usable_startpfn;
6544 }
6545
6546 /*
6547 * The usable PFN range for ZONE_MOVABLE is from
6548 * start_pfn->end_pfn. Calculate size_pages as the
6549 * number of pages used as kernelcore
6550 */
6551 size_pages = end_pfn - start_pfn;
6552 if (size_pages > kernelcore_remaining)
6553 size_pages = kernelcore_remaining;
6554 zone_movable_pfn[nid] = start_pfn + size_pages;
6555
6556 /*
6557 * Some kernelcore has been met, update counts and
6558 * break if the kernelcore for this node has been
6559 * satisfied
6560 */
6561 required_kernelcore -= min(required_kernelcore,
6562 size_pages);
6563 kernelcore_remaining -= size_pages;
6564 if (!kernelcore_remaining)
6565 break;
6566 }
6567 }
6568
6569 /*
6570 * If there is still required_kernelcore, we do another pass with one
6571 * less node in the count. This will push zone_movable_pfn[nid] further
6572 * along on the nodes that still have memory until kernelcore is
6573 * satisfied
6574 */
6575 usable_nodes--;
6576 if (usable_nodes && required_kernelcore > usable_nodes)
6577 goto restart;
6578
6579 out2:
6580 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
6581 for (nid = 0; nid < MAX_NUMNODES; nid++)
6582 zone_movable_pfn[nid] =
6583 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
6584
6585 out:
6586 /* restore the node_state */
6587 node_states[N_MEMORY] = saved_node_state;
6588 }
6589
6590 /* Any regular or high memory on that node ? */
6591 static void check_for_memory(pg_data_t *pgdat, int nid)
6592 {
6593 enum zone_type zone_type;
6594
6595 if (N_MEMORY == N_NORMAL_MEMORY)
6596 return;
6597
6598 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
6599 struct zone *zone = &pgdat->node_zones[zone_type];
6600 if (populated_zone(zone)) {
6601 node_set_state(nid, N_HIGH_MEMORY);
6602 if (N_NORMAL_MEMORY != N_HIGH_MEMORY &&
6603 zone_type <= ZONE_NORMAL)
6604 node_set_state(nid, N_NORMAL_MEMORY);
6605 break;
6606 }
6607 }
6608 }
6609
6610 /**
6611 * free_area_init_nodes - Initialise all pg_data_t and zone data
6612 * @max_zone_pfn: an array of max PFNs for each zone
6613 *
6614 * This will call free_area_init_node() for each active node in the system.
6615 * Using the page ranges provided by memblock_set_node(), the size of each
6616 * zone in each node and their holes is calculated. If the maximum PFN
6617 * between two adjacent zones match, it is assumed that the zone is empty.
6618 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
6619 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
6620 * starts where the previous one ended. For example, ZONE_DMA32 starts
6621 * at arch_max_dma_pfn.
6622 */
6623 void __init free_area_init_nodes(unsigned long *max_zone_pfn)
6624 {
6625 unsigned long start_pfn, end_pfn;
6626 int i, nid;
6627
6628 /* Record where the zone boundaries are */
6629 memset(arch_zone_lowest_possible_pfn, 0,
6630 sizeof(arch_zone_lowest_possible_pfn));
6631 memset(arch_zone_highest_possible_pfn, 0,
6632 sizeof(arch_zone_highest_possible_pfn));
6633
6634 start_pfn = find_min_pfn_with_active_regions();
6635
6636 for (i = 0; i < MAX_NR_ZONES; i++) {
6637 if (i == ZONE_MOVABLE)
6638 continue;
6639
6640 end_pfn = max(max_zone_pfn[i], start_pfn);
6641 arch_zone_lowest_possible_pfn[i] = start_pfn;
6642 arch_zone_highest_possible_pfn[i] = end_pfn;
6643
6644 start_pfn = end_pfn;
6645 }
6646
6647 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
6648 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
6649 find_zone_movable_pfns_for_nodes();
6650
6651 /* Print out the zone ranges */
6652 pr_info("Zone ranges:\n");
6653 for (i = 0; i < MAX_NR_ZONES; i++) {
6654 if (i == ZONE_MOVABLE)
6655 continue;
6656 pr_info(" %-8s ", zone_names[i]);
6657 if (arch_zone_lowest_possible_pfn[i] ==
6658 arch_zone_highest_possible_pfn[i])
6659 pr_cont("empty\n");
6660 else
6661 pr_cont("[mem %#018Lx-%#018Lx]\n",
6662 (u64)arch_zone_lowest_possible_pfn[i]
6663 << PAGE_SHIFT,
6664 ((u64)arch_zone_highest_possible_pfn[i]
6665 << PAGE_SHIFT) - 1);
6666 }
6667
6668 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
6669 pr_info("Movable zone start for each node\n");
6670 for (i = 0; i < MAX_NUMNODES; i++) {
6671 if (zone_movable_pfn[i])
6672 pr_info(" Node %d: %#018Lx\n", i,
6673 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
6674 }
6675
6676 /* Print out the early node map */
6677 pr_info("Early memory node ranges\n");
6678 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
6679 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
6680 (u64)start_pfn << PAGE_SHIFT,
6681 ((u64)end_pfn << PAGE_SHIFT) - 1);
6682
6683 /* Initialise every node */
6684 mminit_verify_pageflags_layout();
6685 setup_nr_node_ids();
6686 for_each_online_node(nid) {
6687 pg_data_t *pgdat = NODE_DATA(nid);
6688 free_area_init_node(nid, NULL,
6689 find_min_pfn_for_node(nid), NULL);
6690
6691 /* Any memory on that node */
6692 if (pgdat->node_present_pages)
6693 node_set_state(nid, N_MEMORY);
6694 check_for_memory(pgdat, nid);
6695 }
6696 zero_resv_unavail();
6697 }
6698
6699 static int __init cmdline_parse_core(char *p, unsigned long *core)
6700 {
6701 unsigned long long coremem;
6702 if (!p)
6703 return -EINVAL;
6704
6705 coremem = memparse(p, &p);
6706 *core = coremem >> PAGE_SHIFT;
6707
6708 /* Paranoid check that UL is enough for the coremem value */
6709 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
6710
6711 return 0;
6712 }
6713
6714 /*
6715 * kernelcore=size sets the amount of memory for use for allocations that
6716 * cannot be reclaimed or migrated.
6717 */
6718 static int __init cmdline_parse_kernelcore(char *p)
6719 {
6720 /* parse kernelcore=mirror */
6721 if (parse_option_str(p, "mirror")) {
6722 mirrored_kernelcore = true;
6723 return 0;
6724 }
6725
6726 return cmdline_parse_core(p, &required_kernelcore);
6727 }
6728
6729 /*
6730 * movablecore=size sets the amount of memory for use for allocations that
6731 * can be reclaimed or migrated.
6732 */
6733 static int __init cmdline_parse_movablecore(char *p)
6734 {
6735 return cmdline_parse_core(p, &required_movablecore);
6736 }
6737
6738 early_param("kernelcore", cmdline_parse_kernelcore);
6739 early_param("movablecore", cmdline_parse_movablecore);
6740
6741 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
6742
6743 void adjust_managed_page_count(struct page *page, long count)
6744 {
6745 spin_lock(&managed_page_count_lock);
6746 page_zone(page)->managed_pages += count;
6747 totalram_pages += count;
6748 #ifdef CONFIG_HIGHMEM
6749 if (PageHighMem(page))
6750 totalhigh_pages += count;
6751 #endif
6752 spin_unlock(&managed_page_count_lock);
6753 }
6754 EXPORT_SYMBOL(adjust_managed_page_count);
6755
6756 unsigned long free_reserved_area(void *start, void *end, int poison, char *s)
6757 {
6758 void *pos;
6759 unsigned long pages = 0;
6760
6761 start = (void *)PAGE_ALIGN((unsigned long)start);
6762 end = (void *)((unsigned long)end & PAGE_MASK);
6763 for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
6764 if ((unsigned int)poison <= 0xFF)
6765 memset(pos, poison, PAGE_SIZE);
6766 free_reserved_page(virt_to_page(pos));
6767 }
6768
6769 if (pages && s)
6770 pr_info("Freeing %s memory: %ldK\n",
6771 s, pages << (PAGE_SHIFT - 10));
6772
6773 return pages;
6774 }
6775 EXPORT_SYMBOL(free_reserved_area);
6776
6777 #ifdef CONFIG_HIGHMEM
6778 void free_highmem_page(struct page *page)
6779 {
6780 __free_reserved_page(page);
6781 totalram_pages++;
6782 page_zone(page)->managed_pages++;
6783 totalhigh_pages++;
6784 }
6785 #endif
6786
6787
6788 void __init mem_init_print_info(const char *str)
6789 {
6790 unsigned long physpages, codesize, datasize, rosize, bss_size;
6791 unsigned long init_code_size, init_data_size;
6792
6793 physpages = get_num_physpages();
6794 codesize = _etext - _stext;
6795 datasize = _edata - _sdata;
6796 rosize = __end_rodata - __start_rodata;
6797 bss_size = __bss_stop - __bss_start;
6798 init_data_size = __init_end - __init_begin;
6799 init_code_size = _einittext - _sinittext;
6800
6801 /*
6802 * Detect special cases and adjust section sizes accordingly:
6803 * 1) .init.* may be embedded into .data sections
6804 * 2) .init.text.* may be out of [__init_begin, __init_end],
6805 * please refer to arch/tile/kernel/vmlinux.lds.S.
6806 * 3) .rodata.* may be embedded into .text or .data sections.
6807 */
6808 #define adj_init_size(start, end, size, pos, adj) \
6809 do { \
6810 if (start <= pos && pos < end && size > adj) \
6811 size -= adj; \
6812 } while (0)
6813
6814 adj_init_size(__init_begin, __init_end, init_data_size,
6815 _sinittext, init_code_size);
6816 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
6817 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
6818 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
6819 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
6820
6821 #undef adj_init_size
6822
6823 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
6824 #ifdef CONFIG_HIGHMEM
6825 ", %luK highmem"
6826 #endif
6827 "%s%s)\n",
6828 nr_free_pages() << (PAGE_SHIFT - 10),
6829 physpages << (PAGE_SHIFT - 10),
6830 codesize >> 10, datasize >> 10, rosize >> 10,
6831 (init_data_size + init_code_size) >> 10, bss_size >> 10,
6832 (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10),
6833 totalcma_pages << (PAGE_SHIFT - 10),
6834 #ifdef CONFIG_HIGHMEM
6835 totalhigh_pages << (PAGE_SHIFT - 10),
6836 #endif
6837 str ? ", " : "", str ? str : "");
6838 }
6839
6840 /**
6841 * set_dma_reserve - set the specified number of pages reserved in the first zone
6842 * @new_dma_reserve: The number of pages to mark reserved
6843 *
6844 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
6845 * In the DMA zone, a significant percentage may be consumed by kernel image
6846 * and other unfreeable allocations which can skew the watermarks badly. This
6847 * function may optionally be used to account for unfreeable pages in the
6848 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
6849 * smaller per-cpu batchsize.
6850 */
6851 void __init set_dma_reserve(unsigned long new_dma_reserve)
6852 {
6853 dma_reserve = new_dma_reserve;
6854 }
6855
6856 void __init free_area_init(unsigned long *zones_size)
6857 {
6858 free_area_init_node(0, zones_size,
6859 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
6860 zero_resv_unavail();
6861 }
6862
6863 static int page_alloc_cpu_dead(unsigned int cpu)
6864 {
6865
6866 lru_add_drain_cpu(cpu);
6867 drain_pages(cpu);
6868
6869 /*
6870 * Spill the event counters of the dead processor
6871 * into the current processors event counters.
6872 * This artificially elevates the count of the current
6873 * processor.
6874 */
6875 vm_events_fold_cpu(cpu);
6876
6877 /*
6878 * Zero the differential counters of the dead processor
6879 * so that the vm statistics are consistent.
6880 *
6881 * This is only okay since the processor is dead and cannot
6882 * race with what we are doing.
6883 */
6884 cpu_vm_stats_fold(cpu);
6885 return 0;
6886 }
6887
6888 void __init page_alloc_init(void)
6889 {
6890 int ret;
6891
6892 ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
6893 "mm/page_alloc:dead", NULL,
6894 page_alloc_cpu_dead);
6895 WARN_ON(ret < 0);
6896 }
6897
6898 /*
6899 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
6900 * or min_free_kbytes changes.
6901 */
6902 static void calculate_totalreserve_pages(void)
6903 {
6904 struct pglist_data *pgdat;
6905 unsigned long reserve_pages = 0;
6906 enum zone_type i, j;
6907
6908 for_each_online_pgdat(pgdat) {
6909
6910 pgdat->totalreserve_pages = 0;
6911
6912 for (i = 0; i < MAX_NR_ZONES; i++) {
6913 struct zone *zone = pgdat->node_zones + i;
6914 long max = 0;
6915
6916 /* Find valid and maximum lowmem_reserve in the zone */
6917 for (j = i; j < MAX_NR_ZONES; j++) {
6918 if (zone->lowmem_reserve[j] > max)
6919 max = zone->lowmem_reserve[j];
6920 }
6921
6922 /* we treat the high watermark as reserved pages. */
6923 max += high_wmark_pages(zone);
6924
6925 if (max > zone->managed_pages)
6926 max = zone->managed_pages;
6927
6928 pgdat->totalreserve_pages += max;
6929
6930 reserve_pages += max;
6931 }
6932 }
6933 totalreserve_pages = reserve_pages;
6934 }
6935
6936 /*
6937 * setup_per_zone_lowmem_reserve - called whenever
6938 * sysctl_lowmem_reserve_ratio changes. Ensures that each zone
6939 * has a correct pages reserved value, so an adequate number of
6940 * pages are left in the zone after a successful __alloc_pages().
6941 */
6942 static void setup_per_zone_lowmem_reserve(void)
6943 {
6944 struct pglist_data *pgdat;
6945 enum zone_type j, idx;
6946
6947 for_each_online_pgdat(pgdat) {
6948 for (j = 0; j < MAX_NR_ZONES; j++) {
6949 struct zone *zone = pgdat->node_zones + j;
6950 unsigned long managed_pages = zone->managed_pages;
6951
6952 zone->lowmem_reserve[j] = 0;
6953
6954 idx = j;
6955 while (idx) {
6956 struct zone *lower_zone;
6957
6958 idx--;
6959
6960 if (sysctl_lowmem_reserve_ratio[idx] < 1)
6961 sysctl_lowmem_reserve_ratio[idx] = 1;
6962
6963 lower_zone = pgdat->node_zones + idx;
6964 lower_zone->lowmem_reserve[j] = managed_pages /
6965 sysctl_lowmem_reserve_ratio[idx];
6966 managed_pages += lower_zone->managed_pages;
6967 }
6968 }
6969 }
6970
6971 /* update totalreserve_pages */
6972 calculate_totalreserve_pages();
6973 }
6974
6975 static void __setup_per_zone_wmarks(void)
6976 {
6977 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
6978 unsigned long lowmem_pages = 0;
6979 struct zone *zone;
6980 unsigned long flags;
6981
6982 /* Calculate total number of !ZONE_HIGHMEM pages */
6983 for_each_zone(zone) {
6984 if (!is_highmem(zone))
6985 lowmem_pages += zone->managed_pages;
6986 }
6987
6988 for_each_zone(zone) {
6989 u64 tmp;
6990
6991 spin_lock_irqsave(&zone->lock, flags);
6992 tmp = (u64)pages_min * zone->managed_pages;
6993 do_div(tmp, lowmem_pages);
6994 if (is_highmem(zone)) {
6995 /*
6996 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
6997 * need highmem pages, so cap pages_min to a small
6998 * value here.
6999 *
7000 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
7001 * deltas control asynch page reclaim, and so should
7002 * not be capped for highmem.
7003 */
7004 unsigned long min_pages;
7005
7006 min_pages = zone->managed_pages / 1024;
7007 min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
7008 zone->watermark[WMARK_MIN] = min_pages;
7009 } else {
7010 /*
7011 * If it's a lowmem zone, reserve a number of pages
7012 * proportionate to the zone's size.
7013 */
7014 zone->watermark[WMARK_MIN] = tmp;
7015 }
7016
7017 /*
7018 * Set the kswapd watermarks distance according to the
7019 * scale factor in proportion to available memory, but
7020 * ensure a minimum size on small systems.
7021 */
7022 tmp = max_t(u64, tmp >> 2,
7023 mult_frac(zone->managed_pages,
7024 watermark_scale_factor, 10000));
7025
7026 zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
7027 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + tmp * 2;
7028
7029 spin_unlock_irqrestore(&zone->lock, flags);
7030 }
7031
7032 /* update totalreserve_pages */
7033 calculate_totalreserve_pages();
7034 }
7035
7036 /**
7037 * setup_per_zone_wmarks - called when min_free_kbytes changes
7038 * or when memory is hot-{added|removed}
7039 *
7040 * Ensures that the watermark[min,low,high] values for each zone are set
7041 * correctly with respect to min_free_kbytes.
7042 */
7043 void setup_per_zone_wmarks(void)
7044 {
7045 static DEFINE_SPINLOCK(lock);
7046
7047 spin_lock(&lock);
7048 __setup_per_zone_wmarks();
7049 spin_unlock(&lock);
7050 }
7051
7052 /*
7053 * Initialise min_free_kbytes.
7054 *
7055 * For small machines we want it small (128k min). For large machines
7056 * we want it large (64MB max). But it is not linear, because network
7057 * bandwidth does not increase linearly with machine size. We use
7058 *
7059 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
7060 * min_free_kbytes = sqrt(lowmem_kbytes * 16)
7061 *
7062 * which yields
7063 *
7064 * 16MB: 512k
7065 * 32MB: 724k
7066 * 64MB: 1024k
7067 * 128MB: 1448k
7068 * 256MB: 2048k
7069 * 512MB: 2896k
7070 * 1024MB: 4096k
7071 * 2048MB: 5792k
7072 * 4096MB: 8192k
7073 * 8192MB: 11584k
7074 * 16384MB: 16384k
7075 */
7076 int __meminit init_per_zone_wmark_min(void)
7077 {
7078 unsigned long lowmem_kbytes;
7079 int new_min_free_kbytes;
7080
7081 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
7082 new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
7083
7084 if (new_min_free_kbytes > user_min_free_kbytes) {
7085 min_free_kbytes = new_min_free_kbytes;
7086 if (min_free_kbytes < 128)
7087 min_free_kbytes = 128;
7088 if (min_free_kbytes > 65536)
7089 min_free_kbytes = 65536;
7090 } else {
7091 pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
7092 new_min_free_kbytes, user_min_free_kbytes);
7093 }
7094 setup_per_zone_wmarks();
7095 refresh_zone_stat_thresholds();
7096 setup_per_zone_lowmem_reserve();
7097
7098 #ifdef CONFIG_NUMA
7099 setup_min_unmapped_ratio();
7100 setup_min_slab_ratio();
7101 #endif
7102
7103 return 0;
7104 }
7105 core_initcall(init_per_zone_wmark_min)
7106
7107 /*
7108 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
7109 * that we can call two helper functions whenever min_free_kbytes
7110 * changes.
7111 */
7112 int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
7113 void __user *buffer, size_t *length, loff_t *ppos)
7114 {
7115 int rc;
7116
7117 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7118 if (rc)
7119 return rc;
7120
7121 if (write) {
7122 user_min_free_kbytes = min_free_kbytes;
7123 setup_per_zone_wmarks();
7124 }
7125 return 0;
7126 }
7127
7128 int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
7129 void __user *buffer, size_t *length, loff_t *ppos)
7130 {
7131 int rc;
7132
7133 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7134 if (rc)
7135 return rc;
7136
7137 if (write)
7138 setup_per_zone_wmarks();
7139
7140 return 0;
7141 }
7142
7143 #ifdef CONFIG_NUMA
7144 static void setup_min_unmapped_ratio(void)
7145 {
7146 pg_data_t *pgdat;
7147 struct zone *zone;
7148
7149 for_each_online_pgdat(pgdat)
7150 pgdat->min_unmapped_pages = 0;
7151
7152 for_each_zone(zone)
7153 zone->zone_pgdat->min_unmapped_pages += (zone->managed_pages *
7154 sysctl_min_unmapped_ratio) / 100;
7155 }
7156
7157
7158 int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
7159 void __user *buffer, size_t *length, loff_t *ppos)
7160 {
7161 int rc;
7162
7163 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7164 if (rc)
7165 return rc;
7166
7167 setup_min_unmapped_ratio();
7168
7169 return 0;
7170 }
7171
7172 static void setup_min_slab_ratio(void)
7173 {
7174 pg_data_t *pgdat;
7175 struct zone *zone;
7176
7177 for_each_online_pgdat(pgdat)
7178 pgdat->min_slab_pages = 0;
7179
7180 for_each_zone(zone)
7181 zone->zone_pgdat->min_slab_pages += (zone->managed_pages *
7182 sysctl_min_slab_ratio) / 100;
7183 }
7184
7185 int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
7186 void __user *buffer, size_t *length, loff_t *ppos)
7187 {
7188 int rc;
7189
7190 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
7191 if (rc)
7192 return rc;
7193
7194 setup_min_slab_ratio();
7195
7196 return 0;
7197 }
7198 #endif
7199
7200 /*
7201 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
7202 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
7203 * whenever sysctl_lowmem_reserve_ratio changes.
7204 *
7205 * The reserve ratio obviously has absolutely no relation with the
7206 * minimum watermarks. The lowmem reserve ratio can only make sense
7207 * if in function of the boot time zone sizes.
7208 */
7209 int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
7210 void __user *buffer, size_t *length, loff_t *ppos)
7211 {
7212 proc_dointvec_minmax(table, write, buffer, length, ppos);
7213 setup_per_zone_lowmem_reserve();
7214 return 0;
7215 }
7216
7217 /*
7218 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
7219 * cpu. It is the fraction of total pages in each zone that a hot per cpu
7220 * pagelist can have before it gets flushed back to buddy allocator.
7221 */
7222 int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write,
7223 void __user *buffer, size_t *length, loff_t *ppos)
7224 {
7225 struct zone *zone;
7226 int old_percpu_pagelist_fraction;
7227 int ret;
7228
7229 mutex_lock(&pcp_batch_high_lock);
7230 old_percpu_pagelist_fraction = percpu_pagelist_fraction;
7231
7232 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
7233 if (!write || ret < 0)
7234 goto out;
7235
7236 /* Sanity checking to avoid pcp imbalance */
7237 if (percpu_pagelist_fraction &&
7238 percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) {
7239 percpu_pagelist_fraction = old_percpu_pagelist_fraction;
7240 ret = -EINVAL;
7241 goto out;
7242 }
7243
7244 /* No change? */
7245 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
7246 goto out;
7247
7248 for_each_populated_zone(zone) {
7249 unsigned int cpu;
7250
7251 for_each_possible_cpu(cpu)
7252 pageset_set_high_and_batch(zone,
7253 per_cpu_ptr(zone->pageset, cpu));
7254 }
7255 out:
7256 mutex_unlock(&pcp_batch_high_lock);
7257 return ret;
7258 }
7259
7260 #ifdef CONFIG_NUMA
7261 int hashdist = HASHDIST_DEFAULT;
7262
7263 static int __init set_hashdist(char *str)
7264 {
7265 if (!str)
7266 return 0;
7267 hashdist = simple_strtoul(str, &str, 0);
7268 return 1;
7269 }
7270 __setup("hashdist=", set_hashdist);
7271 #endif
7272
7273 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
7274 /*
7275 * Returns the number of pages that arch has reserved but
7276 * is not known to alloc_large_system_hash().
7277 */
7278 static unsigned long __init arch_reserved_kernel_pages(void)
7279 {
7280 return 0;
7281 }
7282 #endif
7283
7284 /*
7285 * Adaptive scale is meant to reduce sizes of hash tables on large memory
7286 * machines. As memory size is increased the scale is also increased but at
7287 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
7288 * quadruples the scale is increased by one, which means the size of hash table
7289 * only doubles, instead of quadrupling as well.
7290 * Because 32-bit systems cannot have large physical memory, where this scaling
7291 * makes sense, it is disabled on such platforms.
7292 */
7293 #if __BITS_PER_LONG > 32
7294 #define ADAPT_SCALE_BASE (64ul << 30)
7295 #define ADAPT_SCALE_SHIFT 2
7296 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
7297 #endif
7298
7299 /*
7300 * allocate a large system hash table from bootmem
7301 * - it is assumed that the hash table must contain an exact power-of-2
7302 * quantity of entries
7303 * - limit is the number of hash buckets, not the total allocation size
7304 */
7305 void *__init alloc_large_system_hash(const char *tablename,
7306 unsigned long bucketsize,
7307 unsigned long numentries,
7308 int scale,
7309 int flags,
7310 unsigned int *_hash_shift,
7311 unsigned int *_hash_mask,
7312 unsigned long low_limit,
7313 unsigned long high_limit)
7314 {
7315 unsigned long long max = high_limit;
7316 unsigned long log2qty, size;
7317 void *table = NULL;
7318 gfp_t gfp_flags;
7319
7320 /* allow the kernel cmdline to have a say */
7321 if (!numentries) {
7322 /* round applicable memory size up to nearest megabyte */
7323 numentries = nr_kernel_pages;
7324 numentries -= arch_reserved_kernel_pages();
7325
7326 /* It isn't necessary when PAGE_SIZE >= 1MB */
7327 if (PAGE_SHIFT < 20)
7328 numentries = round_up(numentries, (1<<20)/PAGE_SIZE);
7329
7330 #if __BITS_PER_LONG > 32
7331 if (!high_limit) {
7332 unsigned long adapt;
7333
7334 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
7335 adapt <<= ADAPT_SCALE_SHIFT)
7336 scale++;
7337 }
7338 #endif
7339
7340 /* limit to 1 bucket per 2^scale bytes of low memory */
7341 if (scale > PAGE_SHIFT)
7342 numentries >>= (scale - PAGE_SHIFT);
7343 else
7344 numentries <<= (PAGE_SHIFT - scale);
7345
7346 /* Make sure we've got at least a 0-order allocation.. */
7347 if (unlikely(flags & HASH_SMALL)) {
7348 /* Makes no sense without HASH_EARLY */
7349 WARN_ON(!(flags & HASH_EARLY));
7350 if (!(numentries >> *_hash_shift)) {
7351 numentries = 1UL << *_hash_shift;
7352 BUG_ON(!numentries);
7353 }
7354 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
7355 numentries = PAGE_SIZE / bucketsize;
7356 }
7357 numentries = roundup_pow_of_two(numentries);
7358
7359 /* limit allocation size to 1/16 total memory by default */
7360 if (max == 0) {
7361 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
7362 do_div(max, bucketsize);
7363 }
7364 max = min(max, 0x80000000ULL);
7365
7366 if (numentries < low_limit)
7367 numentries = low_limit;
7368 if (numentries > max)
7369 numentries = max;
7370
7371 log2qty = ilog2(numentries);
7372
7373 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
7374 do {
7375 size = bucketsize << log2qty;
7376 if (flags & HASH_EARLY) {
7377 if (flags & HASH_ZERO)
7378 table = memblock_virt_alloc_nopanic(size, 0);
7379 else
7380 table = memblock_virt_alloc_raw(size, 0);
7381 } else if (hashdist) {
7382 table = __vmalloc(size, gfp_flags, PAGE_KERNEL);
7383 } else {
7384 /*
7385 * If bucketsize is not a power-of-two, we may free
7386 * some pages at the end of hash table which
7387 * alloc_pages_exact() automatically does
7388 */
7389 if (get_order(size) < MAX_ORDER) {
7390 table = alloc_pages_exact(size, gfp_flags);
7391 kmemleak_alloc(table, size, 1, gfp_flags);
7392 }
7393 }
7394 } while (!table && size > PAGE_SIZE && --log2qty);
7395
7396 if (!table)
7397 panic("Failed to allocate %s hash table\n", tablename);
7398
7399 pr_info("%s hash table entries: %ld (order: %d, %lu bytes)\n",
7400 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size);
7401
7402 if (_hash_shift)
7403 *_hash_shift = log2qty;
7404 if (_hash_mask)
7405 *_hash_mask = (1 << log2qty) - 1;
7406
7407 return table;
7408 }
7409
7410 /*
7411 * This function checks whether pageblock includes unmovable pages or not.
7412 * If @count is not zero, it is okay to include less @count unmovable pages
7413 *
7414 * PageLRU check without isolation or lru_lock could race so that
7415 * MIGRATE_MOVABLE block might include unmovable pages. And __PageMovable
7416 * check without lock_page also may miss some movable non-lru pages at
7417 * race condition. So you can't expect this function should be exact.
7418 */
7419 bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
7420 int migratetype,
7421 bool skip_hwpoisoned_pages)
7422 {
7423 unsigned long pfn, iter, found;
7424
7425 /*
7426 * For avoiding noise data, lru_add_drain_all() should be called
7427 * If ZONE_MOVABLE, the zone never contains unmovable pages
7428 */
7429 if (zone_idx(zone) == ZONE_MOVABLE)
7430 return false;
7431
7432 /*
7433 * CMA allocations (alloc_contig_range) really need to mark isolate
7434 * CMA pageblocks even when they are not movable in fact so consider
7435 * them movable here.
7436 */
7437 if (is_migrate_cma(migratetype) &&
7438 is_migrate_cma(get_pageblock_migratetype(page)))
7439 return false;
7440
7441 pfn = page_to_pfn(page);
7442 for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
7443 unsigned long check = pfn + iter;
7444
7445 if (!pfn_valid_within(check))
7446 continue;
7447
7448 page = pfn_to_page(check);
7449
7450 if (PageReserved(page))
7451 return true;
7452
7453 /*
7454 * Hugepages are not in LRU lists, but they're movable.
7455 * We need not scan over tail pages bacause we don't
7456 * handle each tail page individually in migration.
7457 */
7458 if (PageHuge(page)) {
7459 iter = round_up(iter + 1, 1<<compound_order(page)) - 1;
7460 continue;
7461 }
7462
7463 /*
7464 * We can't use page_count without pin a page
7465 * because another CPU can free compound page.
7466 * This check already skips compound tails of THP
7467 * because their page->_refcount is zero at all time.
7468 */
7469 if (!page_ref_count(page)) {
7470 if (PageBuddy(page))
7471 iter += (1 << page_order(page)) - 1;
7472 continue;
7473 }
7474
7475 /*
7476 * The HWPoisoned page may be not in buddy system, and
7477 * page_count() is not 0.
7478 */
7479 if (skip_hwpoisoned_pages && PageHWPoison(page))
7480 continue;
7481
7482 if (__PageMovable(page))
7483 continue;
7484
7485 if (!PageLRU(page))
7486 found++;
7487 /*
7488 * If there are RECLAIMABLE pages, we need to check
7489 * it. But now, memory offline itself doesn't call
7490 * shrink_node_slabs() and it still to be fixed.
7491 */
7492 /*
7493 * If the page is not RAM, page_count()should be 0.
7494 * we don't need more check. This is an _used_ not-movable page.
7495 *
7496 * The problematic thing here is PG_reserved pages. PG_reserved
7497 * is set to both of a memory hole page and a _used_ kernel
7498 * page at boot.
7499 */
7500 if (found > count)
7501 return true;
7502 }
7503 return false;
7504 }
7505
7506 bool is_pageblock_removable_nolock(struct page *page)
7507 {
7508 struct zone *zone;
7509 unsigned long pfn;
7510
7511 /*
7512 * We have to be careful here because we are iterating over memory
7513 * sections which are not zone aware so we might end up outside of
7514 * the zone but still within the section.
7515 * We have to take care about the node as well. If the node is offline
7516 * its NODE_DATA will be NULL - see page_zone.
7517 */
7518 if (!node_online(page_to_nid(page)))
7519 return false;
7520
7521 zone = page_zone(page);
7522 pfn = page_to_pfn(page);
7523 if (!zone_spans_pfn(zone, pfn))
7524 return false;
7525
7526 return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE, true);
7527 }
7528
7529 #if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
7530
7531 static unsigned long pfn_max_align_down(unsigned long pfn)
7532 {
7533 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
7534 pageblock_nr_pages) - 1);
7535 }
7536
7537 static unsigned long pfn_max_align_up(unsigned long pfn)
7538 {
7539 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
7540 pageblock_nr_pages));
7541 }
7542
7543 /* [start, end) must belong to a single zone. */
7544 static int __alloc_contig_migrate_range(struct compact_control *cc,
7545 unsigned long start, unsigned long end)
7546 {
7547 /* This function is based on compact_zone() from compaction.c. */
7548 unsigned long nr_reclaimed;
7549 unsigned long pfn = start;
7550 unsigned int tries = 0;
7551 int ret = 0;
7552
7553 migrate_prep();
7554
7555 while (pfn < end || !list_empty(&cc->migratepages)) {
7556 if (fatal_signal_pending(current)) {
7557 ret = -EINTR;
7558 break;
7559 }
7560
7561 if (list_empty(&cc->migratepages)) {
7562 cc->nr_migratepages = 0;
7563 pfn = isolate_migratepages_range(cc, pfn, end);
7564 if (!pfn) {
7565 ret = -EINTR;
7566 break;
7567 }
7568 tries = 0;
7569 } else if (++tries == 5) {
7570 ret = ret < 0 ? ret : -EBUSY;
7571 break;
7572 }
7573
7574 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
7575 &cc->migratepages);
7576 cc->nr_migratepages -= nr_reclaimed;
7577
7578 ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
7579 NULL, 0, cc->mode, MR_CMA);
7580 }
7581 if (ret < 0) {
7582 putback_movable_pages(&cc->migratepages);
7583 return ret;
7584 }
7585 return 0;
7586 }
7587
7588 /**
7589 * alloc_contig_range() -- tries to allocate given range of pages
7590 * @start: start PFN to allocate
7591 * @end: one-past-the-last PFN to allocate
7592 * @migratetype: migratetype of the underlaying pageblocks (either
7593 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
7594 * in range must have the same migratetype and it must
7595 * be either of the two.
7596 * @gfp_mask: GFP mask to use during compaction
7597 *
7598 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
7599 * aligned, however it's the caller's responsibility to guarantee that
7600 * we are the only thread that changes migrate type of pageblocks the
7601 * pages fall in.
7602 *
7603 * The PFN range must belong to a single zone.
7604 *
7605 * Returns zero on success or negative error code. On success all
7606 * pages which PFN is in [start, end) are allocated for the caller and
7607 * need to be freed with free_contig_range().
7608 */
7609 int alloc_contig_range(unsigned long start, unsigned long end,
7610 unsigned migratetype, gfp_t gfp_mask)
7611 {
7612 unsigned long outer_start, outer_end;
7613 unsigned int order;
7614 int ret = 0;
7615
7616 struct compact_control cc = {
7617 .nr_migratepages = 0,
7618 .order = -1,
7619 .zone = page_zone(pfn_to_page(start)),
7620 .mode = MIGRATE_SYNC,
7621 .ignore_skip_hint = true,
7622 .no_set_skip_hint = true,
7623 .gfp_mask = current_gfp_context(gfp_mask),
7624 };
7625 INIT_LIST_HEAD(&cc.migratepages);
7626
7627 /*
7628 * What we do here is we mark all pageblocks in range as
7629 * MIGRATE_ISOLATE. Because pageblock and max order pages may
7630 * have different sizes, and due to the way page allocator
7631 * work, we align the range to biggest of the two pages so
7632 * that page allocator won't try to merge buddies from
7633 * different pageblocks and change MIGRATE_ISOLATE to some
7634 * other migration type.
7635 *
7636 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
7637 * migrate the pages from an unaligned range (ie. pages that
7638 * we are interested in). This will put all the pages in
7639 * range back to page allocator as MIGRATE_ISOLATE.
7640 *
7641 * When this is done, we take the pages in range from page
7642 * allocator removing them from the buddy system. This way
7643 * page allocator will never consider using them.
7644 *
7645 * This lets us mark the pageblocks back as
7646 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
7647 * aligned range but not in the unaligned, original range are
7648 * put back to page allocator so that buddy can use them.
7649 */
7650
7651 ret = start_isolate_page_range(pfn_max_align_down(start),
7652 pfn_max_align_up(end), migratetype,
7653 false);
7654 if (ret)
7655 return ret;
7656
7657 /*
7658 * In case of -EBUSY, we'd like to know which page causes problem.
7659 * So, just fall through. We will check it in test_pages_isolated().
7660 */
7661 ret = __alloc_contig_migrate_range(&cc, start, end);
7662 if (ret && ret != -EBUSY)
7663 goto done;
7664
7665 /*
7666 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
7667 * aligned blocks that are marked as MIGRATE_ISOLATE. What's
7668 * more, all pages in [start, end) are free in page allocator.
7669 * What we are going to do is to allocate all pages from
7670 * [start, end) (that is remove them from page allocator).
7671 *
7672 * The only problem is that pages at the beginning and at the
7673 * end of interesting range may be not aligned with pages that
7674 * page allocator holds, ie. they can be part of higher order
7675 * pages. Because of this, we reserve the bigger range and
7676 * once this is done free the pages we are not interested in.
7677 *
7678 * We don't have to hold zone->lock here because the pages are
7679 * isolated thus they won't get removed from buddy.
7680 */
7681
7682 lru_add_drain_all();
7683 drain_all_pages(cc.zone);
7684
7685 order = 0;
7686 outer_start = start;
7687 while (!PageBuddy(pfn_to_page(outer_start))) {
7688 if (++order >= MAX_ORDER) {
7689 outer_start = start;
7690 break;
7691 }
7692 outer_start &= ~0UL << order;
7693 }
7694
7695 if (outer_start != start) {
7696 order = page_order(pfn_to_page(outer_start));
7697
7698 /*
7699 * outer_start page could be small order buddy page and
7700 * it doesn't include start page. Adjust outer_start
7701 * in this case to report failed page properly
7702 * on tracepoint in test_pages_isolated()
7703 */
7704 if (outer_start + (1UL << order) <= start)
7705 outer_start = start;
7706 }
7707
7708 /* Make sure the range is really isolated. */
7709 if (test_pages_isolated(outer_start, end, false)) {
7710 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
7711 __func__, outer_start, end);
7712 ret = -EBUSY;
7713 goto done;
7714 }
7715
7716 /* Grab isolated pages from freelists. */
7717 outer_end = isolate_freepages_range(&cc, outer_start, end);
7718 if (!outer_end) {
7719 ret = -EBUSY;
7720 goto done;
7721 }
7722
7723 /* Free head and tail (if any) */
7724 if (start != outer_start)
7725 free_contig_range(outer_start, start - outer_start);
7726 if (end != outer_end)
7727 free_contig_range(end, outer_end - end);
7728
7729 done:
7730 undo_isolate_page_range(pfn_max_align_down(start),
7731 pfn_max_align_up(end), migratetype);
7732 return ret;
7733 }
7734
7735 void free_contig_range(unsigned long pfn, unsigned nr_pages)
7736 {
7737 unsigned int count = 0;
7738
7739 for (; nr_pages--; pfn++) {
7740 struct page *page = pfn_to_page(pfn);
7741
7742 count += page_count(page) != 1;
7743 __free_page(page);
7744 }
7745 WARN(count != 0, "%d pages are still in use!\n", count);
7746 }
7747 #endif
7748
7749 #ifdef CONFIG_MEMORY_HOTPLUG
7750 /*
7751 * The zone indicated has a new number of managed_pages; batch sizes and percpu
7752 * page high values need to be recalulated.
7753 */
7754 void __meminit zone_pcp_update(struct zone *zone)
7755 {
7756 unsigned cpu;
7757 mutex_lock(&pcp_batch_high_lock);
7758 for_each_possible_cpu(cpu)
7759 pageset_set_high_and_batch(zone,
7760 per_cpu_ptr(zone->pageset, cpu));
7761 mutex_unlock(&pcp_batch_high_lock);
7762 }
7763 #endif
7764
7765 void zone_pcp_reset(struct zone *zone)
7766 {
7767 unsigned long flags;
7768 int cpu;
7769 struct per_cpu_pageset *pset;
7770
7771 /* avoid races with drain_pages() */
7772 local_irq_save(flags);
7773 if (zone->pageset != &boot_pageset) {
7774 for_each_online_cpu(cpu) {
7775 pset = per_cpu_ptr(zone->pageset, cpu);
7776 drain_zonestat(zone, pset);
7777 }
7778 free_percpu(zone->pageset);
7779 zone->pageset = &boot_pageset;
7780 }
7781 local_irq_restore(flags);
7782 }
7783
7784 #ifdef CONFIG_MEMORY_HOTREMOVE
7785 /*
7786 * All pages in the range must be in a single zone and isolated
7787 * before calling this.
7788 */
7789 void
7790 __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
7791 {
7792 struct page *page;
7793 struct zone *zone;
7794 unsigned int order, i;
7795 unsigned long pfn;
7796 unsigned long flags;
7797 /* find the first valid pfn */
7798 for (pfn = start_pfn; pfn < end_pfn; pfn++)
7799 if (pfn_valid(pfn))
7800 break;
7801 if (pfn == end_pfn)
7802 return;
7803 offline_mem_sections(pfn, end_pfn);
7804 zone = page_zone(pfn_to_page(pfn));
7805 spin_lock_irqsave(&zone->lock, flags);
7806 pfn = start_pfn;
7807 while (pfn < end_pfn) {
7808 if (!pfn_valid(pfn)) {
7809 pfn++;
7810 continue;
7811 }
7812 page = pfn_to_page(pfn);
7813 /*
7814 * The HWPoisoned page may be not in buddy system, and
7815 * page_count() is not 0.
7816 */
7817 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
7818 pfn++;
7819 SetPageReserved(page);
7820 continue;
7821 }
7822
7823 BUG_ON(page_count(page));
7824 BUG_ON(!PageBuddy(page));
7825 order = page_order(page);
7826 #ifdef CONFIG_DEBUG_VM
7827 pr_info("remove from free list %lx %d %lx\n",
7828 pfn, 1 << order, end_pfn);
7829 #endif
7830 list_del(&page->lru);
7831 rmv_page_order(page);
7832 zone->free_area[order].nr_free--;
7833 for (i = 0; i < (1 << order); i++)
7834 SetPageReserved((page+i));
7835 pfn += (1 << order);
7836 }
7837 spin_unlock_irqrestore(&zone->lock, flags);
7838 }
7839 #endif
7840
7841 bool is_free_buddy_page(struct page *page)
7842 {
7843 struct zone *zone = page_zone(page);
7844 unsigned long pfn = page_to_pfn(page);
7845 unsigned long flags;
7846 unsigned int order;
7847
7848 spin_lock_irqsave(&zone->lock, flags);
7849 for (order = 0; order < MAX_ORDER; order++) {
7850 struct page *page_head = page - (pfn & ((1 << order) - 1));
7851
7852 if (PageBuddy(page_head) && page_order(page_head) >= order)
7853 break;
7854 }
7855 spin_unlock_irqrestore(&zone->lock, flags);
7856
7857 return order < MAX_ORDER;
7858 }