1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 1993 Linus Torvalds
4 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
5 * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
6 * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
7 * Numa awareness, Christoph Lameter, SGI, June 2005
8 * Improving global KVA allocator, Uladzislau Rezki, Sony, May 2019
11 #include <linux/vmalloc.h>
13 #include <linux/module.h>
14 #include <linux/highmem.h>
15 #include <linux/sched/signal.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/interrupt.h>
19 #include <linux/proc_fs.h>
20 #include <linux/seq_file.h>
21 #include <linux/set_memory.h>
22 #include <linux/debugobjects.h>
23 #include <linux/kallsyms.h>
24 #include <linux/list.h>
25 #include <linux/notifier.h>
26 #include <linux/rbtree.h>
27 #include <linux/xarray.h>
29 #include <linux/rcupdate.h>
30 #include <linux/pfn.h>
31 #include <linux/kmemleak.h>
32 #include <linux/atomic.h>
33 #include <linux/compiler.h>
34 #include <linux/memcontrol.h>
35 #include <linux/llist.h>
36 #include <linux/bitops.h>
37 #include <linux/rbtree_augmented.h>
38 #include <linux/overflow.h>
39 #include <linux/pgtable.h>
40 #include <linux/uaccess.h>
41 #include <linux/hugetlb.h>
42 #include <linux/sched/mm.h>
43 #include <asm/tlbflush.h>
44 #include <asm/shmparam.h>
47 #include "pgalloc-track.h"
49 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP
50 static unsigned int __ro_after_init ioremap_max_page_shift
= BITS_PER_LONG
- 1;
52 static int __init
set_nohugeiomap(char *str
)
54 ioremap_max_page_shift
= PAGE_SHIFT
;
57 early_param("nohugeiomap", set_nohugeiomap
);
58 #else /* CONFIG_HAVE_ARCH_HUGE_VMAP */
59 static const unsigned int ioremap_max_page_shift
= PAGE_SHIFT
;
60 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
62 #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
63 static bool __ro_after_init vmap_allow_huge
= true;
65 static int __init
set_nohugevmalloc(char *str
)
67 vmap_allow_huge
= false;
70 early_param("nohugevmalloc", set_nohugevmalloc
);
71 #else /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */
72 static const bool vmap_allow_huge
= false;
73 #endif /* CONFIG_HAVE_ARCH_HUGE_VMALLOC */
75 bool is_vmalloc_addr(const void *x
)
77 unsigned long addr
= (unsigned long)kasan_reset_tag(x
);
79 return addr
>= VMALLOC_START
&& addr
< VMALLOC_END
;
81 EXPORT_SYMBOL(is_vmalloc_addr
);
83 struct vfree_deferred
{
84 struct llist_head list
;
85 struct work_struct wq
;
87 static DEFINE_PER_CPU(struct vfree_deferred
, vfree_deferred
);
89 static void __vunmap(const void *, int);
91 static void free_work(struct work_struct
*w
)
93 struct vfree_deferred
*p
= container_of(w
, struct vfree_deferred
, wq
);
94 struct llist_node
*t
, *llnode
;
96 llist_for_each_safe(llnode
, t
, llist_del_all(&p
->list
))
97 __vunmap((void *)llnode
, 1);
100 /*** Page table manipulation functions ***/
101 static int vmap_pte_range(pmd_t
*pmd
, unsigned long addr
, unsigned long end
,
102 phys_addr_t phys_addr
, pgprot_t prot
,
103 unsigned int max_page_shift
, pgtbl_mod_mask
*mask
)
107 unsigned long size
= PAGE_SIZE
;
109 pfn
= phys_addr
>> PAGE_SHIFT
;
110 pte
= pte_alloc_kernel_track(pmd
, addr
, mask
);
114 BUG_ON(!pte_none(*pte
));
116 #ifdef CONFIG_HUGETLB_PAGE
117 size
= arch_vmap_pte_range_map_size(addr
, end
, pfn
, max_page_shift
);
118 if (size
!= PAGE_SIZE
) {
119 pte_t entry
= pfn_pte(pfn
, prot
);
121 entry
= arch_make_huge_pte(entry
, ilog2(size
), 0);
122 set_huge_pte_at(&init_mm
, addr
, pte
, entry
);
123 pfn
+= PFN_DOWN(size
);
127 set_pte_at(&init_mm
, addr
, pte
, pfn_pte(pfn
, prot
));
129 } while (pte
+= PFN_DOWN(size
), addr
+= size
, addr
!= end
);
130 *mask
|= PGTBL_PTE_MODIFIED
;
134 static int vmap_try_huge_pmd(pmd_t
*pmd
, unsigned long addr
, unsigned long end
,
135 phys_addr_t phys_addr
, pgprot_t prot
,
136 unsigned int max_page_shift
)
138 if (max_page_shift
< PMD_SHIFT
)
141 if (!arch_vmap_pmd_supported(prot
))
144 if ((end
- addr
) != PMD_SIZE
)
147 if (!IS_ALIGNED(addr
, PMD_SIZE
))
150 if (!IS_ALIGNED(phys_addr
, PMD_SIZE
))
153 if (pmd_present(*pmd
) && !pmd_free_pte_page(pmd
, addr
))
156 return pmd_set_huge(pmd
, phys_addr
, prot
);
159 static int vmap_pmd_range(pud_t
*pud
, unsigned long addr
, unsigned long end
,
160 phys_addr_t phys_addr
, pgprot_t prot
,
161 unsigned int max_page_shift
, pgtbl_mod_mask
*mask
)
166 pmd
= pmd_alloc_track(&init_mm
, pud
, addr
, mask
);
170 next
= pmd_addr_end(addr
, end
);
172 if (vmap_try_huge_pmd(pmd
, addr
, next
, phys_addr
, prot
,
174 *mask
|= PGTBL_PMD_MODIFIED
;
178 if (vmap_pte_range(pmd
, addr
, next
, phys_addr
, prot
, max_page_shift
, mask
))
180 } while (pmd
++, phys_addr
+= (next
- addr
), addr
= next
, addr
!= end
);
184 static int vmap_try_huge_pud(pud_t
*pud
, unsigned long addr
, unsigned long end
,
185 phys_addr_t phys_addr
, pgprot_t prot
,
186 unsigned int max_page_shift
)
188 if (max_page_shift
< PUD_SHIFT
)
191 if (!arch_vmap_pud_supported(prot
))
194 if ((end
- addr
) != PUD_SIZE
)
197 if (!IS_ALIGNED(addr
, PUD_SIZE
))
200 if (!IS_ALIGNED(phys_addr
, PUD_SIZE
))
203 if (pud_present(*pud
) && !pud_free_pmd_page(pud
, addr
))
206 return pud_set_huge(pud
, phys_addr
, prot
);
209 static int vmap_pud_range(p4d_t
*p4d
, unsigned long addr
, unsigned long end
,
210 phys_addr_t phys_addr
, pgprot_t prot
,
211 unsigned int max_page_shift
, pgtbl_mod_mask
*mask
)
216 pud
= pud_alloc_track(&init_mm
, p4d
, addr
, mask
);
220 next
= pud_addr_end(addr
, end
);
222 if (vmap_try_huge_pud(pud
, addr
, next
, phys_addr
, prot
,
224 *mask
|= PGTBL_PUD_MODIFIED
;
228 if (vmap_pmd_range(pud
, addr
, next
, phys_addr
, prot
,
229 max_page_shift
, mask
))
231 } while (pud
++, phys_addr
+= (next
- addr
), addr
= next
, addr
!= end
);
235 static int vmap_try_huge_p4d(p4d_t
*p4d
, unsigned long addr
, unsigned long end
,
236 phys_addr_t phys_addr
, pgprot_t prot
,
237 unsigned int max_page_shift
)
239 if (max_page_shift
< P4D_SHIFT
)
242 if (!arch_vmap_p4d_supported(prot
))
245 if ((end
- addr
) != P4D_SIZE
)
248 if (!IS_ALIGNED(addr
, P4D_SIZE
))
251 if (!IS_ALIGNED(phys_addr
, P4D_SIZE
))
254 if (p4d_present(*p4d
) && !p4d_free_pud_page(p4d
, addr
))
257 return p4d_set_huge(p4d
, phys_addr
, prot
);
260 static int vmap_p4d_range(pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
261 phys_addr_t phys_addr
, pgprot_t prot
,
262 unsigned int max_page_shift
, pgtbl_mod_mask
*mask
)
267 p4d
= p4d_alloc_track(&init_mm
, pgd
, addr
, mask
);
271 next
= p4d_addr_end(addr
, end
);
273 if (vmap_try_huge_p4d(p4d
, addr
, next
, phys_addr
, prot
,
275 *mask
|= PGTBL_P4D_MODIFIED
;
279 if (vmap_pud_range(p4d
, addr
, next
, phys_addr
, prot
,
280 max_page_shift
, mask
))
282 } while (p4d
++, phys_addr
+= (next
- addr
), addr
= next
, addr
!= end
);
286 static int vmap_range_noflush(unsigned long addr
, unsigned long end
,
287 phys_addr_t phys_addr
, pgprot_t prot
,
288 unsigned int max_page_shift
)
294 pgtbl_mod_mask mask
= 0;
300 pgd
= pgd_offset_k(addr
);
302 next
= pgd_addr_end(addr
, end
);
303 err
= vmap_p4d_range(pgd
, addr
, next
, phys_addr
, prot
,
304 max_page_shift
, &mask
);
307 } while (pgd
++, phys_addr
+= (next
- addr
), addr
= next
, addr
!= end
);
309 if (mask
& ARCH_PAGE_TABLE_SYNC_MASK
)
310 arch_sync_kernel_mappings(start
, end
);
315 int ioremap_page_range(unsigned long addr
, unsigned long end
,
316 phys_addr_t phys_addr
, pgprot_t prot
)
320 err
= vmap_range_noflush(addr
, end
, phys_addr
, pgprot_nx(prot
),
321 ioremap_max_page_shift
);
322 flush_cache_vmap(addr
, end
);
326 static void vunmap_pte_range(pmd_t
*pmd
, unsigned long addr
, unsigned long end
,
327 pgtbl_mod_mask
*mask
)
331 pte
= pte_offset_kernel(pmd
, addr
);
333 pte_t ptent
= ptep_get_and_clear(&init_mm
, addr
, pte
);
334 WARN_ON(!pte_none(ptent
) && !pte_present(ptent
));
335 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
336 *mask
|= PGTBL_PTE_MODIFIED
;
339 static void vunmap_pmd_range(pud_t
*pud
, unsigned long addr
, unsigned long end
,
340 pgtbl_mod_mask
*mask
)
346 pmd
= pmd_offset(pud
, addr
);
348 next
= pmd_addr_end(addr
, end
);
350 cleared
= pmd_clear_huge(pmd
);
351 if (cleared
|| pmd_bad(*pmd
))
352 *mask
|= PGTBL_PMD_MODIFIED
;
356 if (pmd_none_or_clear_bad(pmd
))
358 vunmap_pte_range(pmd
, addr
, next
, mask
);
361 } while (pmd
++, addr
= next
, addr
!= end
);
364 static void vunmap_pud_range(p4d_t
*p4d
, unsigned long addr
, unsigned long end
,
365 pgtbl_mod_mask
*mask
)
371 pud
= pud_offset(p4d
, addr
);
373 next
= pud_addr_end(addr
, end
);
375 cleared
= pud_clear_huge(pud
);
376 if (cleared
|| pud_bad(*pud
))
377 *mask
|= PGTBL_PUD_MODIFIED
;
381 if (pud_none_or_clear_bad(pud
))
383 vunmap_pmd_range(pud
, addr
, next
, mask
);
384 } while (pud
++, addr
= next
, addr
!= end
);
387 static void vunmap_p4d_range(pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
388 pgtbl_mod_mask
*mask
)
393 p4d
= p4d_offset(pgd
, addr
);
395 next
= p4d_addr_end(addr
, end
);
399 *mask
|= PGTBL_P4D_MODIFIED
;
401 if (p4d_none_or_clear_bad(p4d
))
403 vunmap_pud_range(p4d
, addr
, next
, mask
);
404 } while (p4d
++, addr
= next
, addr
!= end
);
408 * vunmap_range_noflush is similar to vunmap_range, but does not
409 * flush caches or TLBs.
411 * The caller is responsible for calling flush_cache_vmap() before calling
412 * this function, and flush_tlb_kernel_range after it has returned
413 * successfully (and before the addresses are expected to cause a page fault
414 * or be re-mapped for something else, if TLB flushes are being delayed or
417 * This is an internal function only. Do not use outside mm/.
419 void vunmap_range_noflush(unsigned long start
, unsigned long end
)
423 unsigned long addr
= start
;
424 pgtbl_mod_mask mask
= 0;
427 pgd
= pgd_offset_k(addr
);
429 next
= pgd_addr_end(addr
, end
);
431 mask
|= PGTBL_PGD_MODIFIED
;
432 if (pgd_none_or_clear_bad(pgd
))
434 vunmap_p4d_range(pgd
, addr
, next
, &mask
);
435 } while (pgd
++, addr
= next
, addr
!= end
);
437 if (mask
& ARCH_PAGE_TABLE_SYNC_MASK
)
438 arch_sync_kernel_mappings(start
, end
);
442 * vunmap_range - unmap kernel virtual addresses
443 * @addr: start of the VM area to unmap
444 * @end: end of the VM area to unmap (non-inclusive)
446 * Clears any present PTEs in the virtual address range, flushes TLBs and
447 * caches. Any subsequent access to the address before it has been re-mapped
450 void vunmap_range(unsigned long addr
, unsigned long end
)
452 flush_cache_vunmap(addr
, end
);
453 vunmap_range_noflush(addr
, end
);
454 flush_tlb_kernel_range(addr
, end
);
457 static int vmap_pages_pte_range(pmd_t
*pmd
, unsigned long addr
,
458 unsigned long end
, pgprot_t prot
, struct page
**pages
, int *nr
,
459 pgtbl_mod_mask
*mask
)
464 * nr is a running index into the array which helps higher level
465 * callers keep track of where we're up to.
468 pte
= pte_alloc_kernel_track(pmd
, addr
, mask
);
472 struct page
*page
= pages
[*nr
];
474 if (WARN_ON(!pte_none(*pte
)))
478 if (WARN_ON(!pfn_valid(page_to_pfn(page
))))
481 set_pte_at(&init_mm
, addr
, pte
, mk_pte(page
, prot
));
483 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
484 *mask
|= PGTBL_PTE_MODIFIED
;
488 static int vmap_pages_pmd_range(pud_t
*pud
, unsigned long addr
,
489 unsigned long end
, pgprot_t prot
, struct page
**pages
, int *nr
,
490 pgtbl_mod_mask
*mask
)
495 pmd
= pmd_alloc_track(&init_mm
, pud
, addr
, mask
);
499 next
= pmd_addr_end(addr
, end
);
500 if (vmap_pages_pte_range(pmd
, addr
, next
, prot
, pages
, nr
, mask
))
502 } while (pmd
++, addr
= next
, addr
!= end
);
506 static int vmap_pages_pud_range(p4d_t
*p4d
, unsigned long addr
,
507 unsigned long end
, pgprot_t prot
, struct page
**pages
, int *nr
,
508 pgtbl_mod_mask
*mask
)
513 pud
= pud_alloc_track(&init_mm
, p4d
, addr
, mask
);
517 next
= pud_addr_end(addr
, end
);
518 if (vmap_pages_pmd_range(pud
, addr
, next
, prot
, pages
, nr
, mask
))
520 } while (pud
++, addr
= next
, addr
!= end
);
524 static int vmap_pages_p4d_range(pgd_t
*pgd
, unsigned long addr
,
525 unsigned long end
, pgprot_t prot
, struct page
**pages
, int *nr
,
526 pgtbl_mod_mask
*mask
)
531 p4d
= p4d_alloc_track(&init_mm
, pgd
, addr
, mask
);
535 next
= p4d_addr_end(addr
, end
);
536 if (vmap_pages_pud_range(p4d
, addr
, next
, prot
, pages
, nr
, mask
))
538 } while (p4d
++, addr
= next
, addr
!= end
);
542 static int vmap_small_pages_range_noflush(unsigned long addr
, unsigned long end
,
543 pgprot_t prot
, struct page
**pages
)
545 unsigned long start
= addr
;
550 pgtbl_mod_mask mask
= 0;
553 pgd
= pgd_offset_k(addr
);
555 next
= pgd_addr_end(addr
, end
);
557 mask
|= PGTBL_PGD_MODIFIED
;
558 err
= vmap_pages_p4d_range(pgd
, addr
, next
, prot
, pages
, &nr
, &mask
);
561 } while (pgd
++, addr
= next
, addr
!= end
);
563 if (mask
& ARCH_PAGE_TABLE_SYNC_MASK
)
564 arch_sync_kernel_mappings(start
, end
);
570 * vmap_pages_range_noflush is similar to vmap_pages_range, but does not
573 * The caller is responsible for calling flush_cache_vmap() after this
574 * function returns successfully and before the addresses are accessed.
576 * This is an internal function only. Do not use outside mm/.
578 int vmap_pages_range_noflush(unsigned long addr
, unsigned long end
,
579 pgprot_t prot
, struct page
**pages
, unsigned int page_shift
)
581 unsigned int i
, nr
= (end
- addr
) >> PAGE_SHIFT
;
583 WARN_ON(page_shift
< PAGE_SHIFT
);
585 if (!IS_ENABLED(CONFIG_HAVE_ARCH_HUGE_VMALLOC
) ||
586 page_shift
== PAGE_SHIFT
)
587 return vmap_small_pages_range_noflush(addr
, end
, prot
, pages
);
589 for (i
= 0; i
< nr
; i
+= 1U << (page_shift
- PAGE_SHIFT
)) {
592 err
= vmap_range_noflush(addr
, addr
+ (1UL << page_shift
),
593 __pa(page_address(pages
[i
])), prot
,
598 addr
+= 1UL << page_shift
;
605 * vmap_pages_range - map pages to a kernel virtual address
606 * @addr: start of the VM area to map
607 * @end: end of the VM area to map (non-inclusive)
608 * @prot: page protection flags to use
609 * @pages: pages to map (always PAGE_SIZE pages)
610 * @page_shift: maximum shift that the pages may be mapped with, @pages must
611 * be aligned and contiguous up to at least this shift.
614 * 0 on success, -errno on failure.
616 static int vmap_pages_range(unsigned long addr
, unsigned long end
,
617 pgprot_t prot
, struct page
**pages
, unsigned int page_shift
)
621 err
= vmap_pages_range_noflush(addr
, end
, prot
, pages
, page_shift
);
622 flush_cache_vmap(addr
, end
);
626 int is_vmalloc_or_module_addr(const void *x
)
629 * ARM, x86-64 and sparc64 put modules in a special place,
630 * and fall back on vmalloc() if that fails. Others
631 * just put it in the vmalloc space.
633 #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
634 unsigned long addr
= (unsigned long)kasan_reset_tag(x
);
635 if (addr
>= MODULES_VADDR
&& addr
< MODULES_END
)
638 return is_vmalloc_addr(x
);
642 * Walk a vmap address to the struct page it maps. Huge vmap mappings will
643 * return the tail page that corresponds to the base page address, which
644 * matches small vmap mappings.
646 struct page
*vmalloc_to_page(const void *vmalloc_addr
)
648 unsigned long addr
= (unsigned long) vmalloc_addr
;
649 struct page
*page
= NULL
;
650 pgd_t
*pgd
= pgd_offset_k(addr
);
657 * XXX we might need to change this if we add VIRTUAL_BUG_ON for
658 * architectures that do not vmalloc module space
660 VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr
));
664 if (WARN_ON_ONCE(pgd_leaf(*pgd
)))
665 return NULL
; /* XXX: no allowance for huge pgd */
666 if (WARN_ON_ONCE(pgd_bad(*pgd
)))
669 p4d
= p4d_offset(pgd
, addr
);
673 return p4d_page(*p4d
) + ((addr
& ~P4D_MASK
) >> PAGE_SHIFT
);
674 if (WARN_ON_ONCE(p4d_bad(*p4d
)))
677 pud
= pud_offset(p4d
, addr
);
681 return pud_page(*pud
) + ((addr
& ~PUD_MASK
) >> PAGE_SHIFT
);
682 if (WARN_ON_ONCE(pud_bad(*pud
)))
685 pmd
= pmd_offset(pud
, addr
);
689 return pmd_page(*pmd
) + ((addr
& ~PMD_MASK
) >> PAGE_SHIFT
);
690 if (WARN_ON_ONCE(pmd_bad(*pmd
)))
693 ptep
= pte_offset_map(pmd
, addr
);
695 if (pte_present(pte
))
696 page
= pte_page(pte
);
701 EXPORT_SYMBOL(vmalloc_to_page
);
704 * Map a vmalloc()-space virtual address to the physical page frame number.
706 unsigned long vmalloc_to_pfn(const void *vmalloc_addr
)
708 return page_to_pfn(vmalloc_to_page(vmalloc_addr
));
710 EXPORT_SYMBOL(vmalloc_to_pfn
);
713 /*** Global kva allocator ***/
715 #define DEBUG_AUGMENT_PROPAGATE_CHECK 0
716 #define DEBUG_AUGMENT_LOWEST_MATCH_CHECK 0
719 static DEFINE_SPINLOCK(vmap_area_lock
);
720 static DEFINE_SPINLOCK(free_vmap_area_lock
);
721 /* Export for kexec only */
722 LIST_HEAD(vmap_area_list
);
723 static struct rb_root vmap_area_root
= RB_ROOT
;
724 static bool vmap_initialized __read_mostly
;
726 static struct rb_root purge_vmap_area_root
= RB_ROOT
;
727 static LIST_HEAD(purge_vmap_area_list
);
728 static DEFINE_SPINLOCK(purge_vmap_area_lock
);
731 * This kmem_cache is used for vmap_area objects. Instead of
732 * allocating from slab we reuse an object from this cache to
733 * make things faster. Especially in "no edge" splitting of
736 static struct kmem_cache
*vmap_area_cachep
;
739 * This linked list is used in pair with free_vmap_area_root.
740 * It gives O(1) access to prev/next to perform fast coalescing.
742 static LIST_HEAD(free_vmap_area_list
);
745 * This augment red-black tree represents the free vmap space.
746 * All vmap_area objects in this tree are sorted by va->va_start
747 * address. It is used for allocation and merging when a vmap
748 * object is released.
750 * Each vmap_area node contains a maximum available free block
751 * of its sub-tree, right or left. Therefore it is possible to
752 * find a lowest match of free area.
754 static struct rb_root free_vmap_area_root
= RB_ROOT
;
757 * Preload a CPU with one object for "no edge" split case. The
758 * aim is to get rid of allocations from the atomic context, thus
759 * to use more permissive allocation masks.
761 static DEFINE_PER_CPU(struct vmap_area
*, ne_fit_preload_node
);
763 static __always_inline
unsigned long
764 va_size(struct vmap_area
*va
)
766 return (va
->va_end
- va
->va_start
);
769 static __always_inline
unsigned long
770 get_subtree_max_size(struct rb_node
*node
)
772 struct vmap_area
*va
;
774 va
= rb_entry_safe(node
, struct vmap_area
, rb_node
);
775 return va
? va
->subtree_max_size
: 0;
778 RB_DECLARE_CALLBACKS_MAX(static, free_vmap_area_rb_augment_cb
,
779 struct vmap_area
, rb_node
, unsigned long, subtree_max_size
, va_size
)
781 static void purge_vmap_area_lazy(void);
782 static BLOCKING_NOTIFIER_HEAD(vmap_notify_list
);
783 static void drain_vmap_area_work(struct work_struct
*work
);
784 static DECLARE_WORK(drain_vmap_work
, drain_vmap_area_work
);
786 static atomic_long_t nr_vmalloc_pages
;
788 unsigned long vmalloc_nr_pages(void)
790 return atomic_long_read(&nr_vmalloc_pages
);
793 /* Look up the first VA which satisfies addr < va_end, NULL if none. */
794 static struct vmap_area
*find_vmap_area_exceed_addr(unsigned long addr
)
796 struct vmap_area
*va
= NULL
;
797 struct rb_node
*n
= vmap_area_root
.rb_node
;
799 addr
= (unsigned long)kasan_reset_tag((void *)addr
);
802 struct vmap_area
*tmp
;
804 tmp
= rb_entry(n
, struct vmap_area
, rb_node
);
805 if (tmp
->va_end
> addr
) {
807 if (tmp
->va_start
<= addr
)
818 static struct vmap_area
*__find_vmap_area(unsigned long addr
, struct rb_root
*root
)
820 struct rb_node
*n
= root
->rb_node
;
822 addr
= (unsigned long)kasan_reset_tag((void *)addr
);
825 struct vmap_area
*va
;
827 va
= rb_entry(n
, struct vmap_area
, rb_node
);
828 if (addr
< va
->va_start
)
830 else if (addr
>= va
->va_end
)
840 * This function returns back addresses of parent node
841 * and its left or right link for further processing.
843 * Otherwise NULL is returned. In that case all further
844 * steps regarding inserting of conflicting overlap range
845 * have to be declined and actually considered as a bug.
847 static __always_inline
struct rb_node
**
848 find_va_links(struct vmap_area
*va
,
849 struct rb_root
*root
, struct rb_node
*from
,
850 struct rb_node
**parent
)
852 struct vmap_area
*tmp_va
;
853 struct rb_node
**link
;
856 link
= &root
->rb_node
;
857 if (unlikely(!*link
)) {
866 * Go to the bottom of the tree. When we hit the last point
867 * we end up with parent rb_node and correct direction, i name
868 * it link, where the new va->rb_node will be attached to.
871 tmp_va
= rb_entry(*link
, struct vmap_area
, rb_node
);
874 * During the traversal we also do some sanity check.
875 * Trigger the BUG() if there are sides(left/right)
878 if (va
->va_end
<= tmp_va
->va_start
)
879 link
= &(*link
)->rb_left
;
880 else if (va
->va_start
>= tmp_va
->va_end
)
881 link
= &(*link
)->rb_right
;
883 WARN(1, "vmalloc bug: 0x%lx-0x%lx overlaps with 0x%lx-0x%lx\n",
884 va
->va_start
, va
->va_end
, tmp_va
->va_start
, tmp_va
->va_end
);
890 *parent
= &tmp_va
->rb_node
;
894 static __always_inline
struct list_head
*
895 get_va_next_sibling(struct rb_node
*parent
, struct rb_node
**link
)
897 struct list_head
*list
;
899 if (unlikely(!parent
))
901 * The red-black tree where we try to find VA neighbors
902 * before merging or inserting is empty, i.e. it means
903 * there is no free vmap space. Normally it does not
904 * happen but we handle this case anyway.
908 list
= &rb_entry(parent
, struct vmap_area
, rb_node
)->list
;
909 return (&parent
->rb_right
== link
? list
->next
: list
);
912 static __always_inline
void
913 __link_va(struct vmap_area
*va
, struct rb_root
*root
,
914 struct rb_node
*parent
, struct rb_node
**link
,
915 struct list_head
*head
, bool augment
)
918 * VA is still not in the list, but we can
919 * identify its future previous list_head node.
921 if (likely(parent
)) {
922 head
= &rb_entry(parent
, struct vmap_area
, rb_node
)->list
;
923 if (&parent
->rb_right
!= link
)
927 /* Insert to the rb-tree */
928 rb_link_node(&va
->rb_node
, parent
, link
);
931 * Some explanation here. Just perform simple insertion
932 * to the tree. We do not set va->subtree_max_size to
933 * its current size before calling rb_insert_augmented().
934 * It is because we populate the tree from the bottom
935 * to parent levels when the node _is_ in the tree.
937 * Therefore we set subtree_max_size to zero after insertion,
938 * to let __augment_tree_propagate_from() puts everything to
939 * the correct order later on.
941 rb_insert_augmented(&va
->rb_node
,
942 root
, &free_vmap_area_rb_augment_cb
);
943 va
->subtree_max_size
= 0;
945 rb_insert_color(&va
->rb_node
, root
);
948 /* Address-sort this list */
949 list_add(&va
->list
, head
);
952 static __always_inline
void
953 link_va(struct vmap_area
*va
, struct rb_root
*root
,
954 struct rb_node
*parent
, struct rb_node
**link
,
955 struct list_head
*head
)
957 __link_va(va
, root
, parent
, link
, head
, false);
960 static __always_inline
void
961 link_va_augment(struct vmap_area
*va
, struct rb_root
*root
,
962 struct rb_node
*parent
, struct rb_node
**link
,
963 struct list_head
*head
)
965 __link_va(va
, root
, parent
, link
, head
, true);
968 static __always_inline
void
969 __unlink_va(struct vmap_area
*va
, struct rb_root
*root
, bool augment
)
971 if (WARN_ON(RB_EMPTY_NODE(&va
->rb_node
)))
975 rb_erase_augmented(&va
->rb_node
,
976 root
, &free_vmap_area_rb_augment_cb
);
978 rb_erase(&va
->rb_node
, root
);
980 list_del_init(&va
->list
);
981 RB_CLEAR_NODE(&va
->rb_node
);
984 static __always_inline
void
985 unlink_va(struct vmap_area
*va
, struct rb_root
*root
)
987 __unlink_va(va
, root
, false);
990 static __always_inline
void
991 unlink_va_augment(struct vmap_area
*va
, struct rb_root
*root
)
993 __unlink_va(va
, root
, true);
996 #if DEBUG_AUGMENT_PROPAGATE_CHECK
998 * Gets called when remove the node and rotate.
1000 static __always_inline
unsigned long
1001 compute_subtree_max_size(struct vmap_area
*va
)
1003 return max3(va_size(va
),
1004 get_subtree_max_size(va
->rb_node
.rb_left
),
1005 get_subtree_max_size(va
->rb_node
.rb_right
));
1009 augment_tree_propagate_check(void)
1011 struct vmap_area
*va
;
1012 unsigned long computed_size
;
1014 list_for_each_entry(va
, &free_vmap_area_list
, list
) {
1015 computed_size
= compute_subtree_max_size(va
);
1016 if (computed_size
!= va
->subtree_max_size
)
1017 pr_emerg("tree is corrupted: %lu, %lu\n",
1018 va_size(va
), va
->subtree_max_size
);
1024 * This function populates subtree_max_size from bottom to upper
1025 * levels starting from VA point. The propagation must be done
1026 * when VA size is modified by changing its va_start/va_end. Or
1027 * in case of newly inserting of VA to the tree.
1029 * It means that __augment_tree_propagate_from() must be called:
1030 * - After VA has been inserted to the tree(free path);
1031 * - After VA has been shrunk(allocation path);
1032 * - After VA has been increased(merging path).
1034 * Please note that, it does not mean that upper parent nodes
1035 * and their subtree_max_size are recalculated all the time up
1044 * For example if we modify the node 4, shrinking it to 2, then
1045 * no any modification is required. If we shrink the node 2 to 1
1046 * its subtree_max_size is updated only, and set to 1. If we shrink
1047 * the node 8 to 6, then its subtree_max_size is set to 6 and parent
1048 * node becomes 4--6.
1050 static __always_inline
void
1051 augment_tree_propagate_from(struct vmap_area
*va
)
1054 * Populate the tree from bottom towards the root until
1055 * the calculated maximum available size of checked node
1056 * is equal to its current one.
1058 free_vmap_area_rb_augment_cb_propagate(&va
->rb_node
, NULL
);
1060 #if DEBUG_AUGMENT_PROPAGATE_CHECK
1061 augment_tree_propagate_check();
1066 insert_vmap_area(struct vmap_area
*va
,
1067 struct rb_root
*root
, struct list_head
*head
)
1069 struct rb_node
**link
;
1070 struct rb_node
*parent
;
1072 link
= find_va_links(va
, root
, NULL
, &parent
);
1074 link_va(va
, root
, parent
, link
, head
);
1078 insert_vmap_area_augment(struct vmap_area
*va
,
1079 struct rb_node
*from
, struct rb_root
*root
,
1080 struct list_head
*head
)
1082 struct rb_node
**link
;
1083 struct rb_node
*parent
;
1086 link
= find_va_links(va
, NULL
, from
, &parent
);
1088 link
= find_va_links(va
, root
, NULL
, &parent
);
1091 link_va_augment(va
, root
, parent
, link
, head
);
1092 augment_tree_propagate_from(va
);
1097 * Merge de-allocated chunk of VA memory with previous
1098 * and next free blocks. If coalesce is not done a new
1099 * free area is inserted. If VA has been merged, it is
1102 * Please note, it can return NULL in case of overlap
1103 * ranges, followed by WARN() report. Despite it is a
1104 * buggy behaviour, a system can be alive and keep
1107 static __always_inline
struct vmap_area
*
1108 __merge_or_add_vmap_area(struct vmap_area
*va
,
1109 struct rb_root
*root
, struct list_head
*head
, bool augment
)
1111 struct vmap_area
*sibling
;
1112 struct list_head
*next
;
1113 struct rb_node
**link
;
1114 struct rb_node
*parent
;
1115 bool merged
= false;
1118 * Find a place in the tree where VA potentially will be
1119 * inserted, unless it is merged with its sibling/siblings.
1121 link
= find_va_links(va
, root
, NULL
, &parent
);
1126 * Get next node of VA to check if merging can be done.
1128 next
= get_va_next_sibling(parent
, link
);
1129 if (unlikely(next
== NULL
))
1135 * |<------VA------>|<-----Next----->|
1140 sibling
= list_entry(next
, struct vmap_area
, list
);
1141 if (sibling
->va_start
== va
->va_end
) {
1142 sibling
->va_start
= va
->va_start
;
1144 /* Free vmap_area object. */
1145 kmem_cache_free(vmap_area_cachep
, va
);
1147 /* Point to the new merged area. */
1156 * |<-----Prev----->|<------VA------>|
1160 if (next
->prev
!= head
) {
1161 sibling
= list_entry(next
->prev
, struct vmap_area
, list
);
1162 if (sibling
->va_end
== va
->va_start
) {
1164 * If both neighbors are coalesced, it is important
1165 * to unlink the "next" node first, followed by merging
1166 * with "previous" one. Otherwise the tree might not be
1167 * fully populated if a sibling's augmented value is
1168 * "normalized" because of rotation operations.
1171 __unlink_va(va
, root
, augment
);
1173 sibling
->va_end
= va
->va_end
;
1175 /* Free vmap_area object. */
1176 kmem_cache_free(vmap_area_cachep
, va
);
1178 /* Point to the new merged area. */
1186 __link_va(va
, root
, parent
, link
, head
, augment
);
1191 static __always_inline
struct vmap_area
*
1192 merge_or_add_vmap_area(struct vmap_area
*va
,
1193 struct rb_root
*root
, struct list_head
*head
)
1195 return __merge_or_add_vmap_area(va
, root
, head
, false);
1198 static __always_inline
struct vmap_area
*
1199 merge_or_add_vmap_area_augment(struct vmap_area
*va
,
1200 struct rb_root
*root
, struct list_head
*head
)
1202 va
= __merge_or_add_vmap_area(va
, root
, head
, true);
1204 augment_tree_propagate_from(va
);
1209 static __always_inline
bool
1210 is_within_this_va(struct vmap_area
*va
, unsigned long size
,
1211 unsigned long align
, unsigned long vstart
)
1213 unsigned long nva_start_addr
;
1215 if (va
->va_start
> vstart
)
1216 nva_start_addr
= ALIGN(va
->va_start
, align
);
1218 nva_start_addr
= ALIGN(vstart
, align
);
1220 /* Can be overflowed due to big size or alignment. */
1221 if (nva_start_addr
+ size
< nva_start_addr
||
1222 nva_start_addr
< vstart
)
1225 return (nva_start_addr
+ size
<= va
->va_end
);
1229 * Find the first free block(lowest start address) in the tree,
1230 * that will accomplish the request corresponding to passing
1231 * parameters. Please note, with an alignment bigger than PAGE_SIZE,
1232 * a search length is adjusted to account for worst case alignment
1235 static __always_inline
struct vmap_area
*
1236 find_vmap_lowest_match(struct rb_root
*root
, unsigned long size
,
1237 unsigned long align
, unsigned long vstart
, bool adjust_search_size
)
1239 struct vmap_area
*va
;
1240 struct rb_node
*node
;
1241 unsigned long length
;
1243 /* Start from the root. */
1244 node
= root
->rb_node
;
1246 /* Adjust the search size for alignment overhead. */
1247 length
= adjust_search_size
? size
+ align
- 1 : size
;
1250 va
= rb_entry(node
, struct vmap_area
, rb_node
);
1252 if (get_subtree_max_size(node
->rb_left
) >= length
&&
1253 vstart
< va
->va_start
) {
1254 node
= node
->rb_left
;
1256 if (is_within_this_va(va
, size
, align
, vstart
))
1260 * Does not make sense to go deeper towards the right
1261 * sub-tree if it does not have a free block that is
1262 * equal or bigger to the requested search length.
1264 if (get_subtree_max_size(node
->rb_right
) >= length
) {
1265 node
= node
->rb_right
;
1270 * OK. We roll back and find the first right sub-tree,
1271 * that will satisfy the search criteria. It can happen
1272 * due to "vstart" restriction or an alignment overhead
1273 * that is bigger then PAGE_SIZE.
1275 while ((node
= rb_parent(node
))) {
1276 va
= rb_entry(node
, struct vmap_area
, rb_node
);
1277 if (is_within_this_va(va
, size
, align
, vstart
))
1280 if (get_subtree_max_size(node
->rb_right
) >= length
&&
1281 vstart
<= va
->va_start
) {
1283 * Shift the vstart forward. Please note, we update it with
1284 * parent's start address adding "1" because we do not want
1285 * to enter same sub-tree after it has already been checked
1286 * and no suitable free block found there.
1288 vstart
= va
->va_start
+ 1;
1289 node
= node
->rb_right
;
1299 #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
1300 #include <linux/random.h>
1302 static struct vmap_area
*
1303 find_vmap_lowest_linear_match(unsigned long size
,
1304 unsigned long align
, unsigned long vstart
)
1306 struct vmap_area
*va
;
1308 list_for_each_entry(va
, &free_vmap_area_list
, list
) {
1309 if (!is_within_this_va(va
, size
, align
, vstart
))
1319 find_vmap_lowest_match_check(unsigned long size
, unsigned long align
)
1321 struct vmap_area
*va_1
, *va_2
;
1322 unsigned long vstart
;
1325 get_random_bytes(&rnd
, sizeof(rnd
));
1326 vstart
= VMALLOC_START
+ rnd
;
1328 va_1
= find_vmap_lowest_match(size
, align
, vstart
, false);
1329 va_2
= find_vmap_lowest_linear_match(size
, align
, vstart
);
1332 pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n",
1333 va_1
, va_2
, vstart
);
1339 FL_FIT_TYPE
= 1, /* full fit */
1340 LE_FIT_TYPE
= 2, /* left edge fit */
1341 RE_FIT_TYPE
= 3, /* right edge fit */
1342 NE_FIT_TYPE
= 4 /* no edge fit */
1345 static __always_inline
enum fit_type
1346 classify_va_fit_type(struct vmap_area
*va
,
1347 unsigned long nva_start_addr
, unsigned long size
)
1351 /* Check if it is within VA. */
1352 if (nva_start_addr
< va
->va_start
||
1353 nva_start_addr
+ size
> va
->va_end
)
1357 if (va
->va_start
== nva_start_addr
) {
1358 if (va
->va_end
== nva_start_addr
+ size
)
1362 } else if (va
->va_end
== nva_start_addr
+ size
) {
1371 static __always_inline
int
1372 adjust_va_to_fit_type(struct rb_root
*root
, struct list_head
*head
,
1373 struct vmap_area
*va
, unsigned long nva_start_addr
,
1376 struct vmap_area
*lva
= NULL
;
1377 enum fit_type type
= classify_va_fit_type(va
, nva_start_addr
, size
);
1379 if (type
== FL_FIT_TYPE
) {
1381 * No need to split VA, it fully fits.
1387 unlink_va_augment(va
, root
);
1388 kmem_cache_free(vmap_area_cachep
, va
);
1389 } else if (type
== LE_FIT_TYPE
) {
1391 * Split left edge of fit VA.
1397 va
->va_start
+= size
;
1398 } else if (type
== RE_FIT_TYPE
) {
1400 * Split right edge of fit VA.
1406 va
->va_end
= nva_start_addr
;
1407 } else if (type
== NE_FIT_TYPE
) {
1409 * Split no edge of fit VA.
1415 lva
= __this_cpu_xchg(ne_fit_preload_node
, NULL
);
1416 if (unlikely(!lva
)) {
1418 * For percpu allocator we do not do any pre-allocation
1419 * and leave it as it is. The reason is it most likely
1420 * never ends up with NE_FIT_TYPE splitting. In case of
1421 * percpu allocations offsets and sizes are aligned to
1422 * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE
1423 * are its main fitting cases.
1425 * There are a few exceptions though, as an example it is
1426 * a first allocation (early boot up) when we have "one"
1427 * big free space that has to be split.
1429 * Also we can hit this path in case of regular "vmap"
1430 * allocations, if "this" current CPU was not preloaded.
1431 * See the comment in alloc_vmap_area() why. If so, then
1432 * GFP_NOWAIT is used instead to get an extra object for
1433 * split purpose. That is rare and most time does not
1436 * What happens if an allocation gets failed. Basically,
1437 * an "overflow" path is triggered to purge lazily freed
1438 * areas to free some memory, then, the "retry" path is
1439 * triggered to repeat one more time. See more details
1440 * in alloc_vmap_area() function.
1442 lva
= kmem_cache_alloc(vmap_area_cachep
, GFP_NOWAIT
);
1448 * Build the remainder.
1450 lva
->va_start
= va
->va_start
;
1451 lva
->va_end
= nva_start_addr
;
1454 * Shrink this VA to remaining size.
1456 va
->va_start
= nva_start_addr
+ size
;
1461 if (type
!= FL_FIT_TYPE
) {
1462 augment_tree_propagate_from(va
);
1464 if (lva
) /* type == NE_FIT_TYPE */
1465 insert_vmap_area_augment(lva
, &va
->rb_node
, root
, head
);
1472 * Returns a start address of the newly allocated area, if success.
1473 * Otherwise a vend is returned that indicates failure.
1475 static __always_inline
unsigned long
1476 __alloc_vmap_area(struct rb_root
*root
, struct list_head
*head
,
1477 unsigned long size
, unsigned long align
,
1478 unsigned long vstart
, unsigned long vend
)
1480 bool adjust_search_size
= true;
1481 unsigned long nva_start_addr
;
1482 struct vmap_area
*va
;
1486 * Do not adjust when:
1487 * a) align <= PAGE_SIZE, because it does not make any sense.
1488 * All blocks(their start addresses) are at least PAGE_SIZE
1490 * b) a short range where a requested size corresponds to exactly
1491 * specified [vstart:vend] interval and an alignment > PAGE_SIZE.
1492 * With adjusted search length an allocation would not succeed.
1494 if (align
<= PAGE_SIZE
|| (align
> PAGE_SIZE
&& (vend
- vstart
) == size
))
1495 adjust_search_size
= false;
1497 va
= find_vmap_lowest_match(root
, size
, align
, vstart
, adjust_search_size
);
1501 if (va
->va_start
> vstart
)
1502 nva_start_addr
= ALIGN(va
->va_start
, align
);
1504 nva_start_addr
= ALIGN(vstart
, align
);
1506 /* Check the "vend" restriction. */
1507 if (nva_start_addr
+ size
> vend
)
1510 /* Update the free vmap_area. */
1511 ret
= adjust_va_to_fit_type(root
, head
, va
, nva_start_addr
, size
);
1512 if (WARN_ON_ONCE(ret
))
1515 #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK
1516 find_vmap_lowest_match_check(size
, align
);
1519 return nva_start_addr
;
1523 * Free a region of KVA allocated by alloc_vmap_area
1525 static void free_vmap_area(struct vmap_area
*va
)
1528 * Remove from the busy tree/list.
1530 spin_lock(&vmap_area_lock
);
1531 unlink_va(va
, &vmap_area_root
);
1532 spin_unlock(&vmap_area_lock
);
1535 * Insert/Merge it back to the free tree/list.
1537 spin_lock(&free_vmap_area_lock
);
1538 merge_or_add_vmap_area_augment(va
, &free_vmap_area_root
, &free_vmap_area_list
);
1539 spin_unlock(&free_vmap_area_lock
);
1543 preload_this_cpu_lock(spinlock_t
*lock
, gfp_t gfp_mask
, int node
)
1545 struct vmap_area
*va
= NULL
;
1548 * Preload this CPU with one extra vmap_area object. It is used
1549 * when fit type of free area is NE_FIT_TYPE. It guarantees that
1550 * a CPU that does an allocation is preloaded.
1552 * We do it in non-atomic context, thus it allows us to use more
1553 * permissive allocation masks to be more stable under low memory
1554 * condition and high memory pressure.
1556 if (!this_cpu_read(ne_fit_preload_node
))
1557 va
= kmem_cache_alloc_node(vmap_area_cachep
, gfp_mask
, node
);
1561 if (va
&& __this_cpu_cmpxchg(ne_fit_preload_node
, NULL
, va
))
1562 kmem_cache_free(vmap_area_cachep
, va
);
1566 * Allocate a region of KVA of the specified size and alignment, within the
1569 static struct vmap_area
*alloc_vmap_area(unsigned long size
,
1570 unsigned long align
,
1571 unsigned long vstart
, unsigned long vend
,
1572 int node
, gfp_t gfp_mask
)
1574 struct vmap_area
*va
;
1575 unsigned long freed
;
1581 BUG_ON(offset_in_page(size
));
1582 BUG_ON(!is_power_of_2(align
));
1584 if (unlikely(!vmap_initialized
))
1585 return ERR_PTR(-EBUSY
);
1588 gfp_mask
= gfp_mask
& GFP_RECLAIM_MASK
;
1590 va
= kmem_cache_alloc_node(vmap_area_cachep
, gfp_mask
, node
);
1592 return ERR_PTR(-ENOMEM
);
1595 * Only scan the relevant parts containing pointers to other objects
1596 * to avoid false negatives.
1598 kmemleak_scan_area(&va
->rb_node
, SIZE_MAX
, gfp_mask
);
1601 preload_this_cpu_lock(&free_vmap_area_lock
, gfp_mask
, node
);
1602 addr
= __alloc_vmap_area(&free_vmap_area_root
, &free_vmap_area_list
,
1603 size
, align
, vstart
, vend
);
1604 spin_unlock(&free_vmap_area_lock
);
1607 * If an allocation fails, the "vend" address is
1608 * returned. Therefore trigger the overflow path.
1610 if (unlikely(addr
== vend
))
1613 va
->va_start
= addr
;
1614 va
->va_end
= addr
+ size
;
1617 spin_lock(&vmap_area_lock
);
1618 insert_vmap_area(va
, &vmap_area_root
, &vmap_area_list
);
1619 spin_unlock(&vmap_area_lock
);
1621 BUG_ON(!IS_ALIGNED(va
->va_start
, align
));
1622 BUG_ON(va
->va_start
< vstart
);
1623 BUG_ON(va
->va_end
> vend
);
1625 ret
= kasan_populate_vmalloc(addr
, size
);
1628 return ERR_PTR(ret
);
1635 purge_vmap_area_lazy();
1641 blocking_notifier_call_chain(&vmap_notify_list
, 0, &freed
);
1648 if (!(gfp_mask
& __GFP_NOWARN
) && printk_ratelimit())
1649 pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
1652 kmem_cache_free(vmap_area_cachep
, va
);
1653 return ERR_PTR(-EBUSY
);
1656 int register_vmap_purge_notifier(struct notifier_block
*nb
)
1658 return blocking_notifier_chain_register(&vmap_notify_list
, nb
);
1660 EXPORT_SYMBOL_GPL(register_vmap_purge_notifier
);
1662 int unregister_vmap_purge_notifier(struct notifier_block
*nb
)
1664 return blocking_notifier_chain_unregister(&vmap_notify_list
, nb
);
1666 EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier
);
1669 * lazy_max_pages is the maximum amount of virtual address space we gather up
1670 * before attempting to purge with a TLB flush.
1672 * There is a tradeoff here: a larger number will cover more kernel page tables
1673 * and take slightly longer to purge, but it will linearly reduce the number of
1674 * global TLB flushes that must be performed. It would seem natural to scale
1675 * this number up linearly with the number of CPUs (because vmapping activity
1676 * could also scale linearly with the number of CPUs), however it is likely
1677 * that in practice, workloads might be constrained in other ways that mean
1678 * vmap activity will not scale linearly with CPUs. Also, I want to be
1679 * conservative and not introduce a big latency on huge systems, so go with
1680 * a less aggressive log scale. It will still be an improvement over the old
1681 * code, and it will be simple to change the scale factor if we find that it
1682 * becomes a problem on bigger systems.
1684 static unsigned long lazy_max_pages(void)
1688 log
= fls(num_online_cpus());
1690 return log
* (32UL * 1024 * 1024 / PAGE_SIZE
);
1693 static atomic_long_t vmap_lazy_nr
= ATOMIC_LONG_INIT(0);
1696 * Serialize vmap purging. There is no actual critical section protected
1697 * by this lock, but we want to avoid concurrent calls for performance
1698 * reasons and to make the pcpu_get_vm_areas more deterministic.
1700 static DEFINE_MUTEX(vmap_purge_lock
);
1702 /* for per-CPU blocks */
1703 static void purge_fragmented_blocks_allcpus(void);
1706 * Purges all lazily-freed vmap areas.
1708 static bool __purge_vmap_area_lazy(unsigned long start
, unsigned long end
)
1710 unsigned long resched_threshold
;
1711 struct list_head local_purge_list
;
1712 struct vmap_area
*va
, *n_va
;
1714 lockdep_assert_held(&vmap_purge_lock
);
1716 spin_lock(&purge_vmap_area_lock
);
1717 purge_vmap_area_root
= RB_ROOT
;
1718 list_replace_init(&purge_vmap_area_list
, &local_purge_list
);
1719 spin_unlock(&purge_vmap_area_lock
);
1721 if (unlikely(list_empty(&local_purge_list
)))
1725 list_first_entry(&local_purge_list
,
1726 struct vmap_area
, list
)->va_start
);
1729 list_last_entry(&local_purge_list
,
1730 struct vmap_area
, list
)->va_end
);
1732 flush_tlb_kernel_range(start
, end
);
1733 resched_threshold
= lazy_max_pages() << 1;
1735 spin_lock(&free_vmap_area_lock
);
1736 list_for_each_entry_safe(va
, n_va
, &local_purge_list
, list
) {
1737 unsigned long nr
= (va
->va_end
- va
->va_start
) >> PAGE_SHIFT
;
1738 unsigned long orig_start
= va
->va_start
;
1739 unsigned long orig_end
= va
->va_end
;
1742 * Finally insert or merge lazily-freed area. It is
1743 * detached and there is no need to "unlink" it from
1746 va
= merge_or_add_vmap_area_augment(va
, &free_vmap_area_root
,
1747 &free_vmap_area_list
);
1752 if (is_vmalloc_or_module_addr((void *)orig_start
))
1753 kasan_release_vmalloc(orig_start
, orig_end
,
1754 va
->va_start
, va
->va_end
);
1756 atomic_long_sub(nr
, &vmap_lazy_nr
);
1758 if (atomic_long_read(&vmap_lazy_nr
) < resched_threshold
)
1759 cond_resched_lock(&free_vmap_area_lock
);
1761 spin_unlock(&free_vmap_area_lock
);
1766 * Kick off a purge of the outstanding lazy areas.
1768 static void purge_vmap_area_lazy(void)
1770 mutex_lock(&vmap_purge_lock
);
1771 purge_fragmented_blocks_allcpus();
1772 __purge_vmap_area_lazy(ULONG_MAX
, 0);
1773 mutex_unlock(&vmap_purge_lock
);
1776 static void drain_vmap_area_work(struct work_struct
*work
)
1778 unsigned long nr_lazy
;
1781 mutex_lock(&vmap_purge_lock
);
1782 __purge_vmap_area_lazy(ULONG_MAX
, 0);
1783 mutex_unlock(&vmap_purge_lock
);
1785 /* Recheck if further work is required. */
1786 nr_lazy
= atomic_long_read(&vmap_lazy_nr
);
1787 } while (nr_lazy
> lazy_max_pages());
1791 * Free a vmap area, caller ensuring that the area has been unmapped
1792 * and flush_cache_vunmap had been called for the correct range
1795 static void free_vmap_area_noflush(struct vmap_area
*va
)
1797 unsigned long nr_lazy
;
1799 spin_lock(&vmap_area_lock
);
1800 unlink_va(va
, &vmap_area_root
);
1801 spin_unlock(&vmap_area_lock
);
1803 nr_lazy
= atomic_long_add_return((va
->va_end
- va
->va_start
) >>
1804 PAGE_SHIFT
, &vmap_lazy_nr
);
1807 * Merge or place it to the purge tree/list.
1809 spin_lock(&purge_vmap_area_lock
);
1810 merge_or_add_vmap_area(va
,
1811 &purge_vmap_area_root
, &purge_vmap_area_list
);
1812 spin_unlock(&purge_vmap_area_lock
);
1814 /* After this point, we may free va at any time */
1815 if (unlikely(nr_lazy
> lazy_max_pages()))
1816 schedule_work(&drain_vmap_work
);
1820 * Free and unmap a vmap area
1822 static void free_unmap_vmap_area(struct vmap_area
*va
)
1824 flush_cache_vunmap(va
->va_start
, va
->va_end
);
1825 vunmap_range_noflush(va
->va_start
, va
->va_end
);
1826 if (debug_pagealloc_enabled_static())
1827 flush_tlb_kernel_range(va
->va_start
, va
->va_end
);
1829 free_vmap_area_noflush(va
);
1832 struct vmap_area
*find_vmap_area(unsigned long addr
)
1834 struct vmap_area
*va
;
1836 spin_lock(&vmap_area_lock
);
1837 va
= __find_vmap_area(addr
, &vmap_area_root
);
1838 spin_unlock(&vmap_area_lock
);
1843 /*** Per cpu kva allocator ***/
1846 * vmap space is limited especially on 32 bit architectures. Ensure there is
1847 * room for at least 16 percpu vmap blocks per CPU.
1850 * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
1851 * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
1852 * instead (we just need a rough idea)
1854 #if BITS_PER_LONG == 32
1855 #define VMALLOC_SPACE (128UL*1024*1024)
1857 #define VMALLOC_SPACE (128UL*1024*1024*1024)
1860 #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
1861 #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
1862 #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
1863 #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
1864 #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
1865 #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
1866 #define VMAP_BBMAP_BITS \
1867 VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
1868 VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
1869 VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
1871 #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
1873 struct vmap_block_queue
{
1875 struct list_head free
;
1880 struct vmap_area
*va
;
1881 unsigned long free
, dirty
;
1882 unsigned long dirty_min
, dirty_max
; /*< dirty range */
1883 struct list_head free_list
;
1884 struct rcu_head rcu_head
;
1885 struct list_head purge
;
1888 /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
1889 static DEFINE_PER_CPU(struct vmap_block_queue
, vmap_block_queue
);
1892 * XArray of vmap blocks, indexed by address, to quickly find a vmap block
1893 * in the free path. Could get rid of this if we change the API to return a
1894 * "cookie" from alloc, to be passed to free. But no big deal yet.
1896 static DEFINE_XARRAY(vmap_blocks
);
1899 * We should probably have a fallback mechanism to allocate virtual memory
1900 * out of partially filled vmap blocks. However vmap block sizing should be
1901 * fairly reasonable according to the vmalloc size, so it shouldn't be a
1905 static unsigned long addr_to_vb_idx(unsigned long addr
)
1907 addr
-= VMALLOC_START
& ~(VMAP_BLOCK_SIZE
-1);
1908 addr
/= VMAP_BLOCK_SIZE
;
1912 static void *vmap_block_vaddr(unsigned long va_start
, unsigned long pages_off
)
1916 addr
= va_start
+ (pages_off
<< PAGE_SHIFT
);
1917 BUG_ON(addr_to_vb_idx(addr
) != addr_to_vb_idx(va_start
));
1918 return (void *)addr
;
1922 * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
1923 * block. Of course pages number can't exceed VMAP_BBMAP_BITS
1924 * @order: how many 2^order pages should be occupied in newly allocated block
1925 * @gfp_mask: flags for the page level allocator
1927 * Return: virtual address in a newly allocated block or ERR_PTR(-errno)
1929 static void *new_vmap_block(unsigned int order
, gfp_t gfp_mask
)
1931 struct vmap_block_queue
*vbq
;
1932 struct vmap_block
*vb
;
1933 struct vmap_area
*va
;
1934 unsigned long vb_idx
;
1938 node
= numa_node_id();
1940 vb
= kmalloc_node(sizeof(struct vmap_block
),
1941 gfp_mask
& GFP_RECLAIM_MASK
, node
);
1943 return ERR_PTR(-ENOMEM
);
1945 va
= alloc_vmap_area(VMAP_BLOCK_SIZE
, VMAP_BLOCK_SIZE
,
1946 VMALLOC_START
, VMALLOC_END
,
1950 return ERR_CAST(va
);
1953 vaddr
= vmap_block_vaddr(va
->va_start
, 0);
1954 spin_lock_init(&vb
->lock
);
1956 /* At least something should be left free */
1957 BUG_ON(VMAP_BBMAP_BITS
<= (1UL << order
));
1958 vb
->free
= VMAP_BBMAP_BITS
- (1UL << order
);
1960 vb
->dirty_min
= VMAP_BBMAP_BITS
;
1962 INIT_LIST_HEAD(&vb
->free_list
);
1964 vb_idx
= addr_to_vb_idx(va
->va_start
);
1965 err
= xa_insert(&vmap_blocks
, vb_idx
, vb
, gfp_mask
);
1969 return ERR_PTR(err
);
1972 vbq
= raw_cpu_ptr(&vmap_block_queue
);
1973 spin_lock(&vbq
->lock
);
1974 list_add_tail_rcu(&vb
->free_list
, &vbq
->free
);
1975 spin_unlock(&vbq
->lock
);
1980 static void free_vmap_block(struct vmap_block
*vb
)
1982 struct vmap_block
*tmp
;
1984 tmp
= xa_erase(&vmap_blocks
, addr_to_vb_idx(vb
->va
->va_start
));
1987 free_vmap_area_noflush(vb
->va
);
1988 kfree_rcu(vb
, rcu_head
);
1991 static void purge_fragmented_blocks(int cpu
)
1994 struct vmap_block
*vb
;
1995 struct vmap_block
*n_vb
;
1996 struct vmap_block_queue
*vbq
= &per_cpu(vmap_block_queue
, cpu
);
1999 list_for_each_entry_rcu(vb
, &vbq
->free
, free_list
) {
2001 if (!(vb
->free
+ vb
->dirty
== VMAP_BBMAP_BITS
&& vb
->dirty
!= VMAP_BBMAP_BITS
))
2004 spin_lock(&vb
->lock
);
2005 if (vb
->free
+ vb
->dirty
== VMAP_BBMAP_BITS
&& vb
->dirty
!= VMAP_BBMAP_BITS
) {
2006 vb
->free
= 0; /* prevent further allocs after releasing lock */
2007 vb
->dirty
= VMAP_BBMAP_BITS
; /* prevent purging it again */
2009 vb
->dirty_max
= VMAP_BBMAP_BITS
;
2010 spin_lock(&vbq
->lock
);
2011 list_del_rcu(&vb
->free_list
);
2012 spin_unlock(&vbq
->lock
);
2013 spin_unlock(&vb
->lock
);
2014 list_add_tail(&vb
->purge
, &purge
);
2016 spin_unlock(&vb
->lock
);
2020 list_for_each_entry_safe(vb
, n_vb
, &purge
, purge
) {
2021 list_del(&vb
->purge
);
2022 free_vmap_block(vb
);
2026 static void purge_fragmented_blocks_allcpus(void)
2030 for_each_possible_cpu(cpu
)
2031 purge_fragmented_blocks(cpu
);
2034 static void *vb_alloc(unsigned long size
, gfp_t gfp_mask
)
2036 struct vmap_block_queue
*vbq
;
2037 struct vmap_block
*vb
;
2041 BUG_ON(offset_in_page(size
));
2042 BUG_ON(size
> PAGE_SIZE
*VMAP_MAX_ALLOC
);
2043 if (WARN_ON(size
== 0)) {
2045 * Allocating 0 bytes isn't what caller wants since
2046 * get_order(0) returns funny result. Just warn and terminate
2051 order
= get_order(size
);
2054 vbq
= raw_cpu_ptr(&vmap_block_queue
);
2055 list_for_each_entry_rcu(vb
, &vbq
->free
, free_list
) {
2056 unsigned long pages_off
;
2058 spin_lock(&vb
->lock
);
2059 if (vb
->free
< (1UL << order
)) {
2060 spin_unlock(&vb
->lock
);
2064 pages_off
= VMAP_BBMAP_BITS
- vb
->free
;
2065 vaddr
= vmap_block_vaddr(vb
->va
->va_start
, pages_off
);
2066 vb
->free
-= 1UL << order
;
2067 if (vb
->free
== 0) {
2068 spin_lock(&vbq
->lock
);
2069 list_del_rcu(&vb
->free_list
);
2070 spin_unlock(&vbq
->lock
);
2073 spin_unlock(&vb
->lock
);
2079 /* Allocate new block if nothing was found */
2081 vaddr
= new_vmap_block(order
, gfp_mask
);
2086 static void vb_free(unsigned long addr
, unsigned long size
)
2088 unsigned long offset
;
2090 struct vmap_block
*vb
;
2092 BUG_ON(offset_in_page(size
));
2093 BUG_ON(size
> PAGE_SIZE
*VMAP_MAX_ALLOC
);
2095 flush_cache_vunmap(addr
, addr
+ size
);
2097 order
= get_order(size
);
2098 offset
= (addr
& (VMAP_BLOCK_SIZE
- 1)) >> PAGE_SHIFT
;
2099 vb
= xa_load(&vmap_blocks
, addr_to_vb_idx(addr
));
2101 vunmap_range_noflush(addr
, addr
+ size
);
2103 if (debug_pagealloc_enabled_static())
2104 flush_tlb_kernel_range(addr
, addr
+ size
);
2106 spin_lock(&vb
->lock
);
2108 /* Expand dirty range */
2109 vb
->dirty_min
= min(vb
->dirty_min
, offset
);
2110 vb
->dirty_max
= max(vb
->dirty_max
, offset
+ (1UL << order
));
2112 vb
->dirty
+= 1UL << order
;
2113 if (vb
->dirty
== VMAP_BBMAP_BITS
) {
2115 spin_unlock(&vb
->lock
);
2116 free_vmap_block(vb
);
2118 spin_unlock(&vb
->lock
);
2121 static void _vm_unmap_aliases(unsigned long start
, unsigned long end
, int flush
)
2125 if (unlikely(!vmap_initialized
))
2130 for_each_possible_cpu(cpu
) {
2131 struct vmap_block_queue
*vbq
= &per_cpu(vmap_block_queue
, cpu
);
2132 struct vmap_block
*vb
;
2135 list_for_each_entry_rcu(vb
, &vbq
->free
, free_list
) {
2136 spin_lock(&vb
->lock
);
2137 if (vb
->dirty
&& vb
->dirty
!= VMAP_BBMAP_BITS
) {
2138 unsigned long va_start
= vb
->va
->va_start
;
2141 s
= va_start
+ (vb
->dirty_min
<< PAGE_SHIFT
);
2142 e
= va_start
+ (vb
->dirty_max
<< PAGE_SHIFT
);
2144 start
= min(s
, start
);
2149 spin_unlock(&vb
->lock
);
2154 mutex_lock(&vmap_purge_lock
);
2155 purge_fragmented_blocks_allcpus();
2156 if (!__purge_vmap_area_lazy(start
, end
) && flush
)
2157 flush_tlb_kernel_range(start
, end
);
2158 mutex_unlock(&vmap_purge_lock
);
2162 * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
2164 * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
2165 * to amortize TLB flushing overheads. What this means is that any page you
2166 * have now, may, in a former life, have been mapped into kernel virtual
2167 * address by the vmap layer and so there might be some CPUs with TLB entries
2168 * still referencing that page (additional to the regular 1:1 kernel mapping).
2170 * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
2171 * be sure that none of the pages we have control over will have any aliases
2172 * from the vmap layer.
2174 void vm_unmap_aliases(void)
2176 unsigned long start
= ULONG_MAX
, end
= 0;
2179 _vm_unmap_aliases(start
, end
, flush
);
2181 EXPORT_SYMBOL_GPL(vm_unmap_aliases
);
2184 * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
2185 * @mem: the pointer returned by vm_map_ram
2186 * @count: the count passed to that vm_map_ram call (cannot unmap partial)
2188 void vm_unmap_ram(const void *mem
, unsigned int count
)
2190 unsigned long size
= (unsigned long)count
<< PAGE_SHIFT
;
2191 unsigned long addr
= (unsigned long)kasan_reset_tag(mem
);
2192 struct vmap_area
*va
;
2196 BUG_ON(addr
< VMALLOC_START
);
2197 BUG_ON(addr
> VMALLOC_END
);
2198 BUG_ON(!PAGE_ALIGNED(addr
));
2200 kasan_poison_vmalloc(mem
, size
);
2202 if (likely(count
<= VMAP_MAX_ALLOC
)) {
2203 debug_check_no_locks_freed(mem
, size
);
2204 vb_free(addr
, size
);
2208 va
= find_vmap_area(addr
);
2210 debug_check_no_locks_freed((void *)va
->va_start
,
2211 (va
->va_end
- va
->va_start
));
2212 free_unmap_vmap_area(va
);
2214 EXPORT_SYMBOL(vm_unmap_ram
);
2217 * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
2218 * @pages: an array of pointers to the pages to be mapped
2219 * @count: number of pages
2220 * @node: prefer to allocate data structures on this node
2222 * If you use this function for less than VMAP_MAX_ALLOC pages, it could be
2223 * faster than vmap so it's good. But if you mix long-life and short-life
2224 * objects with vm_map_ram(), it could consume lots of address space through
2225 * fragmentation (especially on a 32bit machine). You could see failures in
2226 * the end. Please use this function for short-lived objects.
2228 * Returns: a pointer to the address that has been mapped, or %NULL on failure
2230 void *vm_map_ram(struct page
**pages
, unsigned int count
, int node
)
2232 unsigned long size
= (unsigned long)count
<< PAGE_SHIFT
;
2236 if (likely(count
<= VMAP_MAX_ALLOC
)) {
2237 mem
= vb_alloc(size
, GFP_KERNEL
);
2240 addr
= (unsigned long)mem
;
2242 struct vmap_area
*va
;
2243 va
= alloc_vmap_area(size
, PAGE_SIZE
,
2244 VMALLOC_START
, VMALLOC_END
, node
, GFP_KERNEL
);
2248 addr
= va
->va_start
;
2252 if (vmap_pages_range(addr
, addr
+ size
, PAGE_KERNEL
,
2253 pages
, PAGE_SHIFT
) < 0) {
2254 vm_unmap_ram(mem
, count
);
2259 * Mark the pages as accessible, now that they are mapped.
2260 * With hardware tag-based KASAN, marking is skipped for
2261 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
2263 mem
= kasan_unpoison_vmalloc(mem
, size
, KASAN_VMALLOC_PROT_NORMAL
);
2267 EXPORT_SYMBOL(vm_map_ram
);
2269 static struct vm_struct
*vmlist __initdata
;
2271 static inline unsigned int vm_area_page_order(struct vm_struct
*vm
)
2273 #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
2274 return vm
->page_order
;
2280 static inline void set_vm_area_page_order(struct vm_struct
*vm
, unsigned int order
)
2282 #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC
2283 vm
->page_order
= order
;
2290 * vm_area_add_early - add vmap area early during boot
2291 * @vm: vm_struct to add
2293 * This function is used to add fixed kernel vm area to vmlist before
2294 * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
2295 * should contain proper values and the other fields should be zero.
2297 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
2299 void __init
vm_area_add_early(struct vm_struct
*vm
)
2301 struct vm_struct
*tmp
, **p
;
2303 BUG_ON(vmap_initialized
);
2304 for (p
= &vmlist
; (tmp
= *p
) != NULL
; p
= &tmp
->next
) {
2305 if (tmp
->addr
>= vm
->addr
) {
2306 BUG_ON(tmp
->addr
< vm
->addr
+ vm
->size
);
2309 BUG_ON(tmp
->addr
+ tmp
->size
> vm
->addr
);
2316 * vm_area_register_early - register vmap area early during boot
2317 * @vm: vm_struct to register
2318 * @align: requested alignment
2320 * This function is used to register kernel vm area before
2321 * vmalloc_init() is called. @vm->size and @vm->flags should contain
2322 * proper values on entry and other fields should be zero. On return,
2323 * vm->addr contains the allocated address.
2325 * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
2327 void __init
vm_area_register_early(struct vm_struct
*vm
, size_t align
)
2329 unsigned long addr
= ALIGN(VMALLOC_START
, align
);
2330 struct vm_struct
*cur
, **p
;
2332 BUG_ON(vmap_initialized
);
2334 for (p
= &vmlist
; (cur
= *p
) != NULL
; p
= &cur
->next
) {
2335 if ((unsigned long)cur
->addr
- addr
>= vm
->size
)
2337 addr
= ALIGN((unsigned long)cur
->addr
+ cur
->size
, align
);
2340 BUG_ON(addr
> VMALLOC_END
- vm
->size
);
2341 vm
->addr
= (void *)addr
;
2344 kasan_populate_early_vm_area_shadow(vm
->addr
, vm
->size
);
2347 static void vmap_init_free_space(void)
2349 unsigned long vmap_start
= 1;
2350 const unsigned long vmap_end
= ULONG_MAX
;
2351 struct vmap_area
*busy
, *free
;
2355 * -|-----|.....|-----|-----|-----|.....|-
2357 * |<--------------------------------->|
2359 list_for_each_entry(busy
, &vmap_area_list
, list
) {
2360 if (busy
->va_start
- vmap_start
> 0) {
2361 free
= kmem_cache_zalloc(vmap_area_cachep
, GFP_NOWAIT
);
2362 if (!WARN_ON_ONCE(!free
)) {
2363 free
->va_start
= vmap_start
;
2364 free
->va_end
= busy
->va_start
;
2366 insert_vmap_area_augment(free
, NULL
,
2367 &free_vmap_area_root
,
2368 &free_vmap_area_list
);
2372 vmap_start
= busy
->va_end
;
2375 if (vmap_end
- vmap_start
> 0) {
2376 free
= kmem_cache_zalloc(vmap_area_cachep
, GFP_NOWAIT
);
2377 if (!WARN_ON_ONCE(!free
)) {
2378 free
->va_start
= vmap_start
;
2379 free
->va_end
= vmap_end
;
2381 insert_vmap_area_augment(free
, NULL
,
2382 &free_vmap_area_root
,
2383 &free_vmap_area_list
);
2388 void __init
vmalloc_init(void)
2390 struct vmap_area
*va
;
2391 struct vm_struct
*tmp
;
2395 * Create the cache for vmap_area objects.
2397 vmap_area_cachep
= KMEM_CACHE(vmap_area
, SLAB_PANIC
);
2399 for_each_possible_cpu(i
) {
2400 struct vmap_block_queue
*vbq
;
2401 struct vfree_deferred
*p
;
2403 vbq
= &per_cpu(vmap_block_queue
, i
);
2404 spin_lock_init(&vbq
->lock
);
2405 INIT_LIST_HEAD(&vbq
->free
);
2406 p
= &per_cpu(vfree_deferred
, i
);
2407 init_llist_head(&p
->list
);
2408 INIT_WORK(&p
->wq
, free_work
);
2411 /* Import existing vmlist entries. */
2412 for (tmp
= vmlist
; tmp
; tmp
= tmp
->next
) {
2413 va
= kmem_cache_zalloc(vmap_area_cachep
, GFP_NOWAIT
);
2414 if (WARN_ON_ONCE(!va
))
2417 va
->va_start
= (unsigned long)tmp
->addr
;
2418 va
->va_end
= va
->va_start
+ tmp
->size
;
2420 insert_vmap_area(va
, &vmap_area_root
, &vmap_area_list
);
2424 * Now we can initialize a free vmap space.
2426 vmap_init_free_space();
2427 vmap_initialized
= true;
2430 static inline void setup_vmalloc_vm_locked(struct vm_struct
*vm
,
2431 struct vmap_area
*va
, unsigned long flags
, const void *caller
)
2434 vm
->addr
= (void *)va
->va_start
;
2435 vm
->size
= va
->va_end
- va
->va_start
;
2436 vm
->caller
= caller
;
2440 static void setup_vmalloc_vm(struct vm_struct
*vm
, struct vmap_area
*va
,
2441 unsigned long flags
, const void *caller
)
2443 spin_lock(&vmap_area_lock
);
2444 setup_vmalloc_vm_locked(vm
, va
, flags
, caller
);
2445 spin_unlock(&vmap_area_lock
);
2448 static void clear_vm_uninitialized_flag(struct vm_struct
*vm
)
2451 * Before removing VM_UNINITIALIZED,
2452 * we should make sure that vm has proper values.
2453 * Pair with smp_rmb() in show_numa_info().
2456 vm
->flags
&= ~VM_UNINITIALIZED
;
2459 static struct vm_struct
*__get_vm_area_node(unsigned long size
,
2460 unsigned long align
, unsigned long shift
, unsigned long flags
,
2461 unsigned long start
, unsigned long end
, int node
,
2462 gfp_t gfp_mask
, const void *caller
)
2464 struct vmap_area
*va
;
2465 struct vm_struct
*area
;
2466 unsigned long requested_size
= size
;
2468 BUG_ON(in_interrupt());
2469 size
= ALIGN(size
, 1ul << shift
);
2470 if (unlikely(!size
))
2473 if (flags
& VM_IOREMAP
)
2474 align
= 1ul << clamp_t(int, get_count_order_long(size
),
2475 PAGE_SHIFT
, IOREMAP_MAX_ORDER
);
2477 area
= kzalloc_node(sizeof(*area
), gfp_mask
& GFP_RECLAIM_MASK
, node
);
2478 if (unlikely(!area
))
2481 if (!(flags
& VM_NO_GUARD
))
2484 va
= alloc_vmap_area(size
, align
, start
, end
, node
, gfp_mask
);
2490 setup_vmalloc_vm(area
, va
, flags
, caller
);
2493 * Mark pages for non-VM_ALLOC mappings as accessible. Do it now as a
2494 * best-effort approach, as they can be mapped outside of vmalloc code.
2495 * For VM_ALLOC mappings, the pages are marked as accessible after
2496 * getting mapped in __vmalloc_node_range().
2497 * With hardware tag-based KASAN, marking is skipped for
2498 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
2500 if (!(flags
& VM_ALLOC
))
2501 area
->addr
= kasan_unpoison_vmalloc(area
->addr
, requested_size
,
2502 KASAN_VMALLOC_PROT_NORMAL
);
2507 struct vm_struct
*__get_vm_area_caller(unsigned long size
, unsigned long flags
,
2508 unsigned long start
, unsigned long end
,
2511 return __get_vm_area_node(size
, 1, PAGE_SHIFT
, flags
, start
, end
,
2512 NUMA_NO_NODE
, GFP_KERNEL
, caller
);
2516 * get_vm_area - reserve a contiguous kernel virtual area
2517 * @size: size of the area
2518 * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
2520 * Search an area of @size in the kernel virtual mapping area,
2521 * and reserved it for out purposes. Returns the area descriptor
2522 * on success or %NULL on failure.
2524 * Return: the area descriptor on success or %NULL on failure.
2526 struct vm_struct
*get_vm_area(unsigned long size
, unsigned long flags
)
2528 return __get_vm_area_node(size
, 1, PAGE_SHIFT
, flags
,
2529 VMALLOC_START
, VMALLOC_END
,
2530 NUMA_NO_NODE
, GFP_KERNEL
,
2531 __builtin_return_address(0));
2534 struct vm_struct
*get_vm_area_caller(unsigned long size
, unsigned long flags
,
2537 return __get_vm_area_node(size
, 1, PAGE_SHIFT
, flags
,
2538 VMALLOC_START
, VMALLOC_END
,
2539 NUMA_NO_NODE
, GFP_KERNEL
, caller
);
2543 * find_vm_area - find a continuous kernel virtual area
2544 * @addr: base address
2546 * Search for the kernel VM area starting at @addr, and return it.
2547 * It is up to the caller to do all required locking to keep the returned
2550 * Return: the area descriptor on success or %NULL on failure.
2552 struct vm_struct
*find_vm_area(const void *addr
)
2554 struct vmap_area
*va
;
2556 va
= find_vmap_area((unsigned long)addr
);
2564 * remove_vm_area - find and remove a continuous kernel virtual area
2565 * @addr: base address
2567 * Search for the kernel VM area starting at @addr, and remove it.
2568 * This function returns the found VM area, but using it is NOT safe
2569 * on SMP machines, except for its size or flags.
2571 * Return: the area descriptor on success or %NULL on failure.
2573 struct vm_struct
*remove_vm_area(const void *addr
)
2575 struct vmap_area
*va
;
2579 spin_lock(&vmap_area_lock
);
2580 va
= __find_vmap_area((unsigned long)addr
, &vmap_area_root
);
2582 struct vm_struct
*vm
= va
->vm
;
2585 spin_unlock(&vmap_area_lock
);
2587 kasan_free_module_shadow(vm
);
2588 free_unmap_vmap_area(va
);
2593 spin_unlock(&vmap_area_lock
);
2597 static inline void set_area_direct_map(const struct vm_struct
*area
,
2598 int (*set_direct_map
)(struct page
*page
))
2602 /* HUGE_VMALLOC passes small pages to set_direct_map */
2603 for (i
= 0; i
< area
->nr_pages
; i
++)
2604 if (page_address(area
->pages
[i
]))
2605 set_direct_map(area
->pages
[i
]);
2608 /* Handle removing and resetting vm mappings related to the vm_struct. */
2609 static void vm_remove_mappings(struct vm_struct
*area
, int deallocate_pages
)
2611 unsigned long start
= ULONG_MAX
, end
= 0;
2612 unsigned int page_order
= vm_area_page_order(area
);
2613 int flush_reset
= area
->flags
& VM_FLUSH_RESET_PERMS
;
2617 remove_vm_area(area
->addr
);
2619 /* If this is not VM_FLUSH_RESET_PERMS memory, no need for the below. */
2624 * If not deallocating pages, just do the flush of the VM area and
2627 if (!deallocate_pages
) {
2633 * If execution gets here, flush the vm mapping and reset the direct
2634 * map. Find the start and end range of the direct mappings to make sure
2635 * the vm_unmap_aliases() flush includes the direct map.
2637 for (i
= 0; i
< area
->nr_pages
; i
+= 1U << page_order
) {
2638 unsigned long addr
= (unsigned long)page_address(area
->pages
[i
]);
2640 unsigned long page_size
;
2642 page_size
= PAGE_SIZE
<< page_order
;
2643 start
= min(addr
, start
);
2644 end
= max(addr
+ page_size
, end
);
2650 * Set direct map to something invalid so that it won't be cached if
2651 * there are any accesses after the TLB flush, then flush the TLB and
2652 * reset the direct map permissions to the default.
2654 set_area_direct_map(area
, set_direct_map_invalid_noflush
);
2655 _vm_unmap_aliases(start
, end
, flush_dmap
);
2656 set_area_direct_map(area
, set_direct_map_default_noflush
);
2659 static void __vunmap(const void *addr
, int deallocate_pages
)
2661 struct vm_struct
*area
;
2666 if (WARN(!PAGE_ALIGNED(addr
), "Trying to vfree() bad address (%p)\n",
2670 area
= find_vm_area(addr
);
2671 if (unlikely(!area
)) {
2672 WARN(1, KERN_ERR
"Trying to vfree() nonexistent vm area (%p)\n",
2677 debug_check_no_locks_freed(area
->addr
, get_vm_area_size(area
));
2678 debug_check_no_obj_freed(area
->addr
, get_vm_area_size(area
));
2680 kasan_poison_vmalloc(area
->addr
, get_vm_area_size(area
));
2682 vm_remove_mappings(area
, deallocate_pages
);
2684 if (deallocate_pages
) {
2687 for (i
= 0; i
< area
->nr_pages
; i
++) {
2688 struct page
*page
= area
->pages
[i
];
2691 mod_memcg_page_state(page
, MEMCG_VMALLOC
, -1);
2693 * High-order allocs for huge vmallocs are split, so
2694 * can be freed as an array of order-0 allocations
2696 __free_pages(page
, 0);
2699 atomic_long_sub(area
->nr_pages
, &nr_vmalloc_pages
);
2701 kvfree(area
->pages
);
2707 static inline void __vfree_deferred(const void *addr
)
2710 * Use raw_cpu_ptr() because this can be called from preemptible
2711 * context. Preemption is absolutely fine here, because the llist_add()
2712 * implementation is lockless, so it works even if we are adding to
2713 * another cpu's list. schedule_work() should be fine with this too.
2715 struct vfree_deferred
*p
= raw_cpu_ptr(&vfree_deferred
);
2717 if (llist_add((struct llist_node
*)addr
, &p
->list
))
2718 schedule_work(&p
->wq
);
2722 * vfree_atomic - release memory allocated by vmalloc()
2723 * @addr: memory base address
2725 * This one is just like vfree() but can be called in any atomic context
2728 void vfree_atomic(const void *addr
)
2732 kmemleak_free(addr
);
2736 __vfree_deferred(addr
);
2739 static void __vfree(const void *addr
)
2741 if (unlikely(in_interrupt()))
2742 __vfree_deferred(addr
);
2748 * vfree - Release memory allocated by vmalloc()
2749 * @addr: Memory base address
2751 * Free the virtually continuous memory area starting at @addr, as obtained
2752 * from one of the vmalloc() family of APIs. This will usually also free the
2753 * physical memory underlying the virtual allocation, but that memory is
2754 * reference counted, so it will not be freed until the last user goes away.
2756 * If @addr is NULL, no operation is performed.
2759 * May sleep if called *not* from interrupt context.
2760 * Must not be called in NMI context (strictly speaking, it could be
2761 * if we have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
2762 * conventions for vfree() arch-dependent would be a really bad idea).
2764 void vfree(const void *addr
)
2768 kmemleak_free(addr
);
2770 might_sleep_if(!in_interrupt());
2777 EXPORT_SYMBOL(vfree
);
2780 * vunmap - release virtual mapping obtained by vmap()
2781 * @addr: memory base address
2783 * Free the virtually contiguous memory area starting at @addr,
2784 * which was created from the page array passed to vmap().
2786 * Must not be called in interrupt context.
2788 void vunmap(const void *addr
)
2790 BUG_ON(in_interrupt());
2795 EXPORT_SYMBOL(vunmap
);
2798 * vmap - map an array of pages into virtually contiguous space
2799 * @pages: array of page pointers
2800 * @count: number of pages to map
2801 * @flags: vm_area->flags
2802 * @prot: page protection for the mapping
2804 * Maps @count pages from @pages into contiguous kernel virtual space.
2805 * If @flags contains %VM_MAP_PUT_PAGES the ownership of the pages array itself
2806 * (which must be kmalloc or vmalloc memory) and one reference per pages in it
2807 * are transferred from the caller to vmap(), and will be freed / dropped when
2808 * vfree() is called on the return value.
2810 * Return: the address of the area or %NULL on failure
2812 void *vmap(struct page
**pages
, unsigned int count
,
2813 unsigned long flags
, pgprot_t prot
)
2815 struct vm_struct
*area
;
2817 unsigned long size
; /* In bytes */
2822 * Your top guard is someone else's bottom guard. Not having a top
2823 * guard compromises someone else's mappings too.
2825 if (WARN_ON_ONCE(flags
& VM_NO_GUARD
))
2826 flags
&= ~VM_NO_GUARD
;
2828 if (count
> totalram_pages())
2831 size
= (unsigned long)count
<< PAGE_SHIFT
;
2832 area
= get_vm_area_caller(size
, flags
, __builtin_return_address(0));
2836 addr
= (unsigned long)area
->addr
;
2837 if (vmap_pages_range(addr
, addr
+ size
, pgprot_nx(prot
),
2838 pages
, PAGE_SHIFT
) < 0) {
2843 if (flags
& VM_MAP_PUT_PAGES
) {
2844 area
->pages
= pages
;
2845 area
->nr_pages
= count
;
2849 EXPORT_SYMBOL(vmap
);
2851 #ifdef CONFIG_VMAP_PFN
2852 struct vmap_pfn_data
{
2853 unsigned long *pfns
;
2858 static int vmap_pfn_apply(pte_t
*pte
, unsigned long addr
, void *private)
2860 struct vmap_pfn_data
*data
= private;
2862 if (WARN_ON_ONCE(pfn_valid(data
->pfns
[data
->idx
])))
2864 *pte
= pte_mkspecial(pfn_pte(data
->pfns
[data
->idx
++], data
->prot
));
2869 * vmap_pfn - map an array of PFNs into virtually contiguous space
2870 * @pfns: array of PFNs
2871 * @count: number of pages to map
2872 * @prot: page protection for the mapping
2874 * Maps @count PFNs from @pfns into contiguous kernel virtual space and returns
2875 * the start address of the mapping.
2877 void *vmap_pfn(unsigned long *pfns
, unsigned int count
, pgprot_t prot
)
2879 struct vmap_pfn_data data
= { .pfns
= pfns
, .prot
= pgprot_nx(prot
) };
2880 struct vm_struct
*area
;
2882 area
= get_vm_area_caller(count
* PAGE_SIZE
, VM_IOREMAP
,
2883 __builtin_return_address(0));
2886 if (apply_to_page_range(&init_mm
, (unsigned long)area
->addr
,
2887 count
* PAGE_SIZE
, vmap_pfn_apply
, &data
)) {
2893 EXPORT_SYMBOL_GPL(vmap_pfn
);
2894 #endif /* CONFIG_VMAP_PFN */
2896 static inline unsigned int
2897 vm_area_alloc_pages(gfp_t gfp
, int nid
,
2898 unsigned int order
, unsigned int nr_pages
, struct page
**pages
)
2900 unsigned int nr_allocated
= 0;
2905 * For order-0 pages we make use of bulk allocator, if
2906 * the page array is partly or not at all populated due
2907 * to fails, fallback to a single page allocator that is
2911 gfp_t bulk_gfp
= gfp
& ~__GFP_NOFAIL
;
2913 while (nr_allocated
< nr_pages
) {
2914 unsigned int nr
, nr_pages_request
;
2917 * A maximum allowed request is hard-coded and is 100
2918 * pages per call. That is done in order to prevent a
2919 * long preemption off scenario in the bulk-allocator
2920 * so the range is [1:100].
2922 nr_pages_request
= min(100U, nr_pages
- nr_allocated
);
2924 /* memory allocation should consider mempolicy, we can't
2925 * wrongly use nearest node when nid == NUMA_NO_NODE,
2926 * otherwise memory may be allocated in only one node,
2927 * but mempolicy wants to alloc memory by interleaving.
2929 if (IS_ENABLED(CONFIG_NUMA
) && nid
== NUMA_NO_NODE
)
2930 nr
= alloc_pages_bulk_array_mempolicy(bulk_gfp
,
2932 pages
+ nr_allocated
);
2935 nr
= alloc_pages_bulk_array_node(bulk_gfp
, nid
,
2937 pages
+ nr_allocated
);
2943 * If zero or pages were obtained partly,
2944 * fallback to a single page allocator.
2946 if (nr
!= nr_pages_request
)
2951 /* High-order pages or fallback path if "bulk" fails. */
2953 while (nr_allocated
< nr_pages
) {
2954 if (fatal_signal_pending(current
))
2957 if (nid
== NUMA_NO_NODE
)
2958 page
= alloc_pages(gfp
, order
);
2960 page
= alloc_pages_node(nid
, gfp
, order
);
2961 if (unlikely(!page
))
2964 * Higher order allocations must be able to be treated as
2965 * indepdenent small pages by callers (as they can with
2966 * small-page vmallocs). Some drivers do their own refcounting
2967 * on vmalloc_to_page() pages, some use page->mapping,
2971 split_page(page
, order
);
2974 * Careful, we allocate and map page-order pages, but
2975 * tracking is done per PAGE_SIZE page so as to keep the
2976 * vm_struct APIs independent of the physical/mapped size.
2978 for (i
= 0; i
< (1U << order
); i
++)
2979 pages
[nr_allocated
+ i
] = page
+ i
;
2982 nr_allocated
+= 1U << order
;
2985 return nr_allocated
;
2988 static void *__vmalloc_area_node(struct vm_struct
*area
, gfp_t gfp_mask
,
2989 pgprot_t prot
, unsigned int page_shift
,
2992 const gfp_t nested_gfp
= (gfp_mask
& GFP_RECLAIM_MASK
) | __GFP_ZERO
;
2993 bool nofail
= gfp_mask
& __GFP_NOFAIL
;
2994 unsigned long addr
= (unsigned long)area
->addr
;
2995 unsigned long size
= get_vm_area_size(area
);
2996 unsigned long array_size
;
2997 unsigned int nr_small_pages
= size
>> PAGE_SHIFT
;
2998 unsigned int page_order
;
3002 array_size
= (unsigned long)nr_small_pages
* sizeof(struct page
*);
3003 gfp_mask
|= __GFP_NOWARN
;
3004 if (!(gfp_mask
& (GFP_DMA
| GFP_DMA32
)))
3005 gfp_mask
|= __GFP_HIGHMEM
;
3007 /* Please note that the recursion is strictly bounded. */
3008 if (array_size
> PAGE_SIZE
) {
3009 area
->pages
= __vmalloc_node(array_size
, 1, nested_gfp
, node
,
3012 area
->pages
= kmalloc_node(array_size
, nested_gfp
, node
);
3016 warn_alloc(gfp_mask
, NULL
,
3017 "vmalloc error: size %lu, failed to allocated page array size %lu",
3018 nr_small_pages
* PAGE_SIZE
, array_size
);
3023 set_vm_area_page_order(area
, page_shift
- PAGE_SHIFT
);
3024 page_order
= vm_area_page_order(area
);
3026 area
->nr_pages
= vm_area_alloc_pages(gfp_mask
| __GFP_NOWARN
,
3027 node
, page_order
, nr_small_pages
, area
->pages
);
3029 atomic_long_add(area
->nr_pages
, &nr_vmalloc_pages
);
3030 if (gfp_mask
& __GFP_ACCOUNT
) {
3033 for (i
= 0; i
< area
->nr_pages
; i
++)
3034 mod_memcg_page_state(area
->pages
[i
], MEMCG_VMALLOC
, 1);
3038 * If not enough pages were obtained to accomplish an
3039 * allocation request, free them via __vfree() if any.
3041 if (area
->nr_pages
!= nr_small_pages
) {
3042 warn_alloc(gfp_mask
, NULL
,
3043 "vmalloc error: size %lu, page order %u, failed to allocate pages",
3044 area
->nr_pages
* PAGE_SIZE
, page_order
);
3049 * page tables allocations ignore external gfp mask, enforce it
3052 if ((gfp_mask
& (__GFP_FS
| __GFP_IO
)) == __GFP_IO
)
3053 flags
= memalloc_nofs_save();
3054 else if ((gfp_mask
& (__GFP_FS
| __GFP_IO
)) == 0)
3055 flags
= memalloc_noio_save();
3058 ret
= vmap_pages_range(addr
, addr
+ size
, prot
, area
->pages
,
3060 if (nofail
&& (ret
< 0))
3061 schedule_timeout_uninterruptible(1);
3062 } while (nofail
&& (ret
< 0));
3064 if ((gfp_mask
& (__GFP_FS
| __GFP_IO
)) == __GFP_IO
)
3065 memalloc_nofs_restore(flags
);
3066 else if ((gfp_mask
& (__GFP_FS
| __GFP_IO
)) == 0)
3067 memalloc_noio_restore(flags
);
3070 warn_alloc(gfp_mask
, NULL
,
3071 "vmalloc error: size %lu, failed to map pages",
3072 area
->nr_pages
* PAGE_SIZE
);
3079 __vfree(area
->addr
);
3084 * __vmalloc_node_range - allocate virtually contiguous memory
3085 * @size: allocation size
3086 * @align: desired alignment
3087 * @start: vm area range start
3088 * @end: vm area range end
3089 * @gfp_mask: flags for the page level allocator
3090 * @prot: protection mask for the allocated pages
3091 * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD)
3092 * @node: node to use for allocation or NUMA_NO_NODE
3093 * @caller: caller's return address
3095 * Allocate enough pages to cover @size from the page level
3096 * allocator with @gfp_mask flags. Please note that the full set of gfp
3097 * flags are not supported. GFP_KERNEL, GFP_NOFS and GFP_NOIO are all
3099 * Zone modifiers are not supported. From the reclaim modifiers
3100 * __GFP_DIRECT_RECLAIM is required (aka GFP_NOWAIT is not supported)
3101 * and only __GFP_NOFAIL is supported (i.e. __GFP_NORETRY and
3102 * __GFP_RETRY_MAYFAIL are not supported).
3104 * __GFP_NOWARN can be used to suppress failures messages.
3106 * Map them into contiguous kernel virtual space, using a pagetable
3107 * protection of @prot.
3109 * Return: the address of the area or %NULL on failure
3111 void *__vmalloc_node_range(unsigned long size
, unsigned long align
,
3112 unsigned long start
, unsigned long end
, gfp_t gfp_mask
,
3113 pgprot_t prot
, unsigned long vm_flags
, int node
,
3116 struct vm_struct
*area
;
3118 kasan_vmalloc_flags_t kasan_flags
= KASAN_VMALLOC_NONE
;
3119 unsigned long real_size
= size
;
3120 unsigned long real_align
= align
;
3121 unsigned int shift
= PAGE_SHIFT
;
3123 if (WARN_ON_ONCE(!size
))
3126 if ((size
>> PAGE_SHIFT
) > totalram_pages()) {
3127 warn_alloc(gfp_mask
, NULL
,
3128 "vmalloc error: size %lu, exceeds total pages",
3133 if (vmap_allow_huge
&& (vm_flags
& VM_ALLOW_HUGE_VMAP
)) {
3134 unsigned long size_per_node
;
3137 * Try huge pages. Only try for PAGE_KERNEL allocations,
3138 * others like modules don't yet expect huge pages in
3139 * their allocations due to apply_to_page_range not
3143 size_per_node
= size
;
3144 if (node
== NUMA_NO_NODE
)
3145 size_per_node
/= num_online_nodes();
3146 if (arch_vmap_pmd_supported(prot
) && size_per_node
>= PMD_SIZE
)
3149 shift
= arch_vmap_pte_supported_shift(size_per_node
);
3151 align
= max(real_align
, 1UL << shift
);
3152 size
= ALIGN(real_size
, 1UL << shift
);
3156 area
= __get_vm_area_node(real_size
, align
, shift
, VM_ALLOC
|
3157 VM_UNINITIALIZED
| vm_flags
, start
, end
, node
,
3160 bool nofail
= gfp_mask
& __GFP_NOFAIL
;
3161 warn_alloc(gfp_mask
, NULL
,
3162 "vmalloc error: size %lu, vm_struct allocation failed%s",
3163 real_size
, (nofail
) ? ". Retrying." : "");
3165 schedule_timeout_uninterruptible(1);
3172 * Prepare arguments for __vmalloc_area_node() and
3173 * kasan_unpoison_vmalloc().
3175 if (pgprot_val(prot
) == pgprot_val(PAGE_KERNEL
)) {
3176 if (kasan_hw_tags_enabled()) {
3178 * Modify protection bits to allow tagging.
3179 * This must be done before mapping.
3181 prot
= arch_vmap_pgprot_tagged(prot
);
3184 * Skip page_alloc poisoning and zeroing for physical
3185 * pages backing VM_ALLOC mapping. Memory is instead
3186 * poisoned and zeroed by kasan_unpoison_vmalloc().
3188 gfp_mask
|= __GFP_SKIP_KASAN_UNPOISON
| __GFP_SKIP_ZERO
;
3191 /* Take note that the mapping is PAGE_KERNEL. */
3192 kasan_flags
|= KASAN_VMALLOC_PROT_NORMAL
;
3195 /* Allocate physical pages and map them into vmalloc space. */
3196 ret
= __vmalloc_area_node(area
, gfp_mask
, prot
, shift
, node
);
3201 * Mark the pages as accessible, now that they are mapped.
3202 * The condition for setting KASAN_VMALLOC_INIT should complement the
3203 * one in post_alloc_hook() with regards to the __GFP_SKIP_ZERO check
3204 * to make sure that memory is initialized under the same conditions.
3205 * Tag-based KASAN modes only assign tags to normal non-executable
3206 * allocations, see __kasan_unpoison_vmalloc().
3208 kasan_flags
|= KASAN_VMALLOC_VM_ALLOC
;
3209 if (!want_init_on_free() && want_init_on_alloc(gfp_mask
) &&
3210 (gfp_mask
& __GFP_SKIP_ZERO
))
3211 kasan_flags
|= KASAN_VMALLOC_INIT
;
3212 /* KASAN_VMALLOC_PROT_NORMAL already set if required. */
3213 area
->addr
= kasan_unpoison_vmalloc(area
->addr
, real_size
, kasan_flags
);
3216 * In this function, newly allocated vm_struct has VM_UNINITIALIZED
3217 * flag. It means that vm_struct is not fully initialized.
3218 * Now, it is fully initialized, so remove this flag here.
3220 clear_vm_uninitialized_flag(area
);
3222 size
= PAGE_ALIGN(size
);
3223 if (!(vm_flags
& VM_DEFER_KMEMLEAK
))
3224 kmemleak_vmalloc(area
, size
, gfp_mask
);
3229 if (shift
> PAGE_SHIFT
) {
3240 * __vmalloc_node - allocate virtually contiguous memory
3241 * @size: allocation size
3242 * @align: desired alignment
3243 * @gfp_mask: flags for the page level allocator
3244 * @node: node to use for allocation or NUMA_NO_NODE
3245 * @caller: caller's return address
3247 * Allocate enough pages to cover @size from the page level allocator with
3248 * @gfp_mask flags. Map them into contiguous kernel virtual space.
3250 * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL
3251 * and __GFP_NOFAIL are not supported
3253 * Any use of gfp flags outside of GFP_KERNEL should be consulted
3256 * Return: pointer to the allocated memory or %NULL on error
3258 void *__vmalloc_node(unsigned long size
, unsigned long align
,
3259 gfp_t gfp_mask
, int node
, const void *caller
)
3261 return __vmalloc_node_range(size
, align
, VMALLOC_START
, VMALLOC_END
,
3262 gfp_mask
, PAGE_KERNEL
, 0, node
, caller
);
3265 * This is only for performance analysis of vmalloc and stress purpose.
3266 * It is required by vmalloc test module, therefore do not use it other
3269 #ifdef CONFIG_TEST_VMALLOC_MODULE
3270 EXPORT_SYMBOL_GPL(__vmalloc_node
);
3273 void *__vmalloc(unsigned long size
, gfp_t gfp_mask
)
3275 return __vmalloc_node(size
, 1, gfp_mask
, NUMA_NO_NODE
,
3276 __builtin_return_address(0));
3278 EXPORT_SYMBOL(__vmalloc
);
3281 * vmalloc - allocate virtually contiguous memory
3282 * @size: allocation size
3284 * Allocate enough pages to cover @size from the page level
3285 * allocator and map them into contiguous kernel virtual space.
3287 * For tight control over page level allocator and protection flags
3288 * use __vmalloc() instead.
3290 * Return: pointer to the allocated memory or %NULL on error
3292 void *vmalloc(unsigned long size
)
3294 return __vmalloc_node(size
, 1, GFP_KERNEL
, NUMA_NO_NODE
,
3295 __builtin_return_address(0));
3297 EXPORT_SYMBOL(vmalloc
);
3300 * vmalloc_huge - allocate virtually contiguous memory, allow huge pages
3301 * @size: allocation size
3302 * @gfp_mask: flags for the page level allocator
3304 * Allocate enough pages to cover @size from the page level
3305 * allocator and map them into contiguous kernel virtual space.
3306 * If @size is greater than or equal to PMD_SIZE, allow using
3307 * huge pages for the memory
3309 * Return: pointer to the allocated memory or %NULL on error
3311 void *vmalloc_huge(unsigned long size
, gfp_t gfp_mask
)
3313 return __vmalloc_node_range(size
, 1, VMALLOC_START
, VMALLOC_END
,
3314 gfp_mask
, PAGE_KERNEL
, VM_ALLOW_HUGE_VMAP
,
3315 NUMA_NO_NODE
, __builtin_return_address(0));
3317 EXPORT_SYMBOL_GPL(vmalloc_huge
);
3320 * vzalloc - allocate virtually contiguous memory with zero fill
3321 * @size: allocation size
3323 * Allocate enough pages to cover @size from the page level
3324 * allocator and map them into contiguous kernel virtual space.
3325 * The memory allocated is set to zero.
3327 * For tight control over page level allocator and protection flags
3328 * use __vmalloc() instead.
3330 * Return: pointer to the allocated memory or %NULL on error
3332 void *vzalloc(unsigned long size
)
3334 return __vmalloc_node(size
, 1, GFP_KERNEL
| __GFP_ZERO
, NUMA_NO_NODE
,
3335 __builtin_return_address(0));
3337 EXPORT_SYMBOL(vzalloc
);
3340 * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
3341 * @size: allocation size
3343 * The resulting memory area is zeroed so it can be mapped to userspace
3344 * without leaking data.
3346 * Return: pointer to the allocated memory or %NULL on error
3348 void *vmalloc_user(unsigned long size
)
3350 return __vmalloc_node_range(size
, SHMLBA
, VMALLOC_START
, VMALLOC_END
,
3351 GFP_KERNEL
| __GFP_ZERO
, PAGE_KERNEL
,
3352 VM_USERMAP
, NUMA_NO_NODE
,
3353 __builtin_return_address(0));
3355 EXPORT_SYMBOL(vmalloc_user
);
3358 * vmalloc_node - allocate memory on a specific node
3359 * @size: allocation size
3362 * Allocate enough pages to cover @size from the page level
3363 * allocator and map them into contiguous kernel virtual space.
3365 * For tight control over page level allocator and protection flags
3366 * use __vmalloc() instead.
3368 * Return: pointer to the allocated memory or %NULL on error
3370 void *vmalloc_node(unsigned long size
, int node
)
3372 return __vmalloc_node(size
, 1, GFP_KERNEL
, node
,
3373 __builtin_return_address(0));
3375 EXPORT_SYMBOL(vmalloc_node
);
3378 * vzalloc_node - allocate memory on a specific node with zero fill
3379 * @size: allocation size
3382 * Allocate enough pages to cover @size from the page level
3383 * allocator and map them into contiguous kernel virtual space.
3384 * The memory allocated is set to zero.
3386 * Return: pointer to the allocated memory or %NULL on error
3388 void *vzalloc_node(unsigned long size
, int node
)
3390 return __vmalloc_node(size
, 1, GFP_KERNEL
| __GFP_ZERO
, node
,
3391 __builtin_return_address(0));
3393 EXPORT_SYMBOL(vzalloc_node
);
3395 #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
3396 #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)
3397 #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
3398 #define GFP_VMALLOC32 (GFP_DMA | GFP_KERNEL)
3401 * 64b systems should always have either DMA or DMA32 zones. For others
3402 * GFP_DMA32 should do the right thing and use the normal zone.
3404 #define GFP_VMALLOC32 (GFP_DMA32 | GFP_KERNEL)
3408 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
3409 * @size: allocation size
3411 * Allocate enough 32bit PA addressable pages to cover @size from the
3412 * page level allocator and map them into contiguous kernel virtual space.
3414 * Return: pointer to the allocated memory or %NULL on error
3416 void *vmalloc_32(unsigned long size
)
3418 return __vmalloc_node(size
, 1, GFP_VMALLOC32
, NUMA_NO_NODE
,
3419 __builtin_return_address(0));
3421 EXPORT_SYMBOL(vmalloc_32
);
3424 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
3425 * @size: allocation size
3427 * The resulting memory area is 32bit addressable and zeroed so it can be
3428 * mapped to userspace without leaking data.
3430 * Return: pointer to the allocated memory or %NULL on error
3432 void *vmalloc_32_user(unsigned long size
)
3434 return __vmalloc_node_range(size
, SHMLBA
, VMALLOC_START
, VMALLOC_END
,
3435 GFP_VMALLOC32
| __GFP_ZERO
, PAGE_KERNEL
,
3436 VM_USERMAP
, NUMA_NO_NODE
,
3437 __builtin_return_address(0));
3439 EXPORT_SYMBOL(vmalloc_32_user
);
3442 * small helper routine , copy contents to buf from addr.
3443 * If the page is not present, fill zero.
3446 static int aligned_vread(char *buf
, char *addr
, unsigned long count
)
3452 unsigned long offset
, length
;
3454 offset
= offset_in_page(addr
);
3455 length
= PAGE_SIZE
- offset
;
3458 p
= vmalloc_to_page(addr
);
3460 * To do safe access to this _mapped_ area, we need
3461 * lock. But adding lock here means that we need to add
3462 * overhead of vmalloc()/vfree() calls for this _debug_
3463 * interface, rarely used. Instead of that, we'll use
3464 * kmap() and get small overhead in this access function.
3467 /* We can expect USER0 is not used -- see vread() */
3468 void *map
= kmap_atomic(p
);
3469 memcpy(buf
, map
+ offset
, length
);
3472 memset(buf
, 0, length
);
3483 * vread() - read vmalloc area in a safe way.
3484 * @buf: buffer for reading data
3485 * @addr: vm address.
3486 * @count: number of bytes to be read.
3488 * This function checks that addr is a valid vmalloc'ed area, and
3489 * copy data from that area to a given buffer. If the given memory range
3490 * of [addr...addr+count) includes some valid address, data is copied to
3491 * proper area of @buf. If there are memory holes, they'll be zero-filled.
3492 * IOREMAP area is treated as memory hole and no copy is done.
3494 * If [addr...addr+count) doesn't includes any intersects with alive
3495 * vm_struct area, returns 0. @buf should be kernel's buffer.
3497 * Note: In usual ops, vread() is never necessary because the caller
3498 * should know vmalloc() area is valid and can use memcpy().
3499 * This is for routines which have to access vmalloc area without
3500 * any information, as /proc/kcore.
3502 * Return: number of bytes for which addr and buf should be increased
3503 * (same number as @count) or %0 if [addr...addr+count) doesn't
3504 * include any intersection with valid vmalloc area
3506 long vread(char *buf
, char *addr
, unsigned long count
)
3508 struct vmap_area
*va
;
3509 struct vm_struct
*vm
;
3510 char *vaddr
, *buf_start
= buf
;
3511 unsigned long buflen
= count
;
3514 addr
= kasan_reset_tag(addr
);
3516 /* Don't allow overflow */
3517 if ((unsigned long) addr
+ count
< count
)
3518 count
= -(unsigned long) addr
;
3520 spin_lock(&vmap_area_lock
);
3521 va
= find_vmap_area_exceed_addr((unsigned long)addr
);
3525 /* no intersects with alive vmap_area */
3526 if ((unsigned long)addr
+ count
<= va
->va_start
)
3529 list_for_each_entry_from(va
, &vmap_area_list
, list
) {
3537 vaddr
= (char *) vm
->addr
;
3538 if (addr
>= vaddr
+ get_vm_area_size(vm
))
3540 while (addr
< vaddr
) {
3548 n
= vaddr
+ get_vm_area_size(vm
) - addr
;
3551 if (!(vm
->flags
& VM_IOREMAP
))
3552 aligned_vread(buf
, addr
, n
);
3553 else /* IOREMAP area is treated as memory hole */
3560 spin_unlock(&vmap_area_lock
);
3562 if (buf
== buf_start
)
3564 /* zero-fill memory holes */
3565 if (buf
!= buf_start
+ buflen
)
3566 memset(buf
, 0, buflen
- (buf
- buf_start
));
3572 * remap_vmalloc_range_partial - map vmalloc pages to userspace
3573 * @vma: vma to cover
3574 * @uaddr: target user address to start at
3575 * @kaddr: virtual address of vmalloc kernel memory
3576 * @pgoff: offset from @kaddr to start at
3577 * @size: size of map area
3579 * Returns: 0 for success, -Exxx on failure
3581 * This function checks that @kaddr is a valid vmalloc'ed area,
3582 * and that it is big enough to cover the range starting at
3583 * @uaddr in @vma. Will return failure if that criteria isn't
3586 * Similar to remap_pfn_range() (see mm/memory.c)
3588 int remap_vmalloc_range_partial(struct vm_area_struct
*vma
, unsigned long uaddr
,
3589 void *kaddr
, unsigned long pgoff
,
3592 struct vm_struct
*area
;
3594 unsigned long end_index
;
3596 if (check_shl_overflow(pgoff
, PAGE_SHIFT
, &off
))
3599 size
= PAGE_ALIGN(size
);
3601 if (!PAGE_ALIGNED(uaddr
) || !PAGE_ALIGNED(kaddr
))
3604 area
= find_vm_area(kaddr
);
3608 if (!(area
->flags
& (VM_USERMAP
| VM_DMA_COHERENT
)))
3611 if (check_add_overflow(size
, off
, &end_index
) ||
3612 end_index
> get_vm_area_size(area
))
3617 struct page
*page
= vmalloc_to_page(kaddr
);
3620 ret
= vm_insert_page(vma
, uaddr
, page
);
3629 vma
->vm_flags
|= VM_DONTEXPAND
| VM_DONTDUMP
;
3635 * remap_vmalloc_range - map vmalloc pages to userspace
3636 * @vma: vma to cover (map full range of vma)
3637 * @addr: vmalloc memory
3638 * @pgoff: number of pages into addr before first page to map
3640 * Returns: 0 for success, -Exxx on failure
3642 * This function checks that addr is a valid vmalloc'ed area, and
3643 * that it is big enough to cover the vma. Will return failure if
3644 * that criteria isn't met.
3646 * Similar to remap_pfn_range() (see mm/memory.c)
3648 int remap_vmalloc_range(struct vm_area_struct
*vma
, void *addr
,
3649 unsigned long pgoff
)
3651 return remap_vmalloc_range_partial(vma
, vma
->vm_start
,
3653 vma
->vm_end
- vma
->vm_start
);
3655 EXPORT_SYMBOL(remap_vmalloc_range
);
3657 void free_vm_area(struct vm_struct
*area
)
3659 struct vm_struct
*ret
;
3660 ret
= remove_vm_area(area
->addr
);
3661 BUG_ON(ret
!= area
);
3664 EXPORT_SYMBOL_GPL(free_vm_area
);
3667 static struct vmap_area
*node_to_va(struct rb_node
*n
)
3669 return rb_entry_safe(n
, struct vmap_area
, rb_node
);
3673 * pvm_find_va_enclose_addr - find the vmap_area @addr belongs to
3674 * @addr: target address
3676 * Returns: vmap_area if it is found. If there is no such area
3677 * the first highest(reverse order) vmap_area is returned
3678 * i.e. va->va_start < addr && va->va_end < addr or NULL
3679 * if there are no any areas before @addr.
3681 static struct vmap_area
*
3682 pvm_find_va_enclose_addr(unsigned long addr
)
3684 struct vmap_area
*va
, *tmp
;
3687 n
= free_vmap_area_root
.rb_node
;
3691 tmp
= rb_entry(n
, struct vmap_area
, rb_node
);
3692 if (tmp
->va_start
<= addr
) {
3694 if (tmp
->va_end
>= addr
)
3707 * pvm_determine_end_from_reverse - find the highest aligned address
3708 * of free block below VMALLOC_END
3710 * in - the VA we start the search(reverse order);
3711 * out - the VA with the highest aligned end address.
3712 * @align: alignment for required highest address
3714 * Returns: determined end address within vmap_area
3716 static unsigned long
3717 pvm_determine_end_from_reverse(struct vmap_area
**va
, unsigned long align
)
3719 unsigned long vmalloc_end
= VMALLOC_END
& ~(align
- 1);
3723 list_for_each_entry_from_reverse((*va
),
3724 &free_vmap_area_list
, list
) {
3725 addr
= min((*va
)->va_end
& ~(align
- 1), vmalloc_end
);
3726 if ((*va
)->va_start
< addr
)
3735 * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
3736 * @offsets: array containing offset of each area
3737 * @sizes: array containing size of each area
3738 * @nr_vms: the number of areas to allocate
3739 * @align: alignment, all entries in @offsets and @sizes must be aligned to this
3741 * Returns: kmalloc'd vm_struct pointer array pointing to allocated
3742 * vm_structs on success, %NULL on failure
3744 * Percpu allocator wants to use congruent vm areas so that it can
3745 * maintain the offsets among percpu areas. This function allocates
3746 * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
3747 * be scattered pretty far, distance between two areas easily going up
3748 * to gigabytes. To avoid interacting with regular vmallocs, these
3749 * areas are allocated from top.
3751 * Despite its complicated look, this allocator is rather simple. It
3752 * does everything top-down and scans free blocks from the end looking
3753 * for matching base. While scanning, if any of the areas do not fit the
3754 * base address is pulled down to fit the area. Scanning is repeated till
3755 * all the areas fit and then all necessary data structures are inserted
3756 * and the result is returned.
3758 struct vm_struct
**pcpu_get_vm_areas(const unsigned long *offsets
,
3759 const size_t *sizes
, int nr_vms
,
3762 const unsigned long vmalloc_start
= ALIGN(VMALLOC_START
, align
);
3763 const unsigned long vmalloc_end
= VMALLOC_END
& ~(align
- 1);
3764 struct vmap_area
**vas
, *va
;
3765 struct vm_struct
**vms
;
3766 int area
, area2
, last_area
, term_area
;
3767 unsigned long base
, start
, size
, end
, last_end
, orig_start
, orig_end
;
3768 bool purged
= false;
3770 /* verify parameters and allocate data structures */
3771 BUG_ON(offset_in_page(align
) || !is_power_of_2(align
));
3772 for (last_area
= 0, area
= 0; area
< nr_vms
; area
++) {
3773 start
= offsets
[area
];
3774 end
= start
+ sizes
[area
];
3776 /* is everything aligned properly? */
3777 BUG_ON(!IS_ALIGNED(offsets
[area
], align
));
3778 BUG_ON(!IS_ALIGNED(sizes
[area
], align
));
3780 /* detect the area with the highest address */
3781 if (start
> offsets
[last_area
])
3784 for (area2
= area
+ 1; area2
< nr_vms
; area2
++) {
3785 unsigned long start2
= offsets
[area2
];
3786 unsigned long end2
= start2
+ sizes
[area2
];
3788 BUG_ON(start2
< end
&& start
< end2
);
3791 last_end
= offsets
[last_area
] + sizes
[last_area
];
3793 if (vmalloc_end
- vmalloc_start
< last_end
) {
3798 vms
= kcalloc(nr_vms
, sizeof(vms
[0]), GFP_KERNEL
);
3799 vas
= kcalloc(nr_vms
, sizeof(vas
[0]), GFP_KERNEL
);
3803 for (area
= 0; area
< nr_vms
; area
++) {
3804 vas
[area
] = kmem_cache_zalloc(vmap_area_cachep
, GFP_KERNEL
);
3805 vms
[area
] = kzalloc(sizeof(struct vm_struct
), GFP_KERNEL
);
3806 if (!vas
[area
] || !vms
[area
])
3810 spin_lock(&free_vmap_area_lock
);
3812 /* start scanning - we scan from the top, begin with the last area */
3813 area
= term_area
= last_area
;
3814 start
= offsets
[area
];
3815 end
= start
+ sizes
[area
];
3817 va
= pvm_find_va_enclose_addr(vmalloc_end
);
3818 base
= pvm_determine_end_from_reverse(&va
, align
) - end
;
3822 * base might have underflowed, add last_end before
3825 if (base
+ last_end
< vmalloc_start
+ last_end
)
3829 * Fitting base has not been found.
3835 * If required width exceeds current VA block, move
3836 * base downwards and then recheck.
3838 if (base
+ end
> va
->va_end
) {
3839 base
= pvm_determine_end_from_reverse(&va
, align
) - end
;
3845 * If this VA does not fit, move base downwards and recheck.
3847 if (base
+ start
< va
->va_start
) {
3848 va
= node_to_va(rb_prev(&va
->rb_node
));
3849 base
= pvm_determine_end_from_reverse(&va
, align
) - end
;
3855 * This area fits, move on to the previous one. If
3856 * the previous one is the terminal one, we're done.
3858 area
= (area
+ nr_vms
- 1) % nr_vms
;
3859 if (area
== term_area
)
3862 start
= offsets
[area
];
3863 end
= start
+ sizes
[area
];
3864 va
= pvm_find_va_enclose_addr(base
+ end
);
3867 /* we've found a fitting base, insert all va's */
3868 for (area
= 0; area
< nr_vms
; area
++) {
3871 start
= base
+ offsets
[area
];
3874 va
= pvm_find_va_enclose_addr(start
);
3875 if (WARN_ON_ONCE(va
== NULL
))
3876 /* It is a BUG(), but trigger recovery instead. */
3879 ret
= adjust_va_to_fit_type(&free_vmap_area_root
,
3880 &free_vmap_area_list
,
3882 if (WARN_ON_ONCE(unlikely(ret
)))
3883 /* It is a BUG(), but trigger recovery instead. */
3886 /* Allocated area. */
3888 va
->va_start
= start
;
3889 va
->va_end
= start
+ size
;
3892 spin_unlock(&free_vmap_area_lock
);
3894 /* populate the kasan shadow space */
3895 for (area
= 0; area
< nr_vms
; area
++) {
3896 if (kasan_populate_vmalloc(vas
[area
]->va_start
, sizes
[area
]))
3897 goto err_free_shadow
;
3900 /* insert all vm's */
3901 spin_lock(&vmap_area_lock
);
3902 for (area
= 0; area
< nr_vms
; area
++) {
3903 insert_vmap_area(vas
[area
], &vmap_area_root
, &vmap_area_list
);
3905 setup_vmalloc_vm_locked(vms
[area
], vas
[area
], VM_ALLOC
,
3908 spin_unlock(&vmap_area_lock
);
3911 * Mark allocated areas as accessible. Do it now as a best-effort
3912 * approach, as they can be mapped outside of vmalloc code.
3913 * With hardware tag-based KASAN, marking is skipped for
3914 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
3916 for (area
= 0; area
< nr_vms
; area
++)
3917 vms
[area
]->addr
= kasan_unpoison_vmalloc(vms
[area
]->addr
,
3918 vms
[area
]->size
, KASAN_VMALLOC_PROT_NORMAL
);
3925 * Remove previously allocated areas. There is no
3926 * need in removing these areas from the busy tree,
3927 * because they are inserted only on the final step
3928 * and when pcpu_get_vm_areas() is success.
3931 orig_start
= vas
[area
]->va_start
;
3932 orig_end
= vas
[area
]->va_end
;
3933 va
= merge_or_add_vmap_area_augment(vas
[area
], &free_vmap_area_root
,
3934 &free_vmap_area_list
);
3936 kasan_release_vmalloc(orig_start
, orig_end
,
3937 va
->va_start
, va
->va_end
);
3942 spin_unlock(&free_vmap_area_lock
);
3944 purge_vmap_area_lazy();
3947 /* Before "retry", check if we recover. */
3948 for (area
= 0; area
< nr_vms
; area
++) {
3952 vas
[area
] = kmem_cache_zalloc(
3953 vmap_area_cachep
, GFP_KERNEL
);
3962 for (area
= 0; area
< nr_vms
; area
++) {
3964 kmem_cache_free(vmap_area_cachep
, vas
[area
]);
3974 spin_lock(&free_vmap_area_lock
);
3976 * We release all the vmalloc shadows, even the ones for regions that
3977 * hadn't been successfully added. This relies on kasan_release_vmalloc
3978 * being able to tolerate this case.
3980 for (area
= 0; area
< nr_vms
; area
++) {
3981 orig_start
= vas
[area
]->va_start
;
3982 orig_end
= vas
[area
]->va_end
;
3983 va
= merge_or_add_vmap_area_augment(vas
[area
], &free_vmap_area_root
,
3984 &free_vmap_area_list
);
3986 kasan_release_vmalloc(orig_start
, orig_end
,
3987 va
->va_start
, va
->va_end
);
3991 spin_unlock(&free_vmap_area_lock
);
3998 * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
3999 * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
4000 * @nr_vms: the number of allocated areas
4002 * Free vm_structs and the array allocated by pcpu_get_vm_areas().
4004 void pcpu_free_vm_areas(struct vm_struct
**vms
, int nr_vms
)
4008 for (i
= 0; i
< nr_vms
; i
++)
4009 free_vm_area(vms
[i
]);
4012 #endif /* CONFIG_SMP */
4014 #ifdef CONFIG_PRINTK
4015 bool vmalloc_dump_obj(void *object
)
4017 struct vm_struct
*vm
;
4018 void *objp
= (void *)PAGE_ALIGN((unsigned long)object
);
4020 vm
= find_vm_area(objp
);
4023 pr_cont(" %u-page vmalloc region starting at %#lx allocated at %pS\n",
4024 vm
->nr_pages
, (unsigned long)vm
->addr
, vm
->caller
);
4029 #ifdef CONFIG_PROC_FS
4030 static void *s_start(struct seq_file
*m
, loff_t
*pos
)
4031 __acquires(&vmap_purge_lock
)
4032 __acquires(&vmap_area_lock
)
4034 mutex_lock(&vmap_purge_lock
);
4035 spin_lock(&vmap_area_lock
);
4037 return seq_list_start(&vmap_area_list
, *pos
);
4040 static void *s_next(struct seq_file
*m
, void *p
, loff_t
*pos
)
4042 return seq_list_next(p
, &vmap_area_list
, pos
);
4045 static void s_stop(struct seq_file
*m
, void *p
)
4046 __releases(&vmap_area_lock
)
4047 __releases(&vmap_purge_lock
)
4049 spin_unlock(&vmap_area_lock
);
4050 mutex_unlock(&vmap_purge_lock
);
4053 static void show_numa_info(struct seq_file
*m
, struct vm_struct
*v
)
4055 if (IS_ENABLED(CONFIG_NUMA
)) {
4056 unsigned int nr
, *counters
= m
->private;
4057 unsigned int step
= 1U << vm_area_page_order(v
);
4062 if (v
->flags
& VM_UNINITIALIZED
)
4064 /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
4067 memset(counters
, 0, nr_node_ids
* sizeof(unsigned int));
4069 for (nr
= 0; nr
< v
->nr_pages
; nr
+= step
)
4070 counters
[page_to_nid(v
->pages
[nr
])] += step
;
4071 for_each_node_state(nr
, N_HIGH_MEMORY
)
4073 seq_printf(m
, " N%u=%u", nr
, counters
[nr
]);
4077 static void show_purge_info(struct seq_file
*m
)
4079 struct vmap_area
*va
;
4081 spin_lock(&purge_vmap_area_lock
);
4082 list_for_each_entry(va
, &purge_vmap_area_list
, list
) {
4083 seq_printf(m
, "0x%pK-0x%pK %7ld unpurged vm_area\n",
4084 (void *)va
->va_start
, (void *)va
->va_end
,
4085 va
->va_end
- va
->va_start
);
4087 spin_unlock(&purge_vmap_area_lock
);
4090 static int s_show(struct seq_file
*m
, void *p
)
4092 struct vmap_area
*va
;
4093 struct vm_struct
*v
;
4095 va
= list_entry(p
, struct vmap_area
, list
);
4098 * s_show can encounter race with remove_vm_area, !vm on behalf
4099 * of vmap area is being tear down or vm_map_ram allocation.
4102 seq_printf(m
, "0x%pK-0x%pK %7ld vm_map_ram\n",
4103 (void *)va
->va_start
, (void *)va
->va_end
,
4104 va
->va_end
- va
->va_start
);
4111 seq_printf(m
, "0x%pK-0x%pK %7ld",
4112 v
->addr
, v
->addr
+ v
->size
, v
->size
);
4115 seq_printf(m
, " %pS", v
->caller
);
4118 seq_printf(m
, " pages=%d", v
->nr_pages
);
4121 seq_printf(m
, " phys=%pa", &v
->phys_addr
);
4123 if (v
->flags
& VM_IOREMAP
)
4124 seq_puts(m
, " ioremap");
4126 if (v
->flags
& VM_ALLOC
)
4127 seq_puts(m
, " vmalloc");
4129 if (v
->flags
& VM_MAP
)
4130 seq_puts(m
, " vmap");
4132 if (v
->flags
& VM_USERMAP
)
4133 seq_puts(m
, " user");
4135 if (v
->flags
& VM_DMA_COHERENT
)
4136 seq_puts(m
, " dma-coherent");
4138 if (is_vmalloc_addr(v
->pages
))
4139 seq_puts(m
, " vpages");
4141 show_numa_info(m
, v
);
4145 * As a final step, dump "unpurged" areas.
4148 if (list_is_last(&va
->list
, &vmap_area_list
))
4154 static const struct seq_operations vmalloc_op
= {
4161 static int __init
proc_vmalloc_init(void)
4163 if (IS_ENABLED(CONFIG_NUMA
))
4164 proc_create_seq_private("vmallocinfo", 0400, NULL
,
4166 nr_node_ids
* sizeof(unsigned int), NULL
);
4168 proc_create_seq("vmallocinfo", 0400, NULL
, &vmalloc_op
);
4171 module_init(proc_vmalloc_init
);