2 #include <linux/slab.h>
3 #include <linux/string.h>
4 #include <linux/compiler.h>
5 #include <linux/export.h>
7 #include <linux/sched.h>
8 #include <linux/security.h>
9 #include <linux/swap.h>
10 #include <linux/swapops.h>
11 #include <linux/mman.h>
12 #include <linux/hugetlb.h>
13 #include <linux/vmalloc.h>
14 #include <linux/userfaultfd_k.h>
16 #include <asm/sections.h>
17 #include <linux/uaccess.h>
21 static inline int is_kernel_rodata(unsigned long addr
)
23 return addr
>= (unsigned long)__start_rodata
&&
24 addr
< (unsigned long)__end_rodata
;
28 * kfree_const - conditionally free memory
29 * @x: pointer to the memory
31 * Function calls kfree only if @x is not in .rodata section.
33 void kfree_const(const void *x
)
35 if (!is_kernel_rodata((unsigned long)x
))
38 EXPORT_SYMBOL(kfree_const
);
41 * kstrdup - allocate space for and copy an existing string
42 * @s: the string to duplicate
43 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
45 char *kstrdup(const char *s
, gfp_t gfp
)
54 buf
= kmalloc_track_caller(len
, gfp
);
59 EXPORT_SYMBOL(kstrdup
);
62 * kstrdup_const - conditionally duplicate an existing const string
63 * @s: the string to duplicate
64 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
66 * Function returns source string if it is in .rodata section otherwise it
67 * fallbacks to kstrdup.
68 * Strings allocated by kstrdup_const should be freed by kfree_const.
70 const char *kstrdup_const(const char *s
, gfp_t gfp
)
72 if (is_kernel_rodata((unsigned long)s
))
75 return kstrdup(s
, gfp
);
77 EXPORT_SYMBOL(kstrdup_const
);
80 * kstrndup - allocate space for and copy an existing string
81 * @s: the string to duplicate
82 * @max: read at most @max chars from @s
83 * @gfp: the GFP mask used in the kmalloc() call when allocating memory
85 char *kstrndup(const char *s
, size_t max
, gfp_t gfp
)
93 len
= strnlen(s
, max
);
94 buf
= kmalloc_track_caller(len
+1, gfp
);
101 EXPORT_SYMBOL(kstrndup
);
104 * kmemdup - duplicate region of memory
106 * @src: memory region to duplicate
107 * @len: memory region length
108 * @gfp: GFP mask to use
110 void *kmemdup(const void *src
, size_t len
, gfp_t gfp
)
114 p
= kmalloc_track_caller(len
, gfp
);
119 EXPORT_SYMBOL(kmemdup
);
122 * memdup_user - duplicate memory region from user space
124 * @src: source address in user space
125 * @len: number of bytes to copy
127 * Returns an ERR_PTR() on failure.
129 void *memdup_user(const void __user
*src
, size_t len
)
134 * Always use GFP_KERNEL, since copy_from_user() can sleep and
135 * cause pagefault, which makes it pointless to use GFP_NOFS
138 p
= kmalloc_track_caller(len
, GFP_KERNEL
);
140 return ERR_PTR(-ENOMEM
);
142 if (copy_from_user(p
, src
, len
)) {
144 return ERR_PTR(-EFAULT
);
149 EXPORT_SYMBOL(memdup_user
);
152 * strndup_user - duplicate an existing string from user space
153 * @s: The string to duplicate
154 * @n: Maximum number of bytes to copy, including the trailing NUL.
156 char *strndup_user(const char __user
*s
, long n
)
161 length
= strnlen_user(s
, n
);
164 return ERR_PTR(-EFAULT
);
167 return ERR_PTR(-EINVAL
);
169 p
= memdup_user(s
, length
);
174 p
[length
- 1] = '\0';
178 EXPORT_SYMBOL(strndup_user
);
181 * memdup_user_nul - duplicate memory region from user space and NUL-terminate
183 * @src: source address in user space
184 * @len: number of bytes to copy
186 * Returns an ERR_PTR() on failure.
188 void *memdup_user_nul(const void __user
*src
, size_t len
)
193 * Always use GFP_KERNEL, since copy_from_user() can sleep and
194 * cause pagefault, which makes it pointless to use GFP_NOFS
197 p
= kmalloc_track_caller(len
+ 1, GFP_KERNEL
);
199 return ERR_PTR(-ENOMEM
);
201 if (copy_from_user(p
, src
, len
)) {
203 return ERR_PTR(-EFAULT
);
209 EXPORT_SYMBOL(memdup_user_nul
);
211 void __vma_link_list(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
212 struct vm_area_struct
*prev
, struct rb_node
*rb_parent
)
214 struct vm_area_struct
*next
;
218 next
= prev
->vm_next
;
223 next
= rb_entry(rb_parent
,
224 struct vm_area_struct
, vm_rb
);
233 /* Check if the vma is being used as a stack by this task */
234 int vma_is_stack_for_current(struct vm_area_struct
*vma
)
236 struct task_struct
* __maybe_unused t
= current
;
238 return (vma
->vm_start
<= KSTK_ESP(t
) && vma
->vm_end
>= KSTK_ESP(t
));
241 #if defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
242 void arch_pick_mmap_layout(struct mm_struct
*mm
)
244 mm
->mmap_base
= TASK_UNMAPPED_BASE
;
245 mm
->get_unmapped_area
= arch_get_unmapped_area
;
250 * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
251 * back to the regular GUP.
252 * If the architecture not support this function, simply return with no
255 int __weak
__get_user_pages_fast(unsigned long start
,
256 int nr_pages
, int write
, struct page
**pages
)
260 EXPORT_SYMBOL_GPL(__get_user_pages_fast
);
263 * get_user_pages_fast() - pin user pages in memory
264 * @start: starting user address
265 * @nr_pages: number of pages from start to pin
266 * @write: whether pages will be written to
267 * @pages: array that receives pointers to the pages pinned.
268 * Should be at least nr_pages long.
270 * Returns number of pages pinned. This may be fewer than the number
271 * requested. If nr_pages is 0 or negative, returns 0. If no pages
272 * were pinned, returns -errno.
274 * get_user_pages_fast provides equivalent functionality to get_user_pages,
275 * operating on current and current->mm, with force=0 and vma=NULL. However
276 * unlike get_user_pages, it must be called without mmap_sem held.
278 * get_user_pages_fast may take mmap_sem and page table locks, so no
279 * assumptions can be made about lack of locking. get_user_pages_fast is to be
280 * implemented in a way that is advantageous (vs get_user_pages()) when the
281 * user memory area is already faulted in and present in ptes. However if the
282 * pages have to be faulted in, it may turn out to be slightly slower so
283 * callers need to carefully consider what to use. On many architectures,
284 * get_user_pages_fast simply falls back to get_user_pages.
286 int __weak
get_user_pages_fast(unsigned long start
,
287 int nr_pages
, int write
, struct page
**pages
)
289 return get_user_pages_unlocked(start
, nr_pages
, pages
,
290 write
? FOLL_WRITE
: 0);
292 EXPORT_SYMBOL_GPL(get_user_pages_fast
);
294 unsigned long vm_mmap_pgoff(struct file
*file
, unsigned long addr
,
295 unsigned long len
, unsigned long prot
,
296 unsigned long flag
, unsigned long pgoff
)
299 struct mm_struct
*mm
= current
->mm
;
300 unsigned long populate
;
303 ret
= security_mmap_file(file
, prot
, flag
);
305 if (down_write_killable(&mm
->mmap_sem
))
307 ret
= do_mmap_pgoff(file
, addr
, len
, prot
, flag
, pgoff
,
309 up_write(&mm
->mmap_sem
);
310 userfaultfd_unmap_complete(mm
, &uf
);
312 mm_populate(ret
, populate
);
317 unsigned long vm_mmap(struct file
*file
, unsigned long addr
,
318 unsigned long len
, unsigned long prot
,
319 unsigned long flag
, unsigned long offset
)
321 if (unlikely(offset
+ PAGE_ALIGN(len
) < offset
))
323 if (unlikely(offset_in_page(offset
)))
326 return vm_mmap_pgoff(file
, addr
, len
, prot
, flag
, offset
>> PAGE_SHIFT
);
328 EXPORT_SYMBOL(vm_mmap
);
330 void kvfree(const void *addr
)
332 if (is_vmalloc_addr(addr
))
337 EXPORT_SYMBOL(kvfree
);
339 static inline void *__page_rmapping(struct page
*page
)
341 unsigned long mapping
;
343 mapping
= (unsigned long)page
->mapping
;
344 mapping
&= ~PAGE_MAPPING_FLAGS
;
346 return (void *)mapping
;
349 /* Neutral page->mapping pointer to address_space or anon_vma or other */
350 void *page_rmapping(struct page
*page
)
352 page
= compound_head(page
);
353 return __page_rmapping(page
);
357 * Return true if this page is mapped into pagetables.
358 * For compound page it returns true if any subpage of compound page is mapped.
360 bool page_mapped(struct page
*page
)
364 if (likely(!PageCompound(page
)))
365 return atomic_read(&page
->_mapcount
) >= 0;
366 page
= compound_head(page
);
367 if (atomic_read(compound_mapcount_ptr(page
)) >= 0)
371 for (i
= 0; i
< hpage_nr_pages(page
); i
++) {
372 if (atomic_read(&page
[i
]._mapcount
) >= 0)
377 EXPORT_SYMBOL(page_mapped
);
379 struct anon_vma
*page_anon_vma(struct page
*page
)
381 unsigned long mapping
;
383 page
= compound_head(page
);
384 mapping
= (unsigned long)page
->mapping
;
385 if ((mapping
& PAGE_MAPPING_FLAGS
) != PAGE_MAPPING_ANON
)
387 return __page_rmapping(page
);
390 struct address_space
*page_mapping(struct page
*page
)
392 struct address_space
*mapping
;
394 page
= compound_head(page
);
396 /* This happens if someone calls flush_dcache_page on slab page */
397 if (unlikely(PageSlab(page
)))
400 if (unlikely(PageSwapCache(page
))) {
403 entry
.val
= page_private(page
);
404 return swap_address_space(entry
);
407 mapping
= page
->mapping
;
408 if ((unsigned long)mapping
& PAGE_MAPPING_ANON
)
411 return (void *)((unsigned long)mapping
& ~PAGE_MAPPING_FLAGS
);
413 EXPORT_SYMBOL(page_mapping
);
415 /* Slow path of page_mapcount() for compound pages */
416 int __page_mapcount(struct page
*page
)
420 ret
= atomic_read(&page
->_mapcount
) + 1;
422 * For file THP page->_mapcount contains total number of mapping
423 * of the page: no need to look into compound_mapcount.
425 if (!PageAnon(page
) && !PageHuge(page
))
427 page
= compound_head(page
);
428 ret
+= atomic_read(compound_mapcount_ptr(page
)) + 1;
429 if (PageDoubleMap(page
))
433 EXPORT_SYMBOL_GPL(__page_mapcount
);
435 int sysctl_overcommit_memory __read_mostly
= OVERCOMMIT_GUESS
;
436 int sysctl_overcommit_ratio __read_mostly
= 50;
437 unsigned long sysctl_overcommit_kbytes __read_mostly
;
438 int sysctl_max_map_count __read_mostly
= DEFAULT_MAX_MAP_COUNT
;
439 unsigned long sysctl_user_reserve_kbytes __read_mostly
= 1UL << 17; /* 128MB */
440 unsigned long sysctl_admin_reserve_kbytes __read_mostly
= 1UL << 13; /* 8MB */
442 int overcommit_ratio_handler(struct ctl_table
*table
, int write
,
443 void __user
*buffer
, size_t *lenp
,
448 ret
= proc_dointvec(table
, write
, buffer
, lenp
, ppos
);
449 if (ret
== 0 && write
)
450 sysctl_overcommit_kbytes
= 0;
454 int overcommit_kbytes_handler(struct ctl_table
*table
, int write
,
455 void __user
*buffer
, size_t *lenp
,
460 ret
= proc_doulongvec_minmax(table
, write
, buffer
, lenp
, ppos
);
461 if (ret
== 0 && write
)
462 sysctl_overcommit_ratio
= 0;
467 * Committed memory limit enforced when OVERCOMMIT_NEVER policy is used
469 unsigned long vm_commit_limit(void)
471 unsigned long allowed
;
473 if (sysctl_overcommit_kbytes
)
474 allowed
= sysctl_overcommit_kbytes
>> (PAGE_SHIFT
- 10);
476 allowed
= ((totalram_pages
- hugetlb_total_pages())
477 * sysctl_overcommit_ratio
/ 100);
478 allowed
+= total_swap_pages
;
484 * Make sure vm_committed_as in one cacheline and not cacheline shared with
485 * other variables. It can be updated by several CPUs frequently.
487 struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp
;
490 * The global memory commitment made in the system can be a metric
491 * that can be used to drive ballooning decisions when Linux is hosted
492 * as a guest. On Hyper-V, the host implements a policy engine for dynamically
493 * balancing memory across competing virtual machines that are hosted.
494 * Several metrics drive this policy engine including the guest reported
497 unsigned long vm_memory_committed(void)
499 return percpu_counter_read_positive(&vm_committed_as
);
501 EXPORT_SYMBOL_GPL(vm_memory_committed
);
504 * Check that a process has enough memory to allocate a new virtual
505 * mapping. 0 means there is enough memory for the allocation to
506 * succeed and -ENOMEM implies there is not.
508 * We currently support three overcommit policies, which are set via the
509 * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
511 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
512 * Additional code 2002 Jul 20 by Robert Love.
514 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
516 * Note this is a helper function intended to be used by LSMs which
517 * wish to use this logic.
519 int __vm_enough_memory(struct mm_struct
*mm
, long pages
, int cap_sys_admin
)
521 long free
, allowed
, reserve
;
523 VM_WARN_ONCE(percpu_counter_read(&vm_committed_as
) <
524 -(s64
)vm_committed_as_batch
* num_online_cpus(),
525 "memory commitment underflow");
527 vm_acct_memory(pages
);
530 * Sometimes we want to use more memory than we have
532 if (sysctl_overcommit_memory
== OVERCOMMIT_ALWAYS
)
535 if (sysctl_overcommit_memory
== OVERCOMMIT_GUESS
) {
536 free
= global_page_state(NR_FREE_PAGES
);
537 free
+= global_node_page_state(NR_FILE_PAGES
);
540 * shmem pages shouldn't be counted as free in this
541 * case, they can't be purged, only swapped out, and
542 * that won't affect the overall amount of available
543 * memory in the system.
545 free
-= global_node_page_state(NR_SHMEM
);
547 free
+= get_nr_swap_pages();
550 * Any slabs which are created with the
551 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
552 * which are reclaimable, under pressure. The dentry
553 * cache and most inode caches should fall into this
555 free
+= global_page_state(NR_SLAB_RECLAIMABLE
);
558 * Leave reserved pages. The pages are not for anonymous pages.
560 if (free
<= totalreserve_pages
)
563 free
-= totalreserve_pages
;
566 * Reserve some for root
569 free
-= sysctl_admin_reserve_kbytes
>> (PAGE_SHIFT
- 10);
577 allowed
= vm_commit_limit();
579 * Reserve some for root
582 allowed
-= sysctl_admin_reserve_kbytes
>> (PAGE_SHIFT
- 10);
585 * Don't let a single process grow so big a user can't recover
588 reserve
= sysctl_user_reserve_kbytes
>> (PAGE_SHIFT
- 10);
589 allowed
-= min_t(long, mm
->total_vm
/ 32, reserve
);
592 if (percpu_counter_read_positive(&vm_committed_as
) < allowed
)
595 vm_unacct_memory(pages
);
601 * get_cmdline() - copy the cmdline value to a buffer.
602 * @task: the task whose cmdline value to copy.
603 * @buffer: the buffer to copy to.
604 * @buflen: the length of the buffer. Larger cmdline values are truncated
606 * Returns the size of the cmdline field copied. Note that the copy does
607 * not guarantee an ending NULL byte.
609 int get_cmdline(struct task_struct
*task
, char *buffer
, int buflen
)
613 struct mm_struct
*mm
= get_task_mm(task
);
614 unsigned long arg_start
, arg_end
, env_start
, env_end
;
618 goto out_mm
; /* Shh! No looking before we're done */
620 down_read(&mm
->mmap_sem
);
621 arg_start
= mm
->arg_start
;
622 arg_end
= mm
->arg_end
;
623 env_start
= mm
->env_start
;
624 env_end
= mm
->env_end
;
625 up_read(&mm
->mmap_sem
);
627 len
= arg_end
- arg_start
;
632 res
= access_process_vm(task
, arg_start
, buffer
, len
, FOLL_FORCE
);
635 * If the nul at the end of args has been overwritten, then
636 * assume application is using setproctitle(3).
638 if (res
> 0 && buffer
[res
-1] != '\0' && len
< buflen
) {
639 len
= strnlen(buffer
, res
);
643 len
= env_end
- env_start
;
644 if (len
> buflen
- res
)
646 res
+= access_process_vm(task
, env_start
,
649 res
= strnlen(buffer
, res
);