1 // SPDX-License-Identifier: GPL-2.0
5 * (C) Copyright 1994 Linus Torvalds
6 * (C) Copyright 2002 Christoph Hellwig
8 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
9 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
12 #include <linux/pagewalk.h>
13 #include <linux/hugetlb.h>
14 #include <linux/shm.h>
15 #include <linux/mman.h>
17 #include <linux/highmem.h>
18 #include <linux/security.h>
19 #include <linux/mempolicy.h>
20 #include <linux/personality.h>
21 #include <linux/syscalls.h>
22 #include <linux/swap.h>
23 #include <linux/swapops.h>
24 #include <linux/mmu_notifier.h>
25 #include <linux/migrate.h>
26 #include <linux/perf_event.h>
27 #include <linux/pkeys.h>
28 #include <linux/ksm.h>
29 #include <linux/uaccess.h>
30 #include <linux/mm_inline.h>
31 #include <asm/pgtable.h>
32 #include <asm/cacheflush.h>
33 #include <asm/mmu_context.h>
34 #include <asm/tlbflush.h>
38 static unsigned long change_pte_range(struct vm_area_struct
*vma
, pmd_t
*pmd
,
39 unsigned long addr
, unsigned long end
, pgprot_t newprot
,
40 int dirty_accountable
, int prot_numa
)
44 unsigned long pages
= 0;
45 int target_node
= NUMA_NO_NODE
;
48 * Can be called with only the mmap_sem for reading by
49 * prot_numa so we must check the pmd isn't constantly
50 * changing from under us from pmd_none to pmd_trans_huge
51 * and/or the other way around.
53 if (pmd_trans_unstable(pmd
))
57 * The pmd points to a regular pte so the pmd can't change
58 * from under us even if the mmap_sem is only hold for
61 pte
= pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, &ptl
);
63 /* Get target node for single threaded private VMAs */
64 if (prot_numa
&& !(vma
->vm_flags
& VM_SHARED
) &&
65 atomic_read(&vma
->vm_mm
->mm_users
) == 1)
66 target_node
= numa_node_id();
68 flush_tlb_batched_pending(vma
->vm_mm
);
69 arch_enter_lazy_mmu_mode();
72 if (pte_present(oldpte
)) {
74 bool preserve_write
= prot_numa
&& pte_write(oldpte
);
77 * Avoid trapping faults against the zero or KSM
78 * pages. See similar comment in change_huge_pmd.
83 /* Avoid TLB flush if possible */
84 if (pte_protnone(oldpte
))
87 page
= vm_normal_page(vma
, addr
, oldpte
);
88 if (!page
|| PageKsm(page
))
91 /* Also skip shared copy-on-write pages */
92 if (is_cow_mapping(vma
->vm_flags
) &&
93 page_mapcount(page
) != 1)
97 * While migration can move some dirty pages,
98 * it cannot move them all from MIGRATE_ASYNC
101 if (page_is_file_cache(page
) && PageDirty(page
))
105 * Don't mess with PTEs if page is already on the node
106 * a single-threaded process is running on.
108 if (target_node
== page_to_nid(page
))
112 oldpte
= ptep_modify_prot_start(vma
, addr
, pte
);
113 ptent
= pte_modify(oldpte
, newprot
);
115 ptent
= pte_mk_savedwrite(ptent
);
117 /* Avoid taking write faults for known dirty pages */
118 if (dirty_accountable
&& pte_dirty(ptent
) &&
119 (pte_soft_dirty(ptent
) ||
120 !(vma
->vm_flags
& VM_SOFTDIRTY
))) {
121 ptent
= pte_mkwrite(ptent
);
123 ptep_modify_prot_commit(vma
, addr
, pte
, oldpte
, ptent
);
125 } else if (IS_ENABLED(CONFIG_MIGRATION
)) {
126 swp_entry_t entry
= pte_to_swp_entry(oldpte
);
128 if (is_write_migration_entry(entry
)) {
131 * A protection check is difficult so
132 * just be safe and disable write
134 make_migration_entry_read(&entry
);
135 newpte
= swp_entry_to_pte(entry
);
136 if (pte_swp_soft_dirty(oldpte
))
137 newpte
= pte_swp_mksoft_dirty(newpte
);
138 set_pte_at(vma
->vm_mm
, addr
, pte
, newpte
);
143 if (is_write_device_private_entry(entry
)) {
147 * We do not preserve soft-dirtiness. See
148 * copy_one_pte() for explanation.
150 make_device_private_entry_read(&entry
);
151 newpte
= swp_entry_to_pte(entry
);
152 set_pte_at(vma
->vm_mm
, addr
, pte
, newpte
);
157 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
158 arch_leave_lazy_mmu_mode();
159 pte_unmap_unlock(pte
- 1, ptl
);
165 * Used when setting automatic NUMA hinting protection where it is
166 * critical that a numa hinting PMD is not confused with a bad PMD.
168 static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t
*pmd
)
170 pmd_t pmdval
= pmd_read_atomic(pmd
);
172 /* See pmd_none_or_trans_huge_or_clear_bad for info on barrier */
173 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
177 if (pmd_none(pmdval
))
179 if (pmd_trans_huge(pmdval
))
181 if (unlikely(pmd_bad(pmdval
))) {
189 static inline unsigned long change_pmd_range(struct vm_area_struct
*vma
,
190 pud_t
*pud
, unsigned long addr
, unsigned long end
,
191 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
195 unsigned long pages
= 0;
196 unsigned long nr_huge_updates
= 0;
197 struct mmu_notifier_range range
;
201 pmd
= pmd_offset(pud
, addr
);
203 unsigned long this_pages
;
205 next
= pmd_addr_end(addr
, end
);
208 * Automatic NUMA balancing walks the tables with mmap_sem
209 * held for read. It's possible a parallel update to occur
210 * between pmd_trans_huge() and a pmd_none_or_clear_bad()
211 * check leading to a false positive and clearing.
212 * Hence, it's necessary to atomically read the PMD value
213 * for all the checks.
215 if (!is_swap_pmd(*pmd
) && !pmd_devmap(*pmd
) &&
216 pmd_none_or_clear_bad_unless_trans_huge(pmd
))
219 /* invoke the mmu notifier if the pmd is populated */
221 mmu_notifier_range_init(&range
,
222 MMU_NOTIFY_PROTECTION_VMA
, 0,
223 vma
, vma
->vm_mm
, addr
, end
);
224 mmu_notifier_invalidate_range_start(&range
);
227 if (is_swap_pmd(*pmd
) || pmd_trans_huge(*pmd
) || pmd_devmap(*pmd
)) {
228 if (next
- addr
!= HPAGE_PMD_SIZE
) {
229 __split_huge_pmd(vma
, pmd
, addr
, false, NULL
);
231 int nr_ptes
= change_huge_pmd(vma
, pmd
, addr
,
235 if (nr_ptes
== HPAGE_PMD_NR
) {
236 pages
+= HPAGE_PMD_NR
;
240 /* huge pmd was handled */
244 /* fall through, the trans huge pmd just split */
246 this_pages
= change_pte_range(vma
, pmd
, addr
, next
, newprot
,
247 dirty_accountable
, prot_numa
);
251 } while (pmd
++, addr
= next
, addr
!= end
);
254 mmu_notifier_invalidate_range_end(&range
);
257 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES
, nr_huge_updates
);
261 static inline unsigned long change_pud_range(struct vm_area_struct
*vma
,
262 p4d_t
*p4d
, unsigned long addr
, unsigned long end
,
263 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
267 unsigned long pages
= 0;
269 pud
= pud_offset(p4d
, addr
);
271 next
= pud_addr_end(addr
, end
);
272 if (pud_none_or_clear_bad(pud
))
274 pages
+= change_pmd_range(vma
, pud
, addr
, next
, newprot
,
275 dirty_accountable
, prot_numa
);
276 } while (pud
++, addr
= next
, addr
!= end
);
281 static inline unsigned long change_p4d_range(struct vm_area_struct
*vma
,
282 pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
283 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
287 unsigned long pages
= 0;
289 p4d
= p4d_offset(pgd
, addr
);
291 next
= p4d_addr_end(addr
, end
);
292 if (p4d_none_or_clear_bad(p4d
))
294 pages
+= change_pud_range(vma
, p4d
, addr
, next
, newprot
,
295 dirty_accountable
, prot_numa
);
296 } while (p4d
++, addr
= next
, addr
!= end
);
301 static unsigned long change_protection_range(struct vm_area_struct
*vma
,
302 unsigned long addr
, unsigned long end
, pgprot_t newprot
,
303 int dirty_accountable
, int prot_numa
)
305 struct mm_struct
*mm
= vma
->vm_mm
;
308 unsigned long start
= addr
;
309 unsigned long pages
= 0;
312 pgd
= pgd_offset(mm
, addr
);
313 flush_cache_range(vma
, addr
, end
);
314 inc_tlb_flush_pending(mm
);
316 next
= pgd_addr_end(addr
, end
);
317 if (pgd_none_or_clear_bad(pgd
))
319 pages
+= change_p4d_range(vma
, pgd
, addr
, next
, newprot
,
320 dirty_accountable
, prot_numa
);
321 } while (pgd
++, addr
= next
, addr
!= end
);
323 /* Only flush the TLB if we actually modified any entries: */
325 flush_tlb_range(vma
, start
, end
);
326 dec_tlb_flush_pending(mm
);
331 unsigned long change_protection(struct vm_area_struct
*vma
, unsigned long start
,
332 unsigned long end
, pgprot_t newprot
,
333 int dirty_accountable
, int prot_numa
)
337 if (is_vm_hugetlb_page(vma
))
338 pages
= hugetlb_change_protection(vma
, start
, end
, newprot
);
340 pages
= change_protection_range(vma
, start
, end
, newprot
, dirty_accountable
, prot_numa
);
345 static int prot_none_pte_entry(pte_t
*pte
, unsigned long addr
,
346 unsigned long next
, struct mm_walk
*walk
)
348 return pfn_modify_allowed(pte_pfn(*pte
), *(pgprot_t
*)(walk
->private)) ?
352 static int prot_none_hugetlb_entry(pte_t
*pte
, unsigned long hmask
,
353 unsigned long addr
, unsigned long next
,
354 struct mm_walk
*walk
)
356 return pfn_modify_allowed(pte_pfn(*pte
), *(pgprot_t
*)(walk
->private)) ?
360 static int prot_none_test(unsigned long addr
, unsigned long next
,
361 struct mm_walk
*walk
)
366 static const struct mm_walk_ops prot_none_walk_ops
= {
367 .pte_entry
= prot_none_pte_entry
,
368 .hugetlb_entry
= prot_none_hugetlb_entry
,
369 .test_walk
= prot_none_test
,
373 mprotect_fixup(struct vm_area_struct
*vma
, struct vm_area_struct
**pprev
,
374 unsigned long start
, unsigned long end
, unsigned long newflags
)
376 struct mm_struct
*mm
= vma
->vm_mm
;
377 unsigned long oldflags
= vma
->vm_flags
;
378 long nrpages
= (end
- start
) >> PAGE_SHIFT
;
379 unsigned long charged
= 0;
382 int dirty_accountable
= 0;
384 if (newflags
== oldflags
) {
390 * Do PROT_NONE PFN permission checks here when we can still
391 * bail out without undoing a lot of state. This is a rather
392 * uncommon case, so doesn't need to be very optimized.
394 if (arch_has_pfn_modify_check() &&
395 (vma
->vm_flags
& (VM_PFNMAP
|VM_MIXEDMAP
)) &&
396 (newflags
& (VM_READ
|VM_WRITE
|VM_EXEC
)) == 0) {
397 pgprot_t new_pgprot
= vm_get_page_prot(newflags
);
399 error
= walk_page_range(current
->mm
, start
, end
,
400 &prot_none_walk_ops
, &new_pgprot
);
406 * If we make a private mapping writable we increase our commit;
407 * but (without finer accounting) cannot reduce our commit if we
408 * make it unwritable again. hugetlb mapping were accounted for
409 * even if read-only so there is no need to account for them here
411 if (newflags
& VM_WRITE
) {
412 /* Check space limits when area turns into data. */
413 if (!may_expand_vm(mm
, newflags
, nrpages
) &&
414 may_expand_vm(mm
, oldflags
, nrpages
))
416 if (!(oldflags
& (VM_ACCOUNT
|VM_WRITE
|VM_HUGETLB
|
417 VM_SHARED
|VM_NORESERVE
))) {
419 if (security_vm_enough_memory_mm(mm
, charged
))
421 newflags
|= VM_ACCOUNT
;
426 * First try to merge with previous and/or next vma.
428 pgoff
= vma
->vm_pgoff
+ ((start
- vma
->vm_start
) >> PAGE_SHIFT
);
429 *pprev
= vma_merge(mm
, *pprev
, start
, end
, newflags
,
430 vma
->anon_vma
, vma
->vm_file
, pgoff
, vma_policy(vma
),
431 vma
->vm_userfaultfd_ctx
);
434 VM_WARN_ON((vma
->vm_flags
^ newflags
) & ~VM_SOFTDIRTY
);
440 if (start
!= vma
->vm_start
) {
441 error
= split_vma(mm
, vma
, start
, 1);
446 if (end
!= vma
->vm_end
) {
447 error
= split_vma(mm
, vma
, end
, 0);
454 * vm_flags and vm_page_prot are protected by the mmap_sem
455 * held in write mode.
457 vma
->vm_flags
= newflags
;
458 dirty_accountable
= vma_wants_writenotify(vma
, vma
->vm_page_prot
);
459 vma_set_page_prot(vma
);
461 change_protection(vma
, start
, end
, vma
->vm_page_prot
,
462 dirty_accountable
, 0);
465 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
468 if ((oldflags
& (VM_WRITE
| VM_SHARED
| VM_LOCKED
)) == VM_LOCKED
&&
469 (newflags
& VM_WRITE
)) {
470 populate_vma_page_range(vma
, start
, end
, NULL
);
473 vm_stat_account(mm
, oldflags
, -nrpages
);
474 vm_stat_account(mm
, newflags
, nrpages
);
475 perf_event_mmap(vma
);
479 vm_unacct_memory(charged
);
484 * pkey==-1 when doing a legacy mprotect()
486 static int do_mprotect_pkey(unsigned long start
, size_t len
,
487 unsigned long prot
, int pkey
)
489 unsigned long nstart
, end
, tmp
, reqprot
;
490 struct vm_area_struct
*vma
, *prev
;
492 const int grows
= prot
& (PROT_GROWSDOWN
|PROT_GROWSUP
);
493 const bool rier
= (current
->personality
& READ_IMPLIES_EXEC
) &&
496 start
= untagged_addr(start
);
498 prot
&= ~(PROT_GROWSDOWN
|PROT_GROWSUP
);
499 if (grows
== (PROT_GROWSDOWN
|PROT_GROWSUP
)) /* can't be both */
502 if (start
& ~PAGE_MASK
)
506 len
= PAGE_ALIGN(len
);
510 if (!arch_validate_prot(prot
, start
))
515 if (down_write_killable(¤t
->mm
->mmap_sem
))
519 * If userspace did not allocate the pkey, do not let
523 if ((pkey
!= -1) && !mm_pkey_is_allocated(current
->mm
, pkey
))
526 vma
= find_vma(current
->mm
, start
);
531 if (unlikely(grows
& PROT_GROWSDOWN
)) {
532 if (vma
->vm_start
>= end
)
534 start
= vma
->vm_start
;
536 if (!(vma
->vm_flags
& VM_GROWSDOWN
))
539 if (vma
->vm_start
> start
)
541 if (unlikely(grows
& PROT_GROWSUP
)) {
544 if (!(vma
->vm_flags
& VM_GROWSUP
))
548 if (start
> vma
->vm_start
)
551 for (nstart
= start
; ; ) {
552 unsigned long mask_off_old_flags
;
553 unsigned long newflags
;
556 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
558 /* Does the application expect PROT_READ to imply PROT_EXEC */
559 if (rier
&& (vma
->vm_flags
& VM_MAYEXEC
))
563 * Each mprotect() call explicitly passes r/w/x permissions.
564 * If a permission is not passed to mprotect(), it must be
565 * cleared from the VMA.
567 mask_off_old_flags
= VM_READ
| VM_WRITE
| VM_EXEC
|
570 new_vma_pkey
= arch_override_mprotect_pkey(vma
, prot
, pkey
);
571 newflags
= calc_vm_prot_bits(prot
, new_vma_pkey
);
572 newflags
|= (vma
->vm_flags
& ~mask_off_old_flags
);
574 /* newflags >> 4 shift VM_MAY% in place of VM_% */
575 if ((newflags
& ~(newflags
>> 4)) & (VM_READ
| VM_WRITE
| VM_EXEC
)) {
580 error
= security_file_mprotect(vma
, reqprot
, prot
);
587 error
= mprotect_fixup(vma
, &prev
, nstart
, tmp
, newflags
);
592 if (nstart
< prev
->vm_end
)
593 nstart
= prev
->vm_end
;
598 if (!vma
|| vma
->vm_start
!= nstart
) {
605 up_write(¤t
->mm
->mmap_sem
);
609 SYSCALL_DEFINE3(mprotect
, unsigned long, start
, size_t, len
,
612 return do_mprotect_pkey(start
, len
, prot
, -1);
615 #ifdef CONFIG_ARCH_HAS_PKEYS
617 SYSCALL_DEFINE4(pkey_mprotect
, unsigned long, start
, size_t, len
,
618 unsigned long, prot
, int, pkey
)
620 return do_mprotect_pkey(start
, len
, prot
, pkey
);
623 SYSCALL_DEFINE2(pkey_alloc
, unsigned long, flags
, unsigned long, init_val
)
628 /* No flags supported yet. */
631 /* check for unsupported init values */
632 if (init_val
& ~PKEY_ACCESS_MASK
)
635 down_write(¤t
->mm
->mmap_sem
);
636 pkey
= mm_pkey_alloc(current
->mm
);
642 ret
= arch_set_user_pkey_access(current
, pkey
, init_val
);
644 mm_pkey_free(current
->mm
, pkey
);
649 up_write(¤t
->mm
->mmap_sem
);
653 SYSCALL_DEFINE1(pkey_free
, int, pkey
)
657 down_write(¤t
->mm
->mmap_sem
);
658 ret
= mm_pkey_free(current
->mm
, pkey
);
659 up_write(¤t
->mm
->mmap_sem
);
662 * We could provie warnings or errors if any VMA still
663 * has the pkey set here.
668 #endif /* CONFIG_ARCH_HAS_PKEYS */