]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - mm/mprotect.c
4 * (C) Copyright 1994 Linus Torvalds
5 * (C) Copyright 2002 Christoph Hellwig
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
12 #include <linux/hugetlb.h>
13 #include <linux/shm.h>
14 #include <linux/mman.h>
16 #include <linux/highmem.h>
17 #include <linux/security.h>
18 #include <linux/mempolicy.h>
19 #include <linux/personality.h>
20 #include <linux/syscalls.h>
21 #include <linux/swap.h>
22 #include <linux/swapops.h>
23 #include <linux/mmu_notifier.h>
24 #include <linux/migrate.h>
25 #include <linux/perf_event.h>
26 #include <linux/pkeys.h>
27 #include <linux/ksm.h>
28 #include <asm/uaccess.h>
29 #include <asm/pgtable.h>
30 #include <asm/cacheflush.h>
31 #include <asm/mmu_context.h>
32 #include <asm/tlbflush.h>
37 * For a prot_numa update we only hold mmap_sem for read so there is a
38 * potential race with faulting where a pmd was temporarily none. This
39 * function checks for a transhuge pmd under the appropriate lock. It
40 * returns a pte if it was successfully locked or NULL if it raced with
41 * a transhuge insertion.
43 static pte_t
*lock_pte_protection(struct vm_area_struct
*vma
, pmd_t
*pmd
,
44 unsigned long addr
, int prot_numa
, spinlock_t
**ptl
)
49 /* !prot_numa is protected by mmap_sem held for write */
51 return pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, ptl
);
53 pmdl
= pmd_lock(vma
->vm_mm
, pmd
);
54 if (unlikely(pmd_trans_huge(*pmd
) || pmd_none(*pmd
))) {
59 pte
= pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, ptl
);
64 static unsigned long change_pte_range(struct vm_area_struct
*vma
, pmd_t
*pmd
,
65 unsigned long addr
, unsigned long end
, pgprot_t newprot
,
66 int dirty_accountable
, int prot_numa
)
68 struct mm_struct
*mm
= vma
->vm_mm
;
71 unsigned long pages
= 0;
73 pte
= lock_pte_protection(vma
, pmd
, addr
, prot_numa
, &ptl
);
77 arch_enter_lazy_mmu_mode();
80 if (pte_present(oldpte
)) {
82 bool preserve_write
= prot_numa
&& pte_write(oldpte
);
85 * Avoid trapping faults against the zero or KSM
86 * pages. See similar comment in change_huge_pmd.
91 page
= vm_normal_page(vma
, addr
, oldpte
);
92 if (!page
|| PageKsm(page
))
95 /* Avoid TLB flush if possible */
96 if (pte_protnone(oldpte
))
100 ptent
= ptep_modify_prot_start(mm
, addr
, pte
);
101 ptent
= pte_modify(ptent
, newprot
);
103 ptent
= pte_mkwrite(ptent
);
105 /* Avoid taking write faults for known dirty pages */
106 if (dirty_accountable
&& pte_dirty(ptent
) &&
107 (pte_soft_dirty(ptent
) ||
108 !(vma
->vm_flags
& VM_SOFTDIRTY
))) {
109 ptent
= pte_mkwrite(ptent
);
111 ptep_modify_prot_commit(mm
, addr
, pte
, ptent
);
113 } else if (IS_ENABLED(CONFIG_MIGRATION
)) {
114 swp_entry_t entry
= pte_to_swp_entry(oldpte
);
116 if (is_write_migration_entry(entry
)) {
119 * A protection check is difficult so
120 * just be safe and disable write
122 make_migration_entry_read(&entry
);
123 newpte
= swp_entry_to_pte(entry
);
124 if (pte_swp_soft_dirty(oldpte
))
125 newpte
= pte_swp_mksoft_dirty(newpte
);
126 set_pte_at(mm
, addr
, pte
, newpte
);
131 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
132 arch_leave_lazy_mmu_mode();
133 pte_unmap_unlock(pte
- 1, ptl
);
138 static inline unsigned long change_pmd_range(struct vm_area_struct
*vma
,
139 pud_t
*pud
, unsigned long addr
, unsigned long end
,
140 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
143 struct mm_struct
*mm
= vma
->vm_mm
;
145 unsigned long pages
= 0;
146 unsigned long nr_huge_updates
= 0;
147 unsigned long mni_start
= 0;
149 pmd
= pmd_offset(pud
, addr
);
151 unsigned long this_pages
;
153 next
= pmd_addr_end(addr
, end
);
154 if (!pmd_trans_huge(*pmd
) && !pmd_devmap(*pmd
)
155 && pmd_none_or_clear_bad(pmd
))
158 /* invoke the mmu notifier if the pmd is populated */
161 mmu_notifier_invalidate_range_start(mm
, mni_start
, end
);
164 if (pmd_trans_huge(*pmd
) || pmd_devmap(*pmd
)) {
165 if (next
- addr
!= HPAGE_PMD_SIZE
) {
166 split_huge_pmd(vma
, pmd
, addr
);
167 if (pmd_trans_unstable(pmd
))
170 int nr_ptes
= change_huge_pmd(vma
, pmd
, addr
,
174 if (nr_ptes
== HPAGE_PMD_NR
) {
175 pages
+= HPAGE_PMD_NR
;
179 /* huge pmd was handled */
183 /* fall through, the trans huge pmd just split */
185 this_pages
= change_pte_range(vma
, pmd
, addr
, next
, newprot
,
186 dirty_accountable
, prot_numa
);
188 } while (pmd
++, addr
= next
, addr
!= end
);
191 mmu_notifier_invalidate_range_end(mm
, mni_start
, end
);
194 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES
, nr_huge_updates
);
198 static inline unsigned long change_pud_range(struct vm_area_struct
*vma
,
199 pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
200 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
204 unsigned long pages
= 0;
206 pud
= pud_offset(pgd
, addr
);
208 next
= pud_addr_end(addr
, end
);
209 if (pud_none_or_clear_bad(pud
))
211 pages
+= change_pmd_range(vma
, pud
, addr
, next
, newprot
,
212 dirty_accountable
, prot_numa
);
213 } while (pud
++, addr
= next
, addr
!= end
);
218 static unsigned long change_protection_range(struct vm_area_struct
*vma
,
219 unsigned long addr
, unsigned long end
, pgprot_t newprot
,
220 int dirty_accountable
, int prot_numa
)
222 struct mm_struct
*mm
= vma
->vm_mm
;
225 unsigned long start
= addr
;
226 unsigned long pages
= 0;
229 pgd
= pgd_offset(mm
, addr
);
230 flush_cache_range(vma
, addr
, end
);
231 set_tlb_flush_pending(mm
);
233 next
= pgd_addr_end(addr
, end
);
234 if (pgd_none_or_clear_bad(pgd
))
236 pages
+= change_pud_range(vma
, pgd
, addr
, next
, newprot
,
237 dirty_accountable
, prot_numa
);
238 } while (pgd
++, addr
= next
, addr
!= end
);
240 /* Only flush the TLB if we actually modified any entries: */
242 flush_tlb_range(vma
, start
, end
);
243 clear_tlb_flush_pending(mm
);
248 unsigned long change_protection(struct vm_area_struct
*vma
, unsigned long start
,
249 unsigned long end
, pgprot_t newprot
,
250 int dirty_accountable
, int prot_numa
)
254 if (is_vm_hugetlb_page(vma
))
255 pages
= hugetlb_change_protection(vma
, start
, end
, newprot
);
257 pages
= change_protection_range(vma
, start
, end
, newprot
, dirty_accountable
, prot_numa
);
263 mprotect_fixup(struct vm_area_struct
*vma
, struct vm_area_struct
**pprev
,
264 unsigned long start
, unsigned long end
, unsigned long newflags
)
266 struct mm_struct
*mm
= vma
->vm_mm
;
267 unsigned long oldflags
= vma
->vm_flags
;
268 long nrpages
= (end
- start
) >> PAGE_SHIFT
;
269 unsigned long charged
= 0;
272 int dirty_accountable
= 0;
274 if (newflags
== oldflags
) {
280 * If we make a private mapping writable we increase our commit;
281 * but (without finer accounting) cannot reduce our commit if we
282 * make it unwritable again. hugetlb mapping were accounted for
283 * even if read-only so there is no need to account for them here
285 if (newflags
& VM_WRITE
) {
286 /* Check space limits when area turns into data. */
287 if (!may_expand_vm(mm
, newflags
, nrpages
) &&
288 may_expand_vm(mm
, oldflags
, nrpages
))
290 if (!(oldflags
& (VM_ACCOUNT
|VM_WRITE
|VM_HUGETLB
|
291 VM_SHARED
|VM_NORESERVE
))) {
293 if (security_vm_enough_memory_mm(mm
, charged
))
295 newflags
|= VM_ACCOUNT
;
300 * First try to merge with previous and/or next vma.
302 pgoff
= vma
->vm_pgoff
+ ((start
- vma
->vm_start
) >> PAGE_SHIFT
);
303 *pprev
= vma_merge(mm
, *pprev
, start
, end
, newflags
,
304 vma
->anon_vma
, vma
->vm_file
, pgoff
, vma_policy(vma
),
305 vma
->vm_userfaultfd_ctx
);
308 VM_WARN_ON((vma
->vm_flags
^ newflags
) & ~VM_SOFTDIRTY
);
314 if (start
!= vma
->vm_start
) {
315 error
= split_vma(mm
, vma
, start
, 1);
320 if (end
!= vma
->vm_end
) {
321 error
= split_vma(mm
, vma
, end
, 0);
328 * vm_flags and vm_page_prot are protected by the mmap_sem
329 * held in write mode.
331 vma
->vm_flags
= newflags
;
332 dirty_accountable
= vma_wants_writenotify(vma
, vma
->vm_page_prot
);
333 vma_set_page_prot(vma
);
335 change_protection(vma
, start
, end
, vma
->vm_page_prot
,
336 dirty_accountable
, 0);
339 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
342 if ((oldflags
& (VM_WRITE
| VM_SHARED
| VM_LOCKED
)) == VM_LOCKED
&&
343 (newflags
& VM_WRITE
)) {
344 populate_vma_page_range(vma
, start
, end
, NULL
);
347 vm_stat_account(mm
, oldflags
, -nrpages
);
348 vm_stat_account(mm
, newflags
, nrpages
);
349 perf_event_mmap(vma
);
353 vm_unacct_memory(charged
);
358 * pkey==-1 when doing a legacy mprotect()
360 static int do_mprotect_pkey(unsigned long start
, size_t len
,
361 unsigned long prot
, int pkey
)
363 unsigned long nstart
, end
, tmp
, reqprot
;
364 struct vm_area_struct
*vma
, *prev
;
366 const int grows
= prot
& (PROT_GROWSDOWN
|PROT_GROWSUP
);
367 const bool rier
= (current
->personality
& READ_IMPLIES_EXEC
) &&
370 prot
&= ~(PROT_GROWSDOWN
|PROT_GROWSUP
);
371 if (grows
== (PROT_GROWSDOWN
|PROT_GROWSUP
)) /* can't be both */
374 if (start
& ~PAGE_MASK
)
378 len
= PAGE_ALIGN(len
);
382 if (!arch_validate_prot(prot
))
387 if (down_write_killable(¤t
->mm
->mmap_sem
))
391 * If userspace did not allocate the pkey, do not let
395 if ((pkey
!= -1) && !mm_pkey_is_allocated(current
->mm
, pkey
))
398 vma
= find_vma(current
->mm
, start
);
403 if (unlikely(grows
& PROT_GROWSDOWN
)) {
404 if (vma
->vm_start
>= end
)
406 start
= vma
->vm_start
;
408 if (!(vma
->vm_flags
& VM_GROWSDOWN
))
411 if (vma
->vm_start
> start
)
413 if (unlikely(grows
& PROT_GROWSUP
)) {
416 if (!(vma
->vm_flags
& VM_GROWSUP
))
420 if (start
> vma
->vm_start
)
423 for (nstart
= start
; ; ) {
424 unsigned long mask_off_old_flags
;
425 unsigned long newflags
;
428 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
430 /* Does the application expect PROT_READ to imply PROT_EXEC */
431 if (rier
&& (vma
->vm_flags
& VM_MAYEXEC
))
435 * Each mprotect() call explicitly passes r/w/x permissions.
436 * If a permission is not passed to mprotect(), it must be
437 * cleared from the VMA.
439 mask_off_old_flags
= VM_READ
| VM_WRITE
| VM_EXEC
|
442 new_vma_pkey
= arch_override_mprotect_pkey(vma
, prot
, pkey
);
443 newflags
= calc_vm_prot_bits(prot
, new_vma_pkey
);
444 newflags
|= (vma
->vm_flags
& ~mask_off_old_flags
);
446 /* newflags >> 4 shift VM_MAY% in place of VM_% */
447 if ((newflags
& ~(newflags
>> 4)) & (VM_READ
| VM_WRITE
| VM_EXEC
)) {
452 error
= security_file_mprotect(vma
, reqprot
, prot
);
459 error
= mprotect_fixup(vma
, &prev
, nstart
, tmp
, newflags
);
464 if (nstart
< prev
->vm_end
)
465 nstart
= prev
->vm_end
;
470 if (!vma
|| vma
->vm_start
!= nstart
) {
477 up_write(¤t
->mm
->mmap_sem
);
481 SYSCALL_DEFINE3(mprotect
, unsigned long, start
, size_t, len
,
484 return do_mprotect_pkey(start
, len
, prot
, -1);
487 SYSCALL_DEFINE4(pkey_mprotect
, unsigned long, start
, size_t, len
,
488 unsigned long, prot
, int, pkey
)
490 return do_mprotect_pkey(start
, len
, prot
, pkey
);
493 SYSCALL_DEFINE2(pkey_alloc
, unsigned long, flags
, unsigned long, init_val
)
498 /* No flags supported yet. */
501 /* check for unsupported init values */
502 if (init_val
& ~PKEY_ACCESS_MASK
)
505 down_write(¤t
->mm
->mmap_sem
);
506 pkey
= mm_pkey_alloc(current
->mm
);
512 ret
= arch_set_user_pkey_access(current
, pkey
, init_val
);
514 mm_pkey_free(current
->mm
, pkey
);
519 up_write(¤t
->mm
->mmap_sem
);
523 SYSCALL_DEFINE1(pkey_free
, int, pkey
)
527 down_write(¤t
->mm
->mmap_sem
);
528 ret
= mm_pkey_free(current
->mm
, pkey
);
529 up_write(¤t
->mm
->mmap_sem
);
532 * We could provie warnings or errors if any VMA still
533 * has the pkey set here.