]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - mm/mprotect.c
x86/mm, mm/hwpoison: Don't unconditionally unmap kernel 1:1 pages
[mirror_ubuntu-bionic-kernel.git] / mm / mprotect.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
1da177e4
LT
2/*
3 * mm/mprotect.c
4 *
5 * (C) Copyright 1994 Linus Torvalds
6 * (C) Copyright 2002 Christoph Hellwig
7 *
046c6884 8 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
1da177e4
LT
9 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
10 */
11
12#include <linux/mm.h>
13#include <linux/hugetlb.h>
1da177e4
LT
14#include <linux/shm.h>
15#include <linux/mman.h>
16#include <linux/fs.h>
17#include <linux/highmem.h>
18#include <linux/security.h>
19#include <linux/mempolicy.h>
20#include <linux/personality.h>
21#include <linux/syscalls.h>
0697212a
CL
22#include <linux/swap.h>
23#include <linux/swapops.h>
cddb8a5c 24#include <linux/mmu_notifier.h>
64cdd548 25#include <linux/migrate.h>
cdd6c482 26#include <linux/perf_event.h>
e8c24d3a 27#include <linux/pkeys.h>
64a9a34e 28#include <linux/ksm.h>
7c0f6ba6 29#include <linux/uaccess.h>
1da177e4
LT
30#include <asm/pgtable.h>
31#include <asm/cacheflush.h>
e8c24d3a 32#include <asm/mmu_context.h>
1da177e4
LT
33#include <asm/tlbflush.h>
34
36f88188
KS
35#include "internal.h"
36
4b10e7d5 37static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
c1e6098b 38 unsigned long addr, unsigned long end, pgprot_t newprot,
0f19c179 39 int dirty_accountable, int prot_numa)
1da177e4 40{
4b10e7d5 41 struct mm_struct *mm = vma->vm_mm;
0697212a 42 pte_t *pte, oldpte;
705e87c0 43 spinlock_t *ptl;
7da4d641 44 unsigned long pages = 0;
3e321587 45 int target_node = NUMA_NO_NODE;
1da177e4 46
175ad4f1
AA
47 /*
48 * Can be called with only the mmap_sem for reading by
49 * prot_numa so we must check the pmd isn't constantly
50 * changing from under us from pmd_none to pmd_trans_huge
51 * and/or the other way around.
52 */
53 if (pmd_trans_unstable(pmd))
54 return 0;
55
56 /*
57 * The pmd points to a regular pte so the pmd can't change
58 * from under us even if the mmap_sem is only hold for
59 * reading.
60 */
61 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1ad9f620 62
3e321587
AK
63 /* Get target node for single threaded private VMAs */
64 if (prot_numa && !(vma->vm_flags & VM_SHARED) &&
65 atomic_read(&vma->vm_mm->mm_users) == 1)
66 target_node = numa_node_id();
67
3ea27719 68 flush_tlb_batched_pending(vma->vm_mm);
6606c3e0 69 arch_enter_lazy_mmu_mode();
1da177e4 70 do {
0697212a
CL
71 oldpte = *pte;
72 if (pte_present(oldpte)) {
1da177e4 73 pte_t ptent;
b191f9b1 74 bool preserve_write = prot_numa && pte_write(oldpte);
1da177e4 75
e944fd67
MG
76 /*
77 * Avoid trapping faults against the zero or KSM
78 * pages. See similar comment in change_huge_pmd.
79 */
80 if (prot_numa) {
81 struct page *page;
82
83 page = vm_normal_page(vma, addr, oldpte);
84 if (!page || PageKsm(page))
85 continue;
10c1045f
MG
86
87 /* Avoid TLB flush if possible */
88 if (pte_protnone(oldpte))
89 continue;
3e321587
AK
90
91 /*
92 * Don't mess with PTEs if page is already on the node
93 * a single-threaded process is running on.
94 */
95 if (target_node == page_to_nid(page))
96 continue;
e944fd67
MG
97 }
98
8a0516ed
MG
99 ptent = ptep_modify_prot_start(mm, addr, pte);
100 ptent = pte_modify(ptent, newprot);
b191f9b1 101 if (preserve_write)
288bc549 102 ptent = pte_mk_savedwrite(ptent);
4b10e7d5 103
8a0516ed
MG
104 /* Avoid taking write faults for known dirty pages */
105 if (dirty_accountable && pte_dirty(ptent) &&
106 (pte_soft_dirty(ptent) ||
107 !(vma->vm_flags & VM_SOFTDIRTY))) {
108 ptent = pte_mkwrite(ptent);
4b10e7d5 109 }
8a0516ed
MG
110 ptep_modify_prot_commit(mm, addr, pte, ptent);
111 pages++;
0661a336 112 } else if (IS_ENABLED(CONFIG_MIGRATION)) {
0697212a
CL
113 swp_entry_t entry = pte_to_swp_entry(oldpte);
114
115 if (is_write_migration_entry(entry)) {
c3d16e16 116 pte_t newpte;
0697212a
CL
117 /*
118 * A protection check is difficult so
119 * just be safe and disable write
120 */
121 make_migration_entry_read(&entry);
c3d16e16
CG
122 newpte = swp_entry_to_pte(entry);
123 if (pte_swp_soft_dirty(oldpte))
124 newpte = pte_swp_mksoft_dirty(newpte);
125 set_pte_at(mm, addr, pte, newpte);
e920e14c
MG
126
127 pages++;
0697212a 128 }
5042db43
JG
129
130 if (is_write_device_private_entry(entry)) {
131 pte_t newpte;
132
133 /*
134 * We do not preserve soft-dirtiness. See
135 * copy_one_pte() for explanation.
136 */
137 make_device_private_entry_read(&entry);
138 newpte = swp_entry_to_pte(entry);
139 set_pte_at(mm, addr, pte, newpte);
140
141 pages++;
142 }
1da177e4
LT
143 }
144 } while (pte++, addr += PAGE_SIZE, addr != end);
6606c3e0 145 arch_leave_lazy_mmu_mode();
705e87c0 146 pte_unmap_unlock(pte - 1, ptl);
7da4d641
PZ
147
148 return pages;
1da177e4
LT
149}
150
7d12efae
AM
151static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
152 pud_t *pud, unsigned long addr, unsigned long end,
153 pgprot_t newprot, int dirty_accountable, int prot_numa)
1da177e4
LT
154{
155 pmd_t *pmd;
a5338093 156 struct mm_struct *mm = vma->vm_mm;
1da177e4 157 unsigned long next;
7da4d641 158 unsigned long pages = 0;
72403b4a 159 unsigned long nr_huge_updates = 0;
a5338093 160 unsigned long mni_start = 0;
1da177e4
LT
161
162 pmd = pmd_offset(pud, addr);
163 do {
25cbbef1
MG
164 unsigned long this_pages;
165
1da177e4 166 next = pmd_addr_end(addr, end);
84c3fc4e 167 if (!is_swap_pmd(*pmd) && !pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)
5c7fb56e 168 && pmd_none_or_clear_bad(pmd))
4991c09c 169 goto next;
a5338093
RR
170
171 /* invoke the mmu notifier if the pmd is populated */
172 if (!mni_start) {
173 mni_start = addr;
174 mmu_notifier_invalidate_range_start(mm, mni_start, end);
175 }
176
84c3fc4e 177 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
6b9116a6 178 if (next - addr != HPAGE_PMD_SIZE) {
fd60775a 179 __split_huge_pmd(vma, pmd, addr, false, NULL);
6b9116a6 180 } else {
f123d74a 181 int nr_ptes = change_huge_pmd(vma, pmd, addr,
e944fd67 182 newprot, prot_numa);
f123d74a
MG
183
184 if (nr_ptes) {
72403b4a
MG
185 if (nr_ptes == HPAGE_PMD_NR) {
186 pages += HPAGE_PMD_NR;
187 nr_huge_updates++;
188 }
1ad9f620
MG
189
190 /* huge pmd was handled */
4991c09c 191 goto next;
f123d74a 192 }
7da4d641 193 }
88a9ab6e 194 /* fall through, the trans huge pmd just split */
cd7548ab 195 }
25cbbef1 196 this_pages = change_pte_range(vma, pmd, addr, next, newprot,
0f19c179 197 dirty_accountable, prot_numa);
25cbbef1 198 pages += this_pages;
4991c09c
AK
199next:
200 cond_resched();
1da177e4 201 } while (pmd++, addr = next, addr != end);
7da4d641 202
a5338093
RR
203 if (mni_start)
204 mmu_notifier_invalidate_range_end(mm, mni_start, end);
205
72403b4a
MG
206 if (nr_huge_updates)
207 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
7da4d641 208 return pages;
1da177e4
LT
209}
210
7d12efae 211static inline unsigned long change_pud_range(struct vm_area_struct *vma,
c2febafc 212 p4d_t *p4d, unsigned long addr, unsigned long end,
7d12efae 213 pgprot_t newprot, int dirty_accountable, int prot_numa)
1da177e4
LT
214{
215 pud_t *pud;
216 unsigned long next;
7da4d641 217 unsigned long pages = 0;
1da177e4 218
c2febafc 219 pud = pud_offset(p4d, addr);
1da177e4
LT
220 do {
221 next = pud_addr_end(addr, end);
222 if (pud_none_or_clear_bad(pud))
223 continue;
7da4d641 224 pages += change_pmd_range(vma, pud, addr, next, newprot,
4b10e7d5 225 dirty_accountable, prot_numa);
1da177e4 226 } while (pud++, addr = next, addr != end);
7da4d641
PZ
227
228 return pages;
1da177e4
LT
229}
230
c2febafc
KS
231static inline unsigned long change_p4d_range(struct vm_area_struct *vma,
232 pgd_t *pgd, unsigned long addr, unsigned long end,
233 pgprot_t newprot, int dirty_accountable, int prot_numa)
234{
235 p4d_t *p4d;
236 unsigned long next;
237 unsigned long pages = 0;
238
239 p4d = p4d_offset(pgd, addr);
240 do {
241 next = p4d_addr_end(addr, end);
242 if (p4d_none_or_clear_bad(p4d))
243 continue;
244 pages += change_pud_range(vma, p4d, addr, next, newprot,
245 dirty_accountable, prot_numa);
246 } while (p4d++, addr = next, addr != end);
247
248 return pages;
249}
250
7da4d641 251static unsigned long change_protection_range(struct vm_area_struct *vma,
c1e6098b 252 unsigned long addr, unsigned long end, pgprot_t newprot,
4b10e7d5 253 int dirty_accountable, int prot_numa)
1da177e4
LT
254{
255 struct mm_struct *mm = vma->vm_mm;
256 pgd_t *pgd;
257 unsigned long next;
258 unsigned long start = addr;
7da4d641 259 unsigned long pages = 0;
1da177e4
LT
260
261 BUG_ON(addr >= end);
262 pgd = pgd_offset(mm, addr);
263 flush_cache_range(vma, addr, end);
16af97dc 264 inc_tlb_flush_pending(mm);
1da177e4
LT
265 do {
266 next = pgd_addr_end(addr, end);
267 if (pgd_none_or_clear_bad(pgd))
268 continue;
c2febafc 269 pages += change_p4d_range(vma, pgd, addr, next, newprot,
4b10e7d5 270 dirty_accountable, prot_numa);
1da177e4 271 } while (pgd++, addr = next, addr != end);
7da4d641 272
1233d588
IM
273 /* Only flush the TLB if we actually modified any entries: */
274 if (pages)
275 flush_tlb_range(vma, start, end);
16af97dc 276 dec_tlb_flush_pending(mm);
7da4d641
PZ
277
278 return pages;
279}
280
281unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
282 unsigned long end, pgprot_t newprot,
4b10e7d5 283 int dirty_accountable, int prot_numa)
7da4d641 284{
7da4d641
PZ
285 unsigned long pages;
286
7da4d641
PZ
287 if (is_vm_hugetlb_page(vma))
288 pages = hugetlb_change_protection(vma, start, end, newprot);
289 else
4b10e7d5 290 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa);
7da4d641
PZ
291
292 return pages;
1da177e4
LT
293}
294
b6a2fea3 295int
1da177e4
LT
296mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
297 unsigned long start, unsigned long end, unsigned long newflags)
298{
299 struct mm_struct *mm = vma->vm_mm;
300 unsigned long oldflags = vma->vm_flags;
301 long nrpages = (end - start) >> PAGE_SHIFT;
302 unsigned long charged = 0;
1da177e4
LT
303 pgoff_t pgoff;
304 int error;
c1e6098b 305 int dirty_accountable = 0;
1da177e4
LT
306
307 if (newflags == oldflags) {
308 *pprev = vma;
309 return 0;
310 }
311
312 /*
313 * If we make a private mapping writable we increase our commit;
314 * but (without finer accounting) cannot reduce our commit if we
5a6fe125
MG
315 * make it unwritable again. hugetlb mapping were accounted for
316 * even if read-only so there is no need to account for them here
1da177e4
LT
317 */
318 if (newflags & VM_WRITE) {
84638335
KK
319 /* Check space limits when area turns into data. */
320 if (!may_expand_vm(mm, newflags, nrpages) &&
321 may_expand_vm(mm, oldflags, nrpages))
322 return -ENOMEM;
5a6fe125 323 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
cdfd4325 324 VM_SHARED|VM_NORESERVE))) {
1da177e4 325 charged = nrpages;
191c5424 326 if (security_vm_enough_memory_mm(mm, charged))
1da177e4
LT
327 return -ENOMEM;
328 newflags |= VM_ACCOUNT;
329 }
330 }
331
1da177e4
LT
332 /*
333 * First try to merge with previous and/or next vma.
334 */
335 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
336 *pprev = vma_merge(mm, *pprev, start, end, newflags,
19a809af
AA
337 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
338 vma->vm_userfaultfd_ctx);
1da177e4
LT
339 if (*pprev) {
340 vma = *pprev;
e86f15ee 341 VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);
1da177e4
LT
342 goto success;
343 }
344
345 *pprev = vma;
346
347 if (start != vma->vm_start) {
348 error = split_vma(mm, vma, start, 1);
349 if (error)
350 goto fail;
351 }
352
353 if (end != vma->vm_end) {
354 error = split_vma(mm, vma, end, 0);
355 if (error)
356 goto fail;
357 }
358
359success:
360 /*
361 * vm_flags and vm_page_prot are protected by the mmap_sem
362 * held in write mode.
363 */
364 vma->vm_flags = newflags;
6d2329f8 365 dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot);
64e45507 366 vma_set_page_prot(vma);
d08b3851 367
7d12efae
AM
368 change_protection(vma, start, end, vma->vm_page_prot,
369 dirty_accountable, 0);
7da4d641 370
36f88188
KS
371 /*
372 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
373 * fault on access.
374 */
375 if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
376 (newflags & VM_WRITE)) {
377 populate_vma_page_range(vma, start, end, NULL);
378 }
379
84638335
KK
380 vm_stat_account(mm, oldflags, -nrpages);
381 vm_stat_account(mm, newflags, nrpages);
63bfd738 382 perf_event_mmap(vma);
1da177e4
LT
383 return 0;
384
385fail:
386 vm_unacct_memory(charged);
387 return error;
388}
389
7d06d9c9
DH
390/*
391 * pkey==-1 when doing a legacy mprotect()
392 */
393static int do_mprotect_pkey(unsigned long start, size_t len,
394 unsigned long prot, int pkey)
1da177e4 395{
62b5f7d0 396 unsigned long nstart, end, tmp, reqprot;
1da177e4
LT
397 struct vm_area_struct *vma, *prev;
398 int error = -EINVAL;
399 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
f138556d
PK
400 const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
401 (prot & PROT_READ);
402
1da177e4
LT
403 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
404 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
405 return -EINVAL;
406
407 if (start & ~PAGE_MASK)
408 return -EINVAL;
409 if (!len)
410 return 0;
411 len = PAGE_ALIGN(len);
412 end = start + len;
413 if (end <= start)
414 return -ENOMEM;
b845f313 415 if (!arch_validate_prot(prot))
1da177e4
LT
416 return -EINVAL;
417
418 reqprot = prot;
1da177e4 419
dc0ef0df
MH
420 if (down_write_killable(&current->mm->mmap_sem))
421 return -EINTR;
1da177e4 422
e8c24d3a
DH
423 /*
424 * If userspace did not allocate the pkey, do not let
425 * them use it here.
426 */
427 error = -EINVAL;
428 if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
429 goto out;
430
097d5910 431 vma = find_vma(current->mm, start);
1da177e4
LT
432 error = -ENOMEM;
433 if (!vma)
434 goto out;
097d5910 435 prev = vma->vm_prev;
1da177e4
LT
436 if (unlikely(grows & PROT_GROWSDOWN)) {
437 if (vma->vm_start >= end)
438 goto out;
439 start = vma->vm_start;
440 error = -EINVAL;
441 if (!(vma->vm_flags & VM_GROWSDOWN))
442 goto out;
7d12efae 443 } else {
1da177e4
LT
444 if (vma->vm_start > start)
445 goto out;
446 if (unlikely(grows & PROT_GROWSUP)) {
447 end = vma->vm_end;
448 error = -EINVAL;
449 if (!(vma->vm_flags & VM_GROWSUP))
450 goto out;
451 }
452 }
453 if (start > vma->vm_start)
454 prev = vma;
455
456 for (nstart = start ; ; ) {
a8502b67 457 unsigned long mask_off_old_flags;
1da177e4 458 unsigned long newflags;
7d06d9c9 459 int new_vma_pkey;
1da177e4 460
7d12efae 461 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
1da177e4 462
f138556d
PK
463 /* Does the application expect PROT_READ to imply PROT_EXEC */
464 if (rier && (vma->vm_flags & VM_MAYEXEC))
465 prot |= PROT_EXEC;
466
a8502b67
DH
467 /*
468 * Each mprotect() call explicitly passes r/w/x permissions.
469 * If a permission is not passed to mprotect(), it must be
470 * cleared from the VMA.
471 */
472 mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC |
473 ARCH_VM_PKEY_FLAGS;
474
7d06d9c9
DH
475 new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);
476 newflags = calc_vm_prot_bits(prot, new_vma_pkey);
a8502b67 477 newflags |= (vma->vm_flags & ~mask_off_old_flags);
1da177e4 478
7e2cff42
PBG
479 /* newflags >> 4 shift VM_MAY% in place of VM_% */
480 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) {
1da177e4
LT
481 error = -EACCES;
482 goto out;
483 }
484
485 error = security_file_mprotect(vma, reqprot, prot);
486 if (error)
487 goto out;
488
489 tmp = vma->vm_end;
490 if (tmp > end)
491 tmp = end;
492 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
493 if (error)
494 goto out;
495 nstart = tmp;
496
497 if (nstart < prev->vm_end)
498 nstart = prev->vm_end;
499 if (nstart >= end)
500 goto out;
501
502 vma = prev->vm_next;
503 if (!vma || vma->vm_start != nstart) {
504 error = -ENOMEM;
505 goto out;
506 }
f138556d 507 prot = reqprot;
1da177e4
LT
508 }
509out:
510 up_write(&current->mm->mmap_sem);
511 return error;
512}
7d06d9c9
DH
513
514SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
515 unsigned long, prot)
516{
517 return do_mprotect_pkey(start, len, prot, -1);
518}
519
c7142aea
HC
520#ifdef CONFIG_ARCH_HAS_PKEYS
521
7d06d9c9
DH
522SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len,
523 unsigned long, prot, int, pkey)
524{
525 return do_mprotect_pkey(start, len, prot, pkey);
526}
e8c24d3a
DH
527
528SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val)
529{
530 int pkey;
531 int ret;
532
533 /* No flags supported yet. */
534 if (flags)
535 return -EINVAL;
536 /* check for unsupported init values */
537 if (init_val & ~PKEY_ACCESS_MASK)
538 return -EINVAL;
539
540 down_write(&current->mm->mmap_sem);
541 pkey = mm_pkey_alloc(current->mm);
542
543 ret = -ENOSPC;
544 if (pkey == -1)
545 goto out;
546
547 ret = arch_set_user_pkey_access(current, pkey, init_val);
548 if (ret) {
549 mm_pkey_free(current->mm, pkey);
550 goto out;
551 }
552 ret = pkey;
553out:
554 up_write(&current->mm->mmap_sem);
555 return ret;
556}
557
558SYSCALL_DEFINE1(pkey_free, int, pkey)
559{
560 int ret;
561
562 down_write(&current->mm->mmap_sem);
563 ret = mm_pkey_free(current->mm, pkey);
564 up_write(&current->mm->mmap_sem);
565
566 /*
567 * We could provie warnings or errors if any VMA still
568 * has the pkey set here.
569 */
570 return ret;
571}
c7142aea
HC
572
573#endif /* CONFIG_ARCH_HAS_PKEYS */