1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2015 Red Hat, Inc.
9 #include <linux/sched/signal.h>
10 #include <linux/pagemap.h>
11 #include <linux/rmap.h>
12 #include <linux/swap.h>
13 #include <linux/swapops.h>
14 #include <linux/userfaultfd_k.h>
15 #include <linux/mmu_notifier.h>
16 #include <linux/hugetlb.h>
17 #include <linux/shmem_fs.h>
18 #include <asm/tlbflush.h>
21 static __always_inline
22 struct vm_area_struct
*find_dst_vma(struct mm_struct
*dst_mm
,
23 unsigned long dst_start
,
27 * Make sure that the dst range is both valid and fully within a
28 * single existing vma.
30 struct vm_area_struct
*dst_vma
;
32 dst_vma
= find_vma(dst_mm
, dst_start
);
36 if (dst_start
< dst_vma
->vm_start
||
37 dst_start
+ len
> dst_vma
->vm_end
)
41 * Check the vma is registered in uffd, this is required to
42 * enforce the VM_MAYWRITE check done at uffd registration
45 if (!dst_vma
->vm_userfaultfd_ctx
.ctx
)
52 * Install PTEs, to map dst_addr (within dst_vma) to page.
54 * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem
55 * and anon, and for both shared and private VMAs.
57 int mfill_atomic_install_pte(struct mm_struct
*dst_mm
, pmd_t
*dst_pmd
,
58 struct vm_area_struct
*dst_vma
,
59 unsigned long dst_addr
, struct page
*page
,
60 bool newly_allocated
, bool wp_copy
)
63 pte_t _dst_pte
, *dst_pte
;
64 bool writable
= dst_vma
->vm_flags
& VM_WRITE
;
65 bool vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
66 bool page_in_cache
= page
->mapping
;
69 pgoff_t offset
, max_off
;
71 _dst_pte
= mk_pte(page
, dst_vma
->vm_page_prot
);
72 _dst_pte
= pte_mkdirty(_dst_pte
);
73 if (page_in_cache
&& !vm_shared
)
77 _dst_pte
= pte_mkuffd_wp(_dst_pte
);
79 _dst_pte
= pte_mkwrite(_dst_pte
);
82 dst_pte
= pte_offset_map_lock(dst_mm
, dst_pmd
, dst_addr
, &ptl
);
84 if (vma_is_shmem(dst_vma
)) {
85 /* serialize against truncate with the page table lock */
86 inode
= dst_vma
->vm_file
->f_inode
;
87 offset
= linear_page_index(dst_vma
, dst_addr
);
88 max_off
= DIV_ROUND_UP(i_size_read(inode
), PAGE_SIZE
);
90 if (unlikely(offset
>= max_off
))
95 if (!pte_none(*dst_pte
))
99 page_add_file_rmap(page
, false);
101 page_add_new_anon_rmap(page
, dst_vma
, dst_addr
, false);
104 * Must happen after rmap, as mm_counter() checks mapping (via
105 * PageAnon()), which is set by __page_set_anon_rmap().
107 inc_mm_counter(dst_mm
, mm_counter(page
));
110 lru_cache_add_inactive_or_unevictable(page
, dst_vma
);
112 set_pte_at(dst_mm
, dst_addr
, dst_pte
, _dst_pte
);
114 /* No need to invalidate - it was non-present before */
115 update_mmu_cache(dst_vma
, dst_addr
, dst_pte
);
118 pte_unmap_unlock(dst_pte
, ptl
);
122 static int mcopy_atomic_pte(struct mm_struct
*dst_mm
,
124 struct vm_area_struct
*dst_vma
,
125 unsigned long dst_addr
,
126 unsigned long src_addr
,
136 page
= alloc_page_vma(GFP_HIGHUSER_MOVABLE
, dst_vma
, dst_addr
);
140 page_kaddr
= kmap_atomic(page
);
141 ret
= copy_from_user(page_kaddr
,
142 (const void __user
*) src_addr
,
144 kunmap_atomic(page_kaddr
);
146 /* fallback to copy_from_user outside mmap_lock */
150 /* don't free the page */
159 * The memory barrier inside __SetPageUptodate makes sure that
160 * preceding stores to the page contents become visible before
161 * the set_pte_at() write.
163 __SetPageUptodate(page
);
166 if (mem_cgroup_charge(page_folio(page
), dst_mm
, GFP_KERNEL
))
169 ret
= mfill_atomic_install_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
170 page
, true, wp_copy
);
180 static int mfill_zeropage_pte(struct mm_struct
*dst_mm
,
182 struct vm_area_struct
*dst_vma
,
183 unsigned long dst_addr
)
185 pte_t _dst_pte
, *dst_pte
;
188 pgoff_t offset
, max_off
;
191 _dst_pte
= pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr
),
192 dst_vma
->vm_page_prot
));
193 dst_pte
= pte_offset_map_lock(dst_mm
, dst_pmd
, dst_addr
, &ptl
);
194 if (dst_vma
->vm_file
) {
195 /* the shmem MAP_PRIVATE case requires checking the i_size */
196 inode
= dst_vma
->vm_file
->f_inode
;
197 offset
= linear_page_index(dst_vma
, dst_addr
);
198 max_off
= DIV_ROUND_UP(i_size_read(inode
), PAGE_SIZE
);
200 if (unlikely(offset
>= max_off
))
204 if (!pte_none(*dst_pte
))
206 set_pte_at(dst_mm
, dst_addr
, dst_pte
, _dst_pte
);
207 /* No need to invalidate - it was non-present before */
208 update_mmu_cache(dst_vma
, dst_addr
, dst_pte
);
211 pte_unmap_unlock(dst_pte
, ptl
);
215 /* Handles UFFDIO_CONTINUE for all shmem VMAs (shared or private). */
216 static int mcontinue_atomic_pte(struct mm_struct
*dst_mm
,
218 struct vm_area_struct
*dst_vma
,
219 unsigned long dst_addr
,
222 struct inode
*inode
= file_inode(dst_vma
->vm_file
);
223 pgoff_t pgoff
= linear_page_index(dst_vma
, dst_addr
);
227 ret
= shmem_getpage(inode
, pgoff
, &page
, SGP_READ
);
235 if (PageHWPoison(page
)) {
240 ret
= mfill_atomic_install_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
241 page
, false, wp_copy
);
255 static pmd_t
*mm_alloc_pmd(struct mm_struct
*mm
, unsigned long address
)
261 pgd
= pgd_offset(mm
, address
);
262 p4d
= p4d_alloc(mm
, pgd
, address
);
265 pud
= pud_alloc(mm
, p4d
, address
);
269 * Note that we didn't run this because the pmd was
270 * missing, the *pmd may be already established and in
271 * turn it may also be a trans_huge_pmd.
273 return pmd_alloc(mm
, pud
, address
);
276 #ifdef CONFIG_HUGETLB_PAGE
278 * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is
279 * called with mmap_lock held, it will release mmap_lock before returning.
281 static __always_inline ssize_t
__mcopy_atomic_hugetlb(struct mm_struct
*dst_mm
,
282 struct vm_area_struct
*dst_vma
,
283 unsigned long dst_start
,
284 unsigned long src_start
,
286 enum mcopy_atomic_mode mode
)
288 int vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
291 unsigned long src_addr
, dst_addr
;
294 unsigned long vma_hpagesize
;
297 struct address_space
*mapping
;
300 * There is no default zero huge page for all huge page sizes as
301 * supported by hugetlb. A PMD_SIZE huge pages may exist as used
302 * by THP. Since we can not reliably insert a zero page, this
303 * feature is not supported.
305 if (mode
== MCOPY_ATOMIC_ZEROPAGE
) {
306 mmap_read_unlock(dst_mm
);
310 src_addr
= src_start
;
311 dst_addr
= dst_start
;
314 vma_hpagesize
= vma_kernel_pagesize(dst_vma
);
317 * Validate alignment based on huge page size
320 if (dst_start
& (vma_hpagesize
- 1) || len
& (vma_hpagesize
- 1))
325 * On routine entry dst_vma is set. If we had to drop mmap_lock and
326 * retry, dst_vma will be set to NULL and we must lookup again.
330 dst_vma
= find_dst_vma(dst_mm
, dst_start
, len
);
331 if (!dst_vma
|| !is_vm_hugetlb_page(dst_vma
))
335 if (vma_hpagesize
!= vma_kernel_pagesize(dst_vma
))
338 vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
342 * If not shared, ensure the dst_vma has a anon_vma.
346 if (unlikely(anon_vma_prepare(dst_vma
)))
350 while (src_addr
< src_start
+ len
) {
351 BUG_ON(dst_addr
>= dst_start
+ len
);
354 * Serialize via i_mmap_rwsem and hugetlb_fault_mutex.
355 * i_mmap_rwsem ensures the dst_pte remains valid even
356 * in the case of shared pmds. fault mutex prevents
357 * races with other faulting threads.
359 mapping
= dst_vma
->vm_file
->f_mapping
;
360 i_mmap_lock_read(mapping
);
361 idx
= linear_page_index(dst_vma
, dst_addr
);
362 hash
= hugetlb_fault_mutex_hash(mapping
, idx
);
363 mutex_lock(&hugetlb_fault_mutex_table
[hash
]);
366 dst_pte
= huge_pte_alloc(dst_mm
, dst_vma
, dst_addr
, vma_hpagesize
);
368 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
369 i_mmap_unlock_read(mapping
);
373 if (mode
!= MCOPY_ATOMIC_CONTINUE
&&
374 !huge_pte_none(huge_ptep_get(dst_pte
))) {
376 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
377 i_mmap_unlock_read(mapping
);
381 err
= hugetlb_mcopy_atomic_pte(dst_mm
, dst_pte
, dst_vma
,
382 dst_addr
, src_addr
, mode
, &page
);
384 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
385 i_mmap_unlock_read(mapping
);
389 if (unlikely(err
== -ENOENT
)) {
390 mmap_read_unlock(dst_mm
);
393 err
= copy_huge_page_from_user(page
,
394 (const void __user
*)src_addr
,
395 vma_hpagesize
/ PAGE_SIZE
,
401 mmap_read_lock(dst_mm
);
409 dst_addr
+= vma_hpagesize
;
410 src_addr
+= vma_hpagesize
;
411 copied
+= vma_hpagesize
;
413 if (fatal_signal_pending(current
))
421 mmap_read_unlock(dst_mm
);
427 BUG_ON(!copied
&& !err
);
428 return copied
? copied
: err
;
430 #else /* !CONFIG_HUGETLB_PAGE */
431 /* fail at build time if gcc attempts to use this */
432 extern ssize_t
__mcopy_atomic_hugetlb(struct mm_struct
*dst_mm
,
433 struct vm_area_struct
*dst_vma
,
434 unsigned long dst_start
,
435 unsigned long src_start
,
437 enum mcopy_atomic_mode mode
);
438 #endif /* CONFIG_HUGETLB_PAGE */
440 static __always_inline ssize_t
mfill_atomic_pte(struct mm_struct
*dst_mm
,
442 struct vm_area_struct
*dst_vma
,
443 unsigned long dst_addr
,
444 unsigned long src_addr
,
446 enum mcopy_atomic_mode mode
,
451 if (mode
== MCOPY_ATOMIC_CONTINUE
) {
452 return mcontinue_atomic_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
457 * The normal page fault path for a shmem will invoke the
458 * fault, fill the hole in the file and COW it right away. The
459 * result generates plain anonymous memory. So when we are
460 * asked to fill an hole in a MAP_PRIVATE shmem mapping, we'll
461 * generate anonymous memory directly without actually filling
462 * the hole. For the MAP_PRIVATE case the robustness check
463 * only happens in the pagetable (to verify it's still none)
464 * and not in the radix tree.
466 if (!(dst_vma
->vm_flags
& VM_SHARED
)) {
467 if (mode
== MCOPY_ATOMIC_NORMAL
)
468 err
= mcopy_atomic_pte(dst_mm
, dst_pmd
, dst_vma
,
469 dst_addr
, src_addr
, page
,
472 err
= mfill_zeropage_pte(dst_mm
, dst_pmd
,
475 VM_WARN_ON_ONCE(wp_copy
);
476 err
= shmem_mfill_atomic_pte(dst_mm
, dst_pmd
, dst_vma
,
478 mode
!= MCOPY_ATOMIC_NORMAL
,
485 static __always_inline ssize_t
__mcopy_atomic(struct mm_struct
*dst_mm
,
486 unsigned long dst_start
,
487 unsigned long src_start
,
489 enum mcopy_atomic_mode mcopy_mode
,
490 atomic_t
*mmap_changing
,
493 struct vm_area_struct
*dst_vma
;
496 unsigned long src_addr
, dst_addr
;
502 * Sanitize the command parameters:
504 BUG_ON(dst_start
& ~PAGE_MASK
);
505 BUG_ON(len
& ~PAGE_MASK
);
507 /* Does the address range wrap, or is the span zero-sized? */
508 BUG_ON(src_start
+ len
<= src_start
);
509 BUG_ON(dst_start
+ len
<= dst_start
);
511 src_addr
= src_start
;
512 dst_addr
= dst_start
;
516 mmap_read_lock(dst_mm
);
519 * If memory mappings are changing because of non-cooperative
520 * operation (e.g. mremap) running in parallel, bail out and
521 * request the user to retry later
524 if (mmap_changing
&& atomic_read(mmap_changing
))
528 * Make sure the vma is not shared, that the dst range is
529 * both valid and fully within a single existing vma.
532 dst_vma
= find_dst_vma(dst_mm
, dst_start
, len
);
538 * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but
539 * it will overwrite vm_ops, so vma_is_anonymous must return false.
541 if (WARN_ON_ONCE(vma_is_anonymous(dst_vma
) &&
542 dst_vma
->vm_flags
& VM_SHARED
))
546 * validate 'mode' now that we know the dst_vma: don't allow
547 * a wrprotect copy if the userfaultfd didn't register as WP.
549 wp_copy
= mode
& UFFDIO_COPY_MODE_WP
;
550 if (wp_copy
&& !(dst_vma
->vm_flags
& VM_UFFD_WP
))
554 * If this is a HUGETLB vma, pass off to appropriate routine
556 if (is_vm_hugetlb_page(dst_vma
))
557 return __mcopy_atomic_hugetlb(dst_mm
, dst_vma
, dst_start
,
558 src_start
, len
, mcopy_mode
);
560 if (!vma_is_anonymous(dst_vma
) && !vma_is_shmem(dst_vma
))
562 if (!vma_is_shmem(dst_vma
) && mcopy_mode
== MCOPY_ATOMIC_CONTINUE
)
566 * Ensure the dst_vma has a anon_vma or this page
567 * would get a NULL anon_vma when moved in the
571 if (!(dst_vma
->vm_flags
& VM_SHARED
) &&
572 unlikely(anon_vma_prepare(dst_vma
)))
575 while (src_addr
< src_start
+ len
) {
578 BUG_ON(dst_addr
>= dst_start
+ len
);
580 dst_pmd
= mm_alloc_pmd(dst_mm
, dst_addr
);
581 if (unlikely(!dst_pmd
)) {
586 dst_pmdval
= pmd_read_atomic(dst_pmd
);
588 * If the dst_pmd is mapped as THP don't
589 * override it and just be strict.
591 if (unlikely(pmd_trans_huge(dst_pmdval
))) {
595 if (unlikely(pmd_none(dst_pmdval
)) &&
596 unlikely(__pte_alloc(dst_mm
, dst_pmd
))) {
600 /* If an huge pmd materialized from under us fail */
601 if (unlikely(pmd_trans_huge(*dst_pmd
))) {
606 BUG_ON(pmd_none(*dst_pmd
));
607 BUG_ON(pmd_trans_huge(*dst_pmd
));
609 err
= mfill_atomic_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
610 src_addr
, &page
, mcopy_mode
, wp_copy
);
613 if (unlikely(err
== -ENOENT
)) {
616 mmap_read_unlock(dst_mm
);
619 page_kaddr
= kmap(page
);
620 err
= copy_from_user(page_kaddr
,
621 (const void __user
*) src_addr
,
633 dst_addr
+= PAGE_SIZE
;
634 src_addr
+= PAGE_SIZE
;
637 if (fatal_signal_pending(current
))
645 mmap_read_unlock(dst_mm
);
651 BUG_ON(!copied
&& !err
);
652 return copied
? copied
: err
;
655 ssize_t
mcopy_atomic(struct mm_struct
*dst_mm
, unsigned long dst_start
,
656 unsigned long src_start
, unsigned long len
,
657 atomic_t
*mmap_changing
, __u64 mode
)
659 return __mcopy_atomic(dst_mm
, dst_start
, src_start
, len
,
660 MCOPY_ATOMIC_NORMAL
, mmap_changing
, mode
);
663 ssize_t
mfill_zeropage(struct mm_struct
*dst_mm
, unsigned long start
,
664 unsigned long len
, atomic_t
*mmap_changing
)
666 return __mcopy_atomic(dst_mm
, start
, 0, len
, MCOPY_ATOMIC_ZEROPAGE
,
670 ssize_t
mcopy_continue(struct mm_struct
*dst_mm
, unsigned long start
,
671 unsigned long len
, atomic_t
*mmap_changing
)
673 return __mcopy_atomic(dst_mm
, start
, 0, len
, MCOPY_ATOMIC_CONTINUE
,
677 int mwriteprotect_range(struct mm_struct
*dst_mm
, unsigned long start
,
678 unsigned long len
, bool enable_wp
,
679 atomic_t
*mmap_changing
)
681 struct vm_area_struct
*dst_vma
;
686 * Sanitize the command parameters:
688 BUG_ON(start
& ~PAGE_MASK
);
689 BUG_ON(len
& ~PAGE_MASK
);
691 /* Does the address range wrap, or is the span zero-sized? */
692 BUG_ON(start
+ len
<= start
);
694 mmap_read_lock(dst_mm
);
697 * If memory mappings are changing because of non-cooperative
698 * operation (e.g. mremap) running in parallel, bail out and
699 * request the user to retry later
702 if (mmap_changing
&& atomic_read(mmap_changing
))
706 dst_vma
= find_dst_vma(dst_mm
, start
, len
);
708 * Make sure the vma is not shared, that the dst range is
709 * both valid and fully within a single existing vma.
711 if (!dst_vma
|| (dst_vma
->vm_flags
& VM_SHARED
))
713 if (!userfaultfd_wp(dst_vma
))
715 if (!vma_is_anonymous(dst_vma
))
719 newprot
= vm_get_page_prot(dst_vma
->vm_flags
& ~(VM_WRITE
));
721 newprot
= vm_get_page_prot(dst_vma
->vm_flags
);
723 change_protection(dst_vma
, start
, start
+ len
, newprot
,
724 enable_wp
? MM_CP_UFFD_WP
: MM_CP_UFFD_WP_RESOLVE
);
728 mmap_read_unlock(dst_mm
);