1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2015 Red Hat, Inc.
9 #include <linux/sched/signal.h>
10 #include <linux/pagemap.h>
11 #include <linux/rmap.h>
12 #include <linux/swap.h>
13 #include <linux/swapops.h>
14 #include <linux/userfaultfd_k.h>
15 #include <linux/mmu_notifier.h>
16 #include <linux/hugetlb.h>
17 #include <linux/shmem_fs.h>
18 #include <asm/tlbflush.h>
21 static __always_inline
22 struct vm_area_struct
*find_dst_vma(struct mm_struct
*dst_mm
,
23 unsigned long dst_start
,
27 * Make sure that the dst range is both valid and fully within a
28 * single existing vma.
30 struct vm_area_struct
*dst_vma
;
32 dst_vma
= find_vma(dst_mm
, dst_start
);
36 if (dst_start
< dst_vma
->vm_start
||
37 dst_start
+ len
> dst_vma
->vm_end
)
41 * Check the vma is registered in uffd, this is required to
42 * enforce the VM_MAYWRITE check done at uffd registration
45 if (!dst_vma
->vm_userfaultfd_ctx
.ctx
)
52 * Install PTEs, to map dst_addr (within dst_vma) to page.
54 * This function handles both MCOPY_ATOMIC_NORMAL and _CONTINUE for both shmem
55 * and anon, and for both shared and private VMAs.
57 int mfill_atomic_install_pte(struct mm_struct
*dst_mm
, pmd_t
*dst_pmd
,
58 struct vm_area_struct
*dst_vma
,
59 unsigned long dst_addr
, struct page
*page
,
60 bool newly_allocated
, bool wp_copy
)
63 pte_t _dst_pte
, *dst_pte
;
64 bool writable
= dst_vma
->vm_flags
& VM_WRITE
;
65 bool vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
66 bool page_in_cache
= page
->mapping
;
69 pgoff_t offset
, max_off
;
71 _dst_pte
= mk_pte(page
, dst_vma
->vm_page_prot
);
72 if (page_in_cache
&& !vm_shared
)
74 if (writable
|| !page_in_cache
)
75 _dst_pte
= pte_mkdirty(_dst_pte
);
78 _dst_pte
= pte_mkuffd_wp(_dst_pte
);
80 _dst_pte
= pte_mkwrite(_dst_pte
);
83 dst_pte
= pte_offset_map_lock(dst_mm
, dst_pmd
, dst_addr
, &ptl
);
85 if (vma_is_shmem(dst_vma
)) {
86 /* serialize against truncate with the page table lock */
87 inode
= dst_vma
->vm_file
->f_inode
;
88 offset
= linear_page_index(dst_vma
, dst_addr
);
89 max_off
= DIV_ROUND_UP(i_size_read(inode
), PAGE_SIZE
);
91 if (unlikely(offset
>= max_off
))
96 if (!pte_none(*dst_pte
))
100 page_add_file_rmap(page
, false);
102 page_add_new_anon_rmap(page
, dst_vma
, dst_addr
, false);
105 * Must happen after rmap, as mm_counter() checks mapping (via
106 * PageAnon()), which is set by __page_set_anon_rmap().
108 inc_mm_counter(dst_mm
, mm_counter(page
));
111 lru_cache_add_inactive_or_unevictable(page
, dst_vma
);
113 set_pte_at(dst_mm
, dst_addr
, dst_pte
, _dst_pte
);
115 /* No need to invalidate - it was non-present before */
116 update_mmu_cache(dst_vma
, dst_addr
, dst_pte
);
119 pte_unmap_unlock(dst_pte
, ptl
);
123 static int mcopy_atomic_pte(struct mm_struct
*dst_mm
,
125 struct vm_area_struct
*dst_vma
,
126 unsigned long dst_addr
,
127 unsigned long src_addr
,
137 page
= alloc_page_vma(GFP_HIGHUSER_MOVABLE
, dst_vma
, dst_addr
);
141 page_kaddr
= kmap_atomic(page
);
142 ret
= copy_from_user(page_kaddr
,
143 (const void __user
*) src_addr
,
145 kunmap_atomic(page_kaddr
);
147 /* fallback to copy_from_user outside mmap_lock */
151 /* don't free the page */
160 * The memory barrier inside __SetPageUptodate makes sure that
161 * preceding stores to the page contents become visible before
162 * the set_pte_at() write.
164 __SetPageUptodate(page
);
167 if (mem_cgroup_charge(page
, dst_mm
, GFP_KERNEL
))
170 ret
= mfill_atomic_install_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
171 page
, true, wp_copy
);
181 static int mfill_zeropage_pte(struct mm_struct
*dst_mm
,
183 struct vm_area_struct
*dst_vma
,
184 unsigned long dst_addr
)
186 pte_t _dst_pte
, *dst_pte
;
189 pgoff_t offset
, max_off
;
192 _dst_pte
= pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr
),
193 dst_vma
->vm_page_prot
));
194 dst_pte
= pte_offset_map_lock(dst_mm
, dst_pmd
, dst_addr
, &ptl
);
195 if (dst_vma
->vm_file
) {
196 /* the shmem MAP_PRIVATE case requires checking the i_size */
197 inode
= dst_vma
->vm_file
->f_inode
;
198 offset
= linear_page_index(dst_vma
, dst_addr
);
199 max_off
= DIV_ROUND_UP(i_size_read(inode
), PAGE_SIZE
);
201 if (unlikely(offset
>= max_off
))
205 if (!pte_none(*dst_pte
))
207 set_pte_at(dst_mm
, dst_addr
, dst_pte
, _dst_pte
);
208 /* No need to invalidate - it was non-present before */
209 update_mmu_cache(dst_vma
, dst_addr
, dst_pte
);
212 pte_unmap_unlock(dst_pte
, ptl
);
216 /* Handles UFFDIO_CONTINUE for all shmem VMAs (shared or private). */
217 static int mcontinue_atomic_pte(struct mm_struct
*dst_mm
,
219 struct vm_area_struct
*dst_vma
,
220 unsigned long dst_addr
,
223 struct inode
*inode
= file_inode(dst_vma
->vm_file
);
224 pgoff_t pgoff
= linear_page_index(dst_vma
, dst_addr
);
228 ret
= shmem_getpage(inode
, pgoff
, &page
, SGP_READ
);
236 ret
= mfill_atomic_install_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
237 page
, false, wp_copy
);
251 static pmd_t
*mm_alloc_pmd(struct mm_struct
*mm
, unsigned long address
)
257 pgd
= pgd_offset(mm
, address
);
258 p4d
= p4d_alloc(mm
, pgd
, address
);
261 pud
= pud_alloc(mm
, p4d
, address
);
265 * Note that we didn't run this because the pmd was
266 * missing, the *pmd may be already established and in
267 * turn it may also be a trans_huge_pmd.
269 return pmd_alloc(mm
, pud
, address
);
272 #ifdef CONFIG_HUGETLB_PAGE
274 * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is
275 * called with mmap_lock held, it will release mmap_lock before returning.
277 static __always_inline ssize_t
__mcopy_atomic_hugetlb(struct mm_struct
*dst_mm
,
278 struct vm_area_struct
*dst_vma
,
279 unsigned long dst_start
,
280 unsigned long src_start
,
282 enum mcopy_atomic_mode mode
)
284 int vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
287 unsigned long src_addr
, dst_addr
;
290 unsigned long vma_hpagesize
;
293 struct address_space
*mapping
;
296 * There is no default zero huge page for all huge page sizes as
297 * supported by hugetlb. A PMD_SIZE huge pages may exist as used
298 * by THP. Since we can not reliably insert a zero page, this
299 * feature is not supported.
301 if (mode
== MCOPY_ATOMIC_ZEROPAGE
) {
302 mmap_read_unlock(dst_mm
);
306 src_addr
= src_start
;
307 dst_addr
= dst_start
;
310 vma_hpagesize
= vma_kernel_pagesize(dst_vma
);
313 * Validate alignment based on huge page size
316 if (dst_start
& (vma_hpagesize
- 1) || len
& (vma_hpagesize
- 1))
321 * On routine entry dst_vma is set. If we had to drop mmap_lock and
322 * retry, dst_vma will be set to NULL and we must lookup again.
326 dst_vma
= find_dst_vma(dst_mm
, dst_start
, len
);
327 if (!dst_vma
|| !is_vm_hugetlb_page(dst_vma
))
331 if (vma_hpagesize
!= vma_kernel_pagesize(dst_vma
))
334 vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
338 * If not shared, ensure the dst_vma has a anon_vma.
342 if (unlikely(anon_vma_prepare(dst_vma
)))
346 while (src_addr
< src_start
+ len
) {
347 BUG_ON(dst_addr
>= dst_start
+ len
);
350 * Serialize via i_mmap_rwsem and hugetlb_fault_mutex.
351 * i_mmap_rwsem ensures the dst_pte remains valid even
352 * in the case of shared pmds. fault mutex prevents
353 * races with other faulting threads.
355 mapping
= dst_vma
->vm_file
->f_mapping
;
356 i_mmap_lock_read(mapping
);
357 idx
= linear_page_index(dst_vma
, dst_addr
);
358 hash
= hugetlb_fault_mutex_hash(mapping
, idx
);
359 mutex_lock(&hugetlb_fault_mutex_table
[hash
]);
362 dst_pte
= huge_pte_alloc(dst_mm
, dst_vma
, dst_addr
, vma_hpagesize
);
364 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
365 i_mmap_unlock_read(mapping
);
369 if (mode
!= MCOPY_ATOMIC_CONTINUE
&&
370 !huge_pte_none(huge_ptep_get(dst_pte
))) {
372 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
373 i_mmap_unlock_read(mapping
);
377 err
= hugetlb_mcopy_atomic_pte(dst_mm
, dst_pte
, dst_vma
,
378 dst_addr
, src_addr
, mode
, &page
);
380 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
381 i_mmap_unlock_read(mapping
);
385 if (unlikely(err
== -ENOENT
)) {
386 mmap_read_unlock(dst_mm
);
389 err
= copy_huge_page_from_user(page
,
390 (const void __user
*)src_addr
,
391 vma_hpagesize
/ PAGE_SIZE
,
397 mmap_read_lock(dst_mm
);
405 dst_addr
+= vma_hpagesize
;
406 src_addr
+= vma_hpagesize
;
407 copied
+= vma_hpagesize
;
409 if (fatal_signal_pending(current
))
417 mmap_read_unlock(dst_mm
);
423 BUG_ON(!copied
&& !err
);
424 return copied
? copied
: err
;
426 #else /* !CONFIG_HUGETLB_PAGE */
427 /* fail at build time if gcc attempts to use this */
428 extern ssize_t
__mcopy_atomic_hugetlb(struct mm_struct
*dst_mm
,
429 struct vm_area_struct
*dst_vma
,
430 unsigned long dst_start
,
431 unsigned long src_start
,
433 enum mcopy_atomic_mode mode
);
434 #endif /* CONFIG_HUGETLB_PAGE */
436 static __always_inline ssize_t
mfill_atomic_pte(struct mm_struct
*dst_mm
,
438 struct vm_area_struct
*dst_vma
,
439 unsigned long dst_addr
,
440 unsigned long src_addr
,
442 enum mcopy_atomic_mode mode
,
447 if (mode
== MCOPY_ATOMIC_CONTINUE
) {
448 return mcontinue_atomic_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
453 * The normal page fault path for a shmem will invoke the
454 * fault, fill the hole in the file and COW it right away. The
455 * result generates plain anonymous memory. So when we are
456 * asked to fill an hole in a MAP_PRIVATE shmem mapping, we'll
457 * generate anonymous memory directly without actually filling
458 * the hole. For the MAP_PRIVATE case the robustness check
459 * only happens in the pagetable (to verify it's still none)
460 * and not in the radix tree.
462 if (!(dst_vma
->vm_flags
& VM_SHARED
)) {
463 if (mode
== MCOPY_ATOMIC_NORMAL
)
464 err
= mcopy_atomic_pte(dst_mm
, dst_pmd
, dst_vma
,
465 dst_addr
, src_addr
, page
,
468 err
= mfill_zeropage_pte(dst_mm
, dst_pmd
,
471 VM_WARN_ON_ONCE(wp_copy
);
472 err
= shmem_mfill_atomic_pte(dst_mm
, dst_pmd
, dst_vma
,
474 mode
!= MCOPY_ATOMIC_NORMAL
,
481 static __always_inline ssize_t
__mcopy_atomic(struct mm_struct
*dst_mm
,
482 unsigned long dst_start
,
483 unsigned long src_start
,
485 enum mcopy_atomic_mode mcopy_mode
,
489 struct vm_area_struct
*dst_vma
;
492 unsigned long src_addr
, dst_addr
;
498 * Sanitize the command parameters:
500 BUG_ON(dst_start
& ~PAGE_MASK
);
501 BUG_ON(len
& ~PAGE_MASK
);
503 /* Does the address range wrap, or is the span zero-sized? */
504 BUG_ON(src_start
+ len
<= src_start
);
505 BUG_ON(dst_start
+ len
<= dst_start
);
507 src_addr
= src_start
;
508 dst_addr
= dst_start
;
512 mmap_read_lock(dst_mm
);
515 * If memory mappings are changing because of non-cooperative
516 * operation (e.g. mremap) running in parallel, bail out and
517 * request the user to retry later
520 if (mmap_changing
&& READ_ONCE(*mmap_changing
))
524 * Make sure the vma is not shared, that the dst range is
525 * both valid and fully within a single existing vma.
528 dst_vma
= find_dst_vma(dst_mm
, dst_start
, len
);
534 * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but
535 * it will overwrite vm_ops, so vma_is_anonymous must return false.
537 if (WARN_ON_ONCE(vma_is_anonymous(dst_vma
) &&
538 dst_vma
->vm_flags
& VM_SHARED
))
542 * validate 'mode' now that we know the dst_vma: don't allow
543 * a wrprotect copy if the userfaultfd didn't register as WP.
545 wp_copy
= mode
& UFFDIO_COPY_MODE_WP
;
546 if (wp_copy
&& !(dst_vma
->vm_flags
& VM_UFFD_WP
))
550 * If this is a HUGETLB vma, pass off to appropriate routine
552 if (is_vm_hugetlb_page(dst_vma
))
553 return __mcopy_atomic_hugetlb(dst_mm
, dst_vma
, dst_start
,
554 src_start
, len
, mcopy_mode
);
556 if (!vma_is_anonymous(dst_vma
) && !vma_is_shmem(dst_vma
))
558 if (!vma_is_shmem(dst_vma
) && mcopy_mode
== MCOPY_ATOMIC_CONTINUE
)
562 * Ensure the dst_vma has a anon_vma or this page
563 * would get a NULL anon_vma when moved in the
567 if (!(dst_vma
->vm_flags
& VM_SHARED
) &&
568 unlikely(anon_vma_prepare(dst_vma
)))
571 while (src_addr
< src_start
+ len
) {
574 BUG_ON(dst_addr
>= dst_start
+ len
);
576 dst_pmd
= mm_alloc_pmd(dst_mm
, dst_addr
);
577 if (unlikely(!dst_pmd
)) {
582 dst_pmdval
= pmd_read_atomic(dst_pmd
);
584 * If the dst_pmd is mapped as THP don't
585 * override it and just be strict.
587 if (unlikely(pmd_trans_huge(dst_pmdval
))) {
591 if (unlikely(pmd_none(dst_pmdval
)) &&
592 unlikely(__pte_alloc(dst_mm
, dst_pmd
))) {
596 /* If an huge pmd materialized from under us fail */
597 if (unlikely(pmd_trans_huge(*dst_pmd
))) {
602 BUG_ON(pmd_none(*dst_pmd
));
603 BUG_ON(pmd_trans_huge(*dst_pmd
));
605 err
= mfill_atomic_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
606 src_addr
, &page
, mcopy_mode
, wp_copy
);
609 if (unlikely(err
== -ENOENT
)) {
612 mmap_read_unlock(dst_mm
);
615 page_kaddr
= kmap(page
);
616 err
= copy_from_user(page_kaddr
,
617 (const void __user
*) src_addr
,
629 dst_addr
+= PAGE_SIZE
;
630 src_addr
+= PAGE_SIZE
;
633 if (fatal_signal_pending(current
))
641 mmap_read_unlock(dst_mm
);
647 BUG_ON(!copied
&& !err
);
648 return copied
? copied
: err
;
651 ssize_t
mcopy_atomic(struct mm_struct
*dst_mm
, unsigned long dst_start
,
652 unsigned long src_start
, unsigned long len
,
653 bool *mmap_changing
, __u64 mode
)
655 return __mcopy_atomic(dst_mm
, dst_start
, src_start
, len
,
656 MCOPY_ATOMIC_NORMAL
, mmap_changing
, mode
);
659 ssize_t
mfill_zeropage(struct mm_struct
*dst_mm
, unsigned long start
,
660 unsigned long len
, bool *mmap_changing
)
662 return __mcopy_atomic(dst_mm
, start
, 0, len
, MCOPY_ATOMIC_ZEROPAGE
,
666 ssize_t
mcopy_continue(struct mm_struct
*dst_mm
, unsigned long start
,
667 unsigned long len
, bool *mmap_changing
)
669 return __mcopy_atomic(dst_mm
, start
, 0, len
, MCOPY_ATOMIC_CONTINUE
,
673 int mwriteprotect_range(struct mm_struct
*dst_mm
, unsigned long start
,
674 unsigned long len
, bool enable_wp
, bool *mmap_changing
)
676 struct vm_area_struct
*dst_vma
;
681 * Sanitize the command parameters:
683 BUG_ON(start
& ~PAGE_MASK
);
684 BUG_ON(len
& ~PAGE_MASK
);
686 /* Does the address range wrap, or is the span zero-sized? */
687 BUG_ON(start
+ len
<= start
);
689 mmap_read_lock(dst_mm
);
692 * If memory mappings are changing because of non-cooperative
693 * operation (e.g. mremap) running in parallel, bail out and
694 * request the user to retry later
697 if (mmap_changing
&& READ_ONCE(*mmap_changing
))
701 dst_vma
= find_dst_vma(dst_mm
, start
, len
);
703 * Make sure the vma is not shared, that the dst range is
704 * both valid and fully within a single existing vma.
706 if (!dst_vma
|| (dst_vma
->vm_flags
& VM_SHARED
))
708 if (!userfaultfd_wp(dst_vma
))
710 if (!vma_is_anonymous(dst_vma
))
714 newprot
= vm_get_page_prot(dst_vma
->vm_flags
& ~(VM_WRITE
));
716 newprot
= vm_get_page_prot(dst_vma
->vm_flags
);
718 change_protection(dst_vma
, start
, start
+ len
, newprot
,
719 enable_wp
? MM_CP_UFFD_WP
: MM_CP_UFFD_WP_RESOLVE
);
723 mmap_read_unlock(dst_mm
);