4 * Copyright (C) 2015 Red Hat, Inc.
6 * This work is licensed under the terms of the GNU GPL, version 2. See
7 * the COPYING file in the top-level directory.
11 #include <linux/sched/signal.h>
12 #include <linux/pagemap.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15 #include <linux/swapops.h>
16 #include <linux/userfaultfd_k.h>
17 #include <linux/mmu_notifier.h>
18 #include <linux/hugetlb.h>
19 #include <linux/pagemap.h>
20 #include <linux/shmem_fs.h>
21 #include <asm/tlbflush.h>
24 static int mcopy_atomic_pte(struct mm_struct
*dst_mm
,
26 struct vm_area_struct
*dst_vma
,
27 unsigned long dst_addr
,
28 unsigned long src_addr
,
31 struct mem_cgroup
*memcg
;
32 pte_t _dst_pte
, *dst_pte
;
40 page
= alloc_page_vma(GFP_HIGHUSER_MOVABLE
, dst_vma
, dst_addr
);
44 page_kaddr
= kmap_atomic(page
);
45 ret
= copy_from_user(page_kaddr
,
46 (const void __user
*) src_addr
,
48 kunmap_atomic(page_kaddr
);
50 /* fallback to copy_from_user outside mmap_sem */
54 /* don't free the page */
63 * The memory barrier inside __SetPageUptodate makes sure that
64 * preceeding stores to the page contents become visible before
65 * the set_pte_at() write.
67 __SetPageUptodate(page
);
70 if (mem_cgroup_try_charge(page
, dst_mm
, GFP_KERNEL
, &memcg
, false))
73 _dst_pte
= mk_pte(page
, dst_vma
->vm_page_prot
);
74 if (dst_vma
->vm_flags
& VM_WRITE
)
75 _dst_pte
= pte_mkwrite(pte_mkdirty(_dst_pte
));
78 dst_pte
= pte_offset_map_lock(dst_mm
, dst_pmd
, dst_addr
, &ptl
);
79 if (!pte_none(*dst_pte
))
80 goto out_release_uncharge_unlock
;
82 inc_mm_counter(dst_mm
, MM_ANONPAGES
);
83 page_add_new_anon_rmap(page
, dst_vma
, dst_addr
, false);
84 mem_cgroup_commit_charge(page
, memcg
, false, false);
85 lru_cache_add_active_or_unevictable(page
, dst_vma
);
87 set_pte_at(dst_mm
, dst_addr
, dst_pte
, _dst_pte
);
89 /* No need to invalidate - it was non-present before */
90 update_mmu_cache(dst_vma
, dst_addr
, dst_pte
);
92 pte_unmap_unlock(dst_pte
, ptl
);
96 out_release_uncharge_unlock
:
97 pte_unmap_unlock(dst_pte
, ptl
);
98 mem_cgroup_cancel_charge(page
, memcg
, false);
104 static int mfill_zeropage_pte(struct mm_struct
*dst_mm
,
106 struct vm_area_struct
*dst_vma
,
107 unsigned long dst_addr
)
109 pte_t _dst_pte
, *dst_pte
;
113 _dst_pte
= pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr
),
114 dst_vma
->vm_page_prot
));
116 dst_pte
= pte_offset_map_lock(dst_mm
, dst_pmd
, dst_addr
, &ptl
);
117 if (!pte_none(*dst_pte
))
119 set_pte_at(dst_mm
, dst_addr
, dst_pte
, _dst_pte
);
120 /* No need to invalidate - it was non-present before */
121 update_mmu_cache(dst_vma
, dst_addr
, dst_pte
);
124 pte_unmap_unlock(dst_pte
, ptl
);
128 static pmd_t
*mm_alloc_pmd(struct mm_struct
*mm
, unsigned long address
)
134 pgd
= pgd_offset(mm
, address
);
135 p4d
= p4d_alloc(mm
, pgd
, address
);
138 pud
= pud_alloc(mm
, p4d
, address
);
142 * Note that we didn't run this because the pmd was
143 * missing, the *pmd may be already established and in
144 * turn it may also be a trans_huge_pmd.
146 return pmd_alloc(mm
, pud
, address
);
149 #ifdef CONFIG_HUGETLB_PAGE
151 * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is
152 * called with mmap_sem held, it will release mmap_sem before returning.
154 static __always_inline ssize_t
__mcopy_atomic_hugetlb(struct mm_struct
*dst_mm
,
155 struct vm_area_struct
*dst_vma
,
156 unsigned long dst_start
,
157 unsigned long src_start
,
161 int vm_alloc_shared
= dst_vma
->vm_flags
& VM_SHARED
;
162 int vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
165 unsigned long src_addr
, dst_addr
;
169 unsigned long vma_hpagesize
;
172 struct address_space
*mapping
;
175 * There is no default zero huge page for all huge page sizes as
176 * supported by hugetlb. A PMD_SIZE huge pages may exist as used
177 * by THP. Since we can not reliably insert a zero page, this
178 * feature is not supported.
181 up_read(&dst_mm
->mmap_sem
);
185 src_addr
= src_start
;
186 dst_addr
= dst_start
;
189 vma_hpagesize
= vma_kernel_pagesize(dst_vma
);
192 * Validate alignment based on huge page size
195 if (dst_start
& (vma_hpagesize
- 1) || len
& (vma_hpagesize
- 1))
200 * On routine entry dst_vma is set. If we had to drop mmap_sem and
201 * retry, dst_vma will be set to NULL and we must lookup again.
205 dst_vma
= find_vma(dst_mm
, dst_start
);
206 if (!dst_vma
|| !is_vm_hugetlb_page(dst_vma
))
209 * Check the vma is registered in uffd, this is
210 * required to enforce the VM_MAYWRITE check done at
211 * uffd registration time.
213 if (!dst_vma
->vm_userfaultfd_ctx
.ctx
)
216 if (dst_start
< dst_vma
->vm_start
||
217 dst_start
+ len
> dst_vma
->vm_end
)
221 if (vma_hpagesize
!= vma_kernel_pagesize(dst_vma
))
224 vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
227 if (WARN_ON(dst_addr
& (vma_hpagesize
- 1) ||
228 (len
- copied
) & (vma_hpagesize
- 1)))
232 * If not shared, ensure the dst_vma has a anon_vma.
236 if (unlikely(anon_vma_prepare(dst_vma
)))
240 h
= hstate_vma(dst_vma
);
242 while (src_addr
< src_start
+ len
) {
245 BUG_ON(dst_addr
>= dst_start
+ len
);
246 VM_BUG_ON(dst_addr
& ~huge_page_mask(h
));
249 * Serialize via hugetlb_fault_mutex
251 idx
= linear_page_index(dst_vma
, dst_addr
);
252 mapping
= dst_vma
->vm_file
->f_mapping
;
253 hash
= hugetlb_fault_mutex_hash(h
, dst_mm
, dst_vma
, mapping
,
255 mutex_lock(&hugetlb_fault_mutex_table
[hash
]);
258 dst_pte
= huge_pte_alloc(dst_mm
, dst_addr
, huge_page_size(h
));
260 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
265 dst_pteval
= huge_ptep_get(dst_pte
);
266 if (!huge_pte_none(dst_pteval
)) {
267 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
271 err
= hugetlb_mcopy_atomic_pte(dst_mm
, dst_pte
, dst_vma
,
272 dst_addr
, src_addr
, &page
);
274 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
275 vm_alloc_shared
= vm_shared
;
279 if (unlikely(err
== -ENOENT
)) {
280 up_read(&dst_mm
->mmap_sem
);
283 err
= copy_huge_page_from_user(page
,
284 (const void __user
*)src_addr
,
285 pages_per_huge_page(h
), true);
290 down_read(&dst_mm
->mmap_sem
);
298 dst_addr
+= vma_hpagesize
;
299 src_addr
+= vma_hpagesize
;
300 copied
+= vma_hpagesize
;
302 if (fatal_signal_pending(current
))
310 up_read(&dst_mm
->mmap_sem
);
314 * We encountered an error and are about to free a newly
315 * allocated huge page.
317 * Reservation handling is very subtle, and is different for
318 * private and shared mappings. See the routine
319 * restore_reserve_on_error for details. Unfortunately, we
320 * can not call restore_reserve_on_error now as it would
321 * require holding mmap_sem.
323 * If a reservation for the page existed in the reservation
324 * map of a private mapping, the map was modified to indicate
325 * the reservation was consumed when the page was allocated.
326 * We clear the PagePrivate flag now so that the global
327 * reserve count will not be incremented in free_huge_page.
328 * The reservation map will still indicate the reservation
329 * was consumed and possibly prevent later page allocation.
330 * This is better than leaking a global reservation. If no
331 * reservation existed, it is still safe to clear PagePrivate
332 * as no adjustments to reservation counts were made during
335 * The reservation map for shared mappings indicates which
336 * pages have reservations. When a huge page is allocated
337 * for an address with a reservation, no change is made to
338 * the reserve map. In this case PagePrivate will be set
339 * to indicate that the global reservation count should be
340 * incremented when the page is freed. This is the desired
341 * behavior. However, when a huge page is allocated for an
342 * address without a reservation a reservation entry is added
343 * to the reservation map, and PagePrivate will not be set.
344 * When the page is freed, the global reserve count will NOT
345 * be incremented and it will appear as though we have leaked
346 * reserved page. In this case, set PagePrivate so that the
347 * global reserve count will be incremented to match the
348 * reservation map entry which was created.
350 * Note that vm_alloc_shared is based on the flags of the vma
351 * for which the page was originally allocated. dst_vma could
352 * be different or NULL on error.
355 SetPagePrivate(page
);
357 ClearPagePrivate(page
);
362 BUG_ON(!copied
&& !err
);
363 return copied
? copied
: err
;
365 #else /* !CONFIG_HUGETLB_PAGE */
366 /* fail at build time if gcc attempts to use this */
367 extern ssize_t
__mcopy_atomic_hugetlb(struct mm_struct
*dst_mm
,
368 struct vm_area_struct
*dst_vma
,
369 unsigned long dst_start
,
370 unsigned long src_start
,
373 #endif /* CONFIG_HUGETLB_PAGE */
375 static __always_inline ssize_t
mfill_atomic_pte(struct mm_struct
*dst_mm
,
377 struct vm_area_struct
*dst_vma
,
378 unsigned long dst_addr
,
379 unsigned long src_addr
,
386 * The normal page fault path for a shmem will invoke the
387 * fault, fill the hole in the file and COW it right away. The
388 * result generates plain anonymous memory. So when we are
389 * asked to fill an hole in a MAP_PRIVATE shmem mapping, we'll
390 * generate anonymous memory directly without actually filling
391 * the hole. For the MAP_PRIVATE case the robustness check
392 * only happens in the pagetable (to verify it's still none)
393 * and not in the radix tree.
395 if (!(dst_vma
->vm_flags
& VM_SHARED
)) {
397 err
= mcopy_atomic_pte(dst_mm
, dst_pmd
, dst_vma
,
398 dst_addr
, src_addr
, page
);
400 err
= mfill_zeropage_pte(dst_mm
, dst_pmd
,
404 err
= shmem_mcopy_atomic_pte(dst_mm
, dst_pmd
,
408 err
= shmem_mfill_zeropage_pte(dst_mm
, dst_pmd
,
415 static __always_inline ssize_t
__mcopy_atomic(struct mm_struct
*dst_mm
,
416 unsigned long dst_start
,
417 unsigned long src_start
,
421 struct vm_area_struct
*dst_vma
;
424 unsigned long src_addr
, dst_addr
;
429 * Sanitize the command parameters:
431 BUG_ON(dst_start
& ~PAGE_MASK
);
432 BUG_ON(len
& ~PAGE_MASK
);
434 /* Does the address range wrap, or is the span zero-sized? */
435 BUG_ON(src_start
+ len
<= src_start
);
436 BUG_ON(dst_start
+ len
<= dst_start
);
438 src_addr
= src_start
;
439 dst_addr
= dst_start
;
443 down_read(&dst_mm
->mmap_sem
);
446 * Make sure the vma is not shared, that the dst range is
447 * both valid and fully within a single existing vma.
450 dst_vma
= find_vma(dst_mm
, dst_start
);
454 * Check the vma is registered in uffd, this is required to
455 * enforce the VM_MAYWRITE check done at uffd registration
458 if (!dst_vma
->vm_userfaultfd_ctx
.ctx
)
461 if (dst_start
< dst_vma
->vm_start
||
462 dst_start
+ len
> dst_vma
->vm_end
)
467 * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but
468 * it will overwrite vm_ops, so vma_is_anonymous must return false.
470 if (WARN_ON_ONCE(vma_is_anonymous(dst_vma
) &&
471 dst_vma
->vm_flags
& VM_SHARED
))
475 * If this is a HUGETLB vma, pass off to appropriate routine
477 if (is_vm_hugetlb_page(dst_vma
))
478 return __mcopy_atomic_hugetlb(dst_mm
, dst_vma
, dst_start
,
479 src_start
, len
, zeropage
);
481 if (!vma_is_anonymous(dst_vma
) && !vma_is_shmem(dst_vma
))
485 * Ensure the dst_vma has a anon_vma or this page
486 * would get a NULL anon_vma when moved in the
490 if (!(dst_vma
->vm_flags
& VM_SHARED
) &&
491 unlikely(anon_vma_prepare(dst_vma
)))
494 while (src_addr
< src_start
+ len
) {
497 BUG_ON(dst_addr
>= dst_start
+ len
);
499 dst_pmd
= mm_alloc_pmd(dst_mm
, dst_addr
);
500 if (unlikely(!dst_pmd
)) {
505 dst_pmdval
= pmd_read_atomic(dst_pmd
);
507 * If the dst_pmd is mapped as THP don't
508 * override it and just be strict.
510 if (unlikely(pmd_trans_huge(dst_pmdval
))) {
514 if (unlikely(pmd_none(dst_pmdval
)) &&
515 unlikely(__pte_alloc(dst_mm
, dst_pmd
, dst_addr
))) {
519 /* If an huge pmd materialized from under us fail */
520 if (unlikely(pmd_trans_huge(*dst_pmd
))) {
525 BUG_ON(pmd_none(*dst_pmd
));
526 BUG_ON(pmd_trans_huge(*dst_pmd
));
528 err
= mfill_atomic_pte(dst_mm
, dst_pmd
, dst_vma
, dst_addr
,
529 src_addr
, &page
, zeropage
);
532 if (unlikely(err
== -ENOENT
)) {
535 up_read(&dst_mm
->mmap_sem
);
538 page_kaddr
= kmap(page
);
539 err
= copy_from_user(page_kaddr
,
540 (const void __user
*) src_addr
,
552 dst_addr
+= PAGE_SIZE
;
553 src_addr
+= PAGE_SIZE
;
556 if (fatal_signal_pending(current
))
564 up_read(&dst_mm
->mmap_sem
);
570 BUG_ON(!copied
&& !err
);
571 return copied
? copied
: err
;
574 ssize_t
mcopy_atomic(struct mm_struct
*dst_mm
, unsigned long dst_start
,
575 unsigned long src_start
, unsigned long len
)
577 return __mcopy_atomic(dst_mm
, dst_start
, src_start
, len
, false);
580 ssize_t
mfill_zeropage(struct mm_struct
*dst_mm
, unsigned long start
,
583 return __mcopy_atomic(dst_mm
, start
, 0, len
, true);