4 * Copyright (C) 2015 Red Hat, Inc.
6 * This work is licensed under the terms of the GNU GPL, version 2. See
7 * the COPYING file in the top-level directory.
11 #include <linux/pagemap.h>
12 #include <linux/rmap.h>
13 #include <linux/swap.h>
14 #include <linux/swapops.h>
15 #include <linux/userfaultfd_k.h>
16 #include <linux/mmu_notifier.h>
17 #include <linux/hugetlb.h>
18 #include <linux/pagemap.h>
19 #include <linux/shmem_fs.h>
20 #include <asm/tlbflush.h>
23 static int mcopy_atomic_pte(struct mm_struct
*dst_mm
,
25 struct vm_area_struct
*dst_vma
,
26 unsigned long dst_addr
,
27 unsigned long src_addr
,
30 struct mem_cgroup
*memcg
;
31 pte_t _dst_pte
, *dst_pte
;
39 page
= alloc_page_vma(GFP_HIGHUSER_MOVABLE
, dst_vma
, dst_addr
);
43 page_kaddr
= kmap_atomic(page
);
44 ret
= copy_from_user(page_kaddr
,
45 (const void __user
*) src_addr
,
47 kunmap_atomic(page_kaddr
);
49 /* fallback to copy_from_user outside mmap_sem */
53 /* don't free the page */
62 * The memory barrier inside __SetPageUptodate makes sure that
63 * preceeding stores to the page contents become visible before
64 * the set_pte_at() write.
66 __SetPageUptodate(page
);
69 if (mem_cgroup_try_charge(page
, dst_mm
, GFP_KERNEL
, &memcg
, false))
72 _dst_pte
= mk_pte(page
, dst_vma
->vm_page_prot
);
73 if (dst_vma
->vm_flags
& VM_WRITE
)
74 _dst_pte
= pte_mkwrite(pte_mkdirty(_dst_pte
));
77 dst_pte
= pte_offset_map_lock(dst_mm
, dst_pmd
, dst_addr
, &ptl
);
78 if (!pte_none(*dst_pte
))
79 goto out_release_uncharge_unlock
;
81 inc_mm_counter(dst_mm
, MM_ANONPAGES
);
82 page_add_new_anon_rmap(page
, dst_vma
, dst_addr
, false);
83 mem_cgroup_commit_charge(page
, memcg
, false, false);
84 lru_cache_add_active_or_unevictable(page
, dst_vma
);
86 set_pte_at(dst_mm
, dst_addr
, dst_pte
, _dst_pte
);
88 /* No need to invalidate - it was non-present before */
89 update_mmu_cache(dst_vma
, dst_addr
, dst_pte
);
91 pte_unmap_unlock(dst_pte
, ptl
);
95 out_release_uncharge_unlock
:
96 pte_unmap_unlock(dst_pte
, ptl
);
97 mem_cgroup_cancel_charge(page
, memcg
, false);
103 static int mfill_zeropage_pte(struct mm_struct
*dst_mm
,
105 struct vm_area_struct
*dst_vma
,
106 unsigned long dst_addr
)
108 pte_t _dst_pte
, *dst_pte
;
112 _dst_pte
= pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr
),
113 dst_vma
->vm_page_prot
));
115 dst_pte
= pte_offset_map_lock(dst_mm
, dst_pmd
, dst_addr
, &ptl
);
116 if (!pte_none(*dst_pte
))
118 set_pte_at(dst_mm
, dst_addr
, dst_pte
, _dst_pte
);
119 /* No need to invalidate - it was non-present before */
120 update_mmu_cache(dst_vma
, dst_addr
, dst_pte
);
123 pte_unmap_unlock(dst_pte
, ptl
);
127 static pmd_t
*mm_alloc_pmd(struct mm_struct
*mm
, unsigned long address
)
133 pgd
= pgd_offset(mm
, address
);
134 pud
= pud_alloc(mm
, pgd
, address
);
137 * Note that we didn't run this because the pmd was
138 * missing, the *pmd may be already established and in
139 * turn it may also be a trans_huge_pmd.
141 pmd
= pmd_alloc(mm
, pud
, address
);
145 #ifdef CONFIG_HUGETLB_PAGE
147 * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is
148 * called with mmap_sem held, it will release mmap_sem before returning.
150 static __always_inline ssize_t
__mcopy_atomic_hugetlb(struct mm_struct
*dst_mm
,
151 struct vm_area_struct
*dst_vma
,
152 unsigned long dst_start
,
153 unsigned long src_start
,
157 int vm_alloc_shared
= dst_vma
->vm_flags
& VM_SHARED
;
158 int vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
161 unsigned long src_addr
, dst_addr
;
165 unsigned long vma_hpagesize
;
168 struct address_space
*mapping
;
171 * There is no default zero huge page for all huge page sizes as
172 * supported by hugetlb. A PMD_SIZE huge pages may exist as used
173 * by THP. Since we can not reliably insert a zero page, this
174 * feature is not supported.
177 up_read(&dst_mm
->mmap_sem
);
181 src_addr
= src_start
;
182 dst_addr
= dst_start
;
185 vma_hpagesize
= vma_kernel_pagesize(dst_vma
);
188 * Validate alignment based on huge page size
191 if (dst_start
& (vma_hpagesize
- 1) || len
& (vma_hpagesize
- 1))
196 * On routine entry dst_vma is set. If we had to drop mmap_sem and
197 * retry, dst_vma will be set to NULL and we must lookup again.
201 dst_vma
= find_vma(dst_mm
, dst_start
);
202 if (!dst_vma
|| !is_vm_hugetlb_page(dst_vma
))
205 * Only allow __mcopy_atomic_hugetlb on userfaultfd
208 if (!dst_vma
->vm_userfaultfd_ctx
.ctx
)
211 if (dst_start
< dst_vma
->vm_start
||
212 dst_start
+ len
> dst_vma
->vm_end
)
216 if (vma_hpagesize
!= vma_kernel_pagesize(dst_vma
))
219 vm_shared
= dst_vma
->vm_flags
& VM_SHARED
;
222 if (WARN_ON(dst_addr
& (vma_hpagesize
- 1) ||
223 (len
- copied
) & (vma_hpagesize
- 1)))
227 * If not shared, ensure the dst_vma has a anon_vma.
231 if (unlikely(anon_vma_prepare(dst_vma
)))
235 h
= hstate_vma(dst_vma
);
237 while (src_addr
< src_start
+ len
) {
240 BUG_ON(dst_addr
>= dst_start
+ len
);
241 VM_BUG_ON(dst_addr
& ~huge_page_mask(h
));
244 * Serialize via hugetlb_fault_mutex
246 idx
= linear_page_index(dst_vma
, dst_addr
);
247 mapping
= dst_vma
->vm_file
->f_mapping
;
248 hash
= hugetlb_fault_mutex_hash(h
, dst_mm
, dst_vma
, mapping
,
250 mutex_lock(&hugetlb_fault_mutex_table
[hash
]);
253 dst_pte
= huge_pte_alloc(dst_mm
, dst_addr
, huge_page_size(h
));
255 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
260 dst_pteval
= huge_ptep_get(dst_pte
);
261 if (!huge_pte_none(dst_pteval
)) {
262 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
266 err
= hugetlb_mcopy_atomic_pte(dst_mm
, dst_pte
, dst_vma
,
267 dst_addr
, src_addr
, &page
);
269 mutex_unlock(&hugetlb_fault_mutex_table
[hash
]);
270 vm_alloc_shared
= vm_shared
;
274 if (unlikely(err
== -EFAULT
)) {
275 up_read(&dst_mm
->mmap_sem
);
278 err
= copy_huge_page_from_user(page
,
279 (const void __user
*)src_addr
,
280 pages_per_huge_page(h
), true);
285 down_read(&dst_mm
->mmap_sem
);
293 dst_addr
+= vma_hpagesize
;
294 src_addr
+= vma_hpagesize
;
295 copied
+= vma_hpagesize
;
297 if (fatal_signal_pending(current
))
305 up_read(&dst_mm
->mmap_sem
);
309 * We encountered an error and are about to free a newly
310 * allocated huge page.
312 * Reservation handling is very subtle, and is different for
313 * private and shared mappings. See the routine
314 * restore_reserve_on_error for details. Unfortunately, we
315 * can not call restore_reserve_on_error now as it would
316 * require holding mmap_sem.
318 * If a reservation for the page existed in the reservation
319 * map of a private mapping, the map was modified to indicate
320 * the reservation was consumed when the page was allocated.
321 * We clear the PagePrivate flag now so that the global
322 * reserve count will not be incremented in free_huge_page.
323 * The reservation map will still indicate the reservation
324 * was consumed and possibly prevent later page allocation.
325 * This is better than leaking a global reservation. If no
326 * reservation existed, it is still safe to clear PagePrivate
327 * as no adjustments to reservation counts were made during
330 * The reservation map for shared mappings indicates which
331 * pages have reservations. When a huge page is allocated
332 * for an address with a reservation, no change is made to
333 * the reserve map. In this case PagePrivate will be set
334 * to indicate that the global reservation count should be
335 * incremented when the page is freed. This is the desired
336 * behavior. However, when a huge page is allocated for an
337 * address without a reservation a reservation entry is added
338 * to the reservation map, and PagePrivate will not be set.
339 * When the page is freed, the global reserve count will NOT
340 * be incremented and it will appear as though we have leaked
341 * reserved page. In this case, set PagePrivate so that the
342 * global reserve count will be incremented to match the
343 * reservation map entry which was created.
345 * Note that vm_alloc_shared is based on the flags of the vma
346 * for which the page was originally allocated. dst_vma could
347 * be different or NULL on error.
350 SetPagePrivate(page
);
352 ClearPagePrivate(page
);
357 BUG_ON(!copied
&& !err
);
358 return copied
? copied
: err
;
360 #else /* !CONFIG_HUGETLB_PAGE */
361 /* fail at build time if gcc attempts to use this */
362 extern ssize_t
__mcopy_atomic_hugetlb(struct mm_struct
*dst_mm
,
363 struct vm_area_struct
*dst_vma
,
364 unsigned long dst_start
,
365 unsigned long src_start
,
368 #endif /* CONFIG_HUGETLB_PAGE */
370 static __always_inline ssize_t
__mcopy_atomic(struct mm_struct
*dst_mm
,
371 unsigned long dst_start
,
372 unsigned long src_start
,
376 struct vm_area_struct
*dst_vma
;
379 unsigned long src_addr
, dst_addr
;
384 * Sanitize the command parameters:
386 BUG_ON(dst_start
& ~PAGE_MASK
);
387 BUG_ON(len
& ~PAGE_MASK
);
389 /* Does the address range wrap, or is the span zero-sized? */
390 BUG_ON(src_start
+ len
<= src_start
);
391 BUG_ON(dst_start
+ len
<= dst_start
);
393 src_addr
= src_start
;
394 dst_addr
= dst_start
;
398 down_read(&dst_mm
->mmap_sem
);
401 * Make sure the vma is not shared, that the dst range is
402 * both valid and fully within a single existing vma.
405 dst_vma
= find_vma(dst_mm
, dst_start
);
409 * Be strict and only allow __mcopy_atomic on userfaultfd
410 * registered ranges to prevent userland errors going
411 * unnoticed. As far as the VM consistency is concerned, it
412 * would be perfectly safe to remove this check, but there's
413 * no useful usage for __mcopy_atomic ouside of userfaultfd
414 * registered ranges. This is after all why these are ioctls
415 * belonging to the userfaultfd and not syscalls.
417 if (!dst_vma
->vm_userfaultfd_ctx
.ctx
)
420 if (dst_start
< dst_vma
->vm_start
||
421 dst_start
+ len
> dst_vma
->vm_end
)
426 * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but
427 * it will overwrite vm_ops, so vma_is_anonymous must return false.
429 if (WARN_ON_ONCE(vma_is_anonymous(dst_vma
) &&
430 dst_vma
->vm_flags
& VM_SHARED
))
434 * If this is a HUGETLB vma, pass off to appropriate routine
436 if (is_vm_hugetlb_page(dst_vma
))
437 return __mcopy_atomic_hugetlb(dst_mm
, dst_vma
, dst_start
,
438 src_start
, len
, zeropage
);
440 if (!vma_is_anonymous(dst_vma
) && !vma_is_shmem(dst_vma
))
444 * Ensure the dst_vma has a anon_vma or this page
445 * would get a NULL anon_vma when moved in the
449 if (vma_is_anonymous(dst_vma
) && unlikely(anon_vma_prepare(dst_vma
)))
452 while (src_addr
< src_start
+ len
) {
455 BUG_ON(dst_addr
>= dst_start
+ len
);
457 dst_pmd
= mm_alloc_pmd(dst_mm
, dst_addr
);
458 if (unlikely(!dst_pmd
)) {
463 dst_pmdval
= pmd_read_atomic(dst_pmd
);
465 * If the dst_pmd is mapped as THP don't
466 * override it and just be strict.
468 if (unlikely(pmd_trans_huge(dst_pmdval
))) {
472 if (unlikely(pmd_none(dst_pmdval
)) &&
473 unlikely(__pte_alloc(dst_mm
, dst_pmd
, dst_addr
))) {
477 /* If an huge pmd materialized from under us fail */
478 if (unlikely(pmd_trans_huge(*dst_pmd
))) {
483 BUG_ON(pmd_none(*dst_pmd
));
484 BUG_ON(pmd_trans_huge(*dst_pmd
));
486 if (vma_is_anonymous(dst_vma
)) {
488 err
= mcopy_atomic_pte(dst_mm
, dst_pmd
, dst_vma
,
492 err
= mfill_zeropage_pte(dst_mm
, dst_pmd
,
495 err
= -EINVAL
; /* if zeropage is true return -EINVAL */
496 if (likely(!zeropage
))
497 err
= shmem_mcopy_atomic_pte(dst_mm
, dst_pmd
,
504 if (unlikely(err
== -EFAULT
)) {
507 up_read(&dst_mm
->mmap_sem
);
510 page_kaddr
= kmap(page
);
511 err
= copy_from_user(page_kaddr
,
512 (const void __user
*) src_addr
,
524 dst_addr
+= PAGE_SIZE
;
525 src_addr
+= PAGE_SIZE
;
528 if (fatal_signal_pending(current
))
536 up_read(&dst_mm
->mmap_sem
);
542 BUG_ON(!copied
&& !err
);
543 return copied
? copied
: err
;
546 ssize_t
mcopy_atomic(struct mm_struct
*dst_mm
, unsigned long dst_start
,
547 unsigned long src_start
, unsigned long len
)
549 return __mcopy_atomic(dst_mm
, dst_start
, src_start
, len
, false);
552 ssize_t
mfill_zeropage(struct mm_struct
*dst_mm
, unsigned long start
,
555 return __mcopy_atomic(dst_mm
, start
, 0, len
, true);