1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright 2013 Red Hat Inc.
5 * Authors: Jérôme Glisse <jglisse@redhat.com>
8 * Refer to include/linux/hmm.h for information about heterogeneous memory
9 * management or HMM for short.
11 #include <linux/pagewalk.h>
12 #include <linux/hmm.h>
13 #include <linux/init.h>
14 #include <linux/rmap.h>
15 #include <linux/swap.h>
16 #include <linux/slab.h>
17 #include <linux/sched.h>
18 #include <linux/mmzone.h>
19 #include <linux/pagemap.h>
20 #include <linux/swapops.h>
21 #include <linux/hugetlb.h>
22 #include <linux/memremap.h>
23 #include <linux/sched/mm.h>
24 #include <linux/jump_label.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/mmu_notifier.h>
27 #include <linux/memory_hotplug.h>
32 struct hmm_range
*range
;
37 HMM_NEED_FAULT
= 1 << 0,
38 HMM_NEED_WRITE_FAULT
= 1 << 1,
39 HMM_NEED_ALL_BITS
= HMM_NEED_FAULT
| HMM_NEED_WRITE_FAULT
,
42 static int hmm_pfns_fill(unsigned long addr
, unsigned long end
,
43 struct hmm_range
*range
, unsigned long cpu_flags
)
45 unsigned long i
= (addr
- range
->start
) >> PAGE_SHIFT
;
47 for (; addr
< end
; addr
+= PAGE_SIZE
, i
++)
48 range
->hmm_pfns
[i
] = cpu_flags
;
53 * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s)
54 * @addr: range virtual start address (inclusive)
55 * @end: range virtual end address (exclusive)
56 * @required_fault: HMM_NEED_* flags
57 * @walk: mm_walk structure
58 * Return: -EBUSY after page fault, or page fault error
60 * This function will be called whenever pmd_none() or pte_none() returns true,
61 * or whenever there is no page directory covering the virtual address range.
63 static int hmm_vma_fault(unsigned long addr
, unsigned long end
,
64 unsigned int required_fault
, struct mm_walk
*walk
)
66 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
67 struct vm_area_struct
*vma
= walk
->vma
;
68 unsigned int fault_flags
= FAULT_FLAG_REMOTE
;
70 WARN_ON_ONCE(!required_fault
);
71 hmm_vma_walk
->last
= addr
;
73 if (required_fault
& HMM_NEED_WRITE_FAULT
) {
74 if (!(vma
->vm_flags
& VM_WRITE
))
76 fault_flags
|= FAULT_FLAG_WRITE
;
79 for (; addr
< end
; addr
+= PAGE_SIZE
)
80 if (handle_mm_fault(vma
, addr
, fault_flags
, NULL
) &
86 static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk
*hmm_vma_walk
,
87 unsigned long pfn_req_flags
,
88 unsigned long cpu_flags
)
90 struct hmm_range
*range
= hmm_vma_walk
->range
;
93 * So we not only consider the individual per page request we also
94 * consider the default flags requested for the range. The API can
95 * be used 2 ways. The first one where the HMM user coalesces
96 * multiple page faults into one request and sets flags per pfn for
97 * those faults. The second one where the HMM user wants to pre-
98 * fault a range with specific flags. For the latter one it is a
99 * waste to have the user pre-fill the pfn arrays with a default
102 pfn_req_flags
&= range
->pfn_flags_mask
;
103 pfn_req_flags
|= range
->default_flags
;
105 /* We aren't ask to do anything ... */
106 if (!(pfn_req_flags
& HMM_PFN_REQ_FAULT
))
109 /* Need to write fault ? */
110 if ((pfn_req_flags
& HMM_PFN_REQ_WRITE
) &&
111 !(cpu_flags
& HMM_PFN_WRITE
))
112 return HMM_NEED_FAULT
| HMM_NEED_WRITE_FAULT
;
114 /* If CPU page table is not valid then we need to fault */
115 if (!(cpu_flags
& HMM_PFN_VALID
))
116 return HMM_NEED_FAULT
;
121 hmm_range_need_fault(const struct hmm_vma_walk
*hmm_vma_walk
,
122 const unsigned long hmm_pfns
[], unsigned long npages
,
123 unsigned long cpu_flags
)
125 struct hmm_range
*range
= hmm_vma_walk
->range
;
126 unsigned int required_fault
= 0;
130 * If the default flags do not request to fault pages, and the mask does
131 * not allow for individual pages to be faulted, then
132 * hmm_pte_need_fault() will always return 0.
134 if (!((range
->default_flags
| range
->pfn_flags_mask
) &
138 for (i
= 0; i
< npages
; ++i
) {
139 required_fault
|= hmm_pte_need_fault(hmm_vma_walk
, hmm_pfns
[i
],
141 if (required_fault
== HMM_NEED_ALL_BITS
)
142 return required_fault
;
144 return required_fault
;
147 static int hmm_vma_walk_hole(unsigned long addr
, unsigned long end
,
148 __always_unused
int depth
, struct mm_walk
*walk
)
150 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
151 struct hmm_range
*range
= hmm_vma_walk
->range
;
152 unsigned int required_fault
;
153 unsigned long i
, npages
;
154 unsigned long *hmm_pfns
;
156 i
= (addr
- range
->start
) >> PAGE_SHIFT
;
157 npages
= (end
- addr
) >> PAGE_SHIFT
;
158 hmm_pfns
= &range
->hmm_pfns
[i
];
160 hmm_range_need_fault(hmm_vma_walk
, hmm_pfns
, npages
, 0);
164 return hmm_pfns_fill(addr
, end
, range
, HMM_PFN_ERROR
);
167 return hmm_vma_fault(addr
, end
, required_fault
, walk
);
168 return hmm_pfns_fill(addr
, end
, range
, 0);
171 static inline unsigned long hmm_pfn_flags_order(unsigned long order
)
173 return order
<< HMM_PFN_ORDER_SHIFT
;
176 static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range
*range
,
179 if (pmd_protnone(pmd
))
181 return (pmd_write(pmd
) ? (HMM_PFN_VALID
| HMM_PFN_WRITE
) :
183 hmm_pfn_flags_order(PMD_SHIFT
- PAGE_SHIFT
);
186 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
187 static int hmm_vma_handle_pmd(struct mm_walk
*walk
, unsigned long addr
,
188 unsigned long end
, unsigned long hmm_pfns
[],
191 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
192 struct hmm_range
*range
= hmm_vma_walk
->range
;
193 unsigned long pfn
, npages
, i
;
194 unsigned int required_fault
;
195 unsigned long cpu_flags
;
197 npages
= (end
- addr
) >> PAGE_SHIFT
;
198 cpu_flags
= pmd_to_hmm_pfn_flags(range
, pmd
);
200 hmm_range_need_fault(hmm_vma_walk
, hmm_pfns
, npages
, cpu_flags
);
202 return hmm_vma_fault(addr
, end
, required_fault
, walk
);
204 pfn
= pmd_pfn(pmd
) + ((addr
& ~PMD_MASK
) >> PAGE_SHIFT
);
205 for (i
= 0; addr
< end
; addr
+= PAGE_SIZE
, i
++, pfn
++)
206 hmm_pfns
[i
] = pfn
| cpu_flags
;
209 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
210 /* stub to allow the code below to compile */
211 int hmm_vma_handle_pmd(struct mm_walk
*walk
, unsigned long addr
,
212 unsigned long end
, unsigned long hmm_pfns
[], pmd_t pmd
);
213 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
215 static inline bool hmm_is_device_private_entry(struct hmm_range
*range
,
218 return is_device_private_entry(entry
) &&
219 pfn_swap_entry_to_page(entry
)->pgmap
->owner
==
220 range
->dev_private_owner
;
223 static inline unsigned long pte_to_hmm_pfn_flags(struct hmm_range
*range
,
226 if (pte_none(pte
) || !pte_present(pte
) || pte_protnone(pte
))
228 return pte_write(pte
) ? (HMM_PFN_VALID
| HMM_PFN_WRITE
) : HMM_PFN_VALID
;
231 static int hmm_vma_handle_pte(struct mm_walk
*walk
, unsigned long addr
,
232 unsigned long end
, pmd_t
*pmdp
, pte_t
*ptep
,
233 unsigned long *hmm_pfn
)
235 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
236 struct hmm_range
*range
= hmm_vma_walk
->range
;
237 unsigned int required_fault
;
238 unsigned long cpu_flags
;
240 uint64_t pfn_req_flags
= *hmm_pfn
;
244 hmm_pte_need_fault(hmm_vma_walk
, pfn_req_flags
, 0);
251 if (!pte_present(pte
)) {
252 swp_entry_t entry
= pte_to_swp_entry(pte
);
255 * Never fault in device private pages, but just report
256 * the PFN even if not present.
258 if (hmm_is_device_private_entry(range
, entry
)) {
259 cpu_flags
= HMM_PFN_VALID
;
260 if (is_writable_device_private_entry(entry
))
261 cpu_flags
|= HMM_PFN_WRITE
;
262 *hmm_pfn
= swp_offset(entry
) | cpu_flags
;
267 hmm_pte_need_fault(hmm_vma_walk
, pfn_req_flags
, 0);
268 if (!required_fault
) {
273 if (!non_swap_entry(entry
))
276 if (is_device_exclusive_entry(entry
))
279 if (is_migration_entry(entry
)) {
281 hmm_vma_walk
->last
= addr
;
282 migration_entry_wait(walk
->mm
, pmdp
, addr
);
286 /* Report error for everything else */
291 cpu_flags
= pte_to_hmm_pfn_flags(range
, pte
);
293 hmm_pte_need_fault(hmm_vma_walk
, pfn_req_flags
, cpu_flags
);
298 * Since each architecture defines a struct page for the zero page, just
299 * fall through and treat it like a normal page.
301 if (pte_special(pte
) && !is_zero_pfn(pte_pfn(pte
))) {
302 if (hmm_pte_need_fault(hmm_vma_walk
, pfn_req_flags
, 0)) {
306 *hmm_pfn
= HMM_PFN_ERROR
;
310 *hmm_pfn
= pte_pfn(pte
) | cpu_flags
;
315 /* Fault any virtual address we were asked to fault */
316 return hmm_vma_fault(addr
, end
, required_fault
, walk
);
319 static int hmm_vma_walk_pmd(pmd_t
*pmdp
,
322 struct mm_walk
*walk
)
324 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
325 struct hmm_range
*range
= hmm_vma_walk
->range
;
326 unsigned long *hmm_pfns
=
327 &range
->hmm_pfns
[(start
- range
->start
) >> PAGE_SHIFT
];
328 unsigned long npages
= (end
- start
) >> PAGE_SHIFT
;
329 unsigned long addr
= start
;
334 pmd
= READ_ONCE(*pmdp
);
336 return hmm_vma_walk_hole(start
, end
, -1, walk
);
338 if (thp_migration_supported() && is_pmd_migration_entry(pmd
)) {
339 if (hmm_range_need_fault(hmm_vma_walk
, hmm_pfns
, npages
, 0)) {
340 hmm_vma_walk
->last
= addr
;
341 pmd_migration_entry_wait(walk
->mm
, pmdp
);
344 return hmm_pfns_fill(start
, end
, range
, 0);
347 if (!pmd_present(pmd
)) {
348 if (hmm_range_need_fault(hmm_vma_walk
, hmm_pfns
, npages
, 0))
350 return hmm_pfns_fill(start
, end
, range
, HMM_PFN_ERROR
);
353 if (pmd_devmap(pmd
) || pmd_trans_huge(pmd
)) {
355 * No need to take pmd_lock here, even if some other thread
356 * is splitting the huge pmd we will get that event through
357 * mmu_notifier callback.
359 * So just read pmd value and check again it's a transparent
360 * huge or device mapping one and compute corresponding pfn
363 pmd
= pmd_read_atomic(pmdp
);
365 if (!pmd_devmap(pmd
) && !pmd_trans_huge(pmd
))
368 return hmm_vma_handle_pmd(walk
, addr
, end
, hmm_pfns
, pmd
);
372 * We have handled all the valid cases above ie either none, migration,
373 * huge or transparent huge. At this point either it is a valid pmd
374 * entry pointing to pte directory or it is a bad pmd that will not
378 if (hmm_range_need_fault(hmm_vma_walk
, hmm_pfns
, npages
, 0))
380 return hmm_pfns_fill(start
, end
, range
, HMM_PFN_ERROR
);
383 ptep
= pte_offset_map(pmdp
, addr
);
384 for (; addr
< end
; addr
+= PAGE_SIZE
, ptep
++, hmm_pfns
++) {
387 r
= hmm_vma_handle_pte(walk
, addr
, end
, pmdp
, ptep
, hmm_pfns
);
389 /* hmm_vma_handle_pte() did pte_unmap() */
397 #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \
398 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
399 static inline unsigned long pud_to_hmm_pfn_flags(struct hmm_range
*range
,
402 if (!pud_present(pud
))
404 return (pud_write(pud
) ? (HMM_PFN_VALID
| HMM_PFN_WRITE
) :
406 hmm_pfn_flags_order(PUD_SHIFT
- PAGE_SHIFT
);
409 static int hmm_vma_walk_pud(pud_t
*pudp
, unsigned long start
, unsigned long end
,
410 struct mm_walk
*walk
)
412 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
413 struct hmm_range
*range
= hmm_vma_walk
->range
;
414 unsigned long addr
= start
;
417 spinlock_t
*ptl
= pud_trans_huge_lock(pudp
, walk
->vma
);
422 /* Normally we don't want to split the huge page */
423 walk
->action
= ACTION_CONTINUE
;
425 pud
= READ_ONCE(*pudp
);
428 return hmm_vma_walk_hole(start
, end
, -1, walk
);
431 if (pud_huge(pud
) && pud_devmap(pud
)) {
432 unsigned long i
, npages
, pfn
;
433 unsigned int required_fault
;
434 unsigned long *hmm_pfns
;
435 unsigned long cpu_flags
;
437 if (!pud_present(pud
)) {
439 return hmm_vma_walk_hole(start
, end
, -1, walk
);
442 i
= (addr
- range
->start
) >> PAGE_SHIFT
;
443 npages
= (end
- addr
) >> PAGE_SHIFT
;
444 hmm_pfns
= &range
->hmm_pfns
[i
];
446 cpu_flags
= pud_to_hmm_pfn_flags(range
, pud
);
447 required_fault
= hmm_range_need_fault(hmm_vma_walk
, hmm_pfns
,
449 if (required_fault
) {
451 return hmm_vma_fault(addr
, end
, required_fault
, walk
);
454 pfn
= pud_pfn(pud
) + ((addr
& ~PUD_MASK
) >> PAGE_SHIFT
);
455 for (i
= 0; i
< npages
; ++i
, ++pfn
)
456 hmm_pfns
[i
] = pfn
| cpu_flags
;
460 /* Ask for the PUD to be split */
461 walk
->action
= ACTION_SUBTREE
;
468 #define hmm_vma_walk_pud NULL
471 #ifdef CONFIG_HUGETLB_PAGE
472 static int hmm_vma_walk_hugetlb_entry(pte_t
*pte
, unsigned long hmask
,
473 unsigned long start
, unsigned long end
,
474 struct mm_walk
*walk
)
476 unsigned long addr
= start
, i
, pfn
;
477 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
478 struct hmm_range
*range
= hmm_vma_walk
->range
;
479 struct vm_area_struct
*vma
= walk
->vma
;
480 unsigned int required_fault
;
481 unsigned long pfn_req_flags
;
482 unsigned long cpu_flags
;
486 ptl
= huge_pte_lock(hstate_vma(vma
), walk
->mm
, pte
);
487 entry
= huge_ptep_get(pte
);
489 i
= (start
- range
->start
) >> PAGE_SHIFT
;
490 pfn_req_flags
= range
->hmm_pfns
[i
];
491 cpu_flags
= pte_to_hmm_pfn_flags(range
, entry
) |
492 hmm_pfn_flags_order(huge_page_order(hstate_vma(vma
)));
494 hmm_pte_need_fault(hmm_vma_walk
, pfn_req_flags
, cpu_flags
);
495 if (required_fault
) {
497 return hmm_vma_fault(addr
, end
, required_fault
, walk
);
500 pfn
= pte_pfn(entry
) + ((start
& ~hmask
) >> PAGE_SHIFT
);
501 for (; addr
< end
; addr
+= PAGE_SIZE
, i
++, pfn
++)
502 range
->hmm_pfns
[i
] = pfn
| cpu_flags
;
508 #define hmm_vma_walk_hugetlb_entry NULL
509 #endif /* CONFIG_HUGETLB_PAGE */
511 static int hmm_vma_walk_test(unsigned long start
, unsigned long end
,
512 struct mm_walk
*walk
)
514 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
515 struct hmm_range
*range
= hmm_vma_walk
->range
;
516 struct vm_area_struct
*vma
= walk
->vma
;
518 if (!(vma
->vm_flags
& (VM_IO
| VM_PFNMAP
| VM_MIXEDMAP
)) &&
519 vma
->vm_flags
& VM_READ
)
523 * vma ranges that don't have struct page backing them or map I/O
524 * devices directly cannot be handled by hmm_range_fault().
526 * If the vma does not allow read access, then assume that it does not
527 * allow write access either. HMM does not support architectures that
528 * allow write without read.
530 * If a fault is requested for an unsupported range then it is a hard
533 if (hmm_range_need_fault(hmm_vma_walk
,
535 ((start
- range
->start
) >> PAGE_SHIFT
),
536 (end
- start
) >> PAGE_SHIFT
, 0))
539 hmm_pfns_fill(start
, end
, range
, HMM_PFN_ERROR
);
541 /* Skip this vma and continue processing the next vma. */
545 static const struct mm_walk_ops hmm_walk_ops
= {
546 .pud_entry
= hmm_vma_walk_pud
,
547 .pmd_entry
= hmm_vma_walk_pmd
,
548 .pte_hole
= hmm_vma_walk_hole
,
549 .hugetlb_entry
= hmm_vma_walk_hugetlb_entry
,
550 .test_walk
= hmm_vma_walk_test
,
554 * hmm_range_fault - try to fault some address in a virtual address range
555 * @range: argument structure
557 * Returns 0 on success or one of the following error codes:
559 * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma
560 * (e.g., device file vma).
561 * -ENOMEM: Out of memory.
562 * -EPERM: Invalid permission (e.g., asking for write and range is read
564 * -EBUSY: The range has been invalidated and the caller needs to wait for
565 * the invalidation to finish.
566 * -EFAULT: A page was requested to be valid and could not be made valid
567 * ie it has no backing VMA or it is illegal to access
569 * This is similar to get_user_pages(), except that it can read the page tables
570 * without mutating them (ie causing faults).
572 int hmm_range_fault(struct hmm_range
*range
)
574 struct hmm_vma_walk hmm_vma_walk
= {
576 .last
= range
->start
,
578 struct mm_struct
*mm
= range
->notifier
->mm
;
581 mmap_assert_locked(mm
);
584 /* If range is no longer valid force retry. */
585 if (mmu_interval_check_retry(range
->notifier
,
586 range
->notifier_seq
))
588 ret
= walk_page_range(mm
, hmm_vma_walk
.last
, range
->end
,
589 &hmm_walk_ops
, &hmm_vma_walk
);
591 * When -EBUSY is returned the loop restarts with
592 * hmm_vma_walk.last set to an address that has not been stored
593 * in pfns. All entries < last in the pfn array are set to their
594 * output, and all >= are still at their input values.
596 } while (ret
== -EBUSY
);
599 EXPORT_SYMBOL(hmm_range_fault
);