2 * Copyright 2013 Red Hat Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * Authors: Jérôme Glisse <jglisse@redhat.com>
17 * Refer to include/linux/hmm.h for information about heterogeneous memory
18 * management or HMM for short.
21 #include <linux/hmm.h>
22 #include <linux/init.h>
23 #include <linux/rmap.h>
24 #include <linux/swap.h>
25 #include <linux/slab.h>
26 #include <linux/sched.h>
27 #include <linux/mmzone.h>
28 #include <linux/pagemap.h>
29 #include <linux/swapops.h>
30 #include <linux/hugetlb.h>
31 #include <linux/memremap.h>
32 #include <linux/jump_label.h>
33 #include <linux/mmu_notifier.h>
34 #include <linux/memory_hotplug.h>
36 #define PA_SECTION_SIZE (1UL << PA_SECTION_SHIFT)
38 #if IS_ENABLED(CONFIG_HMM_MIRROR)
39 static const struct mmu_notifier_ops hmm_mmu_notifier_ops
;
42 * struct hmm - HMM per mm struct
44 * @mm: mm struct this HMM struct is bound to
45 * @lock: lock protecting ranges list
46 * @sequence: we track updates to the CPU page table with a sequence number
47 * @ranges: list of range being snapshotted
48 * @mirrors: list of mirrors for this mm
49 * @mmu_notifier: mmu notifier to track updates to CPU page table
50 * @mirrors_sem: read/write semaphore protecting the mirrors list
56 struct list_head ranges
;
57 struct list_head mirrors
;
58 struct mmu_notifier mmu_notifier
;
59 struct rw_semaphore mirrors_sem
;
63 * hmm_register - register HMM against an mm (HMM internal)
65 * @mm: mm struct to attach to
67 * This is not intended to be used directly by device drivers. It allocates an
68 * HMM struct if mm does not have one, and initializes it.
70 static struct hmm
*hmm_register(struct mm_struct
*mm
)
72 struct hmm
*hmm
= READ_ONCE(mm
->hmm
);
76 * The hmm struct can only be freed once the mm_struct goes away,
77 * hence we should always have pre-allocated an new hmm struct
83 hmm
= kmalloc(sizeof(*hmm
), GFP_KERNEL
);
86 INIT_LIST_HEAD(&hmm
->mirrors
);
87 init_rwsem(&hmm
->mirrors_sem
);
88 atomic_set(&hmm
->sequence
, 0);
89 hmm
->mmu_notifier
.ops
= NULL
;
90 INIT_LIST_HEAD(&hmm
->ranges
);
91 spin_lock_init(&hmm
->lock
);
94 spin_lock(&mm
->page_table_lock
);
99 spin_unlock(&mm
->page_table_lock
);
105 * We should only get here if hold the mmap_sem in write mode ie on
106 * registration of first mirror through hmm_mirror_register()
108 hmm
->mmu_notifier
.ops
= &hmm_mmu_notifier_ops
;
109 if (__mmu_notifier_register(&hmm
->mmu_notifier
, mm
))
115 spin_lock(&mm
->page_table_lock
);
118 spin_unlock(&mm
->page_table_lock
);
124 void hmm_mm_destroy(struct mm_struct
*mm
)
129 static void hmm_invalidate_range(struct hmm
*hmm
,
130 enum hmm_update_type action
,
134 struct hmm_mirror
*mirror
;
135 struct hmm_range
*range
;
137 spin_lock(&hmm
->lock
);
138 list_for_each_entry(range
, &hmm
->ranges
, list
) {
139 unsigned long addr
, idx
, npages
;
141 if (end
< range
->start
|| start
>= range
->end
)
144 range
->valid
= false;
145 addr
= max(start
, range
->start
);
146 idx
= (addr
- range
->start
) >> PAGE_SHIFT
;
147 npages
= (min(range
->end
, end
) - addr
) >> PAGE_SHIFT
;
148 memset(&range
->pfns
[idx
], 0, sizeof(*range
->pfns
) * npages
);
150 spin_unlock(&hmm
->lock
);
152 down_read(&hmm
->mirrors_sem
);
153 list_for_each_entry(mirror
, &hmm
->mirrors
, list
)
154 mirror
->ops
->sync_cpu_device_pagetables(mirror
, action
,
156 up_read(&hmm
->mirrors_sem
);
159 static void hmm_release(struct mmu_notifier
*mn
, struct mm_struct
*mm
)
161 struct hmm_mirror
*mirror
;
162 struct hmm
*hmm
= mm
->hmm
;
164 down_write(&hmm
->mirrors_sem
);
165 mirror
= list_first_entry_or_null(&hmm
->mirrors
, struct hmm_mirror
,
168 list_del_init(&mirror
->list
);
169 if (mirror
->ops
->release
) {
171 * Drop mirrors_sem so callback can wait on any pending
172 * work that might itself trigger mmu_notifier callback
173 * and thus would deadlock with us.
175 up_write(&hmm
->mirrors_sem
);
176 mirror
->ops
->release(mirror
);
177 down_write(&hmm
->mirrors_sem
);
179 mirror
= list_first_entry_or_null(&hmm
->mirrors
,
180 struct hmm_mirror
, list
);
182 up_write(&hmm
->mirrors_sem
);
185 static int hmm_invalidate_range_start(struct mmu_notifier
*mn
,
186 struct mm_struct
*mm
,
191 struct hmm
*hmm
= mm
->hmm
;
195 atomic_inc(&hmm
->sequence
);
200 static void hmm_invalidate_range_end(struct mmu_notifier
*mn
,
201 struct mm_struct
*mm
,
205 struct hmm
*hmm
= mm
->hmm
;
209 hmm_invalidate_range(mm
->hmm
, HMM_UPDATE_INVALIDATE
, start
, end
);
212 static const struct mmu_notifier_ops hmm_mmu_notifier_ops
= {
213 .release
= hmm_release
,
214 .invalidate_range_start
= hmm_invalidate_range_start
,
215 .invalidate_range_end
= hmm_invalidate_range_end
,
219 * hmm_mirror_register() - register a mirror against an mm
221 * @mirror: new mirror struct to register
222 * @mm: mm to register against
224 * To start mirroring a process address space, the device driver must register
225 * an HMM mirror struct.
227 * THE mm->mmap_sem MUST BE HELD IN WRITE MODE !
229 int hmm_mirror_register(struct hmm_mirror
*mirror
, struct mm_struct
*mm
)
232 if (!mm
|| !mirror
|| !mirror
->ops
)
236 mirror
->hmm
= hmm_register(mm
);
240 down_write(&mirror
->hmm
->mirrors_sem
);
241 if (mirror
->hmm
->mm
== NULL
) {
243 * A racing hmm_mirror_unregister() is about to destroy the hmm
244 * struct. Try again to allocate a new one.
246 up_write(&mirror
->hmm
->mirrors_sem
);
250 list_add(&mirror
->list
, &mirror
->hmm
->mirrors
);
251 up_write(&mirror
->hmm
->mirrors_sem
);
256 EXPORT_SYMBOL(hmm_mirror_register
);
259 * hmm_mirror_unregister() - unregister a mirror
261 * @mirror: new mirror struct to register
263 * Stop mirroring a process address space, and cleanup.
265 void hmm_mirror_unregister(struct hmm_mirror
*mirror
)
267 bool should_unregister
= false;
268 struct mm_struct
*mm
;
271 if (mirror
->hmm
== NULL
)
275 down_write(&hmm
->mirrors_sem
);
276 list_del_init(&mirror
->list
);
277 should_unregister
= list_empty(&hmm
->mirrors
);
281 up_write(&hmm
->mirrors_sem
);
283 if (!should_unregister
|| mm
== NULL
)
286 mmu_notifier_unregister_no_release(&hmm
->mmu_notifier
, mm
);
288 spin_lock(&mm
->page_table_lock
);
291 spin_unlock(&mm
->page_table_lock
);
295 EXPORT_SYMBOL(hmm_mirror_unregister
);
297 struct hmm_vma_walk
{
298 struct hmm_range
*range
;
304 static int hmm_vma_do_fault(struct mm_walk
*walk
, unsigned long addr
,
305 bool write_fault
, uint64_t *pfn
)
307 unsigned int flags
= FAULT_FLAG_ALLOW_RETRY
| FAULT_FLAG_REMOTE
;
308 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
309 struct hmm_range
*range
= hmm_vma_walk
->range
;
310 struct vm_area_struct
*vma
= walk
->vma
;
313 flags
|= hmm_vma_walk
->block
? 0 : FAULT_FLAG_ALLOW_RETRY
;
314 flags
|= write_fault
? FAULT_FLAG_WRITE
: 0;
315 ret
= handle_mm_fault(vma
, addr
, flags
);
316 if (ret
& VM_FAULT_RETRY
)
318 if (ret
& VM_FAULT_ERROR
) {
319 *pfn
= range
->values
[HMM_PFN_ERROR
];
326 static int hmm_pfns_bad(unsigned long addr
,
328 struct mm_walk
*walk
)
330 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
331 struct hmm_range
*range
= hmm_vma_walk
->range
;
332 uint64_t *pfns
= range
->pfns
;
335 i
= (addr
- range
->start
) >> PAGE_SHIFT
;
336 for (; addr
< end
; addr
+= PAGE_SIZE
, i
++)
337 pfns
[i
] = range
->values
[HMM_PFN_ERROR
];
343 * hmm_vma_walk_hole() - handle a range lacking valid pmd or pte(s)
344 * @start: range virtual start address (inclusive)
345 * @end: range virtual end address (exclusive)
346 * @fault: should we fault or not ?
347 * @write_fault: write fault ?
348 * @walk: mm_walk structure
349 * Returns: 0 on success, -EAGAIN after page fault, or page fault error
351 * This function will be called whenever pmd_none() or pte_none() returns true,
352 * or whenever there is no page directory covering the virtual address range.
354 static int hmm_vma_walk_hole_(unsigned long addr
, unsigned long end
,
355 bool fault
, bool write_fault
,
356 struct mm_walk
*walk
)
358 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
359 struct hmm_range
*range
= hmm_vma_walk
->range
;
360 uint64_t *pfns
= range
->pfns
;
363 hmm_vma_walk
->last
= addr
;
364 i
= (addr
- range
->start
) >> PAGE_SHIFT
;
365 for (; addr
< end
; addr
+= PAGE_SIZE
, i
++) {
366 pfns
[i
] = range
->values
[HMM_PFN_NONE
];
367 if (fault
|| write_fault
) {
370 ret
= hmm_vma_do_fault(walk
, addr
, write_fault
,
377 return (fault
|| write_fault
) ? -EAGAIN
: 0;
380 static inline void hmm_pte_need_fault(const struct hmm_vma_walk
*hmm_vma_walk
,
381 uint64_t pfns
, uint64_t cpu_flags
,
382 bool *fault
, bool *write_fault
)
384 struct hmm_range
*range
= hmm_vma_walk
->range
;
386 *fault
= *write_fault
= false;
387 if (!hmm_vma_walk
->fault
)
390 /* We aren't ask to do anything ... */
391 if (!(pfns
& range
->flags
[HMM_PFN_VALID
]))
393 /* If this is device memory than only fault if explicitly requested */
394 if ((cpu_flags
& range
->flags
[HMM_PFN_DEVICE_PRIVATE
])) {
395 /* Do we fault on device memory ? */
396 if (pfns
& range
->flags
[HMM_PFN_DEVICE_PRIVATE
]) {
397 *write_fault
= pfns
& range
->flags
[HMM_PFN_WRITE
];
403 /* If CPU page table is not valid then we need to fault */
404 *fault
= !(cpu_flags
& range
->flags
[HMM_PFN_VALID
]);
405 /* Need to write fault ? */
406 if ((pfns
& range
->flags
[HMM_PFN_WRITE
]) &&
407 !(cpu_flags
& range
->flags
[HMM_PFN_WRITE
])) {
413 static void hmm_range_need_fault(const struct hmm_vma_walk
*hmm_vma_walk
,
414 const uint64_t *pfns
, unsigned long npages
,
415 uint64_t cpu_flags
, bool *fault
,
420 if (!hmm_vma_walk
->fault
) {
421 *fault
= *write_fault
= false;
425 for (i
= 0; i
< npages
; ++i
) {
426 hmm_pte_need_fault(hmm_vma_walk
, pfns
[i
], cpu_flags
,
428 if ((*fault
) || (*write_fault
))
433 static int hmm_vma_walk_hole(unsigned long addr
, unsigned long end
,
434 struct mm_walk
*walk
)
436 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
437 struct hmm_range
*range
= hmm_vma_walk
->range
;
438 bool fault
, write_fault
;
439 unsigned long i
, npages
;
442 i
= (addr
- range
->start
) >> PAGE_SHIFT
;
443 npages
= (end
- addr
) >> PAGE_SHIFT
;
444 pfns
= &range
->pfns
[i
];
445 hmm_range_need_fault(hmm_vma_walk
, pfns
, npages
,
446 0, &fault
, &write_fault
);
447 return hmm_vma_walk_hole_(addr
, end
, fault
, write_fault
, walk
);
450 static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range
*range
, pmd_t pmd
)
452 if (pmd_protnone(pmd
))
454 return pmd_write(pmd
) ? range
->flags
[HMM_PFN_VALID
] |
455 range
->flags
[HMM_PFN_WRITE
] :
456 range
->flags
[HMM_PFN_VALID
];
459 static int hmm_vma_handle_pmd(struct mm_walk
*walk
,
465 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
466 struct hmm_range
*range
= hmm_vma_walk
->range
;
467 unsigned long pfn
, npages
, i
;
468 bool fault
, write_fault
;
471 npages
= (end
- addr
) >> PAGE_SHIFT
;
472 cpu_flags
= pmd_to_hmm_pfn_flags(range
, pmd
);
473 hmm_range_need_fault(hmm_vma_walk
, pfns
, npages
, cpu_flags
,
474 &fault
, &write_fault
);
476 if (pmd_protnone(pmd
) || fault
|| write_fault
)
477 return hmm_vma_walk_hole_(addr
, end
, fault
, write_fault
, walk
);
479 pfn
= pmd_pfn(pmd
) + pte_index(addr
);
480 for (i
= 0; addr
< end
; addr
+= PAGE_SIZE
, i
++, pfn
++)
481 pfns
[i
] = hmm_pfn_from_pfn(range
, pfn
) | cpu_flags
;
482 hmm_vma_walk
->last
= end
;
486 static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range
*range
, pte_t pte
)
488 if (pte_none(pte
) || !pte_present(pte
))
490 return pte_write(pte
) ? range
->flags
[HMM_PFN_VALID
] |
491 range
->flags
[HMM_PFN_WRITE
] :
492 range
->flags
[HMM_PFN_VALID
];
495 static int hmm_vma_handle_pte(struct mm_walk
*walk
, unsigned long addr
,
496 unsigned long end
, pmd_t
*pmdp
, pte_t
*ptep
,
499 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
500 struct hmm_range
*range
= hmm_vma_walk
->range
;
501 struct vm_area_struct
*vma
= walk
->vma
;
502 bool fault
, write_fault
;
505 uint64_t orig_pfn
= *pfn
;
507 *pfn
= range
->values
[HMM_PFN_NONE
];
508 cpu_flags
= pte_to_hmm_pfn_flags(range
, pte
);
509 hmm_pte_need_fault(hmm_vma_walk
, orig_pfn
, cpu_flags
,
510 &fault
, &write_fault
);
513 if (fault
|| write_fault
)
518 if (!pte_present(pte
)) {
519 swp_entry_t entry
= pte_to_swp_entry(pte
);
521 if (!non_swap_entry(entry
)) {
522 if (fault
|| write_fault
)
528 * This is a special swap entry, ignore migration, use
529 * device and report anything else as error.
531 if (is_device_private_entry(entry
)) {
532 cpu_flags
= range
->flags
[HMM_PFN_VALID
] |
533 range
->flags
[HMM_PFN_DEVICE_PRIVATE
];
534 cpu_flags
|= is_write_device_private_entry(entry
) ?
535 range
->flags
[HMM_PFN_WRITE
] : 0;
536 hmm_pte_need_fault(hmm_vma_walk
, orig_pfn
, cpu_flags
,
537 &fault
, &write_fault
);
538 if (fault
|| write_fault
)
540 *pfn
= hmm_pfn_from_pfn(range
, swp_offset(entry
));
545 if (is_migration_entry(entry
)) {
546 if (fault
|| write_fault
) {
548 hmm_vma_walk
->last
= addr
;
549 migration_entry_wait(vma
->vm_mm
,
556 /* Report error for everything else */
557 *pfn
= range
->values
[HMM_PFN_ERROR
];
561 if (fault
|| write_fault
)
564 *pfn
= hmm_pfn_from_pfn(range
, pte_pfn(pte
)) | cpu_flags
;
569 /* Fault any virtual address we were asked to fault */
570 return hmm_vma_walk_hole_(addr
, end
, fault
, write_fault
, walk
);
573 static int hmm_vma_walk_pmd(pmd_t
*pmdp
,
576 struct mm_walk
*walk
)
578 struct hmm_vma_walk
*hmm_vma_walk
= walk
->private;
579 struct hmm_range
*range
= hmm_vma_walk
->range
;
580 uint64_t *pfns
= range
->pfns
;
581 unsigned long addr
= start
, i
;
584 i
= (addr
- range
->start
) >> PAGE_SHIFT
;
588 return hmm_vma_walk_hole(start
, end
, walk
);
590 if (pmd_huge(*pmdp
) && (range
->vma
->vm_flags
& VM_HUGETLB
))
591 return hmm_pfns_bad(start
, end
, walk
);
593 if (pmd_devmap(*pmdp
) || pmd_trans_huge(*pmdp
)) {
597 * No need to take pmd_lock here, even if some other threads
598 * is splitting the huge pmd we will get that event through
599 * mmu_notifier callback.
601 * So just read pmd value and check again its a transparent
602 * huge or device mapping one and compute corresponding pfn
605 pmd
= pmd_read_atomic(pmdp
);
607 if (!pmd_devmap(pmd
) && !pmd_trans_huge(pmd
))
610 return hmm_vma_handle_pmd(walk
, addr
, end
, &pfns
[i
], pmd
);
614 return hmm_pfns_bad(start
, end
, walk
);
616 ptep
= pte_offset_map(pmdp
, addr
);
617 for (; addr
< end
; addr
+= PAGE_SIZE
, ptep
++, i
++) {
620 r
= hmm_vma_handle_pte(walk
, addr
, end
, pmdp
, ptep
, &pfns
[i
]);
622 /* hmm_vma_handle_pte() did unmap pte directory */
623 hmm_vma_walk
->last
= addr
;
629 hmm_vma_walk
->last
= addr
;
633 static void hmm_pfns_clear(struct hmm_range
*range
,
638 for (; addr
< end
; addr
+= PAGE_SIZE
, pfns
++)
639 *pfns
= range
->values
[HMM_PFN_NONE
];
642 static void hmm_pfns_special(struct hmm_range
*range
)
644 unsigned long addr
= range
->start
, i
= 0;
646 for (; addr
< range
->end
; addr
+= PAGE_SIZE
, i
++)
647 range
->pfns
[i
] = range
->values
[HMM_PFN_SPECIAL
];
651 * hmm_vma_get_pfns() - snapshot CPU page table for a range of virtual addresses
652 * @range: range being snapshotted
653 * Returns: -EINVAL if invalid argument, -ENOMEM out of memory, -EPERM invalid
654 * vma permission, 0 success
656 * This snapshots the CPU page table for a range of virtual addresses. Snapshot
657 * validity is tracked by range struct. See hmm_vma_range_done() for further
660 * The range struct is initialized here. It tracks the CPU page table, but only
661 * if the function returns success (0), in which case the caller must then call
662 * hmm_vma_range_done() to stop CPU page table update tracking on this range.
664 * NOT CALLING hmm_vma_range_done() IF FUNCTION RETURNS 0 WILL LEAD TO SERIOUS
665 * MEMORY CORRUPTION ! YOU HAVE BEEN WARNED !
667 int hmm_vma_get_pfns(struct hmm_range
*range
)
669 struct vm_area_struct
*vma
= range
->vma
;
670 struct hmm_vma_walk hmm_vma_walk
;
671 struct mm_walk mm_walk
;
674 /* Sanity check, this really should not happen ! */
675 if (range
->start
< vma
->vm_start
|| range
->start
>= vma
->vm_end
)
677 if (range
->end
< vma
->vm_start
|| range
->end
> vma
->vm_end
)
680 hmm
= hmm_register(vma
->vm_mm
);
683 /* Caller must have registered a mirror, via hmm_mirror_register() ! */
684 if (!hmm
->mmu_notifier
.ops
)
687 /* FIXME support hugetlb fs */
688 if (is_vm_hugetlb_page(vma
) || (vma
->vm_flags
& VM_SPECIAL
) ||
690 hmm_pfns_special(range
);
694 if (!(vma
->vm_flags
& VM_READ
)) {
696 * If vma do not allow read access, then assume that it does
697 * not allow write access, either. Architecture that allow
698 * write without read access are not supported by HMM, because
699 * operations such has atomic access would not work.
701 hmm_pfns_clear(range
, range
->pfns
, range
->start
, range
->end
);
705 /* Initialize range to track CPU page table update */
706 spin_lock(&hmm
->lock
);
708 list_add_rcu(&range
->list
, &hmm
->ranges
);
709 spin_unlock(&hmm
->lock
);
711 hmm_vma_walk
.fault
= false;
712 hmm_vma_walk
.range
= range
;
713 mm_walk
.private = &hmm_vma_walk
;
716 mm_walk
.mm
= vma
->vm_mm
;
717 mm_walk
.pte_entry
= NULL
;
718 mm_walk
.test_walk
= NULL
;
719 mm_walk
.hugetlb_entry
= NULL
;
720 mm_walk
.pmd_entry
= hmm_vma_walk_pmd
;
721 mm_walk
.pte_hole
= hmm_vma_walk_hole
;
723 walk_page_range(range
->start
, range
->end
, &mm_walk
);
726 EXPORT_SYMBOL(hmm_vma_get_pfns
);
729 * hmm_vma_range_done() - stop tracking change to CPU page table over a range
730 * @range: range being tracked
731 * Returns: false if range data has been invalidated, true otherwise
733 * Range struct is used to track updates to the CPU page table after a call to
734 * either hmm_vma_get_pfns() or hmm_vma_fault(). Once the device driver is done
735 * using the data, or wants to lock updates to the data it got from those
736 * functions, it must call the hmm_vma_range_done() function, which will then
737 * stop tracking CPU page table updates.
739 * Note that device driver must still implement general CPU page table update
740 * tracking either by using hmm_mirror (see hmm_mirror_register()) or by using
741 * the mmu_notifier API directly.
743 * CPU page table update tracking done through hmm_range is only temporary and
744 * to be used while trying to duplicate CPU page table contents for a range of
747 * There are two ways to use this :
749 * hmm_vma_get_pfns(range); or hmm_vma_fault(...);
750 * trans = device_build_page_table_update_transaction(pfns);
751 * device_page_table_lock();
752 * if (!hmm_vma_range_done(range)) {
753 * device_page_table_unlock();
756 * device_commit_transaction(trans);
757 * device_page_table_unlock();
760 * hmm_vma_get_pfns(range); or hmm_vma_fault(...);
761 * device_page_table_lock();
762 * hmm_vma_range_done(range);
763 * device_update_page_table(range->pfns);
764 * device_page_table_unlock();
766 bool hmm_vma_range_done(struct hmm_range
*range
)
768 unsigned long npages
= (range
->end
- range
->start
) >> PAGE_SHIFT
;
771 if (range
->end
<= range
->start
) {
776 hmm
= hmm_register(range
->vma
->vm_mm
);
778 memset(range
->pfns
, 0, sizeof(*range
->pfns
) * npages
);
782 spin_lock(&hmm
->lock
);
783 list_del_rcu(&range
->list
);
784 spin_unlock(&hmm
->lock
);
788 EXPORT_SYMBOL(hmm_vma_range_done
);
791 * hmm_vma_fault() - try to fault some address in a virtual address range
792 * @range: range being faulted
793 * @block: allow blocking on fault (if true it sleeps and do not drop mmap_sem)
794 * Returns: 0 success, error otherwise (-EAGAIN means mmap_sem have been drop)
796 * This is similar to a regular CPU page fault except that it will not trigger
797 * any memory migration if the memory being faulted is not accessible by CPUs.
799 * On error, for one virtual address in the range, the function will mark the
800 * corresponding HMM pfn entry with an error flag.
802 * Expected use pattern:
804 * down_read(&mm->mmap_sem);
805 * // Find vma and address device wants to fault, initialize hmm_pfn_t
806 * // array accordingly
807 * ret = hmm_vma_fault(range, write, block);
810 * hmm_vma_range_done(range);
811 * // You might want to rate limit or yield to play nicely, you may
812 * // also commit any valid pfn in the array assuming that you are
813 * // getting true from hmm_vma_range_monitor_end()
822 * up_read(&mm->mmap_sem)
825 * // Take device driver lock that serialize device page table update
826 * driver_lock_device_page_table_update();
827 * hmm_vma_range_done(range);
828 * // Commit pfns we got from hmm_vma_fault()
829 * driver_unlock_device_page_table_update();
830 * up_read(&mm->mmap_sem)
832 * YOU MUST CALL hmm_vma_range_done() AFTER THIS FUNCTION RETURN SUCCESS (0)
833 * BEFORE FREEING THE range struct OR YOU WILL HAVE SERIOUS MEMORY CORRUPTION !
835 * YOU HAVE BEEN WARNED !
837 int hmm_vma_fault(struct hmm_range
*range
, bool block
)
839 struct vm_area_struct
*vma
= range
->vma
;
840 unsigned long start
= range
->start
;
841 struct hmm_vma_walk hmm_vma_walk
;
842 struct mm_walk mm_walk
;
846 /* Sanity check, this really should not happen ! */
847 if (range
->start
< vma
->vm_start
|| range
->start
>= vma
->vm_end
)
849 if (range
->end
< vma
->vm_start
|| range
->end
> vma
->vm_end
)
852 hmm
= hmm_register(vma
->vm_mm
);
854 hmm_pfns_clear(range
, range
->pfns
, range
->start
, range
->end
);
857 /* Caller must have registered a mirror using hmm_mirror_register() */
858 if (!hmm
->mmu_notifier
.ops
)
861 /* FIXME support hugetlb fs */
862 if (is_vm_hugetlb_page(vma
) || (vma
->vm_flags
& VM_SPECIAL
) ||
864 hmm_pfns_special(range
);
868 if (!(vma
->vm_flags
& VM_READ
)) {
870 * If vma do not allow read access, then assume that it does
871 * not allow write access, either. Architecture that allow
872 * write without read access are not supported by HMM, because
873 * operations such has atomic access would not work.
875 hmm_pfns_clear(range
, range
->pfns
, range
->start
, range
->end
);
879 /* Initialize range to track CPU page table update */
880 spin_lock(&hmm
->lock
);
882 list_add_rcu(&range
->list
, &hmm
->ranges
);
883 spin_unlock(&hmm
->lock
);
885 hmm_vma_walk
.fault
= true;
886 hmm_vma_walk
.block
= block
;
887 hmm_vma_walk
.range
= range
;
888 mm_walk
.private = &hmm_vma_walk
;
889 hmm_vma_walk
.last
= range
->start
;
892 mm_walk
.mm
= vma
->vm_mm
;
893 mm_walk
.pte_entry
= NULL
;
894 mm_walk
.test_walk
= NULL
;
895 mm_walk
.hugetlb_entry
= NULL
;
896 mm_walk
.pmd_entry
= hmm_vma_walk_pmd
;
897 mm_walk
.pte_hole
= hmm_vma_walk_hole
;
900 ret
= walk_page_range(start
, range
->end
, &mm_walk
);
901 start
= hmm_vma_walk
.last
;
902 } while (ret
== -EAGAIN
);
907 i
= (hmm_vma_walk
.last
- range
->start
) >> PAGE_SHIFT
;
908 hmm_pfns_clear(range
, &range
->pfns
[i
], hmm_vma_walk
.last
,
910 hmm_vma_range_done(range
);
914 EXPORT_SYMBOL(hmm_vma_fault
);
915 #endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
918 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE) || IS_ENABLED(CONFIG_DEVICE_PUBLIC)
919 struct page
*hmm_vma_alloc_locked_page(struct vm_area_struct
*vma
,
924 page
= alloc_page_vma(GFP_HIGHUSER
, vma
, addr
);
930 EXPORT_SYMBOL(hmm_vma_alloc_locked_page
);
933 static void hmm_devmem_ref_release(struct percpu_ref
*ref
)
935 struct hmm_devmem
*devmem
;
937 devmem
= container_of(ref
, struct hmm_devmem
, ref
);
938 complete(&devmem
->completion
);
941 static void hmm_devmem_ref_exit(void *data
)
943 struct percpu_ref
*ref
= data
;
944 struct hmm_devmem
*devmem
;
946 devmem
= container_of(ref
, struct hmm_devmem
, ref
);
947 percpu_ref_exit(ref
);
948 devm_remove_action(devmem
->device
, &hmm_devmem_ref_exit
, data
);
951 static void hmm_devmem_ref_kill(void *data
)
953 struct percpu_ref
*ref
= data
;
954 struct hmm_devmem
*devmem
;
956 devmem
= container_of(ref
, struct hmm_devmem
, ref
);
957 percpu_ref_kill(ref
);
958 wait_for_completion(&devmem
->completion
);
959 devm_remove_action(devmem
->device
, &hmm_devmem_ref_kill
, data
);
962 static int hmm_devmem_fault(struct vm_area_struct
*vma
,
964 const struct page
*page
,
968 struct hmm_devmem
*devmem
= page
->pgmap
->data
;
970 return devmem
->ops
->fault(devmem
, vma
, addr
, page
, flags
, pmdp
);
973 static void hmm_devmem_free(struct page
*page
, void *data
)
975 struct hmm_devmem
*devmem
= data
;
977 page
->mapping
= NULL
;
979 devmem
->ops
->free(devmem
, page
);
982 static DEFINE_MUTEX(hmm_devmem_lock
);
983 static RADIX_TREE(hmm_devmem_radix
, GFP_KERNEL
);
985 static void hmm_devmem_radix_release(struct resource
*resource
)
989 mutex_lock(&hmm_devmem_lock
);
990 for (key
= resource
->start
;
991 key
<= resource
->end
;
992 key
+= PA_SECTION_SIZE
)
993 radix_tree_delete(&hmm_devmem_radix
, key
>> PA_SECTION_SHIFT
);
994 mutex_unlock(&hmm_devmem_lock
);
997 static void hmm_devmem_release(struct device
*dev
, void *data
)
999 struct hmm_devmem
*devmem
= data
;
1000 struct resource
*resource
= devmem
->resource
;
1001 unsigned long start_pfn
, npages
;
1005 if (percpu_ref_tryget_live(&devmem
->ref
)) {
1006 dev_WARN(dev
, "%s: page mapping is still live!\n", __func__
);
1007 percpu_ref_put(&devmem
->ref
);
1010 /* pages are dead and unused, undo the arch mapping */
1011 start_pfn
= (resource
->start
& ~(PA_SECTION_SIZE
- 1)) >> PAGE_SHIFT
;
1012 npages
= ALIGN(resource_size(resource
), PA_SECTION_SIZE
) >> PAGE_SHIFT
;
1014 page
= pfn_to_page(start_pfn
);
1015 zone
= page_zone(page
);
1017 mem_hotplug_begin();
1018 if (resource
->desc
== IORES_DESC_DEVICE_PRIVATE_MEMORY
)
1019 __remove_pages(zone
, start_pfn
, npages
, NULL
);
1021 arch_remove_memory(start_pfn
<< PAGE_SHIFT
,
1022 npages
<< PAGE_SHIFT
, NULL
);
1025 hmm_devmem_radix_release(resource
);
1028 static int hmm_devmem_pages_create(struct hmm_devmem
*devmem
)
1030 resource_size_t key
, align_start
, align_size
, align_end
;
1031 struct device
*device
= devmem
->device
;
1032 int ret
, nid
, is_ram
;
1034 align_start
= devmem
->resource
->start
& ~(PA_SECTION_SIZE
- 1);
1035 align_size
= ALIGN(devmem
->resource
->start
+
1036 resource_size(devmem
->resource
),
1037 PA_SECTION_SIZE
) - align_start
;
1039 is_ram
= region_intersects(align_start
, align_size
,
1040 IORESOURCE_SYSTEM_RAM
,
1042 if (is_ram
== REGION_MIXED
) {
1043 WARN_ONCE(1, "%s attempted on mixed region %pr\n",
1044 __func__
, devmem
->resource
);
1047 if (is_ram
== REGION_INTERSECTS
)
1050 if (devmem
->resource
->desc
== IORES_DESC_DEVICE_PUBLIC_MEMORY
)
1051 devmem
->pagemap
.type
= MEMORY_DEVICE_PUBLIC
;
1053 devmem
->pagemap
.type
= MEMORY_DEVICE_PRIVATE
;
1055 devmem
->pagemap
.res
= *devmem
->resource
;
1056 devmem
->pagemap
.page_fault
= hmm_devmem_fault
;
1057 devmem
->pagemap
.page_free
= hmm_devmem_free
;
1058 devmem
->pagemap
.dev
= devmem
->device
;
1059 devmem
->pagemap
.ref
= &devmem
->ref
;
1060 devmem
->pagemap
.data
= devmem
;
1062 mutex_lock(&hmm_devmem_lock
);
1063 align_end
= align_start
+ align_size
- 1;
1064 for (key
= align_start
; key
<= align_end
; key
+= PA_SECTION_SIZE
) {
1065 struct hmm_devmem
*dup
;
1067 dup
= radix_tree_lookup(&hmm_devmem_radix
,
1068 key
>> PA_SECTION_SHIFT
);
1070 dev_err(device
, "%s: collides with mapping for %s\n",
1071 __func__
, dev_name(dup
->device
));
1072 mutex_unlock(&hmm_devmem_lock
);
1076 ret
= radix_tree_insert(&hmm_devmem_radix
,
1077 key
>> PA_SECTION_SHIFT
,
1080 dev_err(device
, "%s: failed: %d\n", __func__
, ret
);
1081 mutex_unlock(&hmm_devmem_lock
);
1085 mutex_unlock(&hmm_devmem_lock
);
1087 nid
= dev_to_node(device
);
1089 nid
= numa_mem_id();
1091 mem_hotplug_begin();
1093 * For device private memory we call add_pages() as we only need to
1094 * allocate and initialize struct page for the device memory. More-
1095 * over the device memory is un-accessible thus we do not want to
1096 * create a linear mapping for the memory like arch_add_memory()
1099 * For device public memory, which is accesible by the CPU, we do
1100 * want the linear mapping and thus use arch_add_memory().
1102 if (devmem
->pagemap
.type
== MEMORY_DEVICE_PUBLIC
)
1103 ret
= arch_add_memory(nid
, align_start
, align_size
, NULL
,
1106 ret
= add_pages(nid
, align_start
>> PAGE_SHIFT
,
1107 align_size
>> PAGE_SHIFT
, NULL
, false);
1110 goto error_add_memory
;
1112 move_pfn_range_to_zone(&NODE_DATA(nid
)->node_zones
[ZONE_DEVICE
],
1113 align_start
>> PAGE_SHIFT
,
1114 align_size
>> PAGE_SHIFT
, NULL
);
1118 * Initialization of the pages has been deferred until now in order
1119 * to allow us to do the work while not holding the hotplug lock.
1121 memmap_init_zone_device(&NODE_DATA(nid
)->node_zones
[ZONE_DEVICE
],
1122 align_start
>> PAGE_SHIFT
,
1123 align_size
>> PAGE_SHIFT
, &devmem
->pagemap
);
1128 untrack_pfn(NULL
, PHYS_PFN(align_start
), align_size
);
1130 hmm_devmem_radix_release(devmem
->resource
);
1135 static int hmm_devmem_match(struct device
*dev
, void *data
, void *match_data
)
1137 struct hmm_devmem
*devmem
= data
;
1139 return devmem
->resource
== match_data
;
1142 static void hmm_devmem_pages_remove(struct hmm_devmem
*devmem
)
1144 devres_release(devmem
->device
, &hmm_devmem_release
,
1145 &hmm_devmem_match
, devmem
->resource
);
1149 * hmm_devmem_add() - hotplug ZONE_DEVICE memory for device memory
1151 * @ops: memory event device driver callback (see struct hmm_devmem_ops)
1152 * @device: device struct to bind the resource too
1153 * @size: size in bytes of the device memory to add
1154 * Returns: pointer to new hmm_devmem struct ERR_PTR otherwise
1156 * This function first finds an empty range of physical address big enough to
1157 * contain the new resource, and then hotplugs it as ZONE_DEVICE memory, which
1158 * in turn allocates struct pages. It does not do anything beyond that; all
1159 * events affecting the memory will go through the various callbacks provided
1160 * by hmm_devmem_ops struct.
1162 * Device driver should call this function during device initialization and
1163 * is then responsible of memory management. HMM only provides helpers.
1165 struct hmm_devmem
*hmm_devmem_add(const struct hmm_devmem_ops
*ops
,
1166 struct device
*device
,
1169 struct hmm_devmem
*devmem
;
1170 resource_size_t addr
;
1173 dev_pagemap_get_ops();
1175 devmem
= devres_alloc_node(&hmm_devmem_release
, sizeof(*devmem
),
1176 GFP_KERNEL
, dev_to_node(device
));
1178 return ERR_PTR(-ENOMEM
);
1180 init_completion(&devmem
->completion
);
1181 devmem
->pfn_first
= -1UL;
1182 devmem
->pfn_last
= -1UL;
1183 devmem
->resource
= NULL
;
1184 devmem
->device
= device
;
1187 ret
= percpu_ref_init(&devmem
->ref
, &hmm_devmem_ref_release
,
1190 goto error_percpu_ref
;
1192 ret
= devm_add_action(device
, hmm_devmem_ref_exit
, &devmem
->ref
);
1194 goto error_devm_add_action
;
1196 size
= ALIGN(size
, PA_SECTION_SIZE
);
1197 addr
= min((unsigned long)iomem_resource
.end
,
1198 (1UL << MAX_PHYSMEM_BITS
) - 1);
1199 addr
= addr
- size
+ 1UL;
1202 * FIXME add a new helper to quickly walk resource tree and find free
1205 * FIXME what about ioport_resource resource ?
1207 for (; addr
> size
&& addr
>= iomem_resource
.start
; addr
-= size
) {
1208 ret
= region_intersects(addr
, size
, 0, IORES_DESC_NONE
);
1209 if (ret
!= REGION_DISJOINT
)
1212 devmem
->resource
= devm_request_mem_region(device
, addr
, size
,
1214 if (!devmem
->resource
) {
1216 goto error_no_resource
;
1220 if (!devmem
->resource
) {
1222 goto error_no_resource
;
1225 devmem
->resource
->desc
= IORES_DESC_DEVICE_PRIVATE_MEMORY
;
1226 devmem
->pfn_first
= devmem
->resource
->start
>> PAGE_SHIFT
;
1227 devmem
->pfn_last
= devmem
->pfn_first
+
1228 (resource_size(devmem
->resource
) >> PAGE_SHIFT
);
1230 ret
= hmm_devmem_pages_create(devmem
);
1234 devres_add(device
, devmem
);
1236 ret
= devm_add_action(device
, hmm_devmem_ref_kill
, &devmem
->ref
);
1238 hmm_devmem_remove(devmem
);
1239 return ERR_PTR(ret
);
1245 devm_release_mem_region(device
, devmem
->resource
->start
,
1246 resource_size(devmem
->resource
));
1248 error_devm_add_action
:
1249 hmm_devmem_ref_kill(&devmem
->ref
);
1250 hmm_devmem_ref_exit(&devmem
->ref
);
1252 devres_free(devmem
);
1253 return ERR_PTR(ret
);
1255 EXPORT_SYMBOL(hmm_devmem_add
);
1257 struct hmm_devmem
*hmm_devmem_add_resource(const struct hmm_devmem_ops
*ops
,
1258 struct device
*device
,
1259 struct resource
*res
)
1261 struct hmm_devmem
*devmem
;
1264 if (res
->desc
!= IORES_DESC_DEVICE_PUBLIC_MEMORY
)
1265 return ERR_PTR(-EINVAL
);
1267 dev_pagemap_get_ops();
1269 devmem
= devres_alloc_node(&hmm_devmem_release
, sizeof(*devmem
),
1270 GFP_KERNEL
, dev_to_node(device
));
1272 return ERR_PTR(-ENOMEM
);
1274 init_completion(&devmem
->completion
);
1275 devmem
->pfn_first
= -1UL;
1276 devmem
->pfn_last
= -1UL;
1277 devmem
->resource
= res
;
1278 devmem
->device
= device
;
1281 ret
= percpu_ref_init(&devmem
->ref
, &hmm_devmem_ref_release
,
1284 goto error_percpu_ref
;
1286 ret
= devm_add_action(device
, hmm_devmem_ref_exit
, &devmem
->ref
);
1288 goto error_devm_add_action
;
1291 devmem
->pfn_first
= devmem
->resource
->start
>> PAGE_SHIFT
;
1292 devmem
->pfn_last
= devmem
->pfn_first
+
1293 (resource_size(devmem
->resource
) >> PAGE_SHIFT
);
1295 ret
= hmm_devmem_pages_create(devmem
);
1297 goto error_devm_add_action
;
1299 devres_add(device
, devmem
);
1301 ret
= devm_add_action(device
, hmm_devmem_ref_kill
, &devmem
->ref
);
1303 hmm_devmem_remove(devmem
);
1304 return ERR_PTR(ret
);
1309 error_devm_add_action
:
1310 hmm_devmem_ref_kill(&devmem
->ref
);
1311 hmm_devmem_ref_exit(&devmem
->ref
);
1313 devres_free(devmem
);
1314 return ERR_PTR(ret
);
1316 EXPORT_SYMBOL(hmm_devmem_add_resource
);
1319 * hmm_devmem_remove() - remove device memory (kill and free ZONE_DEVICE)
1321 * @devmem: hmm_devmem struct use to track and manage the ZONE_DEVICE memory
1323 * This will hot-unplug memory that was hotplugged by hmm_devmem_add on behalf
1324 * of the device driver. It will free struct page and remove the resource that
1325 * reserved the physical address range for this device memory.
1327 void hmm_devmem_remove(struct hmm_devmem
*devmem
)
1329 resource_size_t start
, size
;
1330 struct device
*device
;
1336 device
= devmem
->device
;
1337 start
= devmem
->resource
->start
;
1338 size
= resource_size(devmem
->resource
);
1340 cdm
= devmem
->resource
->desc
== IORES_DESC_DEVICE_PUBLIC_MEMORY
;
1341 hmm_devmem_ref_kill(&devmem
->ref
);
1342 hmm_devmem_ref_exit(&devmem
->ref
);
1343 hmm_devmem_pages_remove(devmem
);
1346 devm_release_mem_region(device
, start
, size
);
1348 EXPORT_SYMBOL(hmm_devmem_remove
);
1351 * A device driver that wants to handle multiple devices memory through a
1352 * single fake device can use hmm_device to do so. This is purely a helper
1353 * and it is not needed to make use of any HMM functionality.
1355 #define HMM_DEVICE_MAX 256
1357 static DECLARE_BITMAP(hmm_device_mask
, HMM_DEVICE_MAX
);
1358 static DEFINE_SPINLOCK(hmm_device_lock
);
1359 static struct class *hmm_device_class
;
1360 static dev_t hmm_device_devt
;
1362 static void hmm_device_release(struct device
*device
)
1364 struct hmm_device
*hmm_device
;
1366 hmm_device
= container_of(device
, struct hmm_device
, device
);
1367 spin_lock(&hmm_device_lock
);
1368 clear_bit(hmm_device
->minor
, hmm_device_mask
);
1369 spin_unlock(&hmm_device_lock
);
1374 struct hmm_device
*hmm_device_new(void *drvdata
)
1376 struct hmm_device
*hmm_device
;
1378 hmm_device
= kzalloc(sizeof(*hmm_device
), GFP_KERNEL
);
1380 return ERR_PTR(-ENOMEM
);
1382 spin_lock(&hmm_device_lock
);
1383 hmm_device
->minor
= find_first_zero_bit(hmm_device_mask
, HMM_DEVICE_MAX
);
1384 if (hmm_device
->minor
>= HMM_DEVICE_MAX
) {
1385 spin_unlock(&hmm_device_lock
);
1387 return ERR_PTR(-EBUSY
);
1389 set_bit(hmm_device
->minor
, hmm_device_mask
);
1390 spin_unlock(&hmm_device_lock
);
1392 dev_set_name(&hmm_device
->device
, "hmm_device%d", hmm_device
->minor
);
1393 hmm_device
->device
.devt
= MKDEV(MAJOR(hmm_device_devt
),
1395 hmm_device
->device
.release
= hmm_device_release
;
1396 dev_set_drvdata(&hmm_device
->device
, drvdata
);
1397 hmm_device
->device
.class = hmm_device_class
;
1398 device_initialize(&hmm_device
->device
);
1402 EXPORT_SYMBOL(hmm_device_new
);
1404 void hmm_device_put(struct hmm_device
*hmm_device
)
1406 put_device(&hmm_device
->device
);
1408 EXPORT_SYMBOL(hmm_device_put
);
1410 static int __init
hmm_init(void)
1414 ret
= alloc_chrdev_region(&hmm_device_devt
, 0,
1420 hmm_device_class
= class_create(THIS_MODULE
, "hmm_device");
1421 if (IS_ERR(hmm_device_class
)) {
1422 unregister_chrdev_region(hmm_device_devt
, HMM_DEVICE_MAX
);
1423 return PTR_ERR(hmm_device_class
);
1428 device_initcall(hmm_init
);
1429 #endif /* CONFIG_DEVICE_PRIVATE || CONFIG_DEVICE_PUBLIC */