]>
git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blob - mm/pagewalk.c
2 #include <linux/highmem.h>
3 #include <linux/sched.h>
4 #include <linux/hugetlb.h>
6 static int walk_pte_range(pmd_t
*pmd
, unsigned long addr
, unsigned long end
,
12 pte
= pte_offset_map(pmd
, addr
);
14 err
= walk
->pte_entry(pte
, addr
, addr
+ PAGE_SIZE
, walk
);
27 static int walk_pmd_range(pud_t
*pud
, unsigned long addr
, unsigned long end
,
34 pmd
= pmd_offset(pud
, addr
);
37 next
= pmd_addr_end(addr
, end
);
40 err
= walk
->pte_hole(addr
, next
, walk
);
46 * This implies that each ->pmd_entry() handler
47 * needs to know about pmd_trans_huge() pmds
50 err
= walk
->pmd_entry(pmd
, addr
, next
, walk
);
55 * Check this here so we only break down trans_huge
56 * pages when we _need_ to
61 split_huge_page_pmd_mm(walk
->mm
, addr
, pmd
);
62 if (pmd_none_or_trans_huge_or_clear_bad(pmd
))
64 err
= walk_pte_range(pmd
, addr
, next
, walk
);
67 } while (pmd
++, addr
= next
, addr
!= end
);
72 static int walk_pud_range(pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
79 pud
= pud_offset(pgd
, addr
);
81 next
= pud_addr_end(addr
, end
);
82 if (pud_none_or_clear_bad(pud
)) {
84 err
= walk
->pte_hole(addr
, next
, walk
);
90 err
= walk
->pud_entry(pud
, addr
, next
, walk
);
91 if (!err
&& (walk
->pmd_entry
|| walk
->pte_entry
))
92 err
= walk_pmd_range(pud
, addr
, next
, walk
);
95 } while (pud
++, addr
= next
, addr
!= end
);
100 #ifdef CONFIG_HUGETLB_PAGE
101 static unsigned long hugetlb_entry_end(struct hstate
*h
, unsigned long addr
,
104 unsigned long boundary
= (addr
& huge_page_mask(h
)) + huge_page_size(h
);
105 return boundary
< end
? boundary
: end
;
108 static int walk_hugetlb_range(struct vm_area_struct
*vma
,
109 unsigned long addr
, unsigned long end
,
110 struct mm_walk
*walk
)
112 struct hstate
*h
= hstate_vma(vma
);
114 unsigned long hmask
= huge_page_mask(h
);
119 next
= hugetlb_entry_end(h
, addr
, end
);
120 pte
= huge_pte_offset(walk
->mm
, addr
& hmask
);
121 if (pte
&& walk
->hugetlb_entry
)
122 err
= walk
->hugetlb_entry(pte
, hmask
, addr
, next
, walk
);
125 } while (addr
= next
, addr
!= end
);
130 #else /* CONFIG_HUGETLB_PAGE */
131 static int walk_hugetlb_range(struct vm_area_struct
*vma
,
132 unsigned long addr
, unsigned long end
,
133 struct mm_walk
*walk
)
138 #endif /* CONFIG_HUGETLB_PAGE */
143 * walk_page_range - walk a memory map's page tables with a callback
144 * @addr: starting address
145 * @end: ending address
146 * @walk: set of callbacks to invoke for each level of the tree
148 * Recursively walk the page table for the memory area in a VMA,
149 * calling supplied callbacks. Callbacks are called in-order (first
150 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
151 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
153 * Each callback receives an entry pointer and the start and end of the
154 * associated range, and a copy of the original mm_walk for access to
155 * the ->private or ->mm fields.
157 * Usually no locks are taken, but splitting transparent huge page may
158 * take page table lock. And the bottom level iterator will map PTE
159 * directories from highmem if necessary.
161 * If any callback returns a non-zero value, the walk is aborted and
162 * the return value is propagated back to the caller. Otherwise 0 is returned.
164 * walk->mm->mmap_sem must be held for at least read if walk->hugetlb_entry
167 int walk_page_range(unsigned long addr
, unsigned long end
,
168 struct mm_walk
*walk
)
180 VM_BUG_ON_MM(!rwsem_is_locked(&walk
->mm
->mmap_sem
), walk
->mm
);
182 pgd
= pgd_offset(walk
->mm
, addr
);
184 struct vm_area_struct
*vma
= NULL
;
186 next
= pgd_addr_end(addr
, end
);
189 * This function was not intended to be vma based.
190 * But there are vma special cases to be handled:
194 vma
= find_vma(walk
->mm
, addr
);
197 * There are no page structures backing a VM_PFNMAP
198 * range, so do not allow split_huge_page_pmd().
200 if ((vma
->vm_start
<= addr
) &&
201 (vma
->vm_flags
& VM_PFNMAP
)) {
203 pgd
= pgd_offset(walk
->mm
, next
);
207 * Handle hugetlb vma individually because pagetable
208 * walk for the hugetlb page is dependent on the
209 * architecture and we can't handled it in the same
210 * manner as non-huge pages.
212 if (walk
->hugetlb_entry
&& (vma
->vm_start
<= addr
) &&
213 is_vm_hugetlb_page(vma
)) {
214 if (vma
->vm_end
< next
)
217 * Hugepage is very tightly coupled with vma,
218 * so walk through hugetlb entries within a
221 err
= walk_hugetlb_range(vma
, addr
, next
, walk
);
224 pgd
= pgd_offset(walk
->mm
, next
);
229 if (pgd_none_or_clear_bad(pgd
)) {
231 err
= walk
->pte_hole(addr
, next
, walk
);
238 err
= walk
->pgd_entry(pgd
, addr
, next
, walk
);
240 (walk
->pud_entry
|| walk
->pmd_entry
|| walk
->pte_entry
))
241 err
= walk_pud_range(pgd
, addr
, next
, walk
);
245 } while (addr
= next
, addr
< end
);