]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - mm/pagewalk.c
2 #include <linux/highmem.h>
3 #include <linux/sched.h>
4 #include <linux/hugetlb.h>
6 static int walk_pte_range(pmd_t
*pmd
, unsigned long addr
, unsigned long end
,
12 pte
= pte_offset_map(pmd
, addr
);
14 err
= walk
->pte_entry(pte
, addr
, addr
+ PAGE_SIZE
, walk
);
27 static int walk_pmd_range(pud_t
*pud
, unsigned long addr
, unsigned long end
,
34 pmd
= pmd_offset(pud
, addr
);
37 next
= pmd_addr_end(addr
, end
);
38 if (pmd_none(*pmd
) || !walk
->vma
) {
40 err
= walk
->pte_hole(addr
, next
, walk
);
46 * This implies that each ->pmd_entry() handler
47 * needs to know about pmd_trans_huge() pmds
50 err
= walk
->pmd_entry(pmd
, addr
, next
, walk
);
55 * Check this here so we only break down trans_huge
56 * pages when we _need_ to
61 split_huge_pmd(walk
->vma
, pmd
, addr
);
62 if (pmd_trans_unstable(pmd
))
64 err
= walk_pte_range(pmd
, addr
, next
, walk
);
67 } while (pmd
++, addr
= next
, addr
!= end
);
72 static int walk_pud_range(p4d_t
*p4d
, unsigned long addr
, unsigned long end
,
79 pud
= pud_offset(p4d
, addr
);
82 next
= pud_addr_end(addr
, end
);
83 if (pud_none(*pud
) || !walk
->vma
) {
85 err
= walk
->pte_hole(addr
, next
, walk
);
91 if (walk
->pud_entry
) {
92 spinlock_t
*ptl
= pud_trans_huge_lock(pud
, walk
->vma
);
95 err
= walk
->pud_entry(pud
, addr
, next
, walk
);
103 split_huge_pud(walk
->vma
, pud
, addr
);
107 if (walk
->pmd_entry
|| walk
->pte_entry
)
108 err
= walk_pmd_range(pud
, addr
, next
, walk
);
111 } while (pud
++, addr
= next
, addr
!= end
);
116 static int walk_p4d_range(pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
117 struct mm_walk
*walk
)
123 p4d
= p4d_offset(pgd
, addr
);
125 next
= p4d_addr_end(addr
, end
);
126 if (p4d_none_or_clear_bad(p4d
)) {
128 err
= walk
->pte_hole(addr
, next
, walk
);
133 if (walk
->pmd_entry
|| walk
->pte_entry
)
134 err
= walk_pud_range(p4d
, addr
, next
, walk
);
137 } while (p4d
++, addr
= next
, addr
!= end
);
142 static int walk_pgd_range(unsigned long addr
, unsigned long end
,
143 struct mm_walk
*walk
)
149 pgd
= pgd_offset(walk
->mm
, addr
);
151 next
= pgd_addr_end(addr
, end
);
152 if (pgd_none_or_clear_bad(pgd
)) {
154 err
= walk
->pte_hole(addr
, next
, walk
);
159 if (walk
->pmd_entry
|| walk
->pte_entry
)
160 err
= walk_p4d_range(pgd
, addr
, next
, walk
);
163 } while (pgd
++, addr
= next
, addr
!= end
);
168 #ifdef CONFIG_HUGETLB_PAGE
169 static unsigned long hugetlb_entry_end(struct hstate
*h
, unsigned long addr
,
172 unsigned long boundary
= (addr
& huge_page_mask(h
)) + huge_page_size(h
);
173 return boundary
< end
? boundary
: end
;
176 static int walk_hugetlb_range(unsigned long addr
, unsigned long end
,
177 struct mm_walk
*walk
)
179 struct vm_area_struct
*vma
= walk
->vma
;
180 struct hstate
*h
= hstate_vma(vma
);
182 unsigned long hmask
= huge_page_mask(h
);
183 unsigned long sz
= huge_page_size(h
);
188 next
= hugetlb_entry_end(h
, addr
, end
);
189 pte
= huge_pte_offset(walk
->mm
, addr
& hmask
, sz
);
190 if (pte
&& walk
->hugetlb_entry
)
191 err
= walk
->hugetlb_entry(pte
, hmask
, addr
, next
, walk
);
194 } while (addr
= next
, addr
!= end
);
199 #else /* CONFIG_HUGETLB_PAGE */
200 static int walk_hugetlb_range(unsigned long addr
, unsigned long end
,
201 struct mm_walk
*walk
)
206 #endif /* CONFIG_HUGETLB_PAGE */
209 * Decide whether we really walk over the current vma on [@start, @end)
210 * or skip it via the returned value. Return 0 if we do walk over the
211 * current vma, and return 1 if we skip the vma. Negative values means
212 * error, where we abort the current walk.
214 static int walk_page_test(unsigned long start
, unsigned long end
,
215 struct mm_walk
*walk
)
217 struct vm_area_struct
*vma
= walk
->vma
;
220 return walk
->test_walk(start
, end
, walk
);
223 * vma(VM_PFNMAP) doesn't have any valid struct pages behind VM_PFNMAP
224 * range, so we don't walk over it as we do for normal vmas. However,
225 * Some callers are interested in handling hole range and they don't
226 * want to just ignore any single address range. Such users certainly
227 * define their ->pte_hole() callbacks, so let's delegate them to handle
230 if (vma
->vm_flags
& VM_PFNMAP
) {
233 err
= walk
->pte_hole(start
, end
, walk
);
234 return err
? err
: 1;
239 static int __walk_page_range(unsigned long start
, unsigned long end
,
240 struct mm_walk
*walk
)
243 struct vm_area_struct
*vma
= walk
->vma
;
245 if (vma
&& is_vm_hugetlb_page(vma
)) {
246 if (walk
->hugetlb_entry
)
247 err
= walk_hugetlb_range(start
, end
, walk
);
249 err
= walk_pgd_range(start
, end
, walk
);
255 * walk_page_range - walk page table with caller specific callbacks
257 * Recursively walk the page table tree of the process represented by @walk->mm
258 * within the virtual address range [@start, @end). During walking, we can do
259 * some caller-specific works for each entry, by setting up pmd_entry(),
260 * pte_entry(), and/or hugetlb_entry(). If you don't set up for some of these
261 * callbacks, the associated entries/pages are just ignored.
262 * The return values of these callbacks are commonly defined like below:
263 * - 0 : succeeded to handle the current entry, and if you don't reach the
264 * end address yet, continue to walk.
265 * - >0 : succeeded to handle the current entry, and return to the caller
266 * with caller specific value.
267 * - <0 : failed to handle the current entry, and return to the caller
270 * Before starting to walk page table, some callers want to check whether
271 * they really want to walk over the current vma, typically by checking
272 * its vm_flags. walk_page_test() and @walk->test_walk() are used for this
275 * struct mm_walk keeps current values of some common data like vma and pmd,
276 * which are useful for the access from callbacks. If you want to pass some
277 * caller-specific data to callbacks, @walk->private should be helpful.
280 * Callers of walk_page_range() and walk_page_vma() should hold
281 * @walk->mm->mmap_sem, because these function traverse vma list and/or
282 * access to vma's data.
284 int walk_page_range(unsigned long start
, unsigned long end
,
285 struct mm_walk
*walk
)
289 struct vm_area_struct
*vma
;
297 VM_BUG_ON_MM(!rwsem_is_locked(&walk
->mm
->mmap_sem
), walk
->mm
);
299 vma
= find_vma(walk
->mm
, start
);
301 if (!vma
) { /* after the last vma */
304 } else if (start
< vma
->vm_start
) { /* outside vma */
306 next
= min(end
, vma
->vm_start
);
307 } else { /* inside vma */
309 next
= min(end
, vma
->vm_end
);
312 err
= walk_page_test(start
, next
, walk
);
315 * positive return values are purely for
316 * controlling the pagewalk, so should never
317 * be passed to the callers.
325 if (walk
->vma
|| walk
->pte_hole
)
326 err
= __walk_page_range(start
, next
, walk
);
329 } while (start
= next
, start
< end
);
333 int walk_page_vma(struct vm_area_struct
*vma
, struct mm_walk
*walk
)
340 VM_BUG_ON(!rwsem_is_locked(&walk
->mm
->mmap_sem
));
343 err
= walk_page_test(vma
->vm_start
, vma
->vm_end
, walk
);
348 return __walk_page_range(vma
->vm_start
, vma
->vm_end
, walk
);