2 #include <linux/rmap.h>
3 #include <linux/hugetlb.h>
4 #include <linux/swap.h>
5 #include <linux/swapops.h>
9 static inline bool not_found(struct page_vma_mapped_walk
*pvmw
)
11 page_vma_mapped_walk_done(pvmw
);
15 static bool map_pte(struct page_vma_mapped_walk
*pvmw
)
17 pvmw
->pte
= pte_offset_map(pvmw
->pmd
, pvmw
->address
);
18 if (!(pvmw
->flags
& PVMW_SYNC
)) {
19 if (pvmw
->flags
& PVMW_MIGRATION
) {
20 if (!is_swap_pte(*pvmw
->pte
))
23 if (!pte_present(*pvmw
->pte
))
27 pvmw
->ptl
= pte_lockptr(pvmw
->vma
->vm_mm
, pvmw
->pmd
);
32 static bool check_pte(struct page_vma_mapped_walk
*pvmw
)
34 if (pvmw
->flags
& PVMW_MIGRATION
) {
35 #ifdef CONFIG_MIGRATION
37 if (!is_swap_pte(*pvmw
->pte
))
39 entry
= pte_to_swp_entry(*pvmw
->pte
);
40 if (!is_migration_entry(entry
))
42 if (migration_entry_to_page(entry
) - pvmw
->page
>=
43 hpage_nr_pages(pvmw
->page
)) {
46 if (migration_entry_to_page(entry
) < pvmw
->page
)
52 if (!pte_present(*pvmw
->pte
))
55 /* THP can be referenced by any subpage */
56 if (pte_page(*pvmw
->pte
) - pvmw
->page
>=
57 hpage_nr_pages(pvmw
->page
)) {
60 if (pte_page(*pvmw
->pte
) < pvmw
->page
)
68 * page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at
70 * @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
71 * must be set. pmd, pte and ptl must be NULL.
73 * Returns true if the page is mapped in the vma. @pvmw->pmd and @pvmw->pte point
74 * to relevant page table entries. @pvmw->ptl is locked. @pvmw->address is
75 * adjusted if needed (for PTE-mapped THPs).
77 * If @pvmw->pmd is set but @pvmw->pte is not, you have found PMD-mapped page
78 * (usually THP). For PTE-mapped THP, you should run page_vma_mapped_walk() in
79 * a loop to find all PTEs that map the THP.
81 * For HugeTLB pages, @pvmw->pte is set to the relevant page table entry
82 * regardless of which page table level the page is mapped at. @pvmw->pmd is
85 * Retruns false if there are no more page table entries for the page in
86 * the vma. @pvmw->ptl is unlocked and @pvmw->pte is unmapped.
88 * If you need to stop the walk before page_vma_mapped_walk() returned false,
89 * use page_vma_mapped_walk_done(). It will do the housekeeping.
91 bool page_vma_mapped_walk(struct page_vma_mapped_walk
*pvmw
)
93 struct mm_struct
*mm
= pvmw
->vma
->vm_mm
;
94 struct page
*page
= pvmw
->page
;
100 /* The only possible pmd mapping has been handled on last iteration */
101 if (pvmw
->pmd
&& !pvmw
->pte
)
102 return not_found(pvmw
);
107 if (unlikely(PageHuge(pvmw
->page
))) {
108 /* when pud is not present, pte will be NULL */
109 pvmw
->pte
= huge_pte_offset(mm
, pvmw
->address
,
110 PAGE_SIZE
<< compound_order(page
));
114 pvmw
->ptl
= huge_pte_lockptr(page_hstate(page
), mm
, pvmw
->pte
);
115 spin_lock(pvmw
->ptl
);
116 if (!check_pte(pvmw
))
117 return not_found(pvmw
);
121 pgd
= pgd_offset(mm
, pvmw
->address
);
122 if (!pgd_present(*pgd
))
124 p4d
= p4d_offset(pgd
, pvmw
->address
);
125 if (!p4d_present(*p4d
))
127 pud
= pud_offset(p4d
, pvmw
->address
);
128 if (!pud_present(*pud
))
130 pvmw
->pmd
= pmd_offset(pud
, pvmw
->address
);
132 * Make sure the pmd value isn't cached in a register by the
133 * compiler and used as a stale value after we've observed a
136 pmde
= READ_ONCE(*pvmw
->pmd
);
137 if (pmd_trans_huge(pmde
)) {
138 pvmw
->ptl
= pmd_lock(mm
, pvmw
->pmd
);
139 if (!pmd_present(*pvmw
->pmd
))
140 return not_found(pvmw
);
141 if (likely(pmd_trans_huge(*pvmw
->pmd
))) {
142 if (pvmw
->flags
& PVMW_MIGRATION
)
143 return not_found(pvmw
);
144 if (pmd_page(*pvmw
->pmd
) != page
)
145 return not_found(pvmw
);
148 /* THP pmd was split under us: handle on pte level */
149 spin_unlock(pvmw
->ptl
);
152 } else if (!pmd_present(pmde
)) {
161 /* Seek to next pte only makes sense for THP */
162 if (!PageTransHuge(pvmw
->page
) || PageHuge(pvmw
->page
))
163 return not_found(pvmw
);
165 pvmw
->address
+= PAGE_SIZE
;
166 if (pvmw
->address
>= pvmw
->vma
->vm_end
||
168 __vma_address(pvmw
->page
, pvmw
->vma
) +
169 hpage_nr_pages(pvmw
->page
) * PAGE_SIZE
)
170 return not_found(pvmw
);
171 /* Did we cross page table boundary? */
172 if (pvmw
->address
% PMD_SIZE
== 0) {
173 pte_unmap(pvmw
->pte
);
175 spin_unlock(pvmw
->ptl
);
182 } while (pte_none(*pvmw
->pte
));
185 pvmw
->ptl
= pte_lockptr(mm
, pvmw
->pmd
);
186 spin_lock(pvmw
->ptl
);
192 * page_mapped_in_vma - check whether a page is really mapped in a VMA
193 * @page: the page to test
194 * @vma: the VMA to test
196 * Returns 1 if the page is mapped into the page tables of the VMA, 0
197 * if the page is not mapped into the page tables of this VMA. Only
198 * valid for normal file or anonymous VMAs.
200 int page_mapped_in_vma(struct page
*page
, struct vm_area_struct
*vma
)
202 struct page_vma_mapped_walk pvmw
= {
207 unsigned long start
, end
;
209 start
= __vma_address(page
, vma
);
210 end
= start
+ PAGE_SIZE
* (hpage_nr_pages(page
) - 1);
212 if (unlikely(end
< vma
->vm_start
|| start
>= vma
->vm_end
))
214 pvmw
.address
= max(start
, vma
->vm_start
);
215 if (!page_vma_mapped_walk(&pvmw
))
217 page_vma_mapped_walk_done(&pvmw
);