2 #include <linux/rmap.h>
3 #include <linux/hugetlb.h>
4 #include <linux/swap.h>
5 #include <linux/swapops.h>
9 static inline bool check_pmd(struct page_vma_mapped_walk
*pvmw
)
13 * Make sure we don't re-load pmd between present and !trans_huge check.
14 * We need a consistent view.
16 pmde
= READ_ONCE(*pvmw
->pmd
);
17 return pmd_present(pmde
) && !pmd_trans_huge(pmde
);
20 static inline bool not_found(struct page_vma_mapped_walk
*pvmw
)
22 page_vma_mapped_walk_done(pvmw
);
26 static bool map_pte(struct page_vma_mapped_walk
*pvmw
)
28 pvmw
->pte
= pte_offset_map(pvmw
->pmd
, pvmw
->address
);
29 if (!(pvmw
->flags
& PVMW_SYNC
)) {
30 if (pvmw
->flags
& PVMW_MIGRATION
) {
31 if (!is_swap_pte(*pvmw
->pte
))
34 if (!pte_present(*pvmw
->pte
))
38 pvmw
->ptl
= pte_lockptr(pvmw
->vma
->vm_mm
, pvmw
->pmd
);
43 static bool check_pte(struct page_vma_mapped_walk
*pvmw
)
45 if (pvmw
->flags
& PVMW_MIGRATION
) {
46 #ifdef CONFIG_MIGRATION
48 if (!is_swap_pte(*pvmw
->pte
))
50 entry
= pte_to_swp_entry(*pvmw
->pte
);
51 if (!is_migration_entry(entry
))
53 if (migration_entry_to_page(entry
) - pvmw
->page
>=
54 hpage_nr_pages(pvmw
->page
)) {
57 if (migration_entry_to_page(entry
) < pvmw
->page
)
63 if (!pte_present(*pvmw
->pte
))
66 /* THP can be referenced by any subpage */
67 if (pte_page(*pvmw
->pte
) - pvmw
->page
>=
68 hpage_nr_pages(pvmw
->page
)) {
71 if (pte_page(*pvmw
->pte
) < pvmw
->page
)
79 * page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at
81 * @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags
82 * must be set. pmd, pte and ptl must be NULL.
84 * Returns true if the page is mapped in the vma. @pvmw->pmd and @pvmw->pte point
85 * to relevant page table entries. @pvmw->ptl is locked. @pvmw->address is
86 * adjusted if needed (for PTE-mapped THPs).
88 * If @pvmw->pmd is set but @pvmw->pte is not, you have found PMD-mapped page
89 * (usually THP). For PTE-mapped THP, you should run page_vma_mapped_walk() in
90 * a loop to find all PTEs that map the THP.
92 * For HugeTLB pages, @pvmw->pte is set to the relevant page table entry
93 * regardless of which page table level the page is mapped at. @pvmw->pmd is
96 * Retruns false if there are no more page table entries for the page in
97 * the vma. @pvmw->ptl is unlocked and @pvmw->pte is unmapped.
99 * If you need to stop the walk before page_vma_mapped_walk() returned false,
100 * use page_vma_mapped_walk_done(). It will do the housekeeping.
102 bool page_vma_mapped_walk(struct page_vma_mapped_walk
*pvmw
)
104 struct mm_struct
*mm
= pvmw
->vma
->vm_mm
;
105 struct page
*page
= pvmw
->page
;
109 /* The only possible pmd mapping has been handled on last iteration */
110 if (pvmw
->pmd
&& !pvmw
->pte
)
111 return not_found(pvmw
);
113 /* Only for THP, seek to next pte entry makes sense */
115 if (!PageTransHuge(pvmw
->page
) || PageHuge(pvmw
->page
))
116 return not_found(pvmw
);
120 if (unlikely(PageHuge(pvmw
->page
))) {
121 /* when pud is not present, pte will be NULL */
122 pvmw
->pte
= huge_pte_offset(mm
, pvmw
->address
);
126 pvmw
->ptl
= huge_pte_lockptr(page_hstate(page
), mm
, pvmw
->pte
);
127 spin_lock(pvmw
->ptl
);
128 if (!check_pte(pvmw
))
129 return not_found(pvmw
);
133 pgd
= pgd_offset(mm
, pvmw
->address
);
134 if (!pgd_present(*pgd
))
136 pud
= pud_offset(pgd
, pvmw
->address
);
137 if (!pud_present(*pud
))
139 pvmw
->pmd
= pmd_offset(pud
, pvmw
->address
);
140 if (pmd_trans_huge(*pvmw
->pmd
)) {
141 pvmw
->ptl
= pmd_lock(mm
, pvmw
->pmd
);
142 if (!pmd_present(*pvmw
->pmd
))
143 return not_found(pvmw
);
144 if (likely(pmd_trans_huge(*pvmw
->pmd
))) {
145 if (pvmw
->flags
& PVMW_MIGRATION
)
146 return not_found(pvmw
);
147 if (pmd_page(*pvmw
->pmd
) != page
)
148 return not_found(pvmw
);
151 /* THP pmd was split under us: handle on pte level */
152 spin_unlock(pvmw
->ptl
);
156 if (!check_pmd(pvmw
))
165 pvmw
->address
+= PAGE_SIZE
;
167 __vma_address(pvmw
->page
, pvmw
->vma
) +
168 hpage_nr_pages(pvmw
->page
) * PAGE_SIZE
)
169 return not_found(pvmw
);
170 /* Did we cross page table boundary? */
171 if (pvmw
->address
% PMD_SIZE
== 0) {
172 pte_unmap(pvmw
->pte
);
174 spin_unlock(pvmw
->ptl
);
181 } while (pte_none(*pvmw
->pte
));
184 pvmw
->ptl
= pte_lockptr(mm
, pvmw
->pmd
);
185 spin_lock(pvmw
->ptl
);
191 * page_mapped_in_vma - check whether a page is really mapped in a VMA
192 * @page: the page to test
193 * @vma: the VMA to test
195 * Returns 1 if the page is mapped into the page tables of the VMA, 0
196 * if the page is not mapped into the page tables of this VMA. Only
197 * valid for normal file or anonymous VMAs.
199 int page_mapped_in_vma(struct page
*page
, struct vm_area_struct
*vma
)
201 struct page_vma_mapped_walk pvmw
= {
206 unsigned long start
, end
;
208 start
= __vma_address(page
, vma
);
209 end
= start
+ PAGE_SIZE
* (hpage_nr_pages(page
) - 1);
211 if (unlikely(end
< vma
->vm_start
|| start
>= vma
->vm_end
))
213 pvmw
.address
= max(start
, vma
->vm_start
);
214 if (!page_vma_mapped_walk(&pvmw
))
216 page_vma_mapped_walk_done(&pvmw
);