1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/init.h>
3 #include <linux/memblock.h>
5 #include <linux/sysfs.h>
6 #include <linux/kobject.h>
7 #include <linux/memory_hotplug.h>
9 #include <linux/mmzone.h>
10 #include <linux/pagemap.h>
11 #include <linux/rmap.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/page_ext.h>
14 #include <linux/page_idle.h>
18 #define BITMAP_CHUNK_SIZE sizeof(u64)
19 #define BITMAP_CHUNK_BITS (BITMAP_CHUNK_SIZE * BITS_PER_BYTE)
22 * Idle page tracking only considers user memory pages, for other types of
23 * pages the idle flag is always unset and an attempt to set it is silently
26 * We treat a page as a user memory page if it is on an LRU list, because it is
27 * always safe to pass such a page to rmap_walk(), which is essential for idle
28 * page tracking. With such an indicator of user pages we can skip isolated
29 * pages, but since there are not usually many of them, it will hardly affect
32 * This function tries to get a user memory page by pfn as described above.
34 static struct page
*page_idle_get_page(unsigned long pfn
)
36 struct page
*page
= pfn_to_online_page(pfn
);
38 if (!page
|| !PageLRU(page
) ||
39 !get_page_unless_zero(page
))
42 if (unlikely(!PageLRU(page
))) {
49 static bool page_idle_clear_pte_refs_one(struct folio
*folio
,
50 struct vm_area_struct
*vma
,
51 unsigned long addr
, void *arg
)
53 DEFINE_FOLIO_VMA_WALK(pvmw
, folio
, vma
, addr
, 0);
54 bool referenced
= false;
56 while (page_vma_mapped_walk(&pvmw
)) {
60 * For PTE-mapped THP, one sub page is referenced,
61 * the whole THP is referenced.
63 if (ptep_clear_young_notify(vma
, addr
, pvmw
.pte
))
65 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE
)) {
66 if (pmdp_clear_young_notify(vma
, addr
, pvmw
.pmd
))
69 /* unexpected pmd-mapped page? */
75 folio_clear_idle(folio
);
77 * We cleared the referenced bit in a mapping to this page. To
78 * avoid interference with page reclaim, mark it young so that
79 * folio_referenced() will return > 0.
81 folio_set_young(folio
);
86 static void page_idle_clear_pte_refs(struct page
*page
)
88 struct folio
*folio
= page_folio(page
);
91 * Since rwc.try_lock is unused, rwc is effectively immutable, so we
92 * can make it static to save some cycles and stack.
94 static struct rmap_walk_control rwc
= {
95 .rmap_one
= page_idle_clear_pte_refs_one
,
96 .anon_lock
= folio_lock_anon_vma_read
,
100 if (!folio_mapped(folio
) || !folio_raw_mapping(folio
))
103 need_lock
= !folio_test_anon(folio
) || folio_test_ksm(folio
);
104 if (need_lock
&& !folio_trylock(folio
))
107 rmap_walk(folio
, &rwc
);
113 static ssize_t
page_idle_bitmap_read(struct file
*file
, struct kobject
*kobj
,
114 struct bin_attribute
*attr
, char *buf
,
115 loff_t pos
, size_t count
)
117 u64
*out
= (u64
*)buf
;
119 unsigned long pfn
, end_pfn
;
122 if (pos
% BITMAP_CHUNK_SIZE
|| count
% BITMAP_CHUNK_SIZE
)
125 pfn
= pos
* BITS_PER_BYTE
;
129 end_pfn
= pfn
+ count
* BITS_PER_BYTE
;
130 if (end_pfn
> max_pfn
)
133 for (; pfn
< end_pfn
; pfn
++) {
134 bit
= pfn
% BITMAP_CHUNK_BITS
;
137 page
= page_idle_get_page(pfn
);
139 if (page_is_idle(page
)) {
141 * The page might have been referenced via a
142 * pte, in which case it is not idle. Clear
145 page_idle_clear_pte_refs(page
);
146 if (page_is_idle(page
))
151 if (bit
== BITMAP_CHUNK_BITS
- 1)
155 return (char *)out
- buf
;
158 static ssize_t
page_idle_bitmap_write(struct file
*file
, struct kobject
*kobj
,
159 struct bin_attribute
*attr
, char *buf
,
160 loff_t pos
, size_t count
)
162 const u64
*in
= (u64
*)buf
;
164 unsigned long pfn
, end_pfn
;
167 if (pos
% BITMAP_CHUNK_SIZE
|| count
% BITMAP_CHUNK_SIZE
)
170 pfn
= pos
* BITS_PER_BYTE
;
174 end_pfn
= pfn
+ count
* BITS_PER_BYTE
;
175 if (end_pfn
> max_pfn
)
178 for (; pfn
< end_pfn
; pfn
++) {
179 bit
= pfn
% BITMAP_CHUNK_BITS
;
180 if ((*in
>> bit
) & 1) {
181 page
= page_idle_get_page(pfn
);
183 page_idle_clear_pte_refs(page
);
188 if (bit
== BITMAP_CHUNK_BITS
- 1)
192 return (char *)in
- buf
;
195 static struct bin_attribute page_idle_bitmap_attr
=
196 __BIN_ATTR(bitmap
, 0600,
197 page_idle_bitmap_read
, page_idle_bitmap_write
, 0);
199 static struct bin_attribute
*page_idle_bin_attrs
[] = {
200 &page_idle_bitmap_attr
,
204 static const struct attribute_group page_idle_attr_group
= {
205 .bin_attrs
= page_idle_bin_attrs
,
209 static int __init
page_idle_init(void)
213 err
= sysfs_create_group(mm_kobj
, &page_idle_attr_group
);
215 pr_err("page_idle: register sysfs failed\n");
220 subsys_initcall(page_idle_init
);