1 // SPDX-License-Identifier: GPL-2.0
3 * DAMON Primitives for Virtual Address Spaces
5 * Author: SeongJae Park <sjpark@amazon.de>
8 #define pr_fmt(fmt) "damon-va: " fmt
10 #include <asm-generic/mman-common.h>
11 #include <linux/highmem.h>
12 #include <linux/hugetlb.h>
13 #include <linux/mmu_notifier.h>
14 #include <linux/page_idle.h>
15 #include <linux/pagewalk.h>
16 #include <linux/sched/mm.h>
18 #include "ops-common.h"
20 #ifdef CONFIG_DAMON_VADDR_KUNIT_TEST
21 #undef DAMON_MIN_REGION
22 #define DAMON_MIN_REGION 1
26 * 't->pid' should be the pointer to the relevant 'struct pid' having reference
27 * count. Caller must put the returned task, unless it is NULL.
29 static inline struct task_struct
*damon_get_task_struct(struct damon_target
*t
)
31 return get_pid_task(t
->pid
, PIDTYPE_PID
);
35 * Get the mm_struct of the given target
37 * Caller _must_ put the mm_struct after use, unless it is NULL.
39 * Returns the mm_struct of the target on success, NULL on failure
41 static struct mm_struct
*damon_get_mm(struct damon_target
*t
)
43 struct task_struct
*task
;
46 task
= damon_get_task_struct(t
);
50 mm
= get_task_mm(task
);
51 put_task_struct(task
);
56 * Functions for the initial monitoring target regions construction
60 * Size-evenly split a region into 'nr_pieces' small regions
62 * Returns 0 on success, or negative error code otherwise.
64 static int damon_va_evenly_split_region(struct damon_target
*t
,
65 struct damon_region
*r
, unsigned int nr_pieces
)
67 unsigned long sz_orig
, sz_piece
, orig_end
;
68 struct damon_region
*n
= NULL
, *next
;
75 sz_orig
= damon_sz_region(r
);
76 sz_piece
= ALIGN_DOWN(sz_orig
/ nr_pieces
, DAMON_MIN_REGION
);
81 r
->ar
.end
= r
->ar
.start
+ sz_piece
;
82 next
= damon_next_region(r
);
83 for (start
= r
->ar
.end
; start
+ sz_piece
<= orig_end
;
85 n
= damon_new_region(start
, start
+ sz_piece
);
88 damon_insert_region(n
, r
, next
, t
);
91 /* complement last region for possible rounding error */
98 static unsigned long sz_range(struct damon_addr_range
*r
)
100 return r
->end
- r
->start
;
104 * Find three regions separated by two biggest unmapped regions
106 * vma the head vma of the target address space
107 * regions an array of three address ranges that results will be saved
109 * This function receives an address space and finds three regions in it which
110 * separated by the two biggest unmapped regions in the space. Please refer to
111 * below comments of '__damon_va_init_regions()' function to know why this is
114 * Returns 0 if success, or negative error code otherwise.
116 static int __damon_va_three_regions(struct mm_struct
*mm
,
117 struct damon_addr_range regions
[3])
119 struct damon_addr_range first_gap
= {0}, second_gap
= {0};
120 VMA_ITERATOR(vmi
, mm
, 0);
121 struct vm_area_struct
*vma
, *prev
= NULL
;
125 * Find the two biggest gaps so that first_gap > second_gap > others.
126 * If this is too slow, it can be optimised to examine the maple
129 for_each_vma(vmi
, vma
) {
133 start
= vma
->vm_start
;
136 gap
= vma
->vm_start
- prev
->vm_end
;
138 if (gap
> sz_range(&first_gap
)) {
139 second_gap
= first_gap
;
140 first_gap
.start
= prev
->vm_end
;
141 first_gap
.end
= vma
->vm_start
;
142 } else if (gap
> sz_range(&second_gap
)) {
143 second_gap
.start
= prev
->vm_end
;
144 second_gap
.end
= vma
->vm_start
;
150 if (!sz_range(&second_gap
) || !sz_range(&first_gap
))
153 /* Sort the two biggest gaps by address */
154 if (first_gap
.start
> second_gap
.start
)
155 swap(first_gap
, second_gap
);
157 /* Store the result */
158 regions
[0].start
= ALIGN(start
, DAMON_MIN_REGION
);
159 regions
[0].end
= ALIGN(first_gap
.start
, DAMON_MIN_REGION
);
160 regions
[1].start
= ALIGN(first_gap
.end
, DAMON_MIN_REGION
);
161 regions
[1].end
= ALIGN(second_gap
.start
, DAMON_MIN_REGION
);
162 regions
[2].start
= ALIGN(second_gap
.end
, DAMON_MIN_REGION
);
163 regions
[2].end
= ALIGN(prev
->vm_end
, DAMON_MIN_REGION
);
169 * Get the three regions in the given target (task)
171 * Returns 0 on success, negative error code otherwise.
173 static int damon_va_three_regions(struct damon_target
*t
,
174 struct damon_addr_range regions
[3])
176 struct mm_struct
*mm
;
179 mm
= damon_get_mm(t
);
184 rc
= __damon_va_three_regions(mm
, regions
);
185 mmap_read_unlock(mm
);
192 * Initialize the monitoring target regions for the given target (task)
196 * Because only a number of small portions of the entire address space
197 * is actually mapped to the memory and accessed, monitoring the unmapped
198 * regions is wasteful. That said, because we can deal with small noises,
199 * tracking every mapping is not strictly required but could even incur a high
200 * overhead if the mapping frequently changes or the number of mappings is
201 * high. The adaptive regions adjustment mechanism will further help to deal
202 * with the noise by simply identifying the unmapped areas as a region that
203 * has no access. Moreover, applying the real mappings that would have many
204 * unmapped areas inside will make the adaptive mechanism quite complex. That
205 * said, too huge unmapped areas inside the monitoring target should be removed
206 * to not take the time for the adaptive mechanism.
208 * For the reason, we convert the complex mappings to three distinct regions
209 * that cover every mapped area of the address space. Also the two gaps
210 * between the three regions are the two biggest unmapped areas in the given
211 * address space. In detail, this function first identifies the start and the
212 * end of the mappings and the two biggest unmapped areas of the address space.
213 * Then, it constructs the three regions as below:
215 * [mappings[0]->start, big_two_unmapped_areas[0]->start)
216 * [big_two_unmapped_areas[0]->end, big_two_unmapped_areas[1]->start)
217 * [big_two_unmapped_areas[1]->end, mappings[nr_mappings - 1]->end)
219 * As usual memory map of processes is as below, the gap between the heap and
220 * the uppermost mmap()-ed region, and the gap between the lowermost mmap()-ed
221 * region and the stack will be two biggest unmapped regions. Because these
222 * gaps are exceptionally huge areas in usual address space, excluding these
223 * two biggest unmapped regions will be sufficient to make a trade-off.
226 * <BIG UNMAPPED REGION 1>
227 * <uppermost mmap()-ed region>
228 * (other mmap()-ed regions and small unmapped regions)
229 * <lowermost mmap()-ed region>
230 * <BIG UNMAPPED REGION 2>
233 static void __damon_va_init_regions(struct damon_ctx
*ctx
,
234 struct damon_target
*t
)
236 struct damon_target
*ti
;
237 struct damon_region
*r
;
238 struct damon_addr_range regions
[3];
239 unsigned long sz
= 0, nr_pieces
;
242 if (damon_va_three_regions(t
, regions
)) {
243 damon_for_each_target(ti
, ctx
) {
248 pr_debug("Failed to get three regions of %dth target\n", tidx
);
252 for (i
= 0; i
< 3; i
++)
253 sz
+= regions
[i
].end
- regions
[i
].start
;
254 if (ctx
->attrs
.min_nr_regions
)
255 sz
/= ctx
->attrs
.min_nr_regions
;
256 if (sz
< DAMON_MIN_REGION
)
257 sz
= DAMON_MIN_REGION
;
259 /* Set the initial three regions of the target */
260 for (i
= 0; i
< 3; i
++) {
261 r
= damon_new_region(regions
[i
].start
, regions
[i
].end
);
263 pr_err("%d'th init region creation failed\n", i
);
266 damon_add_region(r
, t
);
268 nr_pieces
= (regions
[i
].end
- regions
[i
].start
) / sz
;
269 damon_va_evenly_split_region(t
, r
, nr_pieces
);
273 /* Initialize '->regions_list' of every target (task) */
274 static void damon_va_init(struct damon_ctx
*ctx
)
276 struct damon_target
*t
;
278 damon_for_each_target(t
, ctx
) {
279 /* the user may set the target regions as they want */
280 if (!damon_nr_regions(t
))
281 __damon_va_init_regions(ctx
, t
);
286 * Update regions for current memory mappings
288 static void damon_va_update(struct damon_ctx
*ctx
)
290 struct damon_addr_range three_regions
[3];
291 struct damon_target
*t
;
293 damon_for_each_target(t
, ctx
) {
294 if (damon_va_three_regions(t
, three_regions
))
296 damon_set_regions(t
, three_regions
, 3);
300 static int damon_mkold_pmd_entry(pmd_t
*pmd
, unsigned long addr
,
301 unsigned long next
, struct mm_walk
*walk
)
306 if (pmd_trans_huge(*pmd
)) {
307 ptl
= pmd_lock(walk
->mm
, pmd
);
308 if (!pmd_present(*pmd
)) {
313 if (pmd_trans_huge(*pmd
)) {
314 damon_pmdp_mkold(pmd
, walk
->mm
, addr
);
321 if (pmd_none(*pmd
) || unlikely(pmd_bad(*pmd
)))
323 pte
= pte_offset_map_lock(walk
->mm
, pmd
, addr
, &ptl
);
324 if (!pte_present(*pte
))
326 damon_ptep_mkold(pte
, walk
->mm
, addr
);
328 pte_unmap_unlock(pte
, ptl
);
332 #ifdef CONFIG_HUGETLB_PAGE
333 static void damon_hugetlb_mkold(pte_t
*pte
, struct mm_struct
*mm
,
334 struct vm_area_struct
*vma
, unsigned long addr
)
336 bool referenced
= false;
337 pte_t entry
= huge_ptep_get(pte
);
338 struct page
*page
= pte_page(entry
);
342 if (pte_young(entry
)) {
344 entry
= pte_mkold(entry
);
345 set_huge_pte_at(mm
, addr
, pte
, entry
);
348 #ifdef CONFIG_MMU_NOTIFIER
349 if (mmu_notifier_clear_young(mm
, addr
,
350 addr
+ huge_page_size(hstate_vma(vma
))))
352 #endif /* CONFIG_MMU_NOTIFIER */
355 set_page_young(page
);
361 static int damon_mkold_hugetlb_entry(pte_t
*pte
, unsigned long hmask
,
362 unsigned long addr
, unsigned long end
,
363 struct mm_walk
*walk
)
365 struct hstate
*h
= hstate_vma(walk
->vma
);
369 ptl
= huge_pte_lock(h
, walk
->mm
, pte
);
370 entry
= huge_ptep_get(pte
);
371 if (!pte_present(entry
))
374 damon_hugetlb_mkold(pte
, walk
->mm
, walk
->vma
, addr
);
381 #define damon_mkold_hugetlb_entry NULL
382 #endif /* CONFIG_HUGETLB_PAGE */
384 static const struct mm_walk_ops damon_mkold_ops
= {
385 .pmd_entry
= damon_mkold_pmd_entry
,
386 .hugetlb_entry
= damon_mkold_hugetlb_entry
,
389 static void damon_va_mkold(struct mm_struct
*mm
, unsigned long addr
)
392 walk_page_range(mm
, addr
, addr
+ 1, &damon_mkold_ops
, NULL
);
393 mmap_read_unlock(mm
);
397 * Functions for the access checking of the regions
400 static void __damon_va_prepare_access_check(struct mm_struct
*mm
,
401 struct damon_region
*r
)
403 r
->sampling_addr
= damon_rand(r
->ar
.start
, r
->ar
.end
);
405 damon_va_mkold(mm
, r
->sampling_addr
);
408 static void damon_va_prepare_access_checks(struct damon_ctx
*ctx
)
410 struct damon_target
*t
;
411 struct mm_struct
*mm
;
412 struct damon_region
*r
;
414 damon_for_each_target(t
, ctx
) {
415 mm
= damon_get_mm(t
);
418 damon_for_each_region(r
, t
)
419 __damon_va_prepare_access_check(mm
, r
);
424 struct damon_young_walk_private
{
425 unsigned long *page_sz
;
429 static int damon_young_pmd_entry(pmd_t
*pmd
, unsigned long addr
,
430 unsigned long next
, struct mm_walk
*walk
)
435 struct damon_young_walk_private
*priv
= walk
->private;
437 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
438 if (pmd_trans_huge(*pmd
)) {
439 ptl
= pmd_lock(walk
->mm
, pmd
);
440 if (!pmd_present(*pmd
)) {
445 if (!pmd_trans_huge(*pmd
)) {
449 page
= damon_get_page(pmd_pfn(*pmd
));
452 if (pmd_young(*pmd
) || !page_is_idle(page
) ||
453 mmu_notifier_test_young(walk
->mm
,
455 *priv
->page_sz
= HPAGE_PMD_SIZE
;
465 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
467 if (pmd_none(*pmd
) || unlikely(pmd_bad(*pmd
)))
469 pte
= pte_offset_map_lock(walk
->mm
, pmd
, addr
, &ptl
);
470 if (!pte_present(*pte
))
472 page
= damon_get_page(pte_pfn(*pte
));
475 if (pte_young(*pte
) || !page_is_idle(page
) ||
476 mmu_notifier_test_young(walk
->mm
, addr
)) {
477 *priv
->page_sz
= PAGE_SIZE
;
482 pte_unmap_unlock(pte
, ptl
);
486 #ifdef CONFIG_HUGETLB_PAGE
487 static int damon_young_hugetlb_entry(pte_t
*pte
, unsigned long hmask
,
488 unsigned long addr
, unsigned long end
,
489 struct mm_walk
*walk
)
491 struct damon_young_walk_private
*priv
= walk
->private;
492 struct hstate
*h
= hstate_vma(walk
->vma
);
497 ptl
= huge_pte_lock(h
, walk
->mm
, pte
);
498 entry
= huge_ptep_get(pte
);
499 if (!pte_present(entry
))
502 page
= pte_page(entry
);
505 if (pte_young(entry
) || !page_is_idle(page
) ||
506 mmu_notifier_test_young(walk
->mm
, addr
)) {
507 *priv
->page_sz
= huge_page_size(h
);
518 #define damon_young_hugetlb_entry NULL
519 #endif /* CONFIG_HUGETLB_PAGE */
521 static const struct mm_walk_ops damon_young_ops
= {
522 .pmd_entry
= damon_young_pmd_entry
,
523 .hugetlb_entry
= damon_young_hugetlb_entry
,
526 static bool damon_va_young(struct mm_struct
*mm
, unsigned long addr
,
527 unsigned long *page_sz
)
529 struct damon_young_walk_private arg
= {
535 walk_page_range(mm
, addr
, addr
+ 1, &damon_young_ops
, &arg
);
536 mmap_read_unlock(mm
);
541 * Check whether the region was accessed after the last preparation
543 * mm 'mm_struct' for the given virtual address space
544 * r the region to be checked
546 static void __damon_va_check_access(struct mm_struct
*mm
,
547 struct damon_region
*r
, bool same_target
)
549 static unsigned long last_addr
;
550 static unsigned long last_page_sz
= PAGE_SIZE
;
551 static bool last_accessed
;
553 /* If the region is in the last checked page, reuse the result */
554 if (same_target
&& (ALIGN_DOWN(last_addr
, last_page_sz
) ==
555 ALIGN_DOWN(r
->sampling_addr
, last_page_sz
))) {
561 last_accessed
= damon_va_young(mm
, r
->sampling_addr
, &last_page_sz
);
565 last_addr
= r
->sampling_addr
;
568 static unsigned int damon_va_check_accesses(struct damon_ctx
*ctx
)
570 struct damon_target
*t
;
571 struct mm_struct
*mm
;
572 struct damon_region
*r
;
573 unsigned int max_nr_accesses
= 0;
576 damon_for_each_target(t
, ctx
) {
577 mm
= damon_get_mm(t
);
581 damon_for_each_region(r
, t
) {
582 __damon_va_check_access(mm
, r
, same_target
);
583 max_nr_accesses
= max(r
->nr_accesses
, max_nr_accesses
);
589 return max_nr_accesses
;
593 * Functions for the target validity check and cleanup
596 static bool damon_va_target_valid(struct damon_target
*t
)
598 struct task_struct
*task
;
600 task
= damon_get_task_struct(t
);
602 put_task_struct(task
);
609 #ifndef CONFIG_ADVISE_SYSCALLS
610 static unsigned long damos_madvise(struct damon_target
*target
,
611 struct damon_region
*r
, int behavior
)
616 static unsigned long damos_madvise(struct damon_target
*target
,
617 struct damon_region
*r
, int behavior
)
619 struct mm_struct
*mm
;
620 unsigned long start
= PAGE_ALIGN(r
->ar
.start
);
621 unsigned long len
= PAGE_ALIGN(damon_sz_region(r
));
622 unsigned long applied
;
624 mm
= damon_get_mm(target
);
628 applied
= do_madvise(mm
, start
, len
, behavior
) ? 0 : len
;
633 #endif /* CONFIG_ADVISE_SYSCALLS */
635 static unsigned long damon_va_apply_scheme(struct damon_ctx
*ctx
,
636 struct damon_target
*t
, struct damon_region
*r
,
637 struct damos
*scheme
)
641 switch (scheme
->action
) {
643 madv_action
= MADV_WILLNEED
;
646 madv_action
= MADV_COLD
;
649 madv_action
= MADV_PAGEOUT
;
652 madv_action
= MADV_HUGEPAGE
;
654 case DAMOS_NOHUGEPAGE
:
655 madv_action
= MADV_NOHUGEPAGE
;
661 * DAMOS actions that are not yet supported by 'vaddr'.
666 return damos_madvise(t
, r
, madv_action
);
669 static int damon_va_scheme_score(struct damon_ctx
*context
,
670 struct damon_target
*t
, struct damon_region
*r
,
671 struct damos
*scheme
)
674 switch (scheme
->action
) {
676 return damon_cold_score(context
, r
, scheme
);
681 return DAMOS_MAX_SCORE
;
684 static int __init
damon_va_initcall(void)
686 struct damon_operations ops
= {
687 .id
= DAMON_OPS_VADDR
,
688 .init
= damon_va_init
,
689 .update
= damon_va_update
,
690 .prepare_access_checks
= damon_va_prepare_access_checks
,
691 .check_accesses
= damon_va_check_accesses
,
692 .reset_aggregated
= NULL
,
693 .target_valid
= damon_va_target_valid
,
695 .apply_scheme
= damon_va_apply_scheme
,
696 .get_scheme_score
= damon_va_scheme_score
,
698 /* ops for fixed virtual address ranges */
699 struct damon_operations ops_fvaddr
= ops
;
702 /* Don't set the monitoring target regions for the entire mapping */
703 ops_fvaddr
.id
= DAMON_OPS_FVADDR
;
704 ops_fvaddr
.init
= NULL
;
705 ops_fvaddr
.update
= NULL
;
707 err
= damon_register_ops(&ops
);
710 return damon_register_ops(&ops_fvaddr
);
713 subsys_initcall(damon_va_initcall
);
715 #include "vaddr-test.h"