1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5 * Swap reorganised 29.12.95, Stephen Tweedie.
6 * kswapd added: 7.1.96 sct
7 * Removed kswapd_ctl limits, and swap out as many pages as needed
8 * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
9 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
10 * Multiqueue VM started 5.8.00, Rik van Riel.
13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16 #include <linux/sched/mm.h>
17 #include <linux/module.h>
18 #include <linux/gfp.h>
19 #include <linux/kernel_stat.h>
20 #include <linux/swap.h>
21 #include <linux/pagemap.h>
22 #include <linux/init.h>
23 #include <linux/highmem.h>
24 #include <linux/vmpressure.h>
25 #include <linux/vmstat.h>
26 #include <linux/file.h>
27 #include <linux/writeback.h>
28 #include <linux/blkdev.h>
29 #include <linux/buffer_head.h> /* for buffer_heads_over_limit */
30 #include <linux/mm_inline.h>
31 #include <linux/backing-dev.h>
32 #include <linux/rmap.h>
33 #include <linux/topology.h>
34 #include <linux/cpu.h>
35 #include <linux/cpuset.h>
36 #include <linux/compaction.h>
37 #include <linux/notifier.h>
38 #include <linux/rwsem.h>
39 #include <linux/delay.h>
40 #include <linux/kthread.h>
41 #include <linux/freezer.h>
42 #include <linux/memcontrol.h>
43 #include <linux/migrate.h>
44 #include <linux/delayacct.h>
45 #include <linux/sysctl.h>
46 #include <linux/oom.h>
47 #include <linux/pagevec.h>
48 #include <linux/prefetch.h>
49 #include <linux/printk.h>
50 #include <linux/dax.h>
51 #include <linux/psi.h>
53 #include <asm/tlbflush.h>
54 #include <asm/div64.h>
56 #include <linux/swapops.h>
57 #include <linux/balloon_compaction.h>
58 #include <linux/sched/sysctl.h>
63 #define CREATE_TRACE_POINTS
64 #include <trace/events/vmscan.h>
67 /* How many pages shrink_list() should reclaim */
68 unsigned long nr_to_reclaim
;
71 * Nodemask of nodes allowed by the caller. If NULL, all nodes
77 * The memory cgroup that hit its limit and as a result is the
78 * primary target of this reclaim invocation.
80 struct mem_cgroup
*target_mem_cgroup
;
83 * Scan pressure balancing between anon and file LRUs
85 unsigned long anon_cost
;
86 unsigned long file_cost
;
88 /* Can active pages be deactivated as part of reclaim? */
89 #define DEACTIVATE_ANON 1
90 #define DEACTIVATE_FILE 2
91 unsigned int may_deactivate
:2;
92 unsigned int force_deactivate
:1;
93 unsigned int skipped_deactivate
:1;
95 /* Writepage batching in laptop mode; RECLAIM_WRITE */
96 unsigned int may_writepage
:1;
98 /* Can mapped pages be reclaimed? */
99 unsigned int may_unmap
:1;
101 /* Can pages be swapped as part of reclaim? */
102 unsigned int may_swap
:1;
104 /* Proactive reclaim invoked by userspace through memory.reclaim */
105 unsigned int proactive
:1;
108 * Cgroup memory below memory.low is protected as long as we
109 * don't threaten to OOM. If any cgroup is reclaimed at
110 * reduced force or passed over entirely due to its memory.low
111 * setting (memcg_low_skipped), and nothing is reclaimed as a
112 * result, then go back for one more cycle that reclaims the protected
113 * memory (memcg_low_reclaim) to avert OOM.
115 unsigned int memcg_low_reclaim
:1;
116 unsigned int memcg_low_skipped
:1;
118 unsigned int hibernation_mode
:1;
120 /* One of the zones is ready for compaction */
121 unsigned int compaction_ready
:1;
123 /* There is easily reclaimable cold cache in the current node */
124 unsigned int cache_trim_mode
:1;
126 /* The file pages on the current node are dangerously low */
127 unsigned int file_is_tiny
:1;
129 /* Always discard instead of demoting to lower tier memory */
130 unsigned int no_demotion
:1;
132 /* Allocation order */
135 /* Scan (total_size >> priority) pages at once */
138 /* The highest zone to isolate pages for reclaim from */
141 /* This context's GFP mask */
144 /* Incremented by the number of inactive pages that were scanned */
145 unsigned long nr_scanned
;
147 /* Number of pages freed so far during a call to shrink_zones() */
148 unsigned long nr_reclaimed
;
152 unsigned int unqueued_dirty
;
153 unsigned int congested
;
154 unsigned int writeback
;
155 unsigned int immediate
;
156 unsigned int file_taken
;
160 /* for recording the reclaimed slab by now */
161 struct reclaim_state reclaim_state
;
164 #ifdef ARCH_HAS_PREFETCHW
165 #define prefetchw_prev_lru_folio(_folio, _base, _field) \
167 if ((_folio)->lru.prev != _base) { \
168 struct folio *prev; \
170 prev = lru_to_folio(&(_folio->lru)); \
171 prefetchw(&prev->_field); \
175 #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0)
179 * From 0 .. 200. Higher means more swappy.
181 int vm_swappiness
= 60;
183 static void set_task_reclaim_state(struct task_struct
*task
,
184 struct reclaim_state
*rs
)
186 /* Check for an overwrite */
187 WARN_ON_ONCE(rs
&& task
->reclaim_state
);
189 /* Check for the nulling of an already-nulled member */
190 WARN_ON_ONCE(!rs
&& !task
->reclaim_state
);
192 task
->reclaim_state
= rs
;
195 LIST_HEAD(shrinker_list
);
196 DECLARE_RWSEM(shrinker_rwsem
);
199 static int shrinker_nr_max
;
201 /* The shrinker_info is expanded in a batch of BITS_PER_LONG */
202 static inline int shrinker_map_size(int nr_items
)
204 return (DIV_ROUND_UP(nr_items
, BITS_PER_LONG
) * sizeof(unsigned long));
207 static inline int shrinker_defer_size(int nr_items
)
209 return (round_up(nr_items
, BITS_PER_LONG
) * sizeof(atomic_long_t
));
212 static struct shrinker_info
*shrinker_info_protected(struct mem_cgroup
*memcg
,
215 return rcu_dereference_protected(memcg
->nodeinfo
[nid
]->shrinker_info
,
216 lockdep_is_held(&shrinker_rwsem
));
219 static int expand_one_shrinker_info(struct mem_cgroup
*memcg
,
220 int map_size
, int defer_size
,
221 int old_map_size
, int old_defer_size
)
223 struct shrinker_info
*new, *old
;
224 struct mem_cgroup_per_node
*pn
;
226 int size
= map_size
+ defer_size
;
229 pn
= memcg
->nodeinfo
[nid
];
230 old
= shrinker_info_protected(memcg
, nid
);
231 /* Not yet online memcg */
235 new = kvmalloc_node(sizeof(*new) + size
, GFP_KERNEL
, nid
);
239 new->nr_deferred
= (atomic_long_t
*)(new + 1);
240 new->map
= (void *)new->nr_deferred
+ defer_size
;
242 /* map: set all old bits, clear all new bits */
243 memset(new->map
, (int)0xff, old_map_size
);
244 memset((void *)new->map
+ old_map_size
, 0, map_size
- old_map_size
);
245 /* nr_deferred: copy old values, clear all new values */
246 memcpy(new->nr_deferred
, old
->nr_deferred
, old_defer_size
);
247 memset((void *)new->nr_deferred
+ old_defer_size
, 0,
248 defer_size
- old_defer_size
);
250 rcu_assign_pointer(pn
->shrinker_info
, new);
251 kvfree_rcu(old
, rcu
);
257 void free_shrinker_info(struct mem_cgroup
*memcg
)
259 struct mem_cgroup_per_node
*pn
;
260 struct shrinker_info
*info
;
264 pn
= memcg
->nodeinfo
[nid
];
265 info
= rcu_dereference_protected(pn
->shrinker_info
, true);
267 rcu_assign_pointer(pn
->shrinker_info
, NULL
);
271 int alloc_shrinker_info(struct mem_cgroup
*memcg
)
273 struct shrinker_info
*info
;
274 int nid
, size
, ret
= 0;
275 int map_size
, defer_size
= 0;
277 down_write(&shrinker_rwsem
);
278 map_size
= shrinker_map_size(shrinker_nr_max
);
279 defer_size
= shrinker_defer_size(shrinker_nr_max
);
280 size
= map_size
+ defer_size
;
282 info
= kvzalloc_node(sizeof(*info
) + size
, GFP_KERNEL
, nid
);
284 free_shrinker_info(memcg
);
288 info
->nr_deferred
= (atomic_long_t
*)(info
+ 1);
289 info
->map
= (void *)info
->nr_deferred
+ defer_size
;
290 rcu_assign_pointer(memcg
->nodeinfo
[nid
]->shrinker_info
, info
);
292 up_write(&shrinker_rwsem
);
297 static inline bool need_expand(int nr_max
)
299 return round_up(nr_max
, BITS_PER_LONG
) >
300 round_up(shrinker_nr_max
, BITS_PER_LONG
);
303 static int expand_shrinker_info(int new_id
)
306 int new_nr_max
= new_id
+ 1;
307 int map_size
, defer_size
= 0;
308 int old_map_size
, old_defer_size
= 0;
309 struct mem_cgroup
*memcg
;
311 if (!need_expand(new_nr_max
))
314 if (!root_mem_cgroup
)
317 lockdep_assert_held(&shrinker_rwsem
);
319 map_size
= shrinker_map_size(new_nr_max
);
320 defer_size
= shrinker_defer_size(new_nr_max
);
321 old_map_size
= shrinker_map_size(shrinker_nr_max
);
322 old_defer_size
= shrinker_defer_size(shrinker_nr_max
);
324 memcg
= mem_cgroup_iter(NULL
, NULL
, NULL
);
326 ret
= expand_one_shrinker_info(memcg
, map_size
, defer_size
,
327 old_map_size
, old_defer_size
);
329 mem_cgroup_iter_break(NULL
, memcg
);
332 } while ((memcg
= mem_cgroup_iter(NULL
, memcg
, NULL
)) != NULL
);
335 shrinker_nr_max
= new_nr_max
;
340 void set_shrinker_bit(struct mem_cgroup
*memcg
, int nid
, int shrinker_id
)
342 if (shrinker_id
>= 0 && memcg
&& !mem_cgroup_is_root(memcg
)) {
343 struct shrinker_info
*info
;
346 info
= rcu_dereference(memcg
->nodeinfo
[nid
]->shrinker_info
);
347 /* Pairs with smp mb in shrink_slab() */
348 smp_mb__before_atomic();
349 set_bit(shrinker_id
, info
->map
);
354 static DEFINE_IDR(shrinker_idr
);
356 static int prealloc_memcg_shrinker(struct shrinker
*shrinker
)
358 int id
, ret
= -ENOMEM
;
360 if (mem_cgroup_disabled())
363 down_write(&shrinker_rwsem
);
364 /* This may call shrinker, so it must use down_read_trylock() */
365 id
= idr_alloc(&shrinker_idr
, shrinker
, 0, 0, GFP_KERNEL
);
369 if (id
>= shrinker_nr_max
) {
370 if (expand_shrinker_info(id
)) {
371 idr_remove(&shrinker_idr
, id
);
378 up_write(&shrinker_rwsem
);
382 static void unregister_memcg_shrinker(struct shrinker
*shrinker
)
384 int id
= shrinker
->id
;
388 lockdep_assert_held(&shrinker_rwsem
);
390 idr_remove(&shrinker_idr
, id
);
393 static long xchg_nr_deferred_memcg(int nid
, struct shrinker
*shrinker
,
394 struct mem_cgroup
*memcg
)
396 struct shrinker_info
*info
;
398 info
= shrinker_info_protected(memcg
, nid
);
399 return atomic_long_xchg(&info
->nr_deferred
[shrinker
->id
], 0);
402 static long add_nr_deferred_memcg(long nr
, int nid
, struct shrinker
*shrinker
,
403 struct mem_cgroup
*memcg
)
405 struct shrinker_info
*info
;
407 info
= shrinker_info_protected(memcg
, nid
);
408 return atomic_long_add_return(nr
, &info
->nr_deferred
[shrinker
->id
]);
411 void reparent_shrinker_deferred(struct mem_cgroup
*memcg
)
415 struct mem_cgroup
*parent
;
416 struct shrinker_info
*child_info
, *parent_info
;
418 parent
= parent_mem_cgroup(memcg
);
420 parent
= root_mem_cgroup
;
422 /* Prevent from concurrent shrinker_info expand */
423 down_read(&shrinker_rwsem
);
425 child_info
= shrinker_info_protected(memcg
, nid
);
426 parent_info
= shrinker_info_protected(parent
, nid
);
427 for (i
= 0; i
< shrinker_nr_max
; i
++) {
428 nr
= atomic_long_read(&child_info
->nr_deferred
[i
]);
429 atomic_long_add(nr
, &parent_info
->nr_deferred
[i
]);
432 up_read(&shrinker_rwsem
);
435 static bool cgroup_reclaim(struct scan_control
*sc
)
437 return sc
->target_mem_cgroup
;
441 * writeback_throttling_sane - is the usual dirty throttling mechanism available?
442 * @sc: scan_control in question
444 * The normal page dirty throttling mechanism in balance_dirty_pages() is
445 * completely broken with the legacy memcg and direct stalling in
446 * shrink_page_list() is used for throttling instead, which lacks all the
447 * niceties such as fairness, adaptive pausing, bandwidth proportional
448 * allocation and configurability.
450 * This function tests whether the vmscan currently in progress can assume
451 * that the normal dirty throttling mechanism is operational.
453 static bool writeback_throttling_sane(struct scan_control
*sc
)
455 if (!cgroup_reclaim(sc
))
457 #ifdef CONFIG_CGROUP_WRITEBACK
458 if (cgroup_subsys_on_dfl(memory_cgrp_subsys
))
464 static int prealloc_memcg_shrinker(struct shrinker
*shrinker
)
469 static void unregister_memcg_shrinker(struct shrinker
*shrinker
)
473 static long xchg_nr_deferred_memcg(int nid
, struct shrinker
*shrinker
,
474 struct mem_cgroup
*memcg
)
479 static long add_nr_deferred_memcg(long nr
, int nid
, struct shrinker
*shrinker
,
480 struct mem_cgroup
*memcg
)
485 static bool cgroup_reclaim(struct scan_control
*sc
)
490 static bool writeback_throttling_sane(struct scan_control
*sc
)
496 static long xchg_nr_deferred(struct shrinker
*shrinker
,
497 struct shrink_control
*sc
)
501 if (!(shrinker
->flags
& SHRINKER_NUMA_AWARE
))
505 (shrinker
->flags
& SHRINKER_MEMCG_AWARE
))
506 return xchg_nr_deferred_memcg(nid
, shrinker
,
509 return atomic_long_xchg(&shrinker
->nr_deferred
[nid
], 0);
513 static long add_nr_deferred(long nr
, struct shrinker
*shrinker
,
514 struct shrink_control
*sc
)
518 if (!(shrinker
->flags
& SHRINKER_NUMA_AWARE
))
522 (shrinker
->flags
& SHRINKER_MEMCG_AWARE
))
523 return add_nr_deferred_memcg(nr
, nid
, shrinker
,
526 return atomic_long_add_return(nr
, &shrinker
->nr_deferred
[nid
]);
529 static bool can_demote(int nid
, struct scan_control
*sc
)
531 if (!numa_demotion_enabled
)
533 if (sc
&& sc
->no_demotion
)
535 if (next_demotion_node(nid
) == NUMA_NO_NODE
)
541 static inline bool can_reclaim_anon_pages(struct mem_cgroup
*memcg
,
543 struct scan_control
*sc
)
547 * For non-memcg reclaim, is there
548 * space in any swap device?
550 if (get_nr_swap_pages() > 0)
553 /* Is the memcg below its swap limit? */
554 if (mem_cgroup_get_nr_swap_pages(memcg
) > 0)
559 * The page can not be swapped.
561 * Can it be reclaimed from this node via demotion?
563 return can_demote(nid
, sc
);
567 * This misses isolated pages which are not accounted for to save counters.
568 * As the data only determines if reclaim or compaction continues, it is
569 * not expected that isolated pages will be a dominating factor.
571 unsigned long zone_reclaimable_pages(struct zone
*zone
)
575 nr
= zone_page_state_snapshot(zone
, NR_ZONE_INACTIVE_FILE
) +
576 zone_page_state_snapshot(zone
, NR_ZONE_ACTIVE_FILE
);
577 if (can_reclaim_anon_pages(NULL
, zone_to_nid(zone
), NULL
))
578 nr
+= zone_page_state_snapshot(zone
, NR_ZONE_INACTIVE_ANON
) +
579 zone_page_state_snapshot(zone
, NR_ZONE_ACTIVE_ANON
);
585 * lruvec_lru_size - Returns the number of pages on the given LRU list.
586 * @lruvec: lru vector
588 * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list)
590 static unsigned long lruvec_lru_size(struct lruvec
*lruvec
, enum lru_list lru
,
593 unsigned long size
= 0;
596 for (zid
= 0; zid
<= zone_idx
; zid
++) {
597 struct zone
*zone
= &lruvec_pgdat(lruvec
)->node_zones
[zid
];
599 if (!managed_zone(zone
))
602 if (!mem_cgroup_disabled())
603 size
+= mem_cgroup_get_zone_lru_size(lruvec
, lru
, zid
);
605 size
+= zone_page_state(zone
, NR_ZONE_LRU_BASE
+ lru
);
611 * Add a shrinker callback to be called from the vm.
613 static int __prealloc_shrinker(struct shrinker
*shrinker
)
618 if (shrinker
->flags
& SHRINKER_MEMCG_AWARE
) {
619 err
= prealloc_memcg_shrinker(shrinker
);
623 shrinker
->flags
&= ~SHRINKER_MEMCG_AWARE
;
626 size
= sizeof(*shrinker
->nr_deferred
);
627 if (shrinker
->flags
& SHRINKER_NUMA_AWARE
)
630 shrinker
->nr_deferred
= kzalloc(size
, GFP_KERNEL
);
631 if (!shrinker
->nr_deferred
)
637 #ifdef CONFIG_SHRINKER_DEBUG
638 int prealloc_shrinker(struct shrinker
*shrinker
, const char *fmt
, ...)
644 shrinker
->name
= kvasprintf_const(GFP_KERNEL
, fmt
, ap
);
649 err
= __prealloc_shrinker(shrinker
);
651 kfree_const(shrinker
->name
);
652 shrinker
->name
= NULL
;
658 int prealloc_shrinker(struct shrinker
*shrinker
, const char *fmt
, ...)
660 return __prealloc_shrinker(shrinker
);
664 void free_prealloced_shrinker(struct shrinker
*shrinker
)
666 #ifdef CONFIG_SHRINKER_DEBUG
667 kfree_const(shrinker
->name
);
668 shrinker
->name
= NULL
;
670 if (shrinker
->flags
& SHRINKER_MEMCG_AWARE
) {
671 down_write(&shrinker_rwsem
);
672 unregister_memcg_shrinker(shrinker
);
673 up_write(&shrinker_rwsem
);
677 kfree(shrinker
->nr_deferred
);
678 shrinker
->nr_deferred
= NULL
;
681 void register_shrinker_prepared(struct shrinker
*shrinker
)
683 down_write(&shrinker_rwsem
);
684 list_add_tail(&shrinker
->list
, &shrinker_list
);
685 shrinker
->flags
|= SHRINKER_REGISTERED
;
686 shrinker_debugfs_add(shrinker
);
687 up_write(&shrinker_rwsem
);
690 static int __register_shrinker(struct shrinker
*shrinker
)
692 int err
= __prealloc_shrinker(shrinker
);
696 register_shrinker_prepared(shrinker
);
700 #ifdef CONFIG_SHRINKER_DEBUG
701 int register_shrinker(struct shrinker
*shrinker
, const char *fmt
, ...)
707 shrinker
->name
= kvasprintf_const(GFP_KERNEL
, fmt
, ap
);
712 err
= __register_shrinker(shrinker
);
714 kfree_const(shrinker
->name
);
715 shrinker
->name
= NULL
;
720 int register_shrinker(struct shrinker
*shrinker
, const char *fmt
, ...)
722 return __register_shrinker(shrinker
);
725 EXPORT_SYMBOL(register_shrinker
);
730 void unregister_shrinker(struct shrinker
*shrinker
)
732 if (!(shrinker
->flags
& SHRINKER_REGISTERED
))
735 down_write(&shrinker_rwsem
);
736 list_del(&shrinker
->list
);
737 shrinker
->flags
&= ~SHRINKER_REGISTERED
;
738 if (shrinker
->flags
& SHRINKER_MEMCG_AWARE
)
739 unregister_memcg_shrinker(shrinker
);
740 shrinker_debugfs_remove(shrinker
);
741 up_write(&shrinker_rwsem
);
743 kfree(shrinker
->nr_deferred
);
744 shrinker
->nr_deferred
= NULL
;
746 EXPORT_SYMBOL(unregister_shrinker
);
749 * synchronize_shrinkers - Wait for all running shrinkers to complete.
751 * This is equivalent to calling unregister_shrink() and register_shrinker(),
752 * but atomically and with less overhead. This is useful to guarantee that all
753 * shrinker invocations have seen an update, before freeing memory, similar to
756 void synchronize_shrinkers(void)
758 down_write(&shrinker_rwsem
);
759 up_write(&shrinker_rwsem
);
761 EXPORT_SYMBOL(synchronize_shrinkers
);
763 #define SHRINK_BATCH 128
765 static unsigned long do_shrink_slab(struct shrink_control
*shrinkctl
,
766 struct shrinker
*shrinker
, int priority
)
768 unsigned long freed
= 0;
769 unsigned long long delta
;
774 long batch_size
= shrinker
->batch
? shrinker
->batch
776 long scanned
= 0, next_deferred
;
778 freeable
= shrinker
->count_objects(shrinker
, shrinkctl
);
779 if (freeable
== 0 || freeable
== SHRINK_EMPTY
)
783 * copy the current shrinker scan count into a local variable
784 * and zero it so that other concurrent shrinker invocations
785 * don't also do this scanning work.
787 nr
= xchg_nr_deferred(shrinker
, shrinkctl
);
789 if (shrinker
->seeks
) {
790 delta
= freeable
>> priority
;
792 do_div(delta
, shrinker
->seeks
);
795 * These objects don't require any IO to create. Trim
796 * them aggressively under memory pressure to keep
797 * them from causing refetches in the IO caches.
799 delta
= freeable
/ 2;
802 total_scan
= nr
>> priority
;
804 total_scan
= min(total_scan
, (2 * freeable
));
806 trace_mm_shrink_slab_start(shrinker
, shrinkctl
, nr
,
807 freeable
, delta
, total_scan
, priority
);
810 * Normally, we should not scan less than batch_size objects in one
811 * pass to avoid too frequent shrinker calls, but if the slab has less
812 * than batch_size objects in total and we are really tight on memory,
813 * we will try to reclaim all available objects, otherwise we can end
814 * up failing allocations although there are plenty of reclaimable
815 * objects spread over several slabs with usage less than the
818 * We detect the "tight on memory" situations by looking at the total
819 * number of objects we want to scan (total_scan). If it is greater
820 * than the total number of objects on slab (freeable), we must be
821 * scanning at high prio and therefore should try to reclaim as much as
824 while (total_scan
>= batch_size
||
825 total_scan
>= freeable
) {
827 unsigned long nr_to_scan
= min(batch_size
, total_scan
);
829 shrinkctl
->nr_to_scan
= nr_to_scan
;
830 shrinkctl
->nr_scanned
= nr_to_scan
;
831 ret
= shrinker
->scan_objects(shrinker
, shrinkctl
);
832 if (ret
== SHRINK_STOP
)
836 count_vm_events(SLABS_SCANNED
, shrinkctl
->nr_scanned
);
837 total_scan
-= shrinkctl
->nr_scanned
;
838 scanned
+= shrinkctl
->nr_scanned
;
844 * The deferred work is increased by any new work (delta) that wasn't
845 * done, decreased by old deferred work that was done now.
847 * And it is capped to two times of the freeable items.
849 next_deferred
= max_t(long, (nr
+ delta
- scanned
), 0);
850 next_deferred
= min(next_deferred
, (2 * freeable
));
853 * move the unused scan count back into the shrinker in a
854 * manner that handles concurrent updates.
856 new_nr
= add_nr_deferred(next_deferred
, shrinker
, shrinkctl
);
858 trace_mm_shrink_slab_end(shrinker
, shrinkctl
->nid
, freed
, nr
, new_nr
, total_scan
);
863 static unsigned long shrink_slab_memcg(gfp_t gfp_mask
, int nid
,
864 struct mem_cgroup
*memcg
, int priority
)
866 struct shrinker_info
*info
;
867 unsigned long ret
, freed
= 0;
870 if (!mem_cgroup_online(memcg
))
873 if (!down_read_trylock(&shrinker_rwsem
))
876 info
= shrinker_info_protected(memcg
, nid
);
880 for_each_set_bit(i
, info
->map
, shrinker_nr_max
) {
881 struct shrink_control sc
= {
882 .gfp_mask
= gfp_mask
,
886 struct shrinker
*shrinker
;
888 shrinker
= idr_find(&shrinker_idr
, i
);
889 if (unlikely(!shrinker
|| !(shrinker
->flags
& SHRINKER_REGISTERED
))) {
891 clear_bit(i
, info
->map
);
895 /* Call non-slab shrinkers even though kmem is disabled */
896 if (!memcg_kmem_enabled() &&
897 !(shrinker
->flags
& SHRINKER_NONSLAB
))
900 ret
= do_shrink_slab(&sc
, shrinker
, priority
);
901 if (ret
== SHRINK_EMPTY
) {
902 clear_bit(i
, info
->map
);
904 * After the shrinker reported that it had no objects to
905 * free, but before we cleared the corresponding bit in
906 * the memcg shrinker map, a new object might have been
907 * added. To make sure, we have the bit set in this
908 * case, we invoke the shrinker one more time and reset
909 * the bit if it reports that it is not empty anymore.
910 * The memory barrier here pairs with the barrier in
911 * set_shrinker_bit():
913 * list_lru_add() shrink_slab_memcg()
914 * list_add_tail() clear_bit()
916 * set_bit() do_shrink_slab()
918 smp_mb__after_atomic();
919 ret
= do_shrink_slab(&sc
, shrinker
, priority
);
920 if (ret
== SHRINK_EMPTY
)
923 set_shrinker_bit(memcg
, nid
, i
);
927 if (rwsem_is_contended(&shrinker_rwsem
)) {
933 up_read(&shrinker_rwsem
);
936 #else /* CONFIG_MEMCG */
937 static unsigned long shrink_slab_memcg(gfp_t gfp_mask
, int nid
,
938 struct mem_cgroup
*memcg
, int priority
)
942 #endif /* CONFIG_MEMCG */
945 * shrink_slab - shrink slab caches
946 * @gfp_mask: allocation context
947 * @nid: node whose slab caches to target
948 * @memcg: memory cgroup whose slab caches to target
949 * @priority: the reclaim priority
951 * Call the shrink functions to age shrinkable caches.
953 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
954 * unaware shrinkers will receive a node id of 0 instead.
956 * @memcg specifies the memory cgroup to target. Unaware shrinkers
957 * are called only if it is the root cgroup.
959 * @priority is sc->priority, we take the number of objects and >> by priority
960 * in order to get the scan target.
962 * Returns the number of reclaimed slab objects.
964 static unsigned long shrink_slab(gfp_t gfp_mask
, int nid
,
965 struct mem_cgroup
*memcg
,
968 unsigned long ret
, freed
= 0;
969 struct shrinker
*shrinker
;
972 * The root memcg might be allocated even though memcg is disabled
973 * via "cgroup_disable=memory" boot parameter. This could make
974 * mem_cgroup_is_root() return false, then just run memcg slab
975 * shrink, but skip global shrink. This may result in premature
978 if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg
))
979 return shrink_slab_memcg(gfp_mask
, nid
, memcg
, priority
);
981 if (!down_read_trylock(&shrinker_rwsem
))
984 list_for_each_entry(shrinker
, &shrinker_list
, list
) {
985 struct shrink_control sc
= {
986 .gfp_mask
= gfp_mask
,
991 ret
= do_shrink_slab(&sc
, shrinker
, priority
);
992 if (ret
== SHRINK_EMPTY
)
996 * Bail out if someone want to register a new shrinker to
997 * prevent the registration from being stalled for long periods
998 * by parallel ongoing shrinking.
1000 if (rwsem_is_contended(&shrinker_rwsem
)) {
1001 freed
= freed
? : 1;
1006 up_read(&shrinker_rwsem
);
1012 static void drop_slab_node(int nid
)
1014 unsigned long freed
;
1018 struct mem_cgroup
*memcg
= NULL
;
1020 if (fatal_signal_pending(current
))
1024 memcg
= mem_cgroup_iter(NULL
, NULL
, NULL
);
1026 freed
+= shrink_slab(GFP_KERNEL
, nid
, memcg
, 0);
1027 } while ((memcg
= mem_cgroup_iter(NULL
, memcg
, NULL
)) != NULL
);
1028 } while ((freed
>> shift
++) > 1);
1031 void drop_slab(void)
1035 for_each_online_node(nid
)
1036 drop_slab_node(nid
);
1039 static inline int is_page_cache_freeable(struct folio
*folio
)
1042 * A freeable page cache page is referenced only by the caller
1043 * that isolated the page, the page cache and optional buffer
1044 * heads at page->private.
1046 return folio_ref_count(folio
) - folio_test_private(folio
) ==
1047 1 + folio_nr_pages(folio
);
1051 * We detected a synchronous write error writing a folio out. Probably
1052 * -ENOSPC. We need to propagate that into the address_space for a subsequent
1053 * fsync(), msync() or close().
1055 * The tricky part is that after writepage we cannot touch the mapping: nothing
1056 * prevents it from being freed up. But we have a ref on the folio and once
1057 * that folio is locked, the mapping is pinned.
1059 * We're allowed to run sleeping folio_lock() here because we know the caller has
1062 static void handle_write_error(struct address_space
*mapping
,
1063 struct folio
*folio
, int error
)
1066 if (folio_mapping(folio
) == mapping
)
1067 mapping_set_error(mapping
, error
);
1068 folio_unlock(folio
);
1071 static bool skip_throttle_noprogress(pg_data_t
*pgdat
)
1073 int reclaimable
= 0, write_pending
= 0;
1077 * If kswapd is disabled, reschedule if necessary but do not
1078 * throttle as the system is likely near OOM.
1080 if (pgdat
->kswapd_failures
>= MAX_RECLAIM_RETRIES
)
1084 * If there are a lot of dirty/writeback pages then do not
1085 * throttle as throttling will occur when the pages cycle
1086 * towards the end of the LRU if still under writeback.
1088 for (i
= 0; i
< MAX_NR_ZONES
; i
++) {
1089 struct zone
*zone
= pgdat
->node_zones
+ i
;
1091 if (!managed_zone(zone
))
1094 reclaimable
+= zone_reclaimable_pages(zone
);
1095 write_pending
+= zone_page_state_snapshot(zone
,
1096 NR_ZONE_WRITE_PENDING
);
1098 if (2 * write_pending
<= reclaimable
)
1104 void reclaim_throttle(pg_data_t
*pgdat
, enum vmscan_throttle_state reason
)
1106 wait_queue_head_t
*wqh
= &pgdat
->reclaim_wait
[reason
];
1111 * Do not throttle IO workers, kthreads other than kswapd or
1112 * workqueues. They may be required for reclaim to make
1113 * forward progress (e.g. journalling workqueues or kthreads).
1115 if (!current_is_kswapd() &&
1116 current
->flags
& (PF_IO_WORKER
|PF_KTHREAD
)) {
1122 * These figures are pulled out of thin air.
1123 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many
1124 * parallel reclaimers which is a short-lived event so the timeout is
1125 * short. Failing to make progress or waiting on writeback are
1126 * potentially long-lived events so use a longer timeout. This is shaky
1127 * logic as a failure to make progress could be due to anything from
1128 * writeback to a slow device to excessive references pages at the tail
1129 * of the inactive LRU.
1132 case VMSCAN_THROTTLE_WRITEBACK
:
1135 if (atomic_inc_return(&pgdat
->nr_writeback_throttled
) == 1) {
1136 WRITE_ONCE(pgdat
->nr_reclaim_start
,
1137 node_page_state(pgdat
, NR_THROTTLED_WRITTEN
));
1141 case VMSCAN_THROTTLE_CONGESTED
:
1143 case VMSCAN_THROTTLE_NOPROGRESS
:
1144 if (skip_throttle_noprogress(pgdat
)) {
1152 case VMSCAN_THROTTLE_ISOLATED
:
1161 prepare_to_wait(wqh
, &wait
, TASK_UNINTERRUPTIBLE
);
1162 ret
= schedule_timeout(timeout
);
1163 finish_wait(wqh
, &wait
);
1165 if (reason
== VMSCAN_THROTTLE_WRITEBACK
)
1166 atomic_dec(&pgdat
->nr_writeback_throttled
);
1168 trace_mm_vmscan_throttled(pgdat
->node_id
, jiffies_to_usecs(timeout
),
1169 jiffies_to_usecs(timeout
- ret
),
1174 * Account for pages written if tasks are throttled waiting on dirty
1175 * pages to clean. If enough pages have been cleaned since throttling
1176 * started then wakeup the throttled tasks.
1178 void __acct_reclaim_writeback(pg_data_t
*pgdat
, struct folio
*folio
,
1181 unsigned long nr_written
;
1183 node_stat_add_folio(folio
, NR_THROTTLED_WRITTEN
);
1186 * This is an inaccurate read as the per-cpu deltas may not
1187 * be synchronised. However, given that the system is
1188 * writeback throttled, it is not worth taking the penalty
1189 * of getting an accurate count. At worst, the throttle
1190 * timeout guarantees forward progress.
1192 nr_written
= node_page_state(pgdat
, NR_THROTTLED_WRITTEN
) -
1193 READ_ONCE(pgdat
->nr_reclaim_start
);
1195 if (nr_written
> SWAP_CLUSTER_MAX
* nr_throttled
)
1196 wake_up(&pgdat
->reclaim_wait
[VMSCAN_THROTTLE_WRITEBACK
]);
1199 /* possible outcome of pageout() */
1201 /* failed to write page out, page is locked */
1203 /* move page to the active list, page is locked */
1205 /* page has been sent to the disk successfully, page is unlocked */
1207 /* page is clean and locked */
1212 * pageout is called by shrink_page_list() for each dirty page.
1213 * Calls ->writepage().
1215 static pageout_t
pageout(struct folio
*folio
, struct address_space
*mapping
,
1216 struct swap_iocb
**plug
)
1219 * If the folio is dirty, only perform writeback if that write
1220 * will be non-blocking. To prevent this allocation from being
1221 * stalled by pagecache activity. But note that there may be
1222 * stalls if we need to run get_block(). We could test
1223 * PagePrivate for that.
1225 * If this process is currently in __generic_file_write_iter() against
1226 * this folio's queue, we can perform writeback even if that
1229 * If the folio is swapcache, write it back even if that would
1230 * block, for some throttling. This happens by accident, because
1231 * swap_backing_dev_info is bust: it doesn't reflect the
1232 * congestion state of the swapdevs. Easy to fix, if needed.
1234 if (!is_page_cache_freeable(folio
))
1238 * Some data journaling orphaned folios can have
1239 * folio->mapping == NULL while being dirty with clean buffers.
1241 if (folio_test_private(folio
)) {
1242 if (try_to_free_buffers(folio
)) {
1243 folio_clear_dirty(folio
);
1244 pr_info("%s: orphaned folio\n", __func__
);
1250 if (mapping
->a_ops
->writepage
== NULL
)
1251 return PAGE_ACTIVATE
;
1253 if (folio_clear_dirty_for_io(folio
)) {
1255 struct writeback_control wbc
= {
1256 .sync_mode
= WB_SYNC_NONE
,
1257 .nr_to_write
= SWAP_CLUSTER_MAX
,
1259 .range_end
= LLONG_MAX
,
1264 folio_set_reclaim(folio
);
1265 res
= mapping
->a_ops
->writepage(&folio
->page
, &wbc
);
1267 handle_write_error(mapping
, folio
, res
);
1268 if (res
== AOP_WRITEPAGE_ACTIVATE
) {
1269 folio_clear_reclaim(folio
);
1270 return PAGE_ACTIVATE
;
1273 if (!folio_test_writeback(folio
)) {
1274 /* synchronous write or broken a_ops? */
1275 folio_clear_reclaim(folio
);
1277 trace_mm_vmscan_write_folio(folio
);
1278 node_stat_add_folio(folio
, NR_VMSCAN_WRITE
);
1279 return PAGE_SUCCESS
;
1286 * Same as remove_mapping, but if the page is removed from the mapping, it
1287 * gets returned with a refcount of 0.
1289 static int __remove_mapping(struct address_space
*mapping
, struct folio
*folio
,
1290 bool reclaimed
, struct mem_cgroup
*target_memcg
)
1293 void *shadow
= NULL
;
1295 BUG_ON(!folio_test_locked(folio
));
1296 BUG_ON(mapping
!= folio_mapping(folio
));
1298 if (!folio_test_swapcache(folio
))
1299 spin_lock(&mapping
->host
->i_lock
);
1300 xa_lock_irq(&mapping
->i_pages
);
1302 * The non racy check for a busy page.
1304 * Must be careful with the order of the tests. When someone has
1305 * a ref to the page, it may be possible that they dirty it then
1306 * drop the reference. So if PageDirty is tested before page_count
1307 * here, then the following race may occur:
1309 * get_user_pages(&page);
1310 * [user mapping goes away]
1312 * !PageDirty(page) [good]
1313 * SetPageDirty(page);
1315 * !page_count(page) [good, discard it]
1317 * [oops, our write_to data is lost]
1319 * Reversing the order of the tests ensures such a situation cannot
1320 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
1321 * load is not satisfied before that of page->_refcount.
1323 * Note that if SetPageDirty is always performed via set_page_dirty,
1324 * and thus under the i_pages lock, then this ordering is not required.
1326 refcount
= 1 + folio_nr_pages(folio
);
1327 if (!folio_ref_freeze(folio
, refcount
))
1329 /* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
1330 if (unlikely(folio_test_dirty(folio
))) {
1331 folio_ref_unfreeze(folio
, refcount
);
1335 if (folio_test_swapcache(folio
)) {
1336 swp_entry_t swap
= folio_swap_entry(folio
);
1337 mem_cgroup_swapout(folio
, swap
);
1338 if (reclaimed
&& !mapping_exiting(mapping
))
1339 shadow
= workingset_eviction(folio
, target_memcg
);
1340 __delete_from_swap_cache(folio
, swap
, shadow
);
1341 xa_unlock_irq(&mapping
->i_pages
);
1342 put_swap_page(&folio
->page
, swap
);
1344 void (*free_folio
)(struct folio
*);
1346 free_folio
= mapping
->a_ops
->free_folio
;
1348 * Remember a shadow entry for reclaimed file cache in
1349 * order to detect refaults, thus thrashing, later on.
1351 * But don't store shadows in an address space that is
1352 * already exiting. This is not just an optimization,
1353 * inode reclaim needs to empty out the radix tree or
1354 * the nodes are lost. Don't plant shadows behind its
1357 * We also don't store shadows for DAX mappings because the
1358 * only page cache pages found in these are zero pages
1359 * covering holes, and because we don't want to mix DAX
1360 * exceptional entries and shadow exceptional entries in the
1361 * same address_space.
1363 if (reclaimed
&& folio_is_file_lru(folio
) &&
1364 !mapping_exiting(mapping
) && !dax_mapping(mapping
))
1365 shadow
= workingset_eviction(folio
, target_memcg
);
1366 __filemap_remove_folio(folio
, shadow
);
1367 xa_unlock_irq(&mapping
->i_pages
);
1368 if (mapping_shrinkable(mapping
))
1369 inode_add_lru(mapping
->host
);
1370 spin_unlock(&mapping
->host
->i_lock
);
1379 xa_unlock_irq(&mapping
->i_pages
);
1380 if (!folio_test_swapcache(folio
))
1381 spin_unlock(&mapping
->host
->i_lock
);
1386 * remove_mapping() - Attempt to remove a folio from its mapping.
1387 * @mapping: The address space.
1388 * @folio: The folio to remove.
1390 * If the folio is dirty, under writeback or if someone else has a ref
1391 * on it, removal will fail.
1392 * Return: The number of pages removed from the mapping. 0 if the folio
1393 * could not be removed.
1394 * Context: The caller should have a single refcount on the folio and
1397 long remove_mapping(struct address_space
*mapping
, struct folio
*folio
)
1399 if (__remove_mapping(mapping
, folio
, false, NULL
)) {
1401 * Unfreezing the refcount with 1 effectively
1402 * drops the pagecache ref for us without requiring another
1405 folio_ref_unfreeze(folio
, 1);
1406 return folio_nr_pages(folio
);
1412 * folio_putback_lru - Put previously isolated folio onto appropriate LRU list.
1413 * @folio: Folio to be returned to an LRU list.
1415 * Add previously isolated @folio to appropriate LRU list.
1416 * The folio may still be unevictable for other reasons.
1418 * Context: lru_lock must not be held, interrupts must be enabled.
1420 void folio_putback_lru(struct folio
*folio
)
1422 folio_add_lru(folio
);
1423 folio_put(folio
); /* drop ref from isolate */
1426 enum page_references
{
1428 PAGEREF_RECLAIM_CLEAN
,
1433 static enum page_references
folio_check_references(struct folio
*folio
,
1434 struct scan_control
*sc
)
1436 int referenced_ptes
, referenced_folio
;
1437 unsigned long vm_flags
;
1439 referenced_ptes
= folio_referenced(folio
, 1, sc
->target_mem_cgroup
,
1441 referenced_folio
= folio_test_clear_referenced(folio
);
1444 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
1445 * Let the folio, now marked Mlocked, be moved to the unevictable list.
1447 if (vm_flags
& VM_LOCKED
)
1448 return PAGEREF_ACTIVATE
;
1450 /* rmap lock contention: rotate */
1451 if (referenced_ptes
== -1)
1452 return PAGEREF_KEEP
;
1454 if (referenced_ptes
) {
1456 * All mapped folios start out with page table
1457 * references from the instantiating fault, so we need
1458 * to look twice if a mapped file/anon folio is used more
1461 * Mark it and spare it for another trip around the
1462 * inactive list. Another page table reference will
1463 * lead to its activation.
1465 * Note: the mark is set for activated folios as well
1466 * so that recently deactivated but used folios are
1467 * quickly recovered.
1469 folio_set_referenced(folio
);
1471 if (referenced_folio
|| referenced_ptes
> 1)
1472 return PAGEREF_ACTIVATE
;
1475 * Activate file-backed executable folios after first usage.
1477 if ((vm_flags
& VM_EXEC
) && folio_is_file_lru(folio
))
1478 return PAGEREF_ACTIVATE
;
1480 return PAGEREF_KEEP
;
1483 /* Reclaim if clean, defer dirty folios to writeback */
1484 if (referenced_folio
&& folio_is_file_lru(folio
))
1485 return PAGEREF_RECLAIM_CLEAN
;
1487 return PAGEREF_RECLAIM
;
1490 /* Check if a page is dirty or under writeback */
1491 static void folio_check_dirty_writeback(struct folio
*folio
,
1492 bool *dirty
, bool *writeback
)
1494 struct address_space
*mapping
;
1497 * Anonymous pages are not handled by flushers and must be written
1498 * from reclaim context. Do not stall reclaim based on them.
1499 * MADV_FREE anonymous pages are put into inactive file list too.
1500 * They could be mistakenly treated as file lru. So further anon
1503 if (!folio_is_file_lru(folio
) ||
1504 (folio_test_anon(folio
) && !folio_test_swapbacked(folio
))) {
1510 /* By default assume that the folio flags are accurate */
1511 *dirty
= folio_test_dirty(folio
);
1512 *writeback
= folio_test_writeback(folio
);
1514 /* Verify dirty/writeback state if the filesystem supports it */
1515 if (!folio_test_private(folio
))
1518 mapping
= folio_mapping(folio
);
1519 if (mapping
&& mapping
->a_ops
->is_dirty_writeback
)
1520 mapping
->a_ops
->is_dirty_writeback(folio
, dirty
, writeback
);
1523 static struct page
*alloc_demote_page(struct page
*page
, unsigned long node
)
1525 struct migration_target_control mtc
= {
1527 * Allocate from 'node', or fail quickly and quietly.
1528 * When this happens, 'page' will likely just be discarded
1529 * instead of migrated.
1531 .gfp_mask
= (GFP_HIGHUSER_MOVABLE
& ~__GFP_RECLAIM
) |
1532 __GFP_THISNODE
| __GFP_NOWARN
|
1533 __GFP_NOMEMALLOC
| GFP_NOWAIT
,
1537 return alloc_migration_target(page
, (unsigned long)&mtc
);
1541 * Take pages on @demote_list and attempt to demote them to
1542 * another node. Pages which are not demoted are left on
1545 static unsigned int demote_page_list(struct list_head
*demote_pages
,
1546 struct pglist_data
*pgdat
)
1548 int target_nid
= next_demotion_node(pgdat
->node_id
);
1549 unsigned int nr_succeeded
;
1551 if (list_empty(demote_pages
))
1554 if (target_nid
== NUMA_NO_NODE
)
1557 /* Demotion ignores all cpuset and mempolicy settings */
1558 migrate_pages(demote_pages
, alloc_demote_page
, NULL
,
1559 target_nid
, MIGRATE_ASYNC
, MR_DEMOTION
,
1562 if (current_is_kswapd())
1563 __count_vm_events(PGDEMOTE_KSWAPD
, nr_succeeded
);
1565 __count_vm_events(PGDEMOTE_DIRECT
, nr_succeeded
);
1567 return nr_succeeded
;
1570 static bool may_enter_fs(struct folio
*folio
, gfp_t gfp_mask
)
1572 if (gfp_mask
& __GFP_FS
)
1574 if (!folio_test_swapcache(folio
) || !(gfp_mask
& __GFP_IO
))
1577 * We can "enter_fs" for swap-cache with only __GFP_IO
1578 * providing this isn't SWP_FS_OPS.
1579 * ->flags can be updated non-atomicially (scan_swap_map_slots),
1580 * but that will never affect SWP_FS_OPS, so the data_race
1583 return !data_race(folio_swap_flags(folio
) & SWP_FS_OPS
);
1587 * shrink_page_list() returns the number of reclaimed pages
1589 static unsigned int shrink_page_list(struct list_head
*page_list
,
1590 struct pglist_data
*pgdat
,
1591 struct scan_control
*sc
,
1592 struct reclaim_stat
*stat
,
1593 bool ignore_references
)
1595 LIST_HEAD(ret_pages
);
1596 LIST_HEAD(free_pages
);
1597 LIST_HEAD(demote_pages
);
1598 unsigned int nr_reclaimed
= 0;
1599 unsigned int pgactivate
= 0;
1600 bool do_demote_pass
;
1601 struct swap_iocb
*plug
= NULL
;
1603 memset(stat
, 0, sizeof(*stat
));
1605 do_demote_pass
= can_demote(pgdat
->node_id
, sc
);
1608 while (!list_empty(page_list
)) {
1609 struct address_space
*mapping
;
1610 struct folio
*folio
;
1611 enum page_references references
= PAGEREF_RECLAIM
;
1612 bool dirty
, writeback
;
1613 unsigned int nr_pages
;
1617 folio
= lru_to_folio(page_list
);
1618 list_del(&folio
->lru
);
1620 if (!folio_trylock(folio
))
1623 VM_BUG_ON_FOLIO(folio_test_active(folio
), folio
);
1625 nr_pages
= folio_nr_pages(folio
);
1627 /* Account the number of base pages */
1628 sc
->nr_scanned
+= nr_pages
;
1630 if (unlikely(!folio_evictable(folio
)))
1631 goto activate_locked
;
1633 if (!sc
->may_unmap
&& folio_mapped(folio
))
1637 * The number of dirty pages determines if a node is marked
1638 * reclaim_congested. kswapd will stall and start writing
1639 * folios if the tail of the LRU is all dirty unqueued folios.
1641 folio_check_dirty_writeback(folio
, &dirty
, &writeback
);
1642 if (dirty
|| writeback
)
1643 stat
->nr_dirty
+= nr_pages
;
1645 if (dirty
&& !writeback
)
1646 stat
->nr_unqueued_dirty
+= nr_pages
;
1649 * Treat this folio as congested if folios are cycling
1650 * through the LRU so quickly that the folios marked
1651 * for immediate reclaim are making it to the end of
1652 * the LRU a second time.
1654 if (writeback
&& folio_test_reclaim(folio
))
1655 stat
->nr_congested
+= nr_pages
;
1658 * If a folio at the tail of the LRU is under writeback, there
1659 * are three cases to consider.
1661 * 1) If reclaim is encountering an excessive number
1662 * of folios under writeback and this folio has both
1663 * the writeback and reclaim flags set, then it
1664 * indicates that folios are being queued for I/O but
1665 * are being recycled through the LRU before the I/O
1666 * can complete. Waiting on the folio itself risks an
1667 * indefinite stall if it is impossible to writeback
1668 * the folio due to I/O error or disconnected storage
1669 * so instead note that the LRU is being scanned too
1670 * quickly and the caller can stall after the folio
1671 * list has been processed.
1673 * 2) Global or new memcg reclaim encounters a folio that is
1674 * not marked for immediate reclaim, or the caller does not
1675 * have __GFP_FS (or __GFP_IO if it's simply going to swap,
1676 * not to fs). In this case mark the folio for immediate
1677 * reclaim and continue scanning.
1679 * Require may_enter_fs() because we would wait on fs, which
1680 * may not have submitted I/O yet. And the loop driver might
1681 * enter reclaim, and deadlock if it waits on a folio for
1682 * which it is needed to do the write (loop masks off
1683 * __GFP_IO|__GFP_FS for this reason); but more thought
1684 * would probably show more reasons.
1686 * 3) Legacy memcg encounters a folio that already has the
1687 * reclaim flag set. memcg does not have any dirty folio
1688 * throttling so we could easily OOM just because too many
1689 * folios are in writeback and there is nothing else to
1690 * reclaim. Wait for the writeback to complete.
1692 * In cases 1) and 2) we activate the folios to get them out of
1693 * the way while we continue scanning for clean folios on the
1694 * inactive list and refilling from the active list. The
1695 * observation here is that waiting for disk writes is more
1696 * expensive than potentially causing reloads down the line.
1697 * Since they're marked for immediate reclaim, they won't put
1698 * memory pressure on the cache working set any longer than it
1699 * takes to write them to disk.
1701 if (folio_test_writeback(folio
)) {
1703 if (current_is_kswapd() &&
1704 folio_test_reclaim(folio
) &&
1705 test_bit(PGDAT_WRITEBACK
, &pgdat
->flags
)) {
1706 stat
->nr_immediate
+= nr_pages
;
1707 goto activate_locked
;
1710 } else if (writeback_throttling_sane(sc
) ||
1711 !folio_test_reclaim(folio
) ||
1712 !may_enter_fs(folio
, sc
->gfp_mask
)) {
1714 * This is slightly racy -
1715 * folio_end_writeback() might have
1716 * just cleared the reclaim flag, then
1717 * setting the reclaim flag here ends up
1718 * interpreted as the readahead flag - but
1719 * that does not matter enough to care.
1720 * What we do want is for this folio to
1721 * have the reclaim flag set next time
1722 * memcg reclaim reaches the tests above,
1723 * so it will then wait for writeback to
1724 * avoid OOM; and it's also appropriate
1725 * in global reclaim.
1727 folio_set_reclaim(folio
);
1728 stat
->nr_writeback
+= nr_pages
;
1729 goto activate_locked
;
1733 folio_unlock(folio
);
1734 folio_wait_writeback(folio
);
1735 /* then go back and try same folio again */
1736 list_add_tail(&folio
->lru
, page_list
);
1741 if (!ignore_references
)
1742 references
= folio_check_references(folio
, sc
);
1744 switch (references
) {
1745 case PAGEREF_ACTIVATE
:
1746 goto activate_locked
;
1748 stat
->nr_ref_keep
+= nr_pages
;
1750 case PAGEREF_RECLAIM
:
1751 case PAGEREF_RECLAIM_CLEAN
:
1752 ; /* try to reclaim the folio below */
1756 * Before reclaiming the folio, try to relocate
1757 * its contents to another node.
1759 if (do_demote_pass
&&
1760 (thp_migration_supported() || !folio_test_large(folio
))) {
1761 list_add(&folio
->lru
, &demote_pages
);
1762 folio_unlock(folio
);
1767 * Anonymous process memory has backing store?
1768 * Try to allocate it some swap space here.
1769 * Lazyfree folio could be freed directly
1771 if (folio_test_anon(folio
) && folio_test_swapbacked(folio
)) {
1772 if (!folio_test_swapcache(folio
)) {
1773 if (!(sc
->gfp_mask
& __GFP_IO
))
1775 if (folio_maybe_dma_pinned(folio
))
1777 if (folio_test_large(folio
)) {
1778 /* cannot split folio, skip it */
1779 if (!can_split_folio(folio
, NULL
))
1780 goto activate_locked
;
1782 * Split folios without a PMD map right
1783 * away. Chances are some or all of the
1784 * tail pages can be freed without IO.
1786 if (!folio_entire_mapcount(folio
) &&
1787 split_folio_to_list(folio
,
1789 goto activate_locked
;
1791 if (!add_to_swap(folio
)) {
1792 if (!folio_test_large(folio
))
1793 goto activate_locked_split
;
1794 /* Fallback to swap normal pages */
1795 if (split_folio_to_list(folio
,
1797 goto activate_locked
;
1798 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1799 count_vm_event(THP_SWPOUT_FALLBACK
);
1801 if (!add_to_swap(folio
))
1802 goto activate_locked_split
;
1805 } else if (folio_test_swapbacked(folio
) &&
1806 folio_test_large(folio
)) {
1807 /* Split shmem folio */
1808 if (split_folio_to_list(folio
, page_list
))
1813 * If the folio was split above, the tail pages will make
1814 * their own pass through this function and be accounted
1817 if ((nr_pages
> 1) && !folio_test_large(folio
)) {
1818 sc
->nr_scanned
-= (nr_pages
- 1);
1823 * The folio is mapped into the page tables of one or more
1824 * processes. Try to unmap it here.
1826 if (folio_mapped(folio
)) {
1827 enum ttu_flags flags
= TTU_BATCH_FLUSH
;
1828 bool was_swapbacked
= folio_test_swapbacked(folio
);
1830 if (folio_test_pmd_mappable(folio
))
1831 flags
|= TTU_SPLIT_HUGE_PMD
;
1833 try_to_unmap(folio
, flags
);
1834 if (folio_mapped(folio
)) {
1835 stat
->nr_unmap_fail
+= nr_pages
;
1836 if (!was_swapbacked
&&
1837 folio_test_swapbacked(folio
))
1838 stat
->nr_lazyfree_fail
+= nr_pages
;
1839 goto activate_locked
;
1843 mapping
= folio_mapping(folio
);
1844 if (folio_test_dirty(folio
)) {
1846 * Only kswapd can writeback filesystem folios
1847 * to avoid risk of stack overflow. But avoid
1848 * injecting inefficient single-folio I/O into
1849 * flusher writeback as much as possible: only
1850 * write folios when we've encountered many
1851 * dirty folios, and when we've already scanned
1852 * the rest of the LRU for clean folios and see
1853 * the same dirty folios again (with the reclaim
1856 if (folio_is_file_lru(folio
) &&
1857 (!current_is_kswapd() ||
1858 !folio_test_reclaim(folio
) ||
1859 !test_bit(PGDAT_DIRTY
, &pgdat
->flags
))) {
1861 * Immediately reclaim when written back.
1862 * Similar in principle to deactivate_page()
1863 * except we already have the folio isolated
1864 * and know it's dirty
1866 node_stat_mod_folio(folio
, NR_VMSCAN_IMMEDIATE
,
1868 folio_set_reclaim(folio
);
1870 goto activate_locked
;
1873 if (references
== PAGEREF_RECLAIM_CLEAN
)
1875 if (!may_enter_fs(folio
, sc
->gfp_mask
))
1877 if (!sc
->may_writepage
)
1881 * Folio is dirty. Flush the TLB if a writable entry
1882 * potentially exists to avoid CPU writes after I/O
1883 * starts and then write it out here.
1885 try_to_unmap_flush_dirty();
1886 switch (pageout(folio
, mapping
, &plug
)) {
1890 goto activate_locked
;
1892 stat
->nr_pageout
+= nr_pages
;
1894 if (folio_test_writeback(folio
))
1896 if (folio_test_dirty(folio
))
1900 * A synchronous write - probably a ramdisk. Go
1901 * ahead and try to reclaim the folio.
1903 if (!folio_trylock(folio
))
1905 if (folio_test_dirty(folio
) ||
1906 folio_test_writeback(folio
))
1908 mapping
= folio_mapping(folio
);
1911 ; /* try to free the folio below */
1916 * If the folio has buffers, try to free the buffer
1917 * mappings associated with this folio. If we succeed
1918 * we try to free the folio as well.
1920 * We do this even if the folio is dirty.
1921 * filemap_release_folio() does not perform I/O, but it
1922 * is possible for a folio to have the dirty flag set,
1923 * but it is actually clean (all its buffers are clean).
1924 * This happens if the buffers were written out directly,
1925 * with submit_bh(). ext3 will do this, as well as
1926 * the blockdev mapping. filemap_release_folio() will
1927 * discover that cleanness and will drop the buffers
1928 * and mark the folio clean - it can be freed.
1930 * Rarely, folios can have buffers and no ->mapping.
1931 * These are the folios which were not successfully
1932 * invalidated in truncate_cleanup_folio(). We try to
1933 * drop those buffers here and if that worked, and the
1934 * folio is no longer mapped into process address space
1935 * (refcount == 1) it can be freed. Otherwise, leave
1936 * the folio on the LRU so it is swappable.
1938 if (folio_has_private(folio
)) {
1939 if (!filemap_release_folio(folio
, sc
->gfp_mask
))
1940 goto activate_locked
;
1941 if (!mapping
&& folio_ref_count(folio
) == 1) {
1942 folio_unlock(folio
);
1943 if (folio_put_testzero(folio
))
1947 * rare race with speculative reference.
1948 * the speculative reference will free
1949 * this folio shortly, so we may
1950 * increment nr_reclaimed here (and
1951 * leave it off the LRU).
1953 nr_reclaimed
+= nr_pages
;
1959 if (folio_test_anon(folio
) && !folio_test_swapbacked(folio
)) {
1960 /* follow __remove_mapping for reference */
1961 if (!folio_ref_freeze(folio
, 1))
1964 * The folio has only one reference left, which is
1965 * from the isolation. After the caller puts the
1966 * folio back on the lru and drops the reference, the
1967 * folio will be freed anyway. It doesn't matter
1968 * which lru it goes on. So we don't bother checking
1969 * the dirty flag here.
1971 count_vm_events(PGLAZYFREED
, nr_pages
);
1972 count_memcg_folio_events(folio
, PGLAZYFREED
, nr_pages
);
1973 } else if (!mapping
|| !__remove_mapping(mapping
, folio
, true,
1974 sc
->target_mem_cgroup
))
1977 folio_unlock(folio
);
1980 * Folio may get swapped out as a whole, need to account
1983 nr_reclaimed
+= nr_pages
;
1986 * Is there need to periodically free_page_list? It would
1987 * appear not as the counts should be low
1989 if (unlikely(folio_test_large(folio
)))
1990 destroy_large_folio(folio
);
1992 list_add(&folio
->lru
, &free_pages
);
1995 activate_locked_split
:
1997 * The tail pages that are failed to add into swap cache
1998 * reach here. Fixup nr_scanned and nr_pages.
2001 sc
->nr_scanned
-= (nr_pages
- 1);
2005 /* Not a candidate for swapping, so reclaim swap space. */
2006 if (folio_test_swapcache(folio
) &&
2007 (mem_cgroup_swap_full(&folio
->page
) ||
2008 folio_test_mlocked(folio
)))
2009 try_to_free_swap(&folio
->page
);
2010 VM_BUG_ON_FOLIO(folio_test_active(folio
), folio
);
2011 if (!folio_test_mlocked(folio
)) {
2012 int type
= folio_is_file_lru(folio
);
2013 folio_set_active(folio
);
2014 stat
->nr_activate
[type
] += nr_pages
;
2015 count_memcg_folio_events(folio
, PGACTIVATE
, nr_pages
);
2018 folio_unlock(folio
);
2020 list_add(&folio
->lru
, &ret_pages
);
2021 VM_BUG_ON_FOLIO(folio_test_lru(folio
) ||
2022 folio_test_unevictable(folio
), folio
);
2024 /* 'page_list' is always empty here */
2026 /* Migrate folios selected for demotion */
2027 nr_reclaimed
+= demote_page_list(&demote_pages
, pgdat
);
2028 /* Folios that could not be demoted are still in @demote_pages */
2029 if (!list_empty(&demote_pages
)) {
2030 /* Folios which weren't demoted go back on @page_list for retry: */
2031 list_splice_init(&demote_pages
, page_list
);
2032 do_demote_pass
= false;
2036 pgactivate
= stat
->nr_activate
[0] + stat
->nr_activate
[1];
2038 mem_cgroup_uncharge_list(&free_pages
);
2039 try_to_unmap_flush();
2040 free_unref_page_list(&free_pages
);
2042 list_splice(&ret_pages
, page_list
);
2043 count_vm_events(PGACTIVATE
, pgactivate
);
2046 swap_write_unplug(plug
);
2047 return nr_reclaimed
;
2050 unsigned int reclaim_clean_pages_from_list(struct zone
*zone
,
2051 struct list_head
*folio_list
)
2053 struct scan_control sc
= {
2054 .gfp_mask
= GFP_KERNEL
,
2057 struct reclaim_stat stat
;
2058 unsigned int nr_reclaimed
;
2059 struct folio
*folio
, *next
;
2060 LIST_HEAD(clean_folios
);
2061 unsigned int noreclaim_flag
;
2063 list_for_each_entry_safe(folio
, next
, folio_list
, lru
) {
2064 if (!folio_test_hugetlb(folio
) && folio_is_file_lru(folio
) &&
2065 !folio_test_dirty(folio
) && !__folio_test_movable(folio
) &&
2066 !folio_test_unevictable(folio
)) {
2067 folio_clear_active(folio
);
2068 list_move(&folio
->lru
, &clean_folios
);
2073 * We should be safe here since we are only dealing with file pages and
2074 * we are not kswapd and therefore cannot write dirty file pages. But
2075 * call memalloc_noreclaim_save() anyway, just in case these conditions
2076 * change in the future.
2078 noreclaim_flag
= memalloc_noreclaim_save();
2079 nr_reclaimed
= shrink_page_list(&clean_folios
, zone
->zone_pgdat
, &sc
,
2081 memalloc_noreclaim_restore(noreclaim_flag
);
2083 list_splice(&clean_folios
, folio_list
);
2084 mod_node_page_state(zone
->zone_pgdat
, NR_ISOLATED_FILE
,
2085 -(long)nr_reclaimed
);
2087 * Since lazyfree pages are isolated from file LRU from the beginning,
2088 * they will rotate back to anonymous LRU in the end if it failed to
2089 * discard so isolated count will be mismatched.
2090 * Compensate the isolated count for both LRU lists.
2092 mod_node_page_state(zone
->zone_pgdat
, NR_ISOLATED_ANON
,
2093 stat
.nr_lazyfree_fail
);
2094 mod_node_page_state(zone
->zone_pgdat
, NR_ISOLATED_FILE
,
2095 -(long)stat
.nr_lazyfree_fail
);
2096 return nr_reclaimed
;
2100 * Update LRU sizes after isolating pages. The LRU size updates must
2101 * be complete before mem_cgroup_update_lru_size due to a sanity check.
2103 static __always_inline
void update_lru_sizes(struct lruvec
*lruvec
,
2104 enum lru_list lru
, unsigned long *nr_zone_taken
)
2108 for (zid
= 0; zid
< MAX_NR_ZONES
; zid
++) {
2109 if (!nr_zone_taken
[zid
])
2112 update_lru_size(lruvec
, lru
, zid
, -nr_zone_taken
[zid
]);
2118 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
2120 * lruvec->lru_lock is heavily contended. Some of the functions that
2121 * shrink the lists perform better by taking out a batch of pages
2122 * and working on them outside the LRU lock.
2124 * For pagecache intensive workloads, this function is the hottest
2125 * spot in the kernel (apart from copy_*_user functions).
2127 * Lru_lock must be held before calling this function.
2129 * @nr_to_scan: The number of eligible pages to look through on the list.
2130 * @lruvec: The LRU vector to pull pages from.
2131 * @dst: The temp list to put pages on to.
2132 * @nr_scanned: The number of pages that were scanned.
2133 * @sc: The scan_control struct for this reclaim session
2134 * @lru: LRU list id for isolating
2136 * returns how many pages were moved onto *@dst.
2138 static unsigned long isolate_lru_pages(unsigned long nr_to_scan
,
2139 struct lruvec
*lruvec
, struct list_head
*dst
,
2140 unsigned long *nr_scanned
, struct scan_control
*sc
,
2143 struct list_head
*src
= &lruvec
->lists
[lru
];
2144 unsigned long nr_taken
= 0;
2145 unsigned long nr_zone_taken
[MAX_NR_ZONES
] = { 0 };
2146 unsigned long nr_skipped
[MAX_NR_ZONES
] = { 0, };
2147 unsigned long skipped
= 0;
2148 unsigned long scan
, total_scan
, nr_pages
;
2149 LIST_HEAD(folios_skipped
);
2153 while (scan
< nr_to_scan
&& !list_empty(src
)) {
2154 struct list_head
*move_to
= src
;
2155 struct folio
*folio
;
2157 folio
= lru_to_folio(src
);
2158 prefetchw_prev_lru_folio(folio
, src
, flags
);
2160 nr_pages
= folio_nr_pages(folio
);
2161 total_scan
+= nr_pages
;
2163 if (folio_zonenum(folio
) > sc
->reclaim_idx
) {
2164 nr_skipped
[folio_zonenum(folio
)] += nr_pages
;
2165 move_to
= &folios_skipped
;
2170 * Do not count skipped folios because that makes the function
2171 * return with no isolated folios if the LRU mostly contains
2172 * ineligible folios. This causes the VM to not reclaim any
2173 * folios, triggering a premature OOM.
2174 * Account all pages in a folio.
2178 if (!folio_test_lru(folio
))
2180 if (!sc
->may_unmap
&& folio_mapped(folio
))
2184 * Be careful not to clear the lru flag until after we're
2185 * sure the folio is not being freed elsewhere -- the
2186 * folio release code relies on it.
2188 if (unlikely(!folio_try_get(folio
)))
2191 if (!folio_test_clear_lru(folio
)) {
2192 /* Another thread is already isolating this folio */
2197 nr_taken
+= nr_pages
;
2198 nr_zone_taken
[folio_zonenum(folio
)] += nr_pages
;
2201 list_move(&folio
->lru
, move_to
);
2205 * Splice any skipped folios to the start of the LRU list. Note that
2206 * this disrupts the LRU order when reclaiming for lower zones but
2207 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
2208 * scanning would soon rescan the same folios to skip and waste lots
2211 if (!list_empty(&folios_skipped
)) {
2214 list_splice(&folios_skipped
, src
);
2215 for (zid
= 0; zid
< MAX_NR_ZONES
; zid
++) {
2216 if (!nr_skipped
[zid
])
2219 __count_zid_vm_events(PGSCAN_SKIP
, zid
, nr_skipped
[zid
]);
2220 skipped
+= nr_skipped
[zid
];
2223 *nr_scanned
= total_scan
;
2224 trace_mm_vmscan_lru_isolate(sc
->reclaim_idx
, sc
->order
, nr_to_scan
,
2225 total_scan
, skipped
, nr_taken
,
2226 sc
->may_unmap
? 0 : ISOLATE_UNMAPPED
, lru
);
2227 update_lru_sizes(lruvec
, lru
, nr_zone_taken
);
2232 * folio_isolate_lru() - Try to isolate a folio from its LRU list.
2233 * @folio: Folio to isolate from its LRU list.
2235 * Isolate a @folio from an LRU list and adjust the vmstat statistic
2236 * corresponding to whatever LRU list the folio was on.
2238 * The folio will have its LRU flag cleared. If it was found on the
2239 * active list, it will have the Active flag set. If it was found on the
2240 * unevictable list, it will have the Unevictable flag set. These flags
2241 * may need to be cleared by the caller before letting the page go.
2245 * (1) Must be called with an elevated refcount on the page. This is a
2246 * fundamental difference from isolate_lru_pages() (which is called
2247 * without a stable reference).
2248 * (2) The lru_lock must not be held.
2249 * (3) Interrupts must be enabled.
2251 * Return: 0 if the folio was removed from an LRU list.
2252 * -EBUSY if the folio was not on an LRU list.
2254 int folio_isolate_lru(struct folio
*folio
)
2258 VM_BUG_ON_FOLIO(!folio_ref_count(folio
), folio
);
2260 if (folio_test_clear_lru(folio
)) {
2261 struct lruvec
*lruvec
;
2264 lruvec
= folio_lruvec_lock_irq(folio
);
2265 lruvec_del_folio(lruvec
, folio
);
2266 unlock_page_lruvec_irq(lruvec
);
2274 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
2275 * then get rescheduled. When there are massive number of tasks doing page
2276 * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
2277 * the LRU list will go small and be scanned faster than necessary, leading to
2278 * unnecessary swapping, thrashing and OOM.
2280 static int too_many_isolated(struct pglist_data
*pgdat
, int file
,
2281 struct scan_control
*sc
)
2283 unsigned long inactive
, isolated
;
2286 if (current_is_kswapd())
2289 if (!writeback_throttling_sane(sc
))
2293 inactive
= node_page_state(pgdat
, NR_INACTIVE_FILE
);
2294 isolated
= node_page_state(pgdat
, NR_ISOLATED_FILE
);
2296 inactive
= node_page_state(pgdat
, NR_INACTIVE_ANON
);
2297 isolated
= node_page_state(pgdat
, NR_ISOLATED_ANON
);
2301 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
2302 * won't get blocked by normal direct-reclaimers, forming a circular
2305 if ((sc
->gfp_mask
& (__GFP_IO
| __GFP_FS
)) == (__GFP_IO
| __GFP_FS
))
2308 too_many
= isolated
> inactive
;
2310 /* Wake up tasks throttled due to too_many_isolated. */
2312 wake_throttle_isolated(pgdat
);
2318 * move_pages_to_lru() moves folios from private @list to appropriate LRU list.
2319 * On return, @list is reused as a list of folios to be freed by the caller.
2321 * Returns the number of pages moved to the given lruvec.
2323 static unsigned int move_pages_to_lru(struct lruvec
*lruvec
,
2324 struct list_head
*list
)
2326 int nr_pages
, nr_moved
= 0;
2327 LIST_HEAD(folios_to_free
);
2329 while (!list_empty(list
)) {
2330 struct folio
*folio
= lru_to_folio(list
);
2332 VM_BUG_ON_FOLIO(folio_test_lru(folio
), folio
);
2333 list_del(&folio
->lru
);
2334 if (unlikely(!folio_evictable(folio
))) {
2335 spin_unlock_irq(&lruvec
->lru_lock
);
2336 folio_putback_lru(folio
);
2337 spin_lock_irq(&lruvec
->lru_lock
);
2342 * The folio_set_lru needs to be kept here for list integrity.
2344 * #0 move_pages_to_lru #1 release_pages
2345 * if (!folio_put_testzero())
2346 * if (folio_put_testzero())
2347 * !lru //skip lru_lock
2349 * list_add(&folio->lru,)
2350 * list_add(&folio->lru,)
2352 folio_set_lru(folio
);
2354 if (unlikely(folio_put_testzero(folio
))) {
2355 __folio_clear_lru_flags(folio
);
2357 if (unlikely(folio_test_large(folio
))) {
2358 spin_unlock_irq(&lruvec
->lru_lock
);
2359 destroy_large_folio(folio
);
2360 spin_lock_irq(&lruvec
->lru_lock
);
2362 list_add(&folio
->lru
, &folios_to_free
);
2368 * All pages were isolated from the same lruvec (and isolation
2369 * inhibits memcg migration).
2371 VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio
, lruvec
), folio
);
2372 lruvec_add_folio(lruvec
, folio
);
2373 nr_pages
= folio_nr_pages(folio
);
2374 nr_moved
+= nr_pages
;
2375 if (folio_test_active(folio
))
2376 workingset_age_nonresident(lruvec
, nr_pages
);
2380 * To save our caller's stack, now use input list for pages to free.
2382 list_splice(&folios_to_free
, list
);
2388 * If a kernel thread (such as nfsd for loop-back mounts) services a backing
2389 * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case
2390 * we should not throttle. Otherwise it is safe to do so.
2392 static int current_may_throttle(void)
2394 return !(current
->flags
& PF_LOCAL_THROTTLE
);
2398 * shrink_inactive_list() is a helper for shrink_node(). It returns the number
2399 * of reclaimed pages
2401 static unsigned long
2402 shrink_inactive_list(unsigned long nr_to_scan
, struct lruvec
*lruvec
,
2403 struct scan_control
*sc
, enum lru_list lru
)
2405 LIST_HEAD(page_list
);
2406 unsigned long nr_scanned
;
2407 unsigned int nr_reclaimed
= 0;
2408 unsigned long nr_taken
;
2409 struct reclaim_stat stat
;
2410 bool file
= is_file_lru(lru
);
2411 enum vm_event_item item
;
2412 struct pglist_data
*pgdat
= lruvec_pgdat(lruvec
);
2413 bool stalled
= false;
2415 while (unlikely(too_many_isolated(pgdat
, file
, sc
))) {
2419 /* wait a bit for the reclaimer. */
2421 reclaim_throttle(pgdat
, VMSCAN_THROTTLE_ISOLATED
);
2423 /* We are about to die and free our memory. Return now. */
2424 if (fatal_signal_pending(current
))
2425 return SWAP_CLUSTER_MAX
;
2430 spin_lock_irq(&lruvec
->lru_lock
);
2432 nr_taken
= isolate_lru_pages(nr_to_scan
, lruvec
, &page_list
,
2433 &nr_scanned
, sc
, lru
);
2435 __mod_node_page_state(pgdat
, NR_ISOLATED_ANON
+ file
, nr_taken
);
2436 item
= current_is_kswapd() ? PGSCAN_KSWAPD
: PGSCAN_DIRECT
;
2437 if (!cgroup_reclaim(sc
))
2438 __count_vm_events(item
, nr_scanned
);
2439 __count_memcg_events(lruvec_memcg(lruvec
), item
, nr_scanned
);
2440 __count_vm_events(PGSCAN_ANON
+ file
, nr_scanned
);
2442 spin_unlock_irq(&lruvec
->lru_lock
);
2447 nr_reclaimed
= shrink_page_list(&page_list
, pgdat
, sc
, &stat
, false);
2449 spin_lock_irq(&lruvec
->lru_lock
);
2450 move_pages_to_lru(lruvec
, &page_list
);
2452 __mod_node_page_state(pgdat
, NR_ISOLATED_ANON
+ file
, -nr_taken
);
2453 item
= current_is_kswapd() ? PGSTEAL_KSWAPD
: PGSTEAL_DIRECT
;
2454 if (!cgroup_reclaim(sc
))
2455 __count_vm_events(item
, nr_reclaimed
);
2456 __count_memcg_events(lruvec_memcg(lruvec
), item
, nr_reclaimed
);
2457 __count_vm_events(PGSTEAL_ANON
+ file
, nr_reclaimed
);
2458 spin_unlock_irq(&lruvec
->lru_lock
);
2460 lru_note_cost(lruvec
, file
, stat
.nr_pageout
);
2461 mem_cgroup_uncharge_list(&page_list
);
2462 free_unref_page_list(&page_list
);
2465 * If dirty pages are scanned that are not queued for IO, it
2466 * implies that flushers are not doing their job. This can
2467 * happen when memory pressure pushes dirty pages to the end of
2468 * the LRU before the dirty limits are breached and the dirty
2469 * data has expired. It can also happen when the proportion of
2470 * dirty pages grows not through writes but through memory
2471 * pressure reclaiming all the clean cache. And in some cases,
2472 * the flushers simply cannot keep up with the allocation
2473 * rate. Nudge the flusher threads in case they are asleep.
2475 if (stat
.nr_unqueued_dirty
== nr_taken
)
2476 wakeup_flusher_threads(WB_REASON_VMSCAN
);
2478 sc
->nr
.dirty
+= stat
.nr_dirty
;
2479 sc
->nr
.congested
+= stat
.nr_congested
;
2480 sc
->nr
.unqueued_dirty
+= stat
.nr_unqueued_dirty
;
2481 sc
->nr
.writeback
+= stat
.nr_writeback
;
2482 sc
->nr
.immediate
+= stat
.nr_immediate
;
2483 sc
->nr
.taken
+= nr_taken
;
2485 sc
->nr
.file_taken
+= nr_taken
;
2487 trace_mm_vmscan_lru_shrink_inactive(pgdat
->node_id
,
2488 nr_scanned
, nr_reclaimed
, &stat
, sc
->priority
, file
);
2489 return nr_reclaimed
;
2493 * shrink_active_list() moves folios from the active LRU to the inactive LRU.
2495 * We move them the other way if the folio is referenced by one or more
2498 * If the folios are mostly unmapped, the processing is fast and it is
2499 * appropriate to hold lru_lock across the whole operation. But if
2500 * the folios are mapped, the processing is slow (folio_referenced()), so
2501 * we should drop lru_lock around each folio. It's impossible to balance
2502 * this, so instead we remove the folios from the LRU while processing them.
2503 * It is safe to rely on the active flag against the non-LRU folios in here
2504 * because nobody will play with that bit on a non-LRU folio.
2506 * The downside is that we have to touch folio->_refcount against each folio.
2507 * But we had to alter folio->flags anyway.
2509 static void shrink_active_list(unsigned long nr_to_scan
,
2510 struct lruvec
*lruvec
,
2511 struct scan_control
*sc
,
2514 unsigned long nr_taken
;
2515 unsigned long nr_scanned
;
2516 unsigned long vm_flags
;
2517 LIST_HEAD(l_hold
); /* The folios which were snipped off */
2518 LIST_HEAD(l_active
);
2519 LIST_HEAD(l_inactive
);
2520 unsigned nr_deactivate
, nr_activate
;
2521 unsigned nr_rotated
= 0;
2522 int file
= is_file_lru(lru
);
2523 struct pglist_data
*pgdat
= lruvec_pgdat(lruvec
);
2527 spin_lock_irq(&lruvec
->lru_lock
);
2529 nr_taken
= isolate_lru_pages(nr_to_scan
, lruvec
, &l_hold
,
2530 &nr_scanned
, sc
, lru
);
2532 __mod_node_page_state(pgdat
, NR_ISOLATED_ANON
+ file
, nr_taken
);
2534 if (!cgroup_reclaim(sc
))
2535 __count_vm_events(PGREFILL
, nr_scanned
);
2536 __count_memcg_events(lruvec_memcg(lruvec
), PGREFILL
, nr_scanned
);
2538 spin_unlock_irq(&lruvec
->lru_lock
);
2540 while (!list_empty(&l_hold
)) {
2541 struct folio
*folio
;
2544 folio
= lru_to_folio(&l_hold
);
2545 list_del(&folio
->lru
);
2547 if (unlikely(!folio_evictable(folio
))) {
2548 folio_putback_lru(folio
);
2552 if (unlikely(buffer_heads_over_limit
)) {
2553 if (folio_get_private(folio
) && folio_trylock(folio
)) {
2554 if (folio_get_private(folio
))
2555 filemap_release_folio(folio
, 0);
2556 folio_unlock(folio
);
2560 /* Referenced or rmap lock contention: rotate */
2561 if (folio_referenced(folio
, 0, sc
->target_mem_cgroup
,
2564 * Identify referenced, file-backed active folios and
2565 * give them one more trip around the active list. So
2566 * that executable code get better chances to stay in
2567 * memory under moderate memory pressure. Anon folios
2568 * are not likely to be evicted by use-once streaming
2569 * IO, plus JVM can create lots of anon VM_EXEC folios,
2570 * so we ignore them here.
2572 if ((vm_flags
& VM_EXEC
) && folio_is_file_lru(folio
)) {
2573 nr_rotated
+= folio_nr_pages(folio
);
2574 list_add(&folio
->lru
, &l_active
);
2579 folio_clear_active(folio
); /* we are de-activating */
2580 folio_set_workingset(folio
);
2581 list_add(&folio
->lru
, &l_inactive
);
2585 * Move folios back to the lru list.
2587 spin_lock_irq(&lruvec
->lru_lock
);
2589 nr_activate
= move_pages_to_lru(lruvec
, &l_active
);
2590 nr_deactivate
= move_pages_to_lru(lruvec
, &l_inactive
);
2591 /* Keep all free folios in l_active list */
2592 list_splice(&l_inactive
, &l_active
);
2594 __count_vm_events(PGDEACTIVATE
, nr_deactivate
);
2595 __count_memcg_events(lruvec_memcg(lruvec
), PGDEACTIVATE
, nr_deactivate
);
2597 __mod_node_page_state(pgdat
, NR_ISOLATED_ANON
+ file
, -nr_taken
);
2598 spin_unlock_irq(&lruvec
->lru_lock
);
2600 mem_cgroup_uncharge_list(&l_active
);
2601 free_unref_page_list(&l_active
);
2602 trace_mm_vmscan_lru_shrink_active(pgdat
->node_id
, nr_taken
, nr_activate
,
2603 nr_deactivate
, nr_rotated
, sc
->priority
, file
);
2606 static unsigned int reclaim_page_list(struct list_head
*page_list
,
2607 struct pglist_data
*pgdat
)
2609 struct reclaim_stat dummy_stat
;
2610 unsigned int nr_reclaimed
;
2611 struct folio
*folio
;
2612 struct scan_control sc
= {
2613 .gfp_mask
= GFP_KERNEL
,
2620 nr_reclaimed
= shrink_page_list(page_list
, pgdat
, &sc
, &dummy_stat
, false);
2621 while (!list_empty(page_list
)) {
2622 folio
= lru_to_folio(page_list
);
2623 list_del(&folio
->lru
);
2624 folio_putback_lru(folio
);
2627 return nr_reclaimed
;
2630 unsigned long reclaim_pages(struct list_head
*folio_list
)
2633 unsigned int nr_reclaimed
= 0;
2634 LIST_HEAD(node_folio_list
);
2635 unsigned int noreclaim_flag
;
2637 if (list_empty(folio_list
))
2638 return nr_reclaimed
;
2640 noreclaim_flag
= memalloc_noreclaim_save();
2642 nid
= folio_nid(lru_to_folio(folio_list
));
2644 struct folio
*folio
= lru_to_folio(folio_list
);
2646 if (nid
== folio_nid(folio
)) {
2647 folio_clear_active(folio
);
2648 list_move(&folio
->lru
, &node_folio_list
);
2652 nr_reclaimed
+= reclaim_page_list(&node_folio_list
, NODE_DATA(nid
));
2653 nid
= folio_nid(lru_to_folio(folio_list
));
2654 } while (!list_empty(folio_list
));
2656 nr_reclaimed
+= reclaim_page_list(&node_folio_list
, NODE_DATA(nid
));
2658 memalloc_noreclaim_restore(noreclaim_flag
);
2660 return nr_reclaimed
;
2663 static unsigned long shrink_list(enum lru_list lru
, unsigned long nr_to_scan
,
2664 struct lruvec
*lruvec
, struct scan_control
*sc
)
2666 if (is_active_lru(lru
)) {
2667 if (sc
->may_deactivate
& (1 << is_file_lru(lru
)))
2668 shrink_active_list(nr_to_scan
, lruvec
, sc
, lru
);
2670 sc
->skipped_deactivate
= 1;
2674 return shrink_inactive_list(nr_to_scan
, lruvec
, sc
, lru
);
2678 * The inactive anon list should be small enough that the VM never has
2679 * to do too much work.
2681 * The inactive file list should be small enough to leave most memory
2682 * to the established workingset on the scan-resistant active list,
2683 * but large enough to avoid thrashing the aggregate readahead window.
2685 * Both inactive lists should also be large enough that each inactive
2686 * page has a chance to be referenced again before it is reclaimed.
2688 * If that fails and refaulting is observed, the inactive list grows.
2690 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
2691 * on this LRU, maintained by the pageout code. An inactive_ratio
2692 * of 3 means 3:1 or 25% of the pages are kept on the inactive list.
2695 * memory ratio inactive
2696 * -------------------------------------
2705 static bool inactive_is_low(struct lruvec
*lruvec
, enum lru_list inactive_lru
)
2707 enum lru_list active_lru
= inactive_lru
+ LRU_ACTIVE
;
2708 unsigned long inactive
, active
;
2709 unsigned long inactive_ratio
;
2712 inactive
= lruvec_page_state(lruvec
, NR_LRU_BASE
+ inactive_lru
);
2713 active
= lruvec_page_state(lruvec
, NR_LRU_BASE
+ active_lru
);
2715 gb
= (inactive
+ active
) >> (30 - PAGE_SHIFT
);
2717 inactive_ratio
= int_sqrt(10 * gb
);
2721 return inactive
* inactive_ratio
< active
;
2732 * Determine how aggressively the anon and file LRU lists should be
2735 * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2736 * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2738 static void get_scan_count(struct lruvec
*lruvec
, struct scan_control
*sc
,
2741 struct pglist_data
*pgdat
= lruvec_pgdat(lruvec
);
2742 struct mem_cgroup
*memcg
= lruvec_memcg(lruvec
);
2743 unsigned long anon_cost
, file_cost
, total_cost
;
2744 int swappiness
= mem_cgroup_swappiness(memcg
);
2745 u64 fraction
[ANON_AND_FILE
];
2746 u64 denominator
= 0; /* gcc */
2747 enum scan_balance scan_balance
;
2748 unsigned long ap
, fp
;
2751 /* If we have no swap space, do not bother scanning anon pages. */
2752 if (!sc
->may_swap
|| !can_reclaim_anon_pages(memcg
, pgdat
->node_id
, sc
)) {
2753 scan_balance
= SCAN_FILE
;
2758 * Global reclaim will swap to prevent OOM even with no
2759 * swappiness, but memcg users want to use this knob to
2760 * disable swapping for individual groups completely when
2761 * using the memory controller's swap limit feature would be
2764 if (cgroup_reclaim(sc
) && !swappiness
) {
2765 scan_balance
= SCAN_FILE
;
2770 * Do not apply any pressure balancing cleverness when the
2771 * system is close to OOM, scan both anon and file equally
2772 * (unless the swappiness setting disagrees with swapping).
2774 if (!sc
->priority
&& swappiness
) {
2775 scan_balance
= SCAN_EQUAL
;
2780 * If the system is almost out of file pages, force-scan anon.
2782 if (sc
->file_is_tiny
) {
2783 scan_balance
= SCAN_ANON
;
2788 * If there is enough inactive page cache, we do not reclaim
2789 * anything from the anonymous working right now.
2791 if (sc
->cache_trim_mode
) {
2792 scan_balance
= SCAN_FILE
;
2796 scan_balance
= SCAN_FRACT
;
2798 * Calculate the pressure balance between anon and file pages.
2800 * The amount of pressure we put on each LRU is inversely
2801 * proportional to the cost of reclaiming each list, as
2802 * determined by the share of pages that are refaulting, times
2803 * the relative IO cost of bringing back a swapped out
2804 * anonymous page vs reloading a filesystem page (swappiness).
2806 * Although we limit that influence to ensure no list gets
2807 * left behind completely: at least a third of the pressure is
2808 * applied, before swappiness.
2810 * With swappiness at 100, anon and file have equal IO cost.
2812 total_cost
= sc
->anon_cost
+ sc
->file_cost
;
2813 anon_cost
= total_cost
+ sc
->anon_cost
;
2814 file_cost
= total_cost
+ sc
->file_cost
;
2815 total_cost
= anon_cost
+ file_cost
;
2817 ap
= swappiness
* (total_cost
+ 1);
2818 ap
/= anon_cost
+ 1;
2820 fp
= (200 - swappiness
) * (total_cost
+ 1);
2821 fp
/= file_cost
+ 1;
2825 denominator
= ap
+ fp
;
2827 for_each_evictable_lru(lru
) {
2828 int file
= is_file_lru(lru
);
2829 unsigned long lruvec_size
;
2830 unsigned long low
, min
;
2833 lruvec_size
= lruvec_lru_size(lruvec
, lru
, sc
->reclaim_idx
);
2834 mem_cgroup_protection(sc
->target_mem_cgroup
, memcg
,
2839 * Scale a cgroup's reclaim pressure by proportioning
2840 * its current usage to its memory.low or memory.min
2843 * This is important, as otherwise scanning aggression
2844 * becomes extremely binary -- from nothing as we
2845 * approach the memory protection threshold, to totally
2846 * nominal as we exceed it. This results in requiring
2847 * setting extremely liberal protection thresholds. It
2848 * also means we simply get no protection at all if we
2849 * set it too low, which is not ideal.
2851 * If there is any protection in place, we reduce scan
2852 * pressure by how much of the total memory used is
2853 * within protection thresholds.
2855 * There is one special case: in the first reclaim pass,
2856 * we skip over all groups that are within their low
2857 * protection. If that fails to reclaim enough pages to
2858 * satisfy the reclaim goal, we come back and override
2859 * the best-effort low protection. However, we still
2860 * ideally want to honor how well-behaved groups are in
2861 * that case instead of simply punishing them all
2862 * equally. As such, we reclaim them based on how much
2863 * memory they are using, reducing the scan pressure
2864 * again by how much of the total memory used is under
2867 unsigned long cgroup_size
= mem_cgroup_size(memcg
);
2868 unsigned long protection
;
2870 /* memory.low scaling, make sure we retry before OOM */
2871 if (!sc
->memcg_low_reclaim
&& low
> min
) {
2873 sc
->memcg_low_skipped
= 1;
2878 /* Avoid TOCTOU with earlier protection check */
2879 cgroup_size
= max(cgroup_size
, protection
);
2881 scan
= lruvec_size
- lruvec_size
* protection
/
2885 * Minimally target SWAP_CLUSTER_MAX pages to keep
2886 * reclaim moving forwards, avoiding decrementing
2887 * sc->priority further than desirable.
2889 scan
= max(scan
, SWAP_CLUSTER_MAX
);
2894 scan
>>= sc
->priority
;
2897 * If the cgroup's already been deleted, make sure to
2898 * scrape out the remaining cache.
2900 if (!scan
&& !mem_cgroup_online(memcg
))
2901 scan
= min(lruvec_size
, SWAP_CLUSTER_MAX
);
2903 switch (scan_balance
) {
2905 /* Scan lists relative to size */
2909 * Scan types proportional to swappiness and
2910 * their relative recent reclaim efficiency.
2911 * Make sure we don't miss the last page on
2912 * the offlined memory cgroups because of a
2915 scan
= mem_cgroup_online(memcg
) ?
2916 div64_u64(scan
* fraction
[file
], denominator
) :
2917 DIV64_U64_ROUND_UP(scan
* fraction
[file
],
2922 /* Scan one type exclusively */
2923 if ((scan_balance
== SCAN_FILE
) != file
)
2927 /* Look ma, no brain */
2936 * Anonymous LRU management is a waste if there is
2937 * ultimately no way to reclaim the memory.
2939 static bool can_age_anon_pages(struct pglist_data
*pgdat
,
2940 struct scan_control
*sc
)
2942 /* Aging the anon LRU is valuable if swap is present: */
2943 if (total_swap_pages
> 0)
2946 /* Also valuable if anon pages can be demoted: */
2947 return can_demote(pgdat
->node_id
, sc
);
2950 static void shrink_lruvec(struct lruvec
*lruvec
, struct scan_control
*sc
)
2952 unsigned long nr
[NR_LRU_LISTS
];
2953 unsigned long targets
[NR_LRU_LISTS
];
2954 unsigned long nr_to_scan
;
2956 unsigned long nr_reclaimed
= 0;
2957 unsigned long nr_to_reclaim
= sc
->nr_to_reclaim
;
2958 struct blk_plug plug
;
2961 get_scan_count(lruvec
, sc
, nr
);
2963 /* Record the original scan target for proportional adjustments later */
2964 memcpy(targets
, nr
, sizeof(nr
));
2967 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
2968 * event that can occur when there is little memory pressure e.g.
2969 * multiple streaming readers/writers. Hence, we do not abort scanning
2970 * when the requested number of pages are reclaimed when scanning at
2971 * DEF_PRIORITY on the assumption that the fact we are direct
2972 * reclaiming implies that kswapd is not keeping up and it is best to
2973 * do a batch of work at once. For memcg reclaim one check is made to
2974 * abort proportional reclaim if either the file or anon lru has already
2975 * dropped to zero at the first pass.
2977 scan_adjusted
= (!cgroup_reclaim(sc
) && !current_is_kswapd() &&
2978 sc
->priority
== DEF_PRIORITY
);
2980 blk_start_plug(&plug
);
2981 while (nr
[LRU_INACTIVE_ANON
] || nr
[LRU_ACTIVE_FILE
] ||
2982 nr
[LRU_INACTIVE_FILE
]) {
2983 unsigned long nr_anon
, nr_file
, percentage
;
2984 unsigned long nr_scanned
;
2986 for_each_evictable_lru(lru
) {
2988 nr_to_scan
= min(nr
[lru
], SWAP_CLUSTER_MAX
);
2989 nr
[lru
] -= nr_to_scan
;
2991 nr_reclaimed
+= shrink_list(lru
, nr_to_scan
,
2998 if (nr_reclaimed
< nr_to_reclaim
|| scan_adjusted
)
3002 * For kswapd and memcg, reclaim at least the number of pages
3003 * requested. Ensure that the anon and file LRUs are scanned
3004 * proportionally what was requested by get_scan_count(). We
3005 * stop reclaiming one LRU and reduce the amount scanning
3006 * proportional to the original scan target.
3008 nr_file
= nr
[LRU_INACTIVE_FILE
] + nr
[LRU_ACTIVE_FILE
];
3009 nr_anon
= nr
[LRU_INACTIVE_ANON
] + nr
[LRU_ACTIVE_ANON
];
3012 * It's just vindictive to attack the larger once the smaller
3013 * has gone to zero. And given the way we stop scanning the
3014 * smaller below, this makes sure that we only make one nudge
3015 * towards proportionality once we've got nr_to_reclaim.
3017 if (!nr_file
|| !nr_anon
)
3020 if (nr_file
> nr_anon
) {
3021 unsigned long scan_target
= targets
[LRU_INACTIVE_ANON
] +
3022 targets
[LRU_ACTIVE_ANON
] + 1;
3024 percentage
= nr_anon
* 100 / scan_target
;
3026 unsigned long scan_target
= targets
[LRU_INACTIVE_FILE
] +
3027 targets
[LRU_ACTIVE_FILE
] + 1;
3029 percentage
= nr_file
* 100 / scan_target
;
3032 /* Stop scanning the smaller of the LRU */
3034 nr
[lru
+ LRU_ACTIVE
] = 0;
3037 * Recalculate the other LRU scan count based on its original
3038 * scan target and the percentage scanning already complete
3040 lru
= (lru
== LRU_FILE
) ? LRU_BASE
: LRU_FILE
;
3041 nr_scanned
= targets
[lru
] - nr
[lru
];
3042 nr
[lru
] = targets
[lru
] * (100 - percentage
) / 100;
3043 nr
[lru
] -= min(nr
[lru
], nr_scanned
);
3046 nr_scanned
= targets
[lru
] - nr
[lru
];
3047 nr
[lru
] = targets
[lru
] * (100 - percentage
) / 100;
3048 nr
[lru
] -= min(nr
[lru
], nr_scanned
);
3050 scan_adjusted
= true;
3052 blk_finish_plug(&plug
);
3053 sc
->nr_reclaimed
+= nr_reclaimed
;
3056 * Even if we did not try to evict anon pages at all, we want to
3057 * rebalance the anon lru active/inactive ratio.
3059 if (can_age_anon_pages(lruvec_pgdat(lruvec
), sc
) &&
3060 inactive_is_low(lruvec
, LRU_INACTIVE_ANON
))
3061 shrink_active_list(SWAP_CLUSTER_MAX
, lruvec
,
3062 sc
, LRU_ACTIVE_ANON
);
3065 /* Use reclaim/compaction for costly allocs or under memory pressure */
3066 static bool in_reclaim_compaction(struct scan_control
*sc
)
3068 if (IS_ENABLED(CONFIG_COMPACTION
) && sc
->order
&&
3069 (sc
->order
> PAGE_ALLOC_COSTLY_ORDER
||
3070 sc
->priority
< DEF_PRIORITY
- 2))
3077 * Reclaim/compaction is used for high-order allocation requests. It reclaims
3078 * order-0 pages before compacting the zone. should_continue_reclaim() returns
3079 * true if more pages should be reclaimed such that when the page allocator
3080 * calls try_to_compact_pages() that it will have enough free pages to succeed.
3081 * It will give up earlier than that if there is difficulty reclaiming pages.
3083 static inline bool should_continue_reclaim(struct pglist_data
*pgdat
,
3084 unsigned long nr_reclaimed
,
3085 struct scan_control
*sc
)
3087 unsigned long pages_for_compaction
;
3088 unsigned long inactive_lru_pages
;
3091 /* If not in reclaim/compaction mode, stop */
3092 if (!in_reclaim_compaction(sc
))
3096 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
3097 * number of pages that were scanned. This will return to the caller
3098 * with the risk reclaim/compaction and the resulting allocation attempt
3099 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
3100 * allocations through requiring that the full LRU list has been scanned
3101 * first, by assuming that zero delta of sc->nr_scanned means full LRU
3102 * scan, but that approximation was wrong, and there were corner cases
3103 * where always a non-zero amount of pages were scanned.
3108 /* If compaction would go ahead or the allocation would succeed, stop */
3109 for (z
= 0; z
<= sc
->reclaim_idx
; z
++) {
3110 struct zone
*zone
= &pgdat
->node_zones
[z
];
3111 if (!managed_zone(zone
))
3114 switch (compaction_suitable(zone
, sc
->order
, 0, sc
->reclaim_idx
)) {
3115 case COMPACT_SUCCESS
:
3116 case COMPACT_CONTINUE
:
3119 /* check next zone */
3125 * If we have not reclaimed enough pages for compaction and the
3126 * inactive lists are large enough, continue reclaiming
3128 pages_for_compaction
= compact_gap(sc
->order
);
3129 inactive_lru_pages
= node_page_state(pgdat
, NR_INACTIVE_FILE
);
3130 if (can_reclaim_anon_pages(NULL
, pgdat
->node_id
, sc
))
3131 inactive_lru_pages
+= node_page_state(pgdat
, NR_INACTIVE_ANON
);
3133 return inactive_lru_pages
> pages_for_compaction
;
3136 static void shrink_node_memcgs(pg_data_t
*pgdat
, struct scan_control
*sc
)
3138 struct mem_cgroup
*target_memcg
= sc
->target_mem_cgroup
;
3139 struct mem_cgroup
*memcg
;
3141 memcg
= mem_cgroup_iter(target_memcg
, NULL
, NULL
);
3143 struct lruvec
*lruvec
= mem_cgroup_lruvec(memcg
, pgdat
);
3144 unsigned long reclaimed
;
3145 unsigned long scanned
;
3148 * This loop can become CPU-bound when target memcgs
3149 * aren't eligible for reclaim - either because they
3150 * don't have any reclaimable pages, or because their
3151 * memory is explicitly protected. Avoid soft lockups.
3155 mem_cgroup_calculate_protection(target_memcg
, memcg
);
3157 if (mem_cgroup_below_min(memcg
)) {
3160 * If there is no reclaimable memory, OOM.
3163 } else if (mem_cgroup_below_low(memcg
)) {
3166 * Respect the protection only as long as
3167 * there is an unprotected supply
3168 * of reclaimable memory from other cgroups.
3170 if (!sc
->memcg_low_reclaim
) {
3171 sc
->memcg_low_skipped
= 1;
3174 memcg_memory_event(memcg
, MEMCG_LOW
);
3177 reclaimed
= sc
->nr_reclaimed
;
3178 scanned
= sc
->nr_scanned
;
3180 shrink_lruvec(lruvec
, sc
);
3182 shrink_slab(sc
->gfp_mask
, pgdat
->node_id
, memcg
,
3185 /* Record the group's reclaim efficiency */
3187 vmpressure(sc
->gfp_mask
, memcg
, false,
3188 sc
->nr_scanned
- scanned
,
3189 sc
->nr_reclaimed
- reclaimed
);
3191 } while ((memcg
= mem_cgroup_iter(target_memcg
, memcg
, NULL
)));
3194 static void shrink_node(pg_data_t
*pgdat
, struct scan_control
*sc
)
3196 struct reclaim_state
*reclaim_state
= current
->reclaim_state
;
3197 unsigned long nr_reclaimed
, nr_scanned
;
3198 struct lruvec
*target_lruvec
;
3199 bool reclaimable
= false;
3202 target_lruvec
= mem_cgroup_lruvec(sc
->target_mem_cgroup
, pgdat
);
3206 * Flush the memory cgroup stats, so that we read accurate per-memcg
3207 * lruvec stats for heuristics.
3209 mem_cgroup_flush_stats();
3211 memset(&sc
->nr
, 0, sizeof(sc
->nr
));
3213 nr_reclaimed
= sc
->nr_reclaimed
;
3214 nr_scanned
= sc
->nr_scanned
;
3217 * Determine the scan balance between anon and file LRUs.
3219 spin_lock_irq(&target_lruvec
->lru_lock
);
3220 sc
->anon_cost
= target_lruvec
->anon_cost
;
3221 sc
->file_cost
= target_lruvec
->file_cost
;
3222 spin_unlock_irq(&target_lruvec
->lru_lock
);
3225 * Target desirable inactive:active list ratios for the anon
3226 * and file LRU lists.
3228 if (!sc
->force_deactivate
) {
3229 unsigned long refaults
;
3231 refaults
= lruvec_page_state(target_lruvec
,
3232 WORKINGSET_ACTIVATE_ANON
);
3233 if (refaults
!= target_lruvec
->refaults
[0] ||
3234 inactive_is_low(target_lruvec
, LRU_INACTIVE_ANON
))
3235 sc
->may_deactivate
|= DEACTIVATE_ANON
;
3237 sc
->may_deactivate
&= ~DEACTIVATE_ANON
;
3240 * When refaults are being observed, it means a new
3241 * workingset is being established. Deactivate to get
3242 * rid of any stale active pages quickly.
3244 refaults
= lruvec_page_state(target_lruvec
,
3245 WORKINGSET_ACTIVATE_FILE
);
3246 if (refaults
!= target_lruvec
->refaults
[1] ||
3247 inactive_is_low(target_lruvec
, LRU_INACTIVE_FILE
))
3248 sc
->may_deactivate
|= DEACTIVATE_FILE
;
3250 sc
->may_deactivate
&= ~DEACTIVATE_FILE
;
3252 sc
->may_deactivate
= DEACTIVATE_ANON
| DEACTIVATE_FILE
;
3255 * If we have plenty of inactive file pages that aren't
3256 * thrashing, try to reclaim those first before touching
3259 file
= lruvec_page_state(target_lruvec
, NR_INACTIVE_FILE
);
3260 if (file
>> sc
->priority
&& !(sc
->may_deactivate
& DEACTIVATE_FILE
))
3261 sc
->cache_trim_mode
= 1;
3263 sc
->cache_trim_mode
= 0;
3266 * Prevent the reclaimer from falling into the cache trap: as
3267 * cache pages start out inactive, every cache fault will tip
3268 * the scan balance towards the file LRU. And as the file LRU
3269 * shrinks, so does the window for rotation from references.
3270 * This means we have a runaway feedback loop where a tiny
3271 * thrashing file LRU becomes infinitely more attractive than
3272 * anon pages. Try to detect this based on file LRU size.
3274 if (!cgroup_reclaim(sc
)) {
3275 unsigned long total_high_wmark
= 0;
3276 unsigned long free
, anon
;
3279 free
= sum_zone_node_page_state(pgdat
->node_id
, NR_FREE_PAGES
);
3280 file
= node_page_state(pgdat
, NR_ACTIVE_FILE
) +
3281 node_page_state(pgdat
, NR_INACTIVE_FILE
);
3283 for (z
= 0; z
< MAX_NR_ZONES
; z
++) {
3284 struct zone
*zone
= &pgdat
->node_zones
[z
];
3285 if (!managed_zone(zone
))
3288 total_high_wmark
+= high_wmark_pages(zone
);
3292 * Consider anon: if that's low too, this isn't a
3293 * runaway file reclaim problem, but rather just
3294 * extreme pressure. Reclaim as per usual then.
3296 anon
= node_page_state(pgdat
, NR_INACTIVE_ANON
);
3299 file
+ free
<= total_high_wmark
&&
3300 !(sc
->may_deactivate
& DEACTIVATE_ANON
) &&
3301 anon
>> sc
->priority
;
3304 shrink_node_memcgs(pgdat
, sc
);
3306 if (reclaim_state
) {
3307 sc
->nr_reclaimed
+= reclaim_state
->reclaimed_slab
;
3308 reclaim_state
->reclaimed_slab
= 0;
3311 /* Record the subtree's reclaim efficiency */
3313 vmpressure(sc
->gfp_mask
, sc
->target_mem_cgroup
, true,
3314 sc
->nr_scanned
- nr_scanned
,
3315 sc
->nr_reclaimed
- nr_reclaimed
);
3317 if (sc
->nr_reclaimed
- nr_reclaimed
)
3320 if (current_is_kswapd()) {
3322 * If reclaim is isolating dirty pages under writeback,
3323 * it implies that the long-lived page allocation rate
3324 * is exceeding the page laundering rate. Either the
3325 * global limits are not being effective at throttling
3326 * processes due to the page distribution throughout
3327 * zones or there is heavy usage of a slow backing
3328 * device. The only option is to throttle from reclaim
3329 * context which is not ideal as there is no guarantee
3330 * the dirtying process is throttled in the same way
3331 * balance_dirty_pages() manages.
3333 * Once a node is flagged PGDAT_WRITEBACK, kswapd will
3334 * count the number of pages under pages flagged for
3335 * immediate reclaim and stall if any are encountered
3336 * in the nr_immediate check below.
3338 if (sc
->nr
.writeback
&& sc
->nr
.writeback
== sc
->nr
.taken
)
3339 set_bit(PGDAT_WRITEBACK
, &pgdat
->flags
);
3341 /* Allow kswapd to start writing pages during reclaim.*/
3342 if (sc
->nr
.unqueued_dirty
== sc
->nr
.file_taken
)
3343 set_bit(PGDAT_DIRTY
, &pgdat
->flags
);
3346 * If kswapd scans pages marked for immediate
3347 * reclaim and under writeback (nr_immediate), it
3348 * implies that pages are cycling through the LRU
3349 * faster than they are written so forcibly stall
3350 * until some pages complete writeback.
3352 if (sc
->nr
.immediate
)
3353 reclaim_throttle(pgdat
, VMSCAN_THROTTLE_WRITEBACK
);
3357 * Tag a node/memcg as congested if all the dirty pages were marked
3358 * for writeback and immediate reclaim (counted in nr.congested).
3360 * Legacy memcg will stall in page writeback so avoid forcibly
3361 * stalling in reclaim_throttle().
3363 if ((current_is_kswapd() ||
3364 (cgroup_reclaim(sc
) && writeback_throttling_sane(sc
))) &&
3365 sc
->nr
.dirty
&& sc
->nr
.dirty
== sc
->nr
.congested
)
3366 set_bit(LRUVEC_CONGESTED
, &target_lruvec
->flags
);
3369 * Stall direct reclaim for IO completions if the lruvec is
3370 * node is congested. Allow kswapd to continue until it
3371 * starts encountering unqueued dirty pages or cycling through
3372 * the LRU too quickly.
3374 if (!current_is_kswapd() && current_may_throttle() &&
3375 !sc
->hibernation_mode
&&
3376 test_bit(LRUVEC_CONGESTED
, &target_lruvec
->flags
))
3377 reclaim_throttle(pgdat
, VMSCAN_THROTTLE_CONGESTED
);
3379 if (should_continue_reclaim(pgdat
, sc
->nr_reclaimed
- nr_reclaimed
,
3384 * Kswapd gives up on balancing particular nodes after too
3385 * many failures to reclaim anything from them and goes to
3386 * sleep. On reclaim progress, reset the failure counter. A
3387 * successful direct reclaim run will revive a dormant kswapd.
3390 pgdat
->kswapd_failures
= 0;
3394 * Returns true if compaction should go ahead for a costly-order request, or
3395 * the allocation would already succeed without compaction. Return false if we
3396 * should reclaim first.
3398 static inline bool compaction_ready(struct zone
*zone
, struct scan_control
*sc
)
3400 unsigned long watermark
;
3401 enum compact_result suitable
;
3403 suitable
= compaction_suitable(zone
, sc
->order
, 0, sc
->reclaim_idx
);
3404 if (suitable
== COMPACT_SUCCESS
)
3405 /* Allocation should succeed already. Don't reclaim. */
3407 if (suitable
== COMPACT_SKIPPED
)
3408 /* Compaction cannot yet proceed. Do reclaim. */
3412 * Compaction is already possible, but it takes time to run and there
3413 * are potentially other callers using the pages just freed. So proceed
3414 * with reclaim to make a buffer of free pages available to give
3415 * compaction a reasonable chance of completing and allocating the page.
3416 * Note that we won't actually reclaim the whole buffer in one attempt
3417 * as the target watermark in should_continue_reclaim() is lower. But if
3418 * we are already above the high+gap watermark, don't reclaim at all.
3420 watermark
= high_wmark_pages(zone
) + compact_gap(sc
->order
);
3422 return zone_watermark_ok_safe(zone
, 0, watermark
, sc
->reclaim_idx
);
3425 static void consider_reclaim_throttle(pg_data_t
*pgdat
, struct scan_control
*sc
)
3428 * If reclaim is making progress greater than 12% efficiency then
3429 * wake all the NOPROGRESS throttled tasks.
3431 if (sc
->nr_reclaimed
> (sc
->nr_scanned
>> 3)) {
3432 wait_queue_head_t
*wqh
;
3434 wqh
= &pgdat
->reclaim_wait
[VMSCAN_THROTTLE_NOPROGRESS
];
3435 if (waitqueue_active(wqh
))
3442 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will
3443 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages
3444 * under writeback and marked for immediate reclaim at the tail of the
3447 if (current_is_kswapd() || cgroup_reclaim(sc
))
3450 /* Throttle if making no progress at high prioities. */
3451 if (sc
->priority
== 1 && !sc
->nr_reclaimed
)
3452 reclaim_throttle(pgdat
, VMSCAN_THROTTLE_NOPROGRESS
);
3456 * This is the direct reclaim path, for page-allocating processes. We only
3457 * try to reclaim pages from zones which will satisfy the caller's allocation
3460 * If a zone is deemed to be full of pinned pages then just give it a light
3461 * scan then give up on it.
3463 static void shrink_zones(struct zonelist
*zonelist
, struct scan_control
*sc
)
3467 unsigned long nr_soft_reclaimed
;
3468 unsigned long nr_soft_scanned
;
3470 pg_data_t
*last_pgdat
= NULL
;
3471 pg_data_t
*first_pgdat
= NULL
;
3474 * If the number of buffer_heads in the machine exceeds the maximum
3475 * allowed level, force direct reclaim to scan the highmem zone as
3476 * highmem pages could be pinning lowmem pages storing buffer_heads
3478 orig_mask
= sc
->gfp_mask
;
3479 if (buffer_heads_over_limit
) {
3480 sc
->gfp_mask
|= __GFP_HIGHMEM
;
3481 sc
->reclaim_idx
= gfp_zone(sc
->gfp_mask
);
3484 for_each_zone_zonelist_nodemask(zone
, z
, zonelist
,
3485 sc
->reclaim_idx
, sc
->nodemask
) {
3487 * Take care memory controller reclaiming has small influence
3490 if (!cgroup_reclaim(sc
)) {
3491 if (!cpuset_zone_allowed(zone
,
3492 GFP_KERNEL
| __GFP_HARDWALL
))
3496 * If we already have plenty of memory free for
3497 * compaction in this zone, don't free any more.
3498 * Even though compaction is invoked for any
3499 * non-zero order, only frequent costly order
3500 * reclamation is disruptive enough to become a
3501 * noticeable problem, like transparent huge
3504 if (IS_ENABLED(CONFIG_COMPACTION
) &&
3505 sc
->order
> PAGE_ALLOC_COSTLY_ORDER
&&
3506 compaction_ready(zone
, sc
)) {
3507 sc
->compaction_ready
= true;
3512 * Shrink each node in the zonelist once. If the
3513 * zonelist is ordered by zone (not the default) then a
3514 * node may be shrunk multiple times but in that case
3515 * the user prefers lower zones being preserved.
3517 if (zone
->zone_pgdat
== last_pgdat
)
3521 * This steals pages from memory cgroups over softlimit
3522 * and returns the number of reclaimed pages and
3523 * scanned pages. This works for global memory pressure
3524 * and balancing, not for a memcg's limit.
3526 nr_soft_scanned
= 0;
3527 nr_soft_reclaimed
= mem_cgroup_soft_limit_reclaim(zone
->zone_pgdat
,
3528 sc
->order
, sc
->gfp_mask
,
3530 sc
->nr_reclaimed
+= nr_soft_reclaimed
;
3531 sc
->nr_scanned
+= nr_soft_scanned
;
3532 /* need some check for avoid more shrink_zone() */
3536 first_pgdat
= zone
->zone_pgdat
;
3538 /* See comment about same check for global reclaim above */
3539 if (zone
->zone_pgdat
== last_pgdat
)
3541 last_pgdat
= zone
->zone_pgdat
;
3542 shrink_node(zone
->zone_pgdat
, sc
);
3546 consider_reclaim_throttle(first_pgdat
, sc
);
3549 * Restore to original mask to avoid the impact on the caller if we
3550 * promoted it to __GFP_HIGHMEM.
3552 sc
->gfp_mask
= orig_mask
;
3555 static void snapshot_refaults(struct mem_cgroup
*target_memcg
, pg_data_t
*pgdat
)
3557 struct lruvec
*target_lruvec
;
3558 unsigned long refaults
;
3560 target_lruvec
= mem_cgroup_lruvec(target_memcg
, pgdat
);
3561 refaults
= lruvec_page_state(target_lruvec
, WORKINGSET_ACTIVATE_ANON
);
3562 target_lruvec
->refaults
[0] = refaults
;
3563 refaults
= lruvec_page_state(target_lruvec
, WORKINGSET_ACTIVATE_FILE
);
3564 target_lruvec
->refaults
[1] = refaults
;
3568 * This is the main entry point to direct page reclaim.
3570 * If a full scan of the inactive list fails to free enough memory then we
3571 * are "out of memory" and something needs to be killed.
3573 * If the caller is !__GFP_FS then the probability of a failure is reasonably
3574 * high - the zone may be full of dirty or under-writeback pages, which this
3575 * caller can't do much about. We kick the writeback threads and take explicit
3576 * naps in the hope that some of these pages can be written. But if the
3577 * allocating task holds filesystem locks which prevent writeout this might not
3578 * work, and the allocation attempt will fail.
3580 * returns: 0, if no pages reclaimed
3581 * else, the number of pages reclaimed
3583 static unsigned long do_try_to_free_pages(struct zonelist
*zonelist
,
3584 struct scan_control
*sc
)
3586 int initial_priority
= sc
->priority
;
3587 pg_data_t
*last_pgdat
;
3591 delayacct_freepages_start();
3593 if (!cgroup_reclaim(sc
))
3594 __count_zid_vm_events(ALLOCSTALL
, sc
->reclaim_idx
, 1);
3598 vmpressure_prio(sc
->gfp_mask
, sc
->target_mem_cgroup
,
3601 shrink_zones(zonelist
, sc
);
3603 if (sc
->nr_reclaimed
>= sc
->nr_to_reclaim
)
3606 if (sc
->compaction_ready
)
3610 * If we're getting trouble reclaiming, start doing
3611 * writepage even in laptop mode.
3613 if (sc
->priority
< DEF_PRIORITY
- 2)
3614 sc
->may_writepage
= 1;
3615 } while (--sc
->priority
>= 0);
3618 for_each_zone_zonelist_nodemask(zone
, z
, zonelist
, sc
->reclaim_idx
,
3620 if (zone
->zone_pgdat
== last_pgdat
)
3622 last_pgdat
= zone
->zone_pgdat
;
3624 snapshot_refaults(sc
->target_mem_cgroup
, zone
->zone_pgdat
);
3626 if (cgroup_reclaim(sc
)) {
3627 struct lruvec
*lruvec
;
3629 lruvec
= mem_cgroup_lruvec(sc
->target_mem_cgroup
,
3631 clear_bit(LRUVEC_CONGESTED
, &lruvec
->flags
);
3635 delayacct_freepages_end();
3637 if (sc
->nr_reclaimed
)
3638 return sc
->nr_reclaimed
;
3640 /* Aborted reclaim to try compaction? don't OOM, then */
3641 if (sc
->compaction_ready
)
3645 * We make inactive:active ratio decisions based on the node's
3646 * composition of memory, but a restrictive reclaim_idx or a
3647 * memory.low cgroup setting can exempt large amounts of
3648 * memory from reclaim. Neither of which are very common, so
3649 * instead of doing costly eligibility calculations of the
3650 * entire cgroup subtree up front, we assume the estimates are
3651 * good, and retry with forcible deactivation if that fails.
3653 if (sc
->skipped_deactivate
) {
3654 sc
->priority
= initial_priority
;
3655 sc
->force_deactivate
= 1;
3656 sc
->skipped_deactivate
= 0;
3660 /* Untapped cgroup reserves? Don't OOM, retry. */
3661 if (sc
->memcg_low_skipped
) {
3662 sc
->priority
= initial_priority
;
3663 sc
->force_deactivate
= 0;
3664 sc
->memcg_low_reclaim
= 1;
3665 sc
->memcg_low_skipped
= 0;
3672 static bool allow_direct_reclaim(pg_data_t
*pgdat
)
3675 unsigned long pfmemalloc_reserve
= 0;
3676 unsigned long free_pages
= 0;
3680 if (pgdat
->kswapd_failures
>= MAX_RECLAIM_RETRIES
)
3683 for (i
= 0; i
<= ZONE_NORMAL
; i
++) {
3684 zone
= &pgdat
->node_zones
[i
];
3685 if (!managed_zone(zone
))
3688 if (!zone_reclaimable_pages(zone
))
3691 pfmemalloc_reserve
+= min_wmark_pages(zone
);
3692 free_pages
+= zone_page_state(zone
, NR_FREE_PAGES
);
3695 /* If there are no reserves (unexpected config) then do not throttle */
3696 if (!pfmemalloc_reserve
)
3699 wmark_ok
= free_pages
> pfmemalloc_reserve
/ 2;
3701 /* kswapd must be awake if processes are being throttled */
3702 if (!wmark_ok
&& waitqueue_active(&pgdat
->kswapd_wait
)) {
3703 if (READ_ONCE(pgdat
->kswapd_highest_zoneidx
) > ZONE_NORMAL
)
3704 WRITE_ONCE(pgdat
->kswapd_highest_zoneidx
, ZONE_NORMAL
);
3706 wake_up_interruptible(&pgdat
->kswapd_wait
);
3713 * Throttle direct reclaimers if backing storage is backed by the network
3714 * and the PFMEMALLOC reserve for the preferred node is getting dangerously
3715 * depleted. kswapd will continue to make progress and wake the processes
3716 * when the low watermark is reached.
3718 * Returns true if a fatal signal was delivered during throttling. If this
3719 * happens, the page allocator should not consider triggering the OOM killer.
3721 static bool throttle_direct_reclaim(gfp_t gfp_mask
, struct zonelist
*zonelist
,
3722 nodemask_t
*nodemask
)
3726 pg_data_t
*pgdat
= NULL
;
3729 * Kernel threads should not be throttled as they may be indirectly
3730 * responsible for cleaning pages necessary for reclaim to make forward
3731 * progress. kjournald for example may enter direct reclaim while
3732 * committing a transaction where throttling it could forcing other
3733 * processes to block on log_wait_commit().
3735 if (current
->flags
& PF_KTHREAD
)
3739 * If a fatal signal is pending, this process should not throttle.
3740 * It should return quickly so it can exit and free its memory
3742 if (fatal_signal_pending(current
))
3746 * Check if the pfmemalloc reserves are ok by finding the first node
3747 * with a usable ZONE_NORMAL or lower zone. The expectation is that
3748 * GFP_KERNEL will be required for allocating network buffers when
3749 * swapping over the network so ZONE_HIGHMEM is unusable.
3751 * Throttling is based on the first usable node and throttled processes
3752 * wait on a queue until kswapd makes progress and wakes them. There
3753 * is an affinity then between processes waking up and where reclaim
3754 * progress has been made assuming the process wakes on the same node.
3755 * More importantly, processes running on remote nodes will not compete
3756 * for remote pfmemalloc reserves and processes on different nodes
3757 * should make reasonable progress.
3759 for_each_zone_zonelist_nodemask(zone
, z
, zonelist
,
3760 gfp_zone(gfp_mask
), nodemask
) {
3761 if (zone_idx(zone
) > ZONE_NORMAL
)
3764 /* Throttle based on the first usable node */
3765 pgdat
= zone
->zone_pgdat
;
3766 if (allow_direct_reclaim(pgdat
))
3771 /* If no zone was usable by the allocation flags then do not throttle */
3775 /* Account for the throttling */
3776 count_vm_event(PGSCAN_DIRECT_THROTTLE
);
3779 * If the caller cannot enter the filesystem, it's possible that it
3780 * is due to the caller holding an FS lock or performing a journal
3781 * transaction in the case of a filesystem like ext[3|4]. In this case,
3782 * it is not safe to block on pfmemalloc_wait as kswapd could be
3783 * blocked waiting on the same lock. Instead, throttle for up to a
3784 * second before continuing.
3786 if (!(gfp_mask
& __GFP_FS
))
3787 wait_event_interruptible_timeout(pgdat
->pfmemalloc_wait
,
3788 allow_direct_reclaim(pgdat
), HZ
);
3790 /* Throttle until kswapd wakes the process */
3791 wait_event_killable(zone
->zone_pgdat
->pfmemalloc_wait
,
3792 allow_direct_reclaim(pgdat
));
3794 if (fatal_signal_pending(current
))
3801 unsigned long try_to_free_pages(struct zonelist
*zonelist
, int order
,
3802 gfp_t gfp_mask
, nodemask_t
*nodemask
)
3804 unsigned long nr_reclaimed
;
3805 struct scan_control sc
= {
3806 .nr_to_reclaim
= SWAP_CLUSTER_MAX
,
3807 .gfp_mask
= current_gfp_context(gfp_mask
),
3808 .reclaim_idx
= gfp_zone(gfp_mask
),
3810 .nodemask
= nodemask
,
3811 .priority
= DEF_PRIORITY
,
3812 .may_writepage
= !laptop_mode
,
3818 * scan_control uses s8 fields for order, priority, and reclaim_idx.
3819 * Confirm they are large enough for max values.
3821 BUILD_BUG_ON(MAX_ORDER
> S8_MAX
);
3822 BUILD_BUG_ON(DEF_PRIORITY
> S8_MAX
);
3823 BUILD_BUG_ON(MAX_NR_ZONES
> S8_MAX
);
3826 * Do not enter reclaim if fatal signal was delivered while throttled.
3827 * 1 is returned so that the page allocator does not OOM kill at this
3830 if (throttle_direct_reclaim(sc
.gfp_mask
, zonelist
, nodemask
))
3833 set_task_reclaim_state(current
, &sc
.reclaim_state
);
3834 trace_mm_vmscan_direct_reclaim_begin(order
, sc
.gfp_mask
);
3836 nr_reclaimed
= do_try_to_free_pages(zonelist
, &sc
);
3838 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed
);
3839 set_task_reclaim_state(current
, NULL
);
3841 return nr_reclaimed
;
3846 /* Only used by soft limit reclaim. Do not reuse for anything else. */
3847 unsigned long mem_cgroup_shrink_node(struct mem_cgroup
*memcg
,
3848 gfp_t gfp_mask
, bool noswap
,
3850 unsigned long *nr_scanned
)
3852 struct lruvec
*lruvec
= mem_cgroup_lruvec(memcg
, pgdat
);
3853 struct scan_control sc
= {
3854 .nr_to_reclaim
= SWAP_CLUSTER_MAX
,
3855 .target_mem_cgroup
= memcg
,
3856 .may_writepage
= !laptop_mode
,
3858 .reclaim_idx
= MAX_NR_ZONES
- 1,
3859 .may_swap
= !noswap
,
3862 WARN_ON_ONCE(!current
->reclaim_state
);
3864 sc
.gfp_mask
= (gfp_mask
& GFP_RECLAIM_MASK
) |
3865 (GFP_HIGHUSER_MOVABLE
& ~GFP_RECLAIM_MASK
);
3867 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc
.order
,
3871 * NOTE: Although we can get the priority field, using it
3872 * here is not a good idea, since it limits the pages we can scan.
3873 * if we don't reclaim here, the shrink_node from balance_pgdat
3874 * will pick up pages from other mem cgroup's as well. We hack
3875 * the priority and make it zero.
3877 shrink_lruvec(lruvec
, &sc
);
3879 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc
.nr_reclaimed
);
3881 *nr_scanned
= sc
.nr_scanned
;
3883 return sc
.nr_reclaimed
;
3886 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup
*memcg
,
3887 unsigned long nr_pages
,
3889 unsigned int reclaim_options
)
3891 unsigned long nr_reclaimed
;
3892 unsigned int noreclaim_flag
;
3893 struct scan_control sc
= {
3894 .nr_to_reclaim
= max(nr_pages
, SWAP_CLUSTER_MAX
),
3895 .gfp_mask
= (current_gfp_context(gfp_mask
) & GFP_RECLAIM_MASK
) |
3896 (GFP_HIGHUSER_MOVABLE
& ~GFP_RECLAIM_MASK
),
3897 .reclaim_idx
= MAX_NR_ZONES
- 1,
3898 .target_mem_cgroup
= memcg
,
3899 .priority
= DEF_PRIORITY
,
3900 .may_writepage
= !laptop_mode
,
3902 .may_swap
= !!(reclaim_options
& MEMCG_RECLAIM_MAY_SWAP
),
3903 .proactive
= !!(reclaim_options
& MEMCG_RECLAIM_PROACTIVE
),
3906 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
3907 * equal pressure on all the nodes. This is based on the assumption that
3908 * the reclaim does not bail out early.
3910 struct zonelist
*zonelist
= node_zonelist(numa_node_id(), sc
.gfp_mask
);
3912 set_task_reclaim_state(current
, &sc
.reclaim_state
);
3913 trace_mm_vmscan_memcg_reclaim_begin(0, sc
.gfp_mask
);
3914 noreclaim_flag
= memalloc_noreclaim_save();
3916 nr_reclaimed
= do_try_to_free_pages(zonelist
, &sc
);
3918 memalloc_noreclaim_restore(noreclaim_flag
);
3919 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed
);
3920 set_task_reclaim_state(current
, NULL
);
3922 return nr_reclaimed
;
3926 static void age_active_anon(struct pglist_data
*pgdat
,
3927 struct scan_control
*sc
)
3929 struct mem_cgroup
*memcg
;
3930 struct lruvec
*lruvec
;
3932 if (!can_age_anon_pages(pgdat
, sc
))
3935 lruvec
= mem_cgroup_lruvec(NULL
, pgdat
);
3936 if (!inactive_is_low(lruvec
, LRU_INACTIVE_ANON
))
3939 memcg
= mem_cgroup_iter(NULL
, NULL
, NULL
);
3941 lruvec
= mem_cgroup_lruvec(memcg
, pgdat
);
3942 shrink_active_list(SWAP_CLUSTER_MAX
, lruvec
,
3943 sc
, LRU_ACTIVE_ANON
);
3944 memcg
= mem_cgroup_iter(NULL
, memcg
, NULL
);
3948 static bool pgdat_watermark_boosted(pg_data_t
*pgdat
, int highest_zoneidx
)
3954 * Check for watermark boosts top-down as the higher zones
3955 * are more likely to be boosted. Both watermarks and boosts
3956 * should not be checked at the same time as reclaim would
3957 * start prematurely when there is no boosting and a lower
3960 for (i
= highest_zoneidx
; i
>= 0; i
--) {
3961 zone
= pgdat
->node_zones
+ i
;
3962 if (!managed_zone(zone
))
3965 if (zone
->watermark_boost
)
3973 * Returns true if there is an eligible zone balanced for the request order
3974 * and highest_zoneidx
3976 static bool pgdat_balanced(pg_data_t
*pgdat
, int order
, int highest_zoneidx
)
3979 unsigned long mark
= -1;
3983 * Check watermarks bottom-up as lower zones are more likely to
3986 for (i
= 0; i
<= highest_zoneidx
; i
++) {
3987 zone
= pgdat
->node_zones
+ i
;
3989 if (!managed_zone(zone
))
3992 if (sysctl_numa_balancing_mode
& NUMA_BALANCING_MEMORY_TIERING
)
3993 mark
= wmark_pages(zone
, WMARK_PROMO
);
3995 mark
= high_wmark_pages(zone
);
3996 if (zone_watermark_ok_safe(zone
, order
, mark
, highest_zoneidx
))
4001 * If a node has no managed zone within highest_zoneidx, it does not
4002 * need balancing by definition. This can happen if a zone-restricted
4003 * allocation tries to wake a remote kswapd.
4011 /* Clear pgdat state for congested, dirty or under writeback. */
4012 static void clear_pgdat_congested(pg_data_t
*pgdat
)
4014 struct lruvec
*lruvec
= mem_cgroup_lruvec(NULL
, pgdat
);
4016 clear_bit(LRUVEC_CONGESTED
, &lruvec
->flags
);
4017 clear_bit(PGDAT_DIRTY
, &pgdat
->flags
);
4018 clear_bit(PGDAT_WRITEBACK
, &pgdat
->flags
);
4022 * Prepare kswapd for sleeping. This verifies that there are no processes
4023 * waiting in throttle_direct_reclaim() and that watermarks have been met.
4025 * Returns true if kswapd is ready to sleep
4027 static bool prepare_kswapd_sleep(pg_data_t
*pgdat
, int order
,
4028 int highest_zoneidx
)
4031 * The throttled processes are normally woken up in balance_pgdat() as
4032 * soon as allow_direct_reclaim() is true. But there is a potential
4033 * race between when kswapd checks the watermarks and a process gets
4034 * throttled. There is also a potential race if processes get
4035 * throttled, kswapd wakes, a large process exits thereby balancing the
4036 * zones, which causes kswapd to exit balance_pgdat() before reaching
4037 * the wake up checks. If kswapd is going to sleep, no process should
4038 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
4039 * the wake up is premature, processes will wake kswapd and get
4040 * throttled again. The difference from wake ups in balance_pgdat() is
4041 * that here we are under prepare_to_wait().
4043 if (waitqueue_active(&pgdat
->pfmemalloc_wait
))
4044 wake_up_all(&pgdat
->pfmemalloc_wait
);
4046 /* Hopeless node, leave it to direct reclaim */
4047 if (pgdat
->kswapd_failures
>= MAX_RECLAIM_RETRIES
)
4050 if (pgdat_balanced(pgdat
, order
, highest_zoneidx
)) {
4051 clear_pgdat_congested(pgdat
);
4059 * kswapd shrinks a node of pages that are at or below the highest usable
4060 * zone that is currently unbalanced.
4062 * Returns true if kswapd scanned at least the requested number of pages to
4063 * reclaim or if the lack of progress was due to pages under writeback.
4064 * This is used to determine if the scanning priority needs to be raised.
4066 static bool kswapd_shrink_node(pg_data_t
*pgdat
,
4067 struct scan_control
*sc
)
4072 /* Reclaim a number of pages proportional to the number of zones */
4073 sc
->nr_to_reclaim
= 0;
4074 for (z
= 0; z
<= sc
->reclaim_idx
; z
++) {
4075 zone
= pgdat
->node_zones
+ z
;
4076 if (!managed_zone(zone
))
4079 sc
->nr_to_reclaim
+= max(high_wmark_pages(zone
), SWAP_CLUSTER_MAX
);
4083 * Historically care was taken to put equal pressure on all zones but
4084 * now pressure is applied based on node LRU order.
4086 shrink_node(pgdat
, sc
);
4089 * Fragmentation may mean that the system cannot be rebalanced for
4090 * high-order allocations. If twice the allocation size has been
4091 * reclaimed then recheck watermarks only at order-0 to prevent
4092 * excessive reclaim. Assume that a process requested a high-order
4093 * can direct reclaim/compact.
4095 if (sc
->order
&& sc
->nr_reclaimed
>= compact_gap(sc
->order
))
4098 return sc
->nr_scanned
>= sc
->nr_to_reclaim
;
4101 /* Page allocator PCP high watermark is lowered if reclaim is active. */
4103 update_reclaim_active(pg_data_t
*pgdat
, int highest_zoneidx
, bool active
)
4108 for (i
= 0; i
<= highest_zoneidx
; i
++) {
4109 zone
= pgdat
->node_zones
+ i
;
4111 if (!managed_zone(zone
))
4115 set_bit(ZONE_RECLAIM_ACTIVE
, &zone
->flags
);
4117 clear_bit(ZONE_RECLAIM_ACTIVE
, &zone
->flags
);
4122 set_reclaim_active(pg_data_t
*pgdat
, int highest_zoneidx
)
4124 update_reclaim_active(pgdat
, highest_zoneidx
, true);
4128 clear_reclaim_active(pg_data_t
*pgdat
, int highest_zoneidx
)
4130 update_reclaim_active(pgdat
, highest_zoneidx
, false);
4134 * For kswapd, balance_pgdat() will reclaim pages across a node from zones
4135 * that are eligible for use by the caller until at least one zone is
4138 * Returns the order kswapd finished reclaiming at.
4140 * kswapd scans the zones in the highmem->normal->dma direction. It skips
4141 * zones which have free_pages > high_wmark_pages(zone), but once a zone is
4142 * found to have free_pages <= high_wmark_pages(zone), any page in that zone
4143 * or lower is eligible for reclaim until at least one usable zone is
4146 static int balance_pgdat(pg_data_t
*pgdat
, int order
, int highest_zoneidx
)
4149 unsigned long nr_soft_reclaimed
;
4150 unsigned long nr_soft_scanned
;
4151 unsigned long pflags
;
4152 unsigned long nr_boost_reclaim
;
4153 unsigned long zone_boosts
[MAX_NR_ZONES
] = { 0, };
4156 struct scan_control sc
= {
4157 .gfp_mask
= GFP_KERNEL
,
4162 set_task_reclaim_state(current
, &sc
.reclaim_state
);
4163 psi_memstall_enter(&pflags
);
4164 __fs_reclaim_acquire(_THIS_IP_
);
4166 count_vm_event(PAGEOUTRUN
);
4169 * Account for the reclaim boost. Note that the zone boost is left in
4170 * place so that parallel allocations that are near the watermark will
4171 * stall or direct reclaim until kswapd is finished.
4173 nr_boost_reclaim
= 0;
4174 for (i
= 0; i
<= highest_zoneidx
; i
++) {
4175 zone
= pgdat
->node_zones
+ i
;
4176 if (!managed_zone(zone
))
4179 nr_boost_reclaim
+= zone
->watermark_boost
;
4180 zone_boosts
[i
] = zone
->watermark_boost
;
4182 boosted
= nr_boost_reclaim
;
4185 set_reclaim_active(pgdat
, highest_zoneidx
);
4186 sc
.priority
= DEF_PRIORITY
;
4188 unsigned long nr_reclaimed
= sc
.nr_reclaimed
;
4189 bool raise_priority
= true;
4193 sc
.reclaim_idx
= highest_zoneidx
;
4196 * If the number of buffer_heads exceeds the maximum allowed
4197 * then consider reclaiming from all zones. This has a dual
4198 * purpose -- on 64-bit systems it is expected that
4199 * buffer_heads are stripped during active rotation. On 32-bit
4200 * systems, highmem pages can pin lowmem memory and shrinking
4201 * buffers can relieve lowmem pressure. Reclaim may still not
4202 * go ahead if all eligible zones for the original allocation
4203 * request are balanced to avoid excessive reclaim from kswapd.
4205 if (buffer_heads_over_limit
) {
4206 for (i
= MAX_NR_ZONES
- 1; i
>= 0; i
--) {
4207 zone
= pgdat
->node_zones
+ i
;
4208 if (!managed_zone(zone
))
4217 * If the pgdat is imbalanced then ignore boosting and preserve
4218 * the watermarks for a later time and restart. Note that the
4219 * zone watermarks will be still reset at the end of balancing
4220 * on the grounds that the normal reclaim should be enough to
4221 * re-evaluate if boosting is required when kswapd next wakes.
4223 balanced
= pgdat_balanced(pgdat
, sc
.order
, highest_zoneidx
);
4224 if (!balanced
&& nr_boost_reclaim
) {
4225 nr_boost_reclaim
= 0;
4230 * If boosting is not active then only reclaim if there are no
4231 * eligible zones. Note that sc.reclaim_idx is not used as
4232 * buffer_heads_over_limit may have adjusted it.
4234 if (!nr_boost_reclaim
&& balanced
)
4237 /* Limit the priority of boosting to avoid reclaim writeback */
4238 if (nr_boost_reclaim
&& sc
.priority
== DEF_PRIORITY
- 2)
4239 raise_priority
= false;
4242 * Do not writeback or swap pages for boosted reclaim. The
4243 * intent is to relieve pressure not issue sub-optimal IO
4244 * from reclaim context. If no pages are reclaimed, the
4245 * reclaim will be aborted.
4247 sc
.may_writepage
= !laptop_mode
&& !nr_boost_reclaim
;
4248 sc
.may_swap
= !nr_boost_reclaim
;
4251 * Do some background aging of the anon list, to give
4252 * pages a chance to be referenced before reclaiming. All
4253 * pages are rotated regardless of classzone as this is
4254 * about consistent aging.
4256 age_active_anon(pgdat
, &sc
);
4259 * If we're getting trouble reclaiming, start doing writepage
4260 * even in laptop mode.
4262 if (sc
.priority
< DEF_PRIORITY
- 2)
4263 sc
.may_writepage
= 1;
4265 /* Call soft limit reclaim before calling shrink_node. */
4267 nr_soft_scanned
= 0;
4268 nr_soft_reclaimed
= mem_cgroup_soft_limit_reclaim(pgdat
, sc
.order
,
4269 sc
.gfp_mask
, &nr_soft_scanned
);
4270 sc
.nr_reclaimed
+= nr_soft_reclaimed
;
4273 * There should be no need to raise the scanning priority if
4274 * enough pages are already being scanned that that high
4275 * watermark would be met at 100% efficiency.
4277 if (kswapd_shrink_node(pgdat
, &sc
))
4278 raise_priority
= false;
4281 * If the low watermark is met there is no need for processes
4282 * to be throttled on pfmemalloc_wait as they should not be
4283 * able to safely make forward progress. Wake them
4285 if (waitqueue_active(&pgdat
->pfmemalloc_wait
) &&
4286 allow_direct_reclaim(pgdat
))
4287 wake_up_all(&pgdat
->pfmemalloc_wait
);
4289 /* Check if kswapd should be suspending */
4290 __fs_reclaim_release(_THIS_IP_
);
4291 ret
= try_to_freeze();
4292 __fs_reclaim_acquire(_THIS_IP_
);
4293 if (ret
|| kthread_should_stop())
4297 * Raise priority if scanning rate is too low or there was no
4298 * progress in reclaiming pages
4300 nr_reclaimed
= sc
.nr_reclaimed
- nr_reclaimed
;
4301 nr_boost_reclaim
-= min(nr_boost_reclaim
, nr_reclaimed
);
4304 * If reclaim made no progress for a boost, stop reclaim as
4305 * IO cannot be queued and it could be an infinite loop in
4306 * extreme circumstances.
4308 if (nr_boost_reclaim
&& !nr_reclaimed
)
4311 if (raise_priority
|| !nr_reclaimed
)
4313 } while (sc
.priority
>= 1);
4315 if (!sc
.nr_reclaimed
)
4316 pgdat
->kswapd_failures
++;
4319 clear_reclaim_active(pgdat
, highest_zoneidx
);
4321 /* If reclaim was boosted, account for the reclaim done in this pass */
4323 unsigned long flags
;
4325 for (i
= 0; i
<= highest_zoneidx
; i
++) {
4326 if (!zone_boosts
[i
])
4329 /* Increments are under the zone lock */
4330 zone
= pgdat
->node_zones
+ i
;
4331 spin_lock_irqsave(&zone
->lock
, flags
);
4332 zone
->watermark_boost
-= min(zone
->watermark_boost
, zone_boosts
[i
]);
4333 spin_unlock_irqrestore(&zone
->lock
, flags
);
4337 * As there is now likely space, wakeup kcompact to defragment
4340 wakeup_kcompactd(pgdat
, pageblock_order
, highest_zoneidx
);
4343 snapshot_refaults(NULL
, pgdat
);
4344 __fs_reclaim_release(_THIS_IP_
);
4345 psi_memstall_leave(&pflags
);
4346 set_task_reclaim_state(current
, NULL
);
4349 * Return the order kswapd stopped reclaiming at as
4350 * prepare_kswapd_sleep() takes it into account. If another caller
4351 * entered the allocator slow path while kswapd was awake, order will
4352 * remain at the higher level.
4358 * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
4359 * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
4360 * not a valid index then either kswapd runs for first time or kswapd couldn't
4361 * sleep after previous reclaim attempt (node is still unbalanced). In that
4362 * case return the zone index of the previous kswapd reclaim cycle.
4364 static enum zone_type
kswapd_highest_zoneidx(pg_data_t
*pgdat
,
4365 enum zone_type prev_highest_zoneidx
)
4367 enum zone_type curr_idx
= READ_ONCE(pgdat
->kswapd_highest_zoneidx
);
4369 return curr_idx
== MAX_NR_ZONES
? prev_highest_zoneidx
: curr_idx
;
4372 static void kswapd_try_to_sleep(pg_data_t
*pgdat
, int alloc_order
, int reclaim_order
,
4373 unsigned int highest_zoneidx
)
4378 if (freezing(current
) || kthread_should_stop())
4381 prepare_to_wait(&pgdat
->kswapd_wait
, &wait
, TASK_INTERRUPTIBLE
);
4384 * Try to sleep for a short interval. Note that kcompactd will only be
4385 * woken if it is possible to sleep for a short interval. This is
4386 * deliberate on the assumption that if reclaim cannot keep an
4387 * eligible zone balanced that it's also unlikely that compaction will
4390 if (prepare_kswapd_sleep(pgdat
, reclaim_order
, highest_zoneidx
)) {
4392 * Compaction records what page blocks it recently failed to
4393 * isolate pages from and skips them in the future scanning.
4394 * When kswapd is going to sleep, it is reasonable to assume
4395 * that pages and compaction may succeed so reset the cache.
4397 reset_isolation_suitable(pgdat
);
4400 * We have freed the memory, now we should compact it to make
4401 * allocation of the requested order possible.
4403 wakeup_kcompactd(pgdat
, alloc_order
, highest_zoneidx
);
4405 remaining
= schedule_timeout(HZ
/10);
4408 * If woken prematurely then reset kswapd_highest_zoneidx and
4409 * order. The values will either be from a wakeup request or
4410 * the previous request that slept prematurely.
4413 WRITE_ONCE(pgdat
->kswapd_highest_zoneidx
,
4414 kswapd_highest_zoneidx(pgdat
,
4417 if (READ_ONCE(pgdat
->kswapd_order
) < reclaim_order
)
4418 WRITE_ONCE(pgdat
->kswapd_order
, reclaim_order
);
4421 finish_wait(&pgdat
->kswapd_wait
, &wait
);
4422 prepare_to_wait(&pgdat
->kswapd_wait
, &wait
, TASK_INTERRUPTIBLE
);
4426 * After a short sleep, check if it was a premature sleep. If not, then
4427 * go fully to sleep until explicitly woken up.
4430 prepare_kswapd_sleep(pgdat
, reclaim_order
, highest_zoneidx
)) {
4431 trace_mm_vmscan_kswapd_sleep(pgdat
->node_id
);
4434 * vmstat counters are not perfectly accurate and the estimated
4435 * value for counters such as NR_FREE_PAGES can deviate from the
4436 * true value by nr_online_cpus * threshold. To avoid the zone
4437 * watermarks being breached while under pressure, we reduce the
4438 * per-cpu vmstat threshold while kswapd is awake and restore
4439 * them before going back to sleep.
4441 set_pgdat_percpu_threshold(pgdat
, calculate_normal_threshold
);
4443 if (!kthread_should_stop())
4446 set_pgdat_percpu_threshold(pgdat
, calculate_pressure_threshold
);
4449 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY
);
4451 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY
);
4453 finish_wait(&pgdat
->kswapd_wait
, &wait
);
4457 * The background pageout daemon, started as a kernel thread
4458 * from the init process.
4460 * This basically trickles out pages so that we have _some_
4461 * free memory available even if there is no other activity
4462 * that frees anything up. This is needed for things like routing
4463 * etc, where we otherwise might have all activity going on in
4464 * asynchronous contexts that cannot page things out.
4466 * If there are applications that are active memory-allocators
4467 * (most normal use), this basically shouldn't matter.
4469 static int kswapd(void *p
)
4471 unsigned int alloc_order
, reclaim_order
;
4472 unsigned int highest_zoneidx
= MAX_NR_ZONES
- 1;
4473 pg_data_t
*pgdat
= (pg_data_t
*)p
;
4474 struct task_struct
*tsk
= current
;
4475 const struct cpumask
*cpumask
= cpumask_of_node(pgdat
->node_id
);
4477 if (!cpumask_empty(cpumask
))
4478 set_cpus_allowed_ptr(tsk
, cpumask
);
4481 * Tell the memory management that we're a "memory allocator",
4482 * and that if we need more memory we should get access to it
4483 * regardless (see "__alloc_pages()"). "kswapd" should
4484 * never get caught in the normal page freeing logic.
4486 * (Kswapd normally doesn't need memory anyway, but sometimes
4487 * you need a small amount of memory in order to be able to
4488 * page out something else, and this flag essentially protects
4489 * us from recursively trying to free more memory as we're
4490 * trying to free the first piece of memory in the first place).
4492 tsk
->flags
|= PF_MEMALLOC
| PF_KSWAPD
;
4495 WRITE_ONCE(pgdat
->kswapd_order
, 0);
4496 WRITE_ONCE(pgdat
->kswapd_highest_zoneidx
, MAX_NR_ZONES
);
4497 atomic_set(&pgdat
->nr_writeback_throttled
, 0);
4501 alloc_order
= reclaim_order
= READ_ONCE(pgdat
->kswapd_order
);
4502 highest_zoneidx
= kswapd_highest_zoneidx(pgdat
,
4506 kswapd_try_to_sleep(pgdat
, alloc_order
, reclaim_order
,
4509 /* Read the new order and highest_zoneidx */
4510 alloc_order
= READ_ONCE(pgdat
->kswapd_order
);
4511 highest_zoneidx
= kswapd_highest_zoneidx(pgdat
,
4513 WRITE_ONCE(pgdat
->kswapd_order
, 0);
4514 WRITE_ONCE(pgdat
->kswapd_highest_zoneidx
, MAX_NR_ZONES
);
4516 ret
= try_to_freeze();
4517 if (kthread_should_stop())
4521 * We can speed up thawing tasks if we don't call balance_pgdat
4522 * after returning from the refrigerator
4528 * Reclaim begins at the requested order but if a high-order
4529 * reclaim fails then kswapd falls back to reclaiming for
4530 * order-0. If that happens, kswapd will consider sleeping
4531 * for the order it finished reclaiming at (reclaim_order)
4532 * but kcompactd is woken to compact for the original
4533 * request (alloc_order).
4535 trace_mm_vmscan_kswapd_wake(pgdat
->node_id
, highest_zoneidx
,
4537 reclaim_order
= balance_pgdat(pgdat
, alloc_order
,
4539 if (reclaim_order
< alloc_order
)
4540 goto kswapd_try_sleep
;
4543 tsk
->flags
&= ~(PF_MEMALLOC
| PF_KSWAPD
);
4549 * A zone is low on free memory or too fragmented for high-order memory. If
4550 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
4551 * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim
4552 * has failed or is not needed, still wake up kcompactd if only compaction is
4555 void wakeup_kswapd(struct zone
*zone
, gfp_t gfp_flags
, int order
,
4556 enum zone_type highest_zoneidx
)
4559 enum zone_type curr_idx
;
4561 if (!managed_zone(zone
))
4564 if (!cpuset_zone_allowed(zone
, gfp_flags
))
4567 pgdat
= zone
->zone_pgdat
;
4568 curr_idx
= READ_ONCE(pgdat
->kswapd_highest_zoneidx
);
4570 if (curr_idx
== MAX_NR_ZONES
|| curr_idx
< highest_zoneidx
)
4571 WRITE_ONCE(pgdat
->kswapd_highest_zoneidx
, highest_zoneidx
);
4573 if (READ_ONCE(pgdat
->kswapd_order
) < order
)
4574 WRITE_ONCE(pgdat
->kswapd_order
, order
);
4576 if (!waitqueue_active(&pgdat
->kswapd_wait
))
4579 /* Hopeless node, leave it to direct reclaim if possible */
4580 if (pgdat
->kswapd_failures
>= MAX_RECLAIM_RETRIES
||
4581 (pgdat_balanced(pgdat
, order
, highest_zoneidx
) &&
4582 !pgdat_watermark_boosted(pgdat
, highest_zoneidx
))) {
4584 * There may be plenty of free memory available, but it's too
4585 * fragmented for high-order allocations. Wake up kcompactd
4586 * and rely on compaction_suitable() to determine if it's
4587 * needed. If it fails, it will defer subsequent attempts to
4588 * ratelimit its work.
4590 if (!(gfp_flags
& __GFP_DIRECT_RECLAIM
))
4591 wakeup_kcompactd(pgdat
, order
, highest_zoneidx
);
4595 trace_mm_vmscan_wakeup_kswapd(pgdat
->node_id
, highest_zoneidx
, order
,
4597 wake_up_interruptible(&pgdat
->kswapd_wait
);
4600 #ifdef CONFIG_HIBERNATION
4602 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
4605 * Rather than trying to age LRUs the aim is to preserve the overall
4606 * LRU order by reclaiming preferentially
4607 * inactive > active > active referenced > active mapped
4609 unsigned long shrink_all_memory(unsigned long nr_to_reclaim
)
4611 struct scan_control sc
= {
4612 .nr_to_reclaim
= nr_to_reclaim
,
4613 .gfp_mask
= GFP_HIGHUSER_MOVABLE
,
4614 .reclaim_idx
= MAX_NR_ZONES
- 1,
4615 .priority
= DEF_PRIORITY
,
4619 .hibernation_mode
= 1,
4621 struct zonelist
*zonelist
= node_zonelist(numa_node_id(), sc
.gfp_mask
);
4622 unsigned long nr_reclaimed
;
4623 unsigned int noreclaim_flag
;
4625 fs_reclaim_acquire(sc
.gfp_mask
);
4626 noreclaim_flag
= memalloc_noreclaim_save();
4627 set_task_reclaim_state(current
, &sc
.reclaim_state
);
4629 nr_reclaimed
= do_try_to_free_pages(zonelist
, &sc
);
4631 set_task_reclaim_state(current
, NULL
);
4632 memalloc_noreclaim_restore(noreclaim_flag
);
4633 fs_reclaim_release(sc
.gfp_mask
);
4635 return nr_reclaimed
;
4637 #endif /* CONFIG_HIBERNATION */
4640 * This kswapd start function will be called by init and node-hot-add.
4642 void kswapd_run(int nid
)
4644 pg_data_t
*pgdat
= NODE_DATA(nid
);
4649 pgdat
->kswapd
= kthread_run(kswapd
, pgdat
, "kswapd%d", nid
);
4650 if (IS_ERR(pgdat
->kswapd
)) {
4651 /* failure at boot is fatal */
4652 BUG_ON(system_state
< SYSTEM_RUNNING
);
4653 pr_err("Failed to start kswapd on node %d\n", nid
);
4654 pgdat
->kswapd
= NULL
;
4659 * Called by memory hotplug when all memory in a node is offlined. Caller must
4660 * be holding mem_hotplug_begin/done().
4662 void kswapd_stop(int nid
)
4664 struct task_struct
*kswapd
= NODE_DATA(nid
)->kswapd
;
4667 kthread_stop(kswapd
);
4668 NODE_DATA(nid
)->kswapd
= NULL
;
4672 static int __init
kswapd_init(void)
4677 for_each_node_state(nid
, N_MEMORY
)
4682 module_init(kswapd_init
)
4688 * If non-zero call node_reclaim when the number of free pages falls below
4691 int node_reclaim_mode __read_mostly
;
4694 * Priority for NODE_RECLAIM. This determines the fraction of pages
4695 * of a node considered for each zone_reclaim. 4 scans 1/16th of
4698 #define NODE_RECLAIM_PRIORITY 4
4701 * Percentage of pages in a zone that must be unmapped for node_reclaim to
4704 int sysctl_min_unmapped_ratio
= 1;
4707 * If the number of slab pages in a zone grows beyond this percentage then
4708 * slab reclaim needs to occur.
4710 int sysctl_min_slab_ratio
= 5;
4712 static inline unsigned long node_unmapped_file_pages(struct pglist_data
*pgdat
)
4714 unsigned long file_mapped
= node_page_state(pgdat
, NR_FILE_MAPPED
);
4715 unsigned long file_lru
= node_page_state(pgdat
, NR_INACTIVE_FILE
) +
4716 node_page_state(pgdat
, NR_ACTIVE_FILE
);
4719 * It's possible for there to be more file mapped pages than
4720 * accounted for by the pages on the file LRU lists because
4721 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
4723 return (file_lru
> file_mapped
) ? (file_lru
- file_mapped
) : 0;
4726 /* Work out how many page cache pages we can reclaim in this reclaim_mode */
4727 static unsigned long node_pagecache_reclaimable(struct pglist_data
*pgdat
)
4729 unsigned long nr_pagecache_reclaimable
;
4730 unsigned long delta
= 0;
4733 * If RECLAIM_UNMAP is set, then all file pages are considered
4734 * potentially reclaimable. Otherwise, we have to worry about
4735 * pages like swapcache and node_unmapped_file_pages() provides
4738 if (node_reclaim_mode
& RECLAIM_UNMAP
)
4739 nr_pagecache_reclaimable
= node_page_state(pgdat
, NR_FILE_PAGES
);
4741 nr_pagecache_reclaimable
= node_unmapped_file_pages(pgdat
);
4743 /* If we can't clean pages, remove dirty pages from consideration */
4744 if (!(node_reclaim_mode
& RECLAIM_WRITE
))
4745 delta
+= node_page_state(pgdat
, NR_FILE_DIRTY
);
4747 /* Watch for any possible underflows due to delta */
4748 if (unlikely(delta
> nr_pagecache_reclaimable
))
4749 delta
= nr_pagecache_reclaimable
;
4751 return nr_pagecache_reclaimable
- delta
;
4755 * Try to free up some pages from this node through reclaim.
4757 static int __node_reclaim(struct pglist_data
*pgdat
, gfp_t gfp_mask
, unsigned int order
)
4759 /* Minimum pages needed in order to stay on node */
4760 const unsigned long nr_pages
= 1 << order
;
4761 struct task_struct
*p
= current
;
4762 unsigned int noreclaim_flag
;
4763 struct scan_control sc
= {
4764 .nr_to_reclaim
= max(nr_pages
, SWAP_CLUSTER_MAX
),
4765 .gfp_mask
= current_gfp_context(gfp_mask
),
4767 .priority
= NODE_RECLAIM_PRIORITY
,
4768 .may_writepage
= !!(node_reclaim_mode
& RECLAIM_WRITE
),
4769 .may_unmap
= !!(node_reclaim_mode
& RECLAIM_UNMAP
),
4771 .reclaim_idx
= gfp_zone(gfp_mask
),
4773 unsigned long pflags
;
4775 trace_mm_vmscan_node_reclaim_begin(pgdat
->node_id
, order
,
4779 psi_memstall_enter(&pflags
);
4780 fs_reclaim_acquire(sc
.gfp_mask
);
4782 * We need to be able to allocate from the reserves for RECLAIM_UNMAP
4784 noreclaim_flag
= memalloc_noreclaim_save();
4785 set_task_reclaim_state(p
, &sc
.reclaim_state
);
4787 if (node_pagecache_reclaimable(pgdat
) > pgdat
->min_unmapped_pages
||
4788 node_page_state_pages(pgdat
, NR_SLAB_RECLAIMABLE_B
) > pgdat
->min_slab_pages
) {
4790 * Free memory by calling shrink node with increasing
4791 * priorities until we have enough memory freed.
4794 shrink_node(pgdat
, &sc
);
4795 } while (sc
.nr_reclaimed
< nr_pages
&& --sc
.priority
>= 0);
4798 set_task_reclaim_state(p
, NULL
);
4799 memalloc_noreclaim_restore(noreclaim_flag
);
4800 fs_reclaim_release(sc
.gfp_mask
);
4801 psi_memstall_leave(&pflags
);
4803 trace_mm_vmscan_node_reclaim_end(sc
.nr_reclaimed
);
4805 return sc
.nr_reclaimed
>= nr_pages
;
4808 int node_reclaim(struct pglist_data
*pgdat
, gfp_t gfp_mask
, unsigned int order
)
4813 * Node reclaim reclaims unmapped file backed pages and
4814 * slab pages if we are over the defined limits.
4816 * A small portion of unmapped file backed pages is needed for
4817 * file I/O otherwise pages read by file I/O will be immediately
4818 * thrown out if the node is overallocated. So we do not reclaim
4819 * if less than a specified percentage of the node is used by
4820 * unmapped file backed pages.
4822 if (node_pagecache_reclaimable(pgdat
) <= pgdat
->min_unmapped_pages
&&
4823 node_page_state_pages(pgdat
, NR_SLAB_RECLAIMABLE_B
) <=
4824 pgdat
->min_slab_pages
)
4825 return NODE_RECLAIM_FULL
;
4828 * Do not scan if the allocation should not be delayed.
4830 if (!gfpflags_allow_blocking(gfp_mask
) || (current
->flags
& PF_MEMALLOC
))
4831 return NODE_RECLAIM_NOSCAN
;
4834 * Only run node reclaim on the local node or on nodes that do not
4835 * have associated processors. This will favor the local processor
4836 * over remote processors and spread off node memory allocations
4837 * as wide as possible.
4839 if (node_state(pgdat
->node_id
, N_CPU
) && pgdat
->node_id
!= numa_node_id())
4840 return NODE_RECLAIM_NOSCAN
;
4842 if (test_and_set_bit(PGDAT_RECLAIM_LOCKED
, &pgdat
->flags
))
4843 return NODE_RECLAIM_NOSCAN
;
4845 ret
= __node_reclaim(pgdat
, gfp_mask
, order
);
4846 clear_bit(PGDAT_RECLAIM_LOCKED
, &pgdat
->flags
);
4849 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED
);
4855 void check_move_unevictable_pages(struct pagevec
*pvec
)
4857 struct folio_batch fbatch
;
4860 folio_batch_init(&fbatch
);
4861 for (i
= 0; i
< pvec
->nr
; i
++) {
4862 struct page
*page
= pvec
->pages
[i
];
4864 if (PageTransTail(page
))
4866 folio_batch_add(&fbatch
, page_folio(page
));
4868 check_move_unevictable_folios(&fbatch
);
4870 EXPORT_SYMBOL_GPL(check_move_unevictable_pages
);
4873 * check_move_unevictable_folios - Move evictable folios to appropriate zone
4875 * @fbatch: Batch of lru folios to check.
4877 * Checks folios for evictability, if an evictable folio is in the unevictable
4878 * lru list, moves it to the appropriate evictable lru list. This function
4879 * should be only used for lru folios.
4881 void check_move_unevictable_folios(struct folio_batch
*fbatch
)
4883 struct lruvec
*lruvec
= NULL
;
4888 for (i
= 0; i
< fbatch
->nr
; i
++) {
4889 struct folio
*folio
= fbatch
->folios
[i
];
4890 int nr_pages
= folio_nr_pages(folio
);
4892 pgscanned
+= nr_pages
;
4894 /* block memcg migration while the folio moves between lrus */
4895 if (!folio_test_clear_lru(folio
))
4898 lruvec
= folio_lruvec_relock_irq(folio
, lruvec
);
4899 if (folio_evictable(folio
) && folio_test_unevictable(folio
)) {
4900 lruvec_del_folio(lruvec
, folio
);
4901 folio_clear_unevictable(folio
);
4902 lruvec_add_folio(lruvec
, folio
);
4903 pgrescued
+= nr_pages
;
4905 folio_set_lru(folio
);
4909 __count_vm_events(UNEVICTABLE_PGRESCUED
, pgrescued
);
4910 __count_vm_events(UNEVICTABLE_PGSCANNED
, pgscanned
);
4911 unlock_page_lruvec_irq(lruvec
);
4912 } else if (pgscanned
) {
4913 count_vm_events(UNEVICTABLE_PGSCANNED
, pgscanned
);
4916 EXPORT_SYMBOL_GPL(check_move_unevictable_folios
);