1 // SPDX-License-Identifier: GPL-2.0
3 * Manage cache of swap slots to be used for and returned from
6 * Copyright(c) 2016 Intel Corporation.
8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
10 * We allocate the swap slots from the global pool and put
11 * it into local per cpu caches. This has the advantage
12 * of no needing to acquire the swap_info lock every time
15 * There is also opportunity to simply return the slot
16 * to local caches without needing to acquire swap_info
17 * lock. We do not reuse the returned slots directly but
18 * move them back to the global pool in a batch. This
19 * allows the slots to coalesce and reduce fragmentation.
21 * The swap entry allocated is marked with SWAP_HAS_CACHE
22 * flag in map_count that prevents it from being allocated
23 * again from the global pool.
25 * The swap slots cache is protected by a mutex instead of
26 * a spin lock as when we search for slots with scan_swap_map,
27 * we can possibly sleep.
30 #include <linux/swap_slots.h>
31 #include <linux/cpu.h>
32 #include <linux/cpumask.h>
33 #include <linux/vmalloc.h>
34 #include <linux/mutex.h>
37 static DEFINE_PER_CPU(struct swap_slots_cache
, swp_slots
);
38 static bool swap_slot_cache_active
;
39 bool swap_slot_cache_enabled
;
40 static bool swap_slot_cache_initialized
;
41 static DEFINE_MUTEX(swap_slots_cache_mutex
);
42 /* Serialize swap slots cache enable/disable operations */
43 static DEFINE_MUTEX(swap_slots_cache_enable_mutex
);
45 static void __drain_swap_slots_cache(unsigned int type
);
47 #define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
48 #define SLOTS_CACHE 0x1
49 #define SLOTS_CACHE_RET 0x2
51 static void deactivate_swap_slots_cache(void)
53 mutex_lock(&swap_slots_cache_mutex
);
54 swap_slot_cache_active
= false;
55 __drain_swap_slots_cache(SLOTS_CACHE
|SLOTS_CACHE_RET
);
56 mutex_unlock(&swap_slots_cache_mutex
);
59 static void reactivate_swap_slots_cache(void)
61 mutex_lock(&swap_slots_cache_mutex
);
62 swap_slot_cache_active
= true;
63 mutex_unlock(&swap_slots_cache_mutex
);
66 /* Must not be called with cpu hot plug lock */
67 void disable_swap_slots_cache_lock(void)
69 mutex_lock(&swap_slots_cache_enable_mutex
);
70 swap_slot_cache_enabled
= false;
71 if (swap_slot_cache_initialized
) {
72 /* serialize with cpu hotplug operations */
74 __drain_swap_slots_cache(SLOTS_CACHE
|SLOTS_CACHE_RET
);
79 static void __reenable_swap_slots_cache(void)
81 swap_slot_cache_enabled
= has_usable_swap();
84 void reenable_swap_slots_cache_unlock(void)
86 __reenable_swap_slots_cache();
87 mutex_unlock(&swap_slots_cache_enable_mutex
);
90 static bool check_cache_active(void)
94 if (!swap_slot_cache_enabled
)
97 pages
= get_nr_swap_pages();
98 if (!swap_slot_cache_active
) {
99 if (pages
> num_online_cpus() *
100 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE
)
101 reactivate_swap_slots_cache();
105 /* if global pool of slot caches too low, deactivate cache */
106 if (pages
< num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE
)
107 deactivate_swap_slots_cache();
109 return swap_slot_cache_active
;
112 static int alloc_swap_slot_cache(unsigned int cpu
)
114 struct swap_slots_cache
*cache
;
115 swp_entry_t
*slots
, *slots_ret
;
118 * Do allocation outside swap_slots_cache_mutex
119 * as kvzalloc could trigger reclaim and get_swap_page,
120 * which can lock swap_slots_cache_mutex.
122 slots
= kvcalloc(SWAP_SLOTS_CACHE_SIZE
, sizeof(swp_entry_t
),
127 slots_ret
= kvcalloc(SWAP_SLOTS_CACHE_SIZE
, sizeof(swp_entry_t
),
134 mutex_lock(&swap_slots_cache_mutex
);
135 cache
= &per_cpu(swp_slots
, cpu
);
136 if (cache
->slots
|| cache
->slots_ret
) {
137 /* cache already allocated */
138 mutex_unlock(&swap_slots_cache_mutex
);
146 if (!cache
->lock_initialized
) {
147 mutex_init(&cache
->alloc_lock
);
148 spin_lock_init(&cache
->free_lock
);
149 cache
->lock_initialized
= true;
155 * We initialized alloc_lock and free_lock earlier. We use
156 * !cache->slots or !cache->slots_ret to know if it is safe to acquire
157 * the corresponding lock and use the cache. Memory barrier below
158 * ensures the assumption.
161 cache
->slots
= slots
;
162 cache
->slots_ret
= slots_ret
;
163 mutex_unlock(&swap_slots_cache_mutex
);
167 static void drain_slots_cache_cpu(unsigned int cpu
, unsigned int type
,
170 struct swap_slots_cache
*cache
;
171 swp_entry_t
*slots
= NULL
;
173 cache
= &per_cpu(swp_slots
, cpu
);
174 if ((type
& SLOTS_CACHE
) && cache
->slots
) {
175 mutex_lock(&cache
->alloc_lock
);
176 swapcache_free_entries(cache
->slots
+ cache
->cur
, cache
->nr
);
179 if (free_slots
&& cache
->slots
) {
180 kvfree(cache
->slots
);
183 mutex_unlock(&cache
->alloc_lock
);
185 if ((type
& SLOTS_CACHE_RET
) && cache
->slots_ret
) {
186 spin_lock_irq(&cache
->free_lock
);
187 swapcache_free_entries(cache
->slots_ret
, cache
->n_ret
);
189 if (free_slots
&& cache
->slots_ret
) {
190 slots
= cache
->slots_ret
;
191 cache
->slots_ret
= NULL
;
193 spin_unlock_irq(&cache
->free_lock
);
198 static void __drain_swap_slots_cache(unsigned int type
)
203 * This function is called during
204 * 1) swapoff, when we have to make sure no
205 * left over slots are in cache when we remove
207 * 2) disabling of swap slot cache, when we run low
208 * on swap slots when allocating memory and need
209 * to return swap slots to global pool.
211 * We cannot acquire cpu hot plug lock here as
212 * this function can be invoked in the cpu
214 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
215 * -> memory allocation -> direct reclaim -> get_swap_page
216 * -> drain_swap_slots_cache
218 * Hence the loop over current online cpu below could miss cpu that
219 * is being brought online but not yet marked as online.
220 * That is okay as we do not schedule and run anything on a
221 * cpu before it has been marked online. Hence, we will not
222 * fill any swap slots in slots cache of such cpu.
223 * There are no slots on such cpu that need to be drained.
225 for_each_online_cpu(cpu
)
226 drain_slots_cache_cpu(cpu
, type
, false);
229 static int free_slot_cache(unsigned int cpu
)
231 mutex_lock(&swap_slots_cache_mutex
);
232 drain_slots_cache_cpu(cpu
, SLOTS_CACHE
| SLOTS_CACHE_RET
, true);
233 mutex_unlock(&swap_slots_cache_mutex
);
237 void enable_swap_slots_cache(void)
239 mutex_lock(&swap_slots_cache_enable_mutex
);
240 if (!swap_slot_cache_initialized
) {
243 ret
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "swap_slots_cache",
244 alloc_swap_slot_cache
, free_slot_cache
);
245 if (WARN_ONCE(ret
< 0, "Cache allocation failed (%s), operating "
246 "without swap slots cache.\n", __func__
))
249 swap_slot_cache_initialized
= true;
252 __reenable_swap_slots_cache();
254 mutex_unlock(&swap_slots_cache_enable_mutex
);
257 /* called with swap slot cache's alloc lock held */
258 static int refill_swap_slots_cache(struct swap_slots_cache
*cache
)
260 if (!use_swap_slot_cache
|| cache
->nr
)
264 if (swap_slot_cache_active
)
265 cache
->nr
= get_swap_pages(SWAP_SLOTS_CACHE_SIZE
,
271 int free_swap_slot(swp_entry_t entry
)
273 struct swap_slots_cache
*cache
;
275 cache
= raw_cpu_ptr(&swp_slots
);
276 if (likely(use_swap_slot_cache
&& cache
->slots_ret
)) {
277 spin_lock_irq(&cache
->free_lock
);
278 /* Swap slots cache may be deactivated before acquiring lock */
279 if (!use_swap_slot_cache
|| !cache
->slots_ret
) {
280 spin_unlock_irq(&cache
->free_lock
);
283 if (cache
->n_ret
>= SWAP_SLOTS_CACHE_SIZE
) {
285 * Return slots to global pool.
286 * The current swap_map value is SWAP_HAS_CACHE.
287 * Set it to 0 to indicate it is available for
288 * allocation in global pool
290 swapcache_free_entries(cache
->slots_ret
, cache
->n_ret
);
293 cache
->slots_ret
[cache
->n_ret
++] = entry
;
294 spin_unlock_irq(&cache
->free_lock
);
297 swapcache_free_entries(&entry
, 1);
303 swp_entry_t
get_swap_page(struct page
*page
)
306 struct swap_slots_cache
*cache
;
310 if (PageTransHuge(page
)) {
311 if (IS_ENABLED(CONFIG_THP_SWAP
))
312 get_swap_pages(1, &entry
, HPAGE_PMD_NR
);
317 * Preemption is allowed here, because we may sleep
318 * in refill_swap_slots_cache(). But it is safe, because
319 * accesses to the per-CPU data structure are protected by the
320 * mutex cache->alloc_lock.
322 * The alloc path here does not touch cache->slots_ret
323 * so cache->free_lock is not taken.
325 cache
= raw_cpu_ptr(&swp_slots
);
327 if (likely(check_cache_active() && cache
->slots
)) {
328 mutex_lock(&cache
->alloc_lock
);
332 entry
= cache
->slots
[cache
->cur
];
333 cache
->slots
[cache
->cur
++].val
= 0;
335 } else if (refill_swap_slots_cache(cache
)) {
339 mutex_unlock(&cache
->alloc_lock
);
344 get_swap_pages(1, &entry
, 1);
346 if (mem_cgroup_try_charge_swap(page
, entry
)) {
347 put_swap_page(page
, entry
);