1 // SPDX-License-Identifier: GPL-2.0
3 * Manage cache of swap slots to be used for and returned from
6 * Copyright(c) 2016 Intel Corporation.
8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
10 * We allocate the swap slots from the global pool and put
11 * it into local per cpu caches. This has the advantage
12 * of no needing to acquire the swap_info lock every time
15 * There is also opportunity to simply return the slot
16 * to local caches without needing to acquire swap_info
17 * lock. We do not reuse the returned slots directly but
18 * move them back to the global pool in a batch. This
19 * allows the slots to coaellesce and reduce fragmentation.
21 * The swap entry allocated is marked with SWAP_HAS_CACHE
22 * flag in map_count that prevents it from being allocated
23 * again from the global pool.
25 * The swap slots cache is protected by a mutex instead of
26 * a spin lock as when we search for slots with scan_swap_map,
27 * we can possibly sleep.
30 #include <linux/swap_slots.h>
31 #include <linux/cpu.h>
32 #include <linux/cpumask.h>
33 #include <linux/vmalloc.h>
34 #include <linux/mutex.h>
37 static DEFINE_PER_CPU(struct swap_slots_cache
, swp_slots
);
38 static bool swap_slot_cache_active
;
39 bool swap_slot_cache_enabled
;
40 static bool swap_slot_cache_initialized
;
41 static DEFINE_MUTEX(swap_slots_cache_mutex
);
42 /* Serialize swap slots cache enable/disable operations */
43 static DEFINE_MUTEX(swap_slots_cache_enable_mutex
);
45 static void __drain_swap_slots_cache(unsigned int type
);
46 static void deactivate_swap_slots_cache(void);
47 static void reactivate_swap_slots_cache(void);
49 #define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
50 #define SLOTS_CACHE 0x1
51 #define SLOTS_CACHE_RET 0x2
53 static void deactivate_swap_slots_cache(void)
55 mutex_lock(&swap_slots_cache_mutex
);
56 swap_slot_cache_active
= false;
57 __drain_swap_slots_cache(SLOTS_CACHE
|SLOTS_CACHE_RET
);
58 mutex_unlock(&swap_slots_cache_mutex
);
61 static void reactivate_swap_slots_cache(void)
63 mutex_lock(&swap_slots_cache_mutex
);
64 swap_slot_cache_active
= true;
65 mutex_unlock(&swap_slots_cache_mutex
);
68 /* Must not be called with cpu hot plug lock */
69 void disable_swap_slots_cache_lock(void)
71 mutex_lock(&swap_slots_cache_enable_mutex
);
72 swap_slot_cache_enabled
= false;
73 if (swap_slot_cache_initialized
) {
74 /* serialize with cpu hotplug operations */
76 __drain_swap_slots_cache(SLOTS_CACHE
|SLOTS_CACHE_RET
);
81 static void __reenable_swap_slots_cache(void)
83 swap_slot_cache_enabled
= has_usable_swap();
86 void reenable_swap_slots_cache_unlock(void)
88 __reenable_swap_slots_cache();
89 mutex_unlock(&swap_slots_cache_enable_mutex
);
92 static bool check_cache_active(void)
96 if (!swap_slot_cache_enabled
)
99 pages
= get_nr_swap_pages();
100 if (!swap_slot_cache_active
) {
101 if (pages
> num_online_cpus() *
102 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE
)
103 reactivate_swap_slots_cache();
107 /* if global pool of slot caches too low, deactivate cache */
108 if (pages
< num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE
)
109 deactivate_swap_slots_cache();
111 return swap_slot_cache_active
;
114 static int alloc_swap_slot_cache(unsigned int cpu
)
116 struct swap_slots_cache
*cache
;
117 swp_entry_t
*slots
, *slots_ret
;
120 * Do allocation outside swap_slots_cache_mutex
121 * as kvzalloc could trigger reclaim and get_swap_page,
122 * which can lock swap_slots_cache_mutex.
124 slots
= kvcalloc(SWAP_SLOTS_CACHE_SIZE
, sizeof(swp_entry_t
),
129 slots_ret
= kvcalloc(SWAP_SLOTS_CACHE_SIZE
, sizeof(swp_entry_t
),
136 mutex_lock(&swap_slots_cache_mutex
);
137 cache
= &per_cpu(swp_slots
, cpu
);
138 if (cache
->slots
|| cache
->slots_ret
) {
139 /* cache already allocated */
140 mutex_unlock(&swap_slots_cache_mutex
);
148 if (!cache
->lock_initialized
) {
149 mutex_init(&cache
->alloc_lock
);
150 spin_lock_init(&cache
->free_lock
);
151 cache
->lock_initialized
= true;
157 * We initialized alloc_lock and free_lock earlier. We use
158 * !cache->slots or !cache->slots_ret to know if it is safe to acquire
159 * the corresponding lock and use the cache. Memory barrier below
160 * ensures the assumption.
163 cache
->slots
= slots
;
164 cache
->slots_ret
= slots_ret
;
165 mutex_unlock(&swap_slots_cache_mutex
);
169 static void drain_slots_cache_cpu(unsigned int cpu
, unsigned int type
,
172 struct swap_slots_cache
*cache
;
173 swp_entry_t
*slots
= NULL
;
175 cache
= &per_cpu(swp_slots
, cpu
);
176 if ((type
& SLOTS_CACHE
) && cache
->slots
) {
177 mutex_lock(&cache
->alloc_lock
);
178 swapcache_free_entries(cache
->slots
+ cache
->cur
, cache
->nr
);
181 if (free_slots
&& cache
->slots
) {
182 kvfree(cache
->slots
);
185 mutex_unlock(&cache
->alloc_lock
);
187 if ((type
& SLOTS_CACHE_RET
) && cache
->slots_ret
) {
188 spin_lock_irq(&cache
->free_lock
);
189 swapcache_free_entries(cache
->slots_ret
, cache
->n_ret
);
191 if (free_slots
&& cache
->slots_ret
) {
192 slots
= cache
->slots_ret
;
193 cache
->slots_ret
= NULL
;
195 spin_unlock_irq(&cache
->free_lock
);
200 static void __drain_swap_slots_cache(unsigned int type
)
205 * This function is called during
206 * 1) swapoff, when we have to make sure no
207 * left over slots are in cache when we remove
209 * 2) disabling of swap slot cache, when we run low
210 * on swap slots when allocating memory and need
211 * to return swap slots to global pool.
213 * We cannot acquire cpu hot plug lock here as
214 * this function can be invoked in the cpu
216 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
217 * -> memory allocation -> direct reclaim -> get_swap_page
218 * -> drain_swap_slots_cache
220 * Hence the loop over current online cpu below could miss cpu that
221 * is being brought online but not yet marked as online.
222 * That is okay as we do not schedule and run anything on a
223 * cpu before it has been marked online. Hence, we will not
224 * fill any swap slots in slots cache of such cpu.
225 * There are no slots on such cpu that need to be drained.
227 for_each_online_cpu(cpu
)
228 drain_slots_cache_cpu(cpu
, type
, false);
231 static int free_slot_cache(unsigned int cpu
)
233 mutex_lock(&swap_slots_cache_mutex
);
234 drain_slots_cache_cpu(cpu
, SLOTS_CACHE
| SLOTS_CACHE_RET
, true);
235 mutex_unlock(&swap_slots_cache_mutex
);
239 void enable_swap_slots_cache(void)
241 mutex_lock(&swap_slots_cache_enable_mutex
);
242 if (!swap_slot_cache_initialized
) {
245 ret
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "swap_slots_cache",
246 alloc_swap_slot_cache
, free_slot_cache
);
247 if (WARN_ONCE(ret
< 0, "Cache allocation failed (%s), operating "
248 "without swap slots cache.\n", __func__
))
251 swap_slot_cache_initialized
= true;
254 __reenable_swap_slots_cache();
256 mutex_unlock(&swap_slots_cache_enable_mutex
);
259 /* called with swap slot cache's alloc lock held */
260 static int refill_swap_slots_cache(struct swap_slots_cache
*cache
)
262 if (!use_swap_slot_cache
|| cache
->nr
)
266 if (swap_slot_cache_active
)
267 cache
->nr
= get_swap_pages(SWAP_SLOTS_CACHE_SIZE
,
273 int free_swap_slot(swp_entry_t entry
)
275 struct swap_slots_cache
*cache
;
277 cache
= raw_cpu_ptr(&swp_slots
);
278 if (likely(use_swap_slot_cache
&& cache
->slots_ret
)) {
279 spin_lock_irq(&cache
->free_lock
);
280 /* Swap slots cache may be deactivated before acquiring lock */
281 if (!use_swap_slot_cache
|| !cache
->slots_ret
) {
282 spin_unlock_irq(&cache
->free_lock
);
285 if (cache
->n_ret
>= SWAP_SLOTS_CACHE_SIZE
) {
287 * Return slots to global pool.
288 * The current swap_map value is SWAP_HAS_CACHE.
289 * Set it to 0 to indicate it is available for
290 * allocation in global pool
292 swapcache_free_entries(cache
->slots_ret
, cache
->n_ret
);
295 cache
->slots_ret
[cache
->n_ret
++] = entry
;
296 spin_unlock_irq(&cache
->free_lock
);
299 swapcache_free_entries(&entry
, 1);
305 swp_entry_t
get_swap_page(struct page
*page
)
308 struct swap_slots_cache
*cache
;
312 if (PageTransHuge(page
)) {
313 if (IS_ENABLED(CONFIG_THP_SWAP
))
314 get_swap_pages(1, &entry
, HPAGE_PMD_NR
);
319 * Preemption is allowed here, because we may sleep
320 * in refill_swap_slots_cache(). But it is safe, because
321 * accesses to the per-CPU data structure are protected by the
322 * mutex cache->alloc_lock.
324 * The alloc path here does not touch cache->slots_ret
325 * so cache->free_lock is not taken.
327 cache
= raw_cpu_ptr(&swp_slots
);
329 if (likely(check_cache_active() && cache
->slots
)) {
330 mutex_lock(&cache
->alloc_lock
);
334 entry
= cache
->slots
[cache
->cur
];
335 cache
->slots
[cache
->cur
++].val
= 0;
337 } else if (refill_swap_slots_cache(cache
)) {
341 mutex_unlock(&cache
->alloc_lock
);
346 get_swap_pages(1, &entry
, 1);
348 if (mem_cgroup_try_charge_swap(page
, entry
)) {
349 put_swap_page(page
, entry
);