]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - mm/swap_slots.c
Merge tag 'pci-v4.12-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/helgaa...
[mirror_ubuntu-bionic-kernel.git] / mm / swap_slots.c
1 /*
2 * Manage cache of swap slots to be used for and returned from
3 * swap.
4 *
5 * Copyright(c) 2016 Intel Corporation.
6 *
7 * Author: Tim Chen <tim.c.chen@linux.intel.com>
8 *
9 * We allocate the swap slots from the global pool and put
10 * it into local per cpu caches. This has the advantage
11 * of no needing to acquire the swap_info lock every time
12 * we need a new slot.
13 *
14 * There is also opportunity to simply return the slot
15 * to local caches without needing to acquire swap_info
16 * lock. We do not reuse the returned slots directly but
17 * move them back to the global pool in a batch. This
18 * allows the slots to coaellesce and reduce fragmentation.
19 *
20 * The swap entry allocated is marked with SWAP_HAS_CACHE
21 * flag in map_count that prevents it from being allocated
22 * again from the global pool.
23 *
24 * The swap slots cache is protected by a mutex instead of
25 * a spin lock as when we search for slots with scan_swap_map,
26 * we can possibly sleep.
27 */
28
29 #include <linux/swap_slots.h>
30 #include <linux/cpu.h>
31 #include <linux/cpumask.h>
32 #include <linux/vmalloc.h>
33 #include <linux/mutex.h>
34 #include <linux/mm.h>
35
36 #ifdef CONFIG_SWAP
37
38 static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
39 static bool swap_slot_cache_active;
40 bool swap_slot_cache_enabled;
41 static bool swap_slot_cache_initialized;
42 DEFINE_MUTEX(swap_slots_cache_mutex);
43 /* Serialize swap slots cache enable/disable operations */
44 DEFINE_MUTEX(swap_slots_cache_enable_mutex);
45
46 static void __drain_swap_slots_cache(unsigned int type);
47 static void deactivate_swap_slots_cache(void);
48 static void reactivate_swap_slots_cache(void);
49
50 #define use_swap_slot_cache (swap_slot_cache_active && \
51 swap_slot_cache_enabled && swap_slot_cache_initialized)
52 #define SLOTS_CACHE 0x1
53 #define SLOTS_CACHE_RET 0x2
54
55 static void deactivate_swap_slots_cache(void)
56 {
57 mutex_lock(&swap_slots_cache_mutex);
58 swap_slot_cache_active = false;
59 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
60 mutex_unlock(&swap_slots_cache_mutex);
61 }
62
63 static void reactivate_swap_slots_cache(void)
64 {
65 mutex_lock(&swap_slots_cache_mutex);
66 swap_slot_cache_active = true;
67 mutex_unlock(&swap_slots_cache_mutex);
68 }
69
70 /* Must not be called with cpu hot plug lock */
71 void disable_swap_slots_cache_lock(void)
72 {
73 mutex_lock(&swap_slots_cache_enable_mutex);
74 swap_slot_cache_enabled = false;
75 if (swap_slot_cache_initialized) {
76 /* serialize with cpu hotplug operations */
77 get_online_cpus();
78 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
79 put_online_cpus();
80 }
81 }
82
83 static void __reenable_swap_slots_cache(void)
84 {
85 swap_slot_cache_enabled = has_usable_swap();
86 }
87
88 void reenable_swap_slots_cache_unlock(void)
89 {
90 __reenable_swap_slots_cache();
91 mutex_unlock(&swap_slots_cache_enable_mutex);
92 }
93
94 static bool check_cache_active(void)
95 {
96 long pages;
97
98 if (!swap_slot_cache_enabled || !swap_slot_cache_initialized)
99 return false;
100
101 pages = get_nr_swap_pages();
102 if (!swap_slot_cache_active) {
103 if (pages > num_online_cpus() *
104 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
105 reactivate_swap_slots_cache();
106 goto out;
107 }
108
109 /* if global pool of slot caches too low, deactivate cache */
110 if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
111 deactivate_swap_slots_cache();
112 out:
113 return swap_slot_cache_active;
114 }
115
116 static int alloc_swap_slot_cache(unsigned int cpu)
117 {
118 struct swap_slots_cache *cache;
119 swp_entry_t *slots, *slots_ret;
120
121 /*
122 * Do allocation outside swap_slots_cache_mutex
123 * as kvzalloc could trigger reclaim and get_swap_page,
124 * which can lock swap_slots_cache_mutex.
125 */
126 slots = kvzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE,
127 GFP_KERNEL);
128 if (!slots)
129 return -ENOMEM;
130
131 slots_ret = kvzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE,
132 GFP_KERNEL);
133 if (!slots_ret) {
134 kvfree(slots);
135 return -ENOMEM;
136 }
137
138 mutex_lock(&swap_slots_cache_mutex);
139 cache = &per_cpu(swp_slots, cpu);
140 if (cache->slots || cache->slots_ret)
141 /* cache already allocated */
142 goto out;
143 if (!cache->lock_initialized) {
144 mutex_init(&cache->alloc_lock);
145 spin_lock_init(&cache->free_lock);
146 cache->lock_initialized = true;
147 }
148 cache->nr = 0;
149 cache->cur = 0;
150 cache->n_ret = 0;
151 cache->slots = slots;
152 slots = NULL;
153 cache->slots_ret = slots_ret;
154 slots_ret = NULL;
155 out:
156 mutex_unlock(&swap_slots_cache_mutex);
157 if (slots)
158 kvfree(slots);
159 if (slots_ret)
160 kvfree(slots_ret);
161 return 0;
162 }
163
164 static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
165 bool free_slots)
166 {
167 struct swap_slots_cache *cache;
168 swp_entry_t *slots = NULL;
169
170 cache = &per_cpu(swp_slots, cpu);
171 if ((type & SLOTS_CACHE) && cache->slots) {
172 mutex_lock(&cache->alloc_lock);
173 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
174 cache->cur = 0;
175 cache->nr = 0;
176 if (free_slots && cache->slots) {
177 kvfree(cache->slots);
178 cache->slots = NULL;
179 }
180 mutex_unlock(&cache->alloc_lock);
181 }
182 if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
183 spin_lock_irq(&cache->free_lock);
184 swapcache_free_entries(cache->slots_ret, cache->n_ret);
185 cache->n_ret = 0;
186 if (free_slots && cache->slots_ret) {
187 slots = cache->slots_ret;
188 cache->slots_ret = NULL;
189 }
190 spin_unlock_irq(&cache->free_lock);
191 if (slots)
192 kvfree(slots);
193 }
194 }
195
196 static void __drain_swap_slots_cache(unsigned int type)
197 {
198 unsigned int cpu;
199
200 /*
201 * This function is called during
202 * 1) swapoff, when we have to make sure no
203 * left over slots are in cache when we remove
204 * a swap device;
205 * 2) disabling of swap slot cache, when we run low
206 * on swap slots when allocating memory and need
207 * to return swap slots to global pool.
208 *
209 * We cannot acquire cpu hot plug lock here as
210 * this function can be invoked in the cpu
211 * hot plug path:
212 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
213 * -> memory allocation -> direct reclaim -> get_swap_page
214 * -> drain_swap_slots_cache
215 *
216 * Hence the loop over current online cpu below could miss cpu that
217 * is being brought online but not yet marked as online.
218 * That is okay as we do not schedule and run anything on a
219 * cpu before it has been marked online. Hence, we will not
220 * fill any swap slots in slots cache of such cpu.
221 * There are no slots on such cpu that need to be drained.
222 */
223 for_each_online_cpu(cpu)
224 drain_slots_cache_cpu(cpu, type, false);
225 }
226
227 static int free_slot_cache(unsigned int cpu)
228 {
229 mutex_lock(&swap_slots_cache_mutex);
230 drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
231 mutex_unlock(&swap_slots_cache_mutex);
232 return 0;
233 }
234
235 int enable_swap_slots_cache(void)
236 {
237 int ret = 0;
238
239 mutex_lock(&swap_slots_cache_enable_mutex);
240 if (swap_slot_cache_initialized) {
241 __reenable_swap_slots_cache();
242 goto out_unlock;
243 }
244
245 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
246 alloc_swap_slot_cache, free_slot_cache);
247 if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
248 "without swap slots cache.\n", __func__))
249 goto out_unlock;
250
251 swap_slot_cache_initialized = true;
252 __reenable_swap_slots_cache();
253 out_unlock:
254 mutex_unlock(&swap_slots_cache_enable_mutex);
255 return 0;
256 }
257
258 /* called with swap slot cache's alloc lock held */
259 static int refill_swap_slots_cache(struct swap_slots_cache *cache)
260 {
261 if (!use_swap_slot_cache || cache->nr)
262 return 0;
263
264 cache->cur = 0;
265 if (swap_slot_cache_active)
266 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE, cache->slots);
267
268 return cache->nr;
269 }
270
271 int free_swap_slot(swp_entry_t entry)
272 {
273 struct swap_slots_cache *cache;
274
275 cache = &get_cpu_var(swp_slots);
276 if (use_swap_slot_cache && cache->slots_ret) {
277 spin_lock_irq(&cache->free_lock);
278 /* Swap slots cache may be deactivated before acquiring lock */
279 if (!use_swap_slot_cache) {
280 spin_unlock_irq(&cache->free_lock);
281 goto direct_free;
282 }
283 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
284 /*
285 * Return slots to global pool.
286 * The current swap_map value is SWAP_HAS_CACHE.
287 * Set it to 0 to indicate it is available for
288 * allocation in global pool
289 */
290 swapcache_free_entries(cache->slots_ret, cache->n_ret);
291 cache->n_ret = 0;
292 }
293 cache->slots_ret[cache->n_ret++] = entry;
294 spin_unlock_irq(&cache->free_lock);
295 } else {
296 direct_free:
297 swapcache_free_entries(&entry, 1);
298 }
299 put_cpu_var(swp_slots);
300
301 return 0;
302 }
303
304 swp_entry_t get_swap_page(void)
305 {
306 swp_entry_t entry, *pentry;
307 struct swap_slots_cache *cache;
308
309 /*
310 * Preemption is allowed here, because we may sleep
311 * in refill_swap_slots_cache(). But it is safe, because
312 * accesses to the per-CPU data structure are protected by the
313 * mutex cache->alloc_lock.
314 *
315 * The alloc path here does not touch cache->slots_ret
316 * so cache->free_lock is not taken.
317 */
318 cache = raw_cpu_ptr(&swp_slots);
319
320 entry.val = 0;
321 if (check_cache_active()) {
322 mutex_lock(&cache->alloc_lock);
323 if (cache->slots) {
324 repeat:
325 if (cache->nr) {
326 pentry = &cache->slots[cache->cur++];
327 entry = *pentry;
328 pentry->val = 0;
329 cache->nr--;
330 } else {
331 if (refill_swap_slots_cache(cache))
332 goto repeat;
333 }
334 }
335 mutex_unlock(&cache->alloc_lock);
336 if (entry.val)
337 return entry;
338 }
339
340 get_swap_pages(1, &entry);
341
342 return entry;
343 }
344
345 #endif /* CONFIG_SWAP */