]>
Commit | Line | Data |
---|---|---|
fd534e9b | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1e214a5c SL |
2 | /* |
3 | * Virtio balloon implementation, inspired by Dor Laor and Marcelo | |
6b35e407 RR |
4 | * Tosatti's implementations. |
5 | * | |
6 | * Copyright 2008 Rusty Russell IBM Corporation | |
6b35e407 | 7 | */ |
1e214a5c | 8 | |
6b35e407 RR |
9 | #include <linux/virtio.h> |
10 | #include <linux/virtio_balloon.h> | |
11 | #include <linux/swap.h> | |
fad7b7b2 | 12 | #include <linux/workqueue.h> |
6659a0f0 | 13 | #include <linux/delay.h> |
5a0e3ad6 | 14 | #include <linux/slab.h> |
b5a2c4f1 | 15 | #include <linux/module.h> |
e2250429 | 16 | #include <linux/balloon_compaction.h> |
3d2a3774 | 17 | #include <linux/wait.h> |
5057dcd0 | 18 | #include <linux/mm.h> |
b1123ea6 | 19 | #include <linux/mount.h> |
50d34394 | 20 | #include <linux/magic.h> |
99558d20 | 21 | #include <linux/pseudo_fs.h> |
6b35e407 | 22 | |
3ccc9372 MT |
23 | /* |
24 | * Balloon device works in 4K page units. So each page is pointed to by | |
25 | * multiple balloon pages. All memory counters in this driver are in balloon | |
26 | * page units. | |
27 | */ | |
e2250429 RA |
28 | #define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) |
29 | #define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256 | |
5a10b7db RM |
30 | #define VIRTBALLOON_OOM_NOTIFY_PRIORITY 80 |
31 | ||
86a55978 WW |
32 | #define VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG (__GFP_NORETRY | __GFP_NOWARN | \ |
33 | __GFP_NOMEMALLOC) | |
34 | /* The order of free page blocks to report to host */ | |
2a946fa1 | 35 | #define VIRTIO_BALLOON_HINT_BLOCK_ORDER (MAX_ORDER - 1) |
86a55978 | 36 | /* The size of a free page block in bytes */ |
2a946fa1 MT |
37 | #define VIRTIO_BALLOON_HINT_BLOCK_BYTES \ |
38 | (1 << (VIRTIO_BALLOON_HINT_BLOCK_ORDER + PAGE_SHIFT)) | |
63b9b80e | 39 | #define VIRTIO_BALLOON_HINT_BLOCK_PAGES (1 << VIRTIO_BALLOON_HINT_BLOCK_ORDER) |
86a55978 | 40 | |
b1123ea6 MK |
41 | #ifdef CONFIG_BALLOON_COMPACTION |
42 | static struct vfsmount *balloon_mnt; | |
43 | #endif | |
44 | ||
86a55978 WW |
45 | enum virtio_balloon_vq { |
46 | VIRTIO_BALLOON_VQ_INFLATE, | |
47 | VIRTIO_BALLOON_VQ_DEFLATE, | |
48 | VIRTIO_BALLOON_VQ_STATS, | |
49 | VIRTIO_BALLOON_VQ_FREE_PAGE, | |
50 | VIRTIO_BALLOON_VQ_MAX | |
51 | }; | |
52 | ||
bf4dc0b2 WW |
53 | enum virtio_balloon_config_read { |
54 | VIRTIO_BALLOON_CONFIG_READ_CMD_ID = 0, | |
55 | }; | |
56 | ||
25e65e4e | 57 | struct virtio_balloon { |
6b35e407 | 58 | struct virtio_device *vdev; |
86a55978 WW |
59 | struct virtqueue *inflate_vq, *deflate_vq, *stats_vq, *free_page_vq; |
60 | ||
61 | /* Balloon's own wq for cpu-intensive work items */ | |
62 | struct workqueue_struct *balloon_wq; | |
63 | /* The free page reporting work item submitted to the balloon wq */ | |
64 | struct work_struct report_free_page_work; | |
6b35e407 | 65 | |
fad7b7b2 | 66 | /* The balloon servicing is delegated to a freezable workqueue. */ |
fd0e21c3 PM |
67 | struct work_struct update_balloon_stats_work; |
68 | struct work_struct update_balloon_size_work; | |
6b35e407 | 69 | |
fad7b7b2 PM |
70 | /* Prevent updating balloon when it is being canceled. */ |
71 | spinlock_t stop_update_lock; | |
72 | bool stop_update; | |
bf4dc0b2 WW |
73 | /* Bitmap to indicate if reading the related config fields are needed */ |
74 | unsigned long config_read_bitmap; | |
6b35e407 | 75 | |
86a55978 WW |
76 | /* The list of allocated free pages, waiting to be given back to mm */ |
77 | struct list_head free_page_list; | |
78 | spinlock_t free_page_list_lock; | |
79 | /* The number of free page blocks on the above list */ | |
80 | unsigned long num_free_page_blocks; | |
bf4dc0b2 WW |
81 | /* |
82 | * The cmd id received from host. | |
83 | * Read it via virtio_balloon_cmd_id_received to get the latest value | |
84 | * sent from host. | |
85 | */ | |
86 | u32 cmd_id_received_cache; | |
86a55978 WW |
87 | /* The cmd id that is actively in use */ |
88 | __virtio32 cmd_id_active; | |
89 | /* Buffer to store the stop sign */ | |
90 | __virtio32 cmd_id_stop; | |
91 | ||
6b35e407 | 92 | /* Waiting for host to ack the pages we released. */ |
9c378abc | 93 | wait_queue_head_t acked; |
6b35e407 | 94 | |
3ccc9372 | 95 | /* Number of balloon pages we've told the Host we're not using. */ |
6b35e407 | 96 | unsigned int num_pages; |
3ccc9372 | 97 | /* |
e2250429 RA |
98 | * The pages we've told the Host we're not using are enqueued |
99 | * at vb_dev_info->pages list. | |
3ccc9372 MT |
100 | * Each page on this list adds VIRTIO_BALLOON_PAGES_PER_PAGE |
101 | * to num_pages above. | |
102 | */ | |
9d1ba805 | 103 | struct balloon_dev_info vb_dev_info; |
e2250429 RA |
104 | |
105 | /* Synchronize access/update to this struct virtio_balloon elements */ | |
106 | struct mutex balloon_lock; | |
6b35e407 RR |
107 | |
108 | /* The array of pfns we tell the Host about. */ | |
109 | unsigned int num_pfns; | |
87c9403b | 110 | __virtio32 pfns[VIRTIO_BALLOON_ARRAY_PFNS_MAX]; |
9564e138 AL |
111 | |
112 | /* Memory statistics */ | |
113 | struct virtio_balloon_stat stats[VIRTIO_BALLOON_S_NR]; | |
5a10b7db | 114 | |
71994620 WW |
115 | /* To register a shrinker to shrink memory upon memory pressure */ |
116 | struct shrinker shrinker; | |
6b35e407 RR |
117 | }; |
118 | ||
119 | static struct virtio_device_id id_table[] = { | |
120 | { VIRTIO_ID_BALLOON, VIRTIO_DEV_ANY_ID }, | |
121 | { 0 }, | |
122 | }; | |
123 | ||
1b4aa2fa HB |
124 | static u32 page_to_balloon_pfn(struct page *page) |
125 | { | |
126 | unsigned long pfn = page_to_pfn(page); | |
127 | ||
128 | BUILD_BUG_ON(PAGE_SHIFT < VIRTIO_BALLOON_PFN_SHIFT); | |
129 | /* Convert pfn from Linux page size to balloon page size. */ | |
3ccc9372 MT |
130 | return pfn * VIRTIO_BALLOON_PAGES_PER_PAGE; |
131 | } | |
132 | ||
6b35e407 RR |
133 | static void balloon_ack(struct virtqueue *vq) |
134 | { | |
9c378abc | 135 | struct virtio_balloon *vb = vq->vdev->priv; |
6b35e407 | 136 | |
9c378abc | 137 | wake_up(&vb->acked); |
6b35e407 RR |
138 | } |
139 | ||
140 | static void tell_host(struct virtio_balloon *vb, struct virtqueue *vq) | |
141 | { | |
142 | struct scatterlist sg; | |
9c378abc | 143 | unsigned int len; |
6b35e407 RR |
144 | |
145 | sg_init_one(&sg, vb->pfns, sizeof(vb->pfns[0]) * vb->num_pfns); | |
146 | ||
6b35e407 | 147 | /* We should always be able to add one buffer to an empty queue. */ |
4951cc90 | 148 | virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL); |
946cfe0e | 149 | virtqueue_kick(vq); |
6b35e407 RR |
150 | |
151 | /* When host has read buffer, this completes via balloon_ack */ | |
9c378abc | 152 | wait_event(vb->acked, virtqueue_get_buf(vq, &len)); |
fd0e21c3 | 153 | |
6b35e407 RR |
154 | } |
155 | ||
87c9403b MT |
156 | static void set_page_pfns(struct virtio_balloon *vb, |
157 | __virtio32 pfns[], struct page *page) | |
3ccc9372 MT |
158 | { |
159 | unsigned int i; | |
160 | ||
f9aada5f WW |
161 | /* |
162 | * Set balloon pfns pointing at this page. | |
163 | * Note that the first pfn points at start of the page. | |
164 | */ | |
3ccc9372 | 165 | for (i = 0; i < VIRTIO_BALLOON_PAGES_PER_PAGE; i++) |
87c9403b MT |
166 | pfns[i] = cpu_to_virtio32(vb->vdev, |
167 | page_to_balloon_pfn(page) + i); | |
3ccc9372 MT |
168 | } |
169 | ||
fad7b7b2 | 170 | static unsigned fill_balloon(struct virtio_balloon *vb, size_t num) |
6b35e407 | 171 | { |
fad7b7b2 | 172 | unsigned num_allocated_pages; |
c7cdff0e MT |
173 | unsigned num_pfns; |
174 | struct page *page; | |
175 | LIST_HEAD(pages); | |
e2250429 | 176 | |
6b35e407 RR |
177 | /* We can only do one array worth at a time. */ |
178 | num = min(num, ARRAY_SIZE(vb->pfns)); | |
179 | ||
c7cdff0e MT |
180 | for (num_pfns = 0; num_pfns < num; |
181 | num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { | |
182 | struct page *page = balloon_page_alloc(); | |
e2250429 | 183 | |
6b35e407 | 184 | if (!page) { |
800ba5ea | 185 | dev_info_ratelimited(&vb->vdev->dev, |
b7dfde95 LT |
186 | "Out of puff! Can't get %u pages\n", |
187 | VIRTIO_BALLOON_PAGES_PER_PAGE); | |
6b35e407 RR |
188 | /* Sleep for at least 1/5 of a second before retry. */ |
189 | msleep(200); | |
190 | break; | |
191 | } | |
c7cdff0e MT |
192 | |
193 | balloon_page_push(&pages, page); | |
194 | } | |
195 | ||
196 | mutex_lock(&vb->balloon_lock); | |
197 | ||
198 | vb->num_pfns = 0; | |
199 | ||
200 | while ((page = balloon_page_pop(&pages))) { | |
201 | balloon_page_enqueue(&vb->vb_dev_info, page); | |
202 | ||
87c9403b | 203 | set_page_pfns(vb, vb->pfns + vb->num_pfns, page); |
3ccc9372 | 204 | vb->num_pages += VIRTIO_BALLOON_PAGES_PER_PAGE; |
997e1208 DL |
205 | if (!virtio_has_feature(vb->vdev, |
206 | VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) | |
207 | adjust_managed_page_count(page, -1); | |
d9e427f6 | 208 | vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE; |
6b35e407 RR |
209 | } |
210 | ||
fad7b7b2 | 211 | num_allocated_pages = vb->num_pfns; |
e2250429 RA |
212 | /* Did we get any? */ |
213 | if (vb->num_pfns != 0) | |
214 | tell_host(vb, vb->inflate_vq); | |
215 | mutex_unlock(&vb->balloon_lock); | |
fad7b7b2 PM |
216 | |
217 | return num_allocated_pages; | |
6b35e407 RR |
218 | } |
219 | ||
195a8c43 LL |
220 | static void release_pages_balloon(struct virtio_balloon *vb, |
221 | struct list_head *pages) | |
6b35e407 | 222 | { |
195a8c43 | 223 | struct page *page, *next; |
6b35e407 | 224 | |
195a8c43 | 225 | list_for_each_entry_safe(page, next, pages, lru) { |
997e1208 DL |
226 | if (!virtio_has_feature(vb->vdev, |
227 | VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) | |
228 | adjust_managed_page_count(page, 1); | |
195a8c43 | 229 | list_del(&page->lru); |
d6d86c0a | 230 | put_page(page); /* balloon reference */ |
6b35e407 RR |
231 | } |
232 | } | |
233 | ||
1fd9c672 | 234 | static unsigned leak_balloon(struct virtio_balloon *vb, size_t num) |
6b35e407 | 235 | { |
1fd9c672 | 236 | unsigned num_freed_pages; |
6b35e407 | 237 | struct page *page; |
9d1ba805 | 238 | struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info; |
195a8c43 | 239 | LIST_HEAD(pages); |
6b35e407 RR |
240 | |
241 | /* We can only do one array worth at a time. */ | |
242 | num = min(num, ARRAY_SIZE(vb->pfns)); | |
243 | ||
e2250429 | 244 | mutex_lock(&vb->balloon_lock); |
37cf99e0 KN |
245 | /* We can't release more pages than taken */ |
246 | num = min(num, (size_t)vb->num_pages); | |
3ccc9372 MT |
247 | for (vb->num_pfns = 0; vb->num_pfns < num; |
248 | vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { | |
e2250429 RA |
249 | page = balloon_page_dequeue(vb_dev_info); |
250 | if (!page) | |
251 | break; | |
87c9403b | 252 | set_page_pfns(vb, vb->pfns + vb->num_pfns, page); |
195a8c43 | 253 | list_add(&page->lru, &pages); |
3ccc9372 | 254 | vb->num_pages -= VIRTIO_BALLOON_PAGES_PER_PAGE; |
6b35e407 RR |
255 | } |
256 | ||
1fd9c672 | 257 | num_freed_pages = vb->num_pfns; |
bf50e69f DH |
258 | /* |
259 | * Note that if | |
260 | * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST); | |
261 | * is true, we *have* to do it in this order | |
262 | */ | |
8c6bab4f LC |
263 | if (vb->num_pfns != 0) |
264 | tell_host(vb, vb->deflate_vq); | |
195a8c43 | 265 | release_pages_balloon(vb, &pages); |
f68b992b | 266 | mutex_unlock(&vb->balloon_lock); |
1fd9c672 | 267 | return num_freed_pages; |
6b35e407 RR |
268 | } |
269 | ||
9564e138 AL |
270 | static inline void update_stat(struct virtio_balloon *vb, int idx, |
271 | u16 tag, u64 val) | |
272 | { | |
273 | BUG_ON(idx >= VIRTIO_BALLOON_S_NR); | |
df81b29c MT |
274 | vb->stats[idx].tag = cpu_to_virtio16(vb->vdev, tag); |
275 | vb->stats[idx].val = cpu_to_virtio64(vb->vdev, val); | |
9564e138 AL |
276 | } |
277 | ||
278 | #define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT) | |
279 | ||
9646b26e | 280 | static unsigned int update_balloon_stats(struct virtio_balloon *vb) |
9564e138 AL |
281 | { |
282 | unsigned long events[NR_VM_EVENT_ITEMS]; | |
283 | struct sysinfo i; | |
9646b26e | 284 | unsigned int idx = 0; |
5057dcd0 | 285 | long available; |
4d32029b | 286 | unsigned long caches; |
9564e138 AL |
287 | |
288 | all_vm_events(events); | |
289 | si_meminfo(&i); | |
290 | ||
5057dcd0 | 291 | available = si_mem_available(); |
4d32029b | 292 | caches = global_node_page_state(NR_FILE_PAGES); |
5057dcd0 | 293 | |
f0bb2d50 | 294 | #ifdef CONFIG_VM_EVENT_COUNTERS |
9564e138 AL |
295 | update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN, |
296 | pages_to_bytes(events[PSWPIN])); | |
297 | update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT, | |
298 | pages_to_bytes(events[PSWPOUT])); | |
299 | update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]); | |
300 | update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]); | |
6c64fe7f JH |
301 | #ifdef CONFIG_HUGETLB_PAGE |
302 | update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC, | |
303 | events[HTLB_BUDDY_PGALLOC]); | |
304 | update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL, | |
305 | events[HTLB_BUDDY_PGALLOC_FAIL]); | |
306 | #endif | |
f0bb2d50 | 307 | #endif |
9564e138 AL |
308 | update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE, |
309 | pages_to_bytes(i.freeram)); | |
310 | update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT, | |
311 | pages_to_bytes(i.totalram)); | |
5057dcd0 IR |
312 | update_stat(vb, idx++, VIRTIO_BALLOON_S_AVAIL, |
313 | pages_to_bytes(available)); | |
4d32029b TG |
314 | update_stat(vb, idx++, VIRTIO_BALLOON_S_CACHES, |
315 | pages_to_bytes(caches)); | |
9646b26e LP |
316 | |
317 | return idx; | |
9564e138 AL |
318 | } |
319 | ||
320 | /* | |
321 | * While most virtqueues communicate guest-initiated requests to the hypervisor, | |
322 | * the stats queue operates in reverse. The driver initializes the virtqueue | |
323 | * with a single buffer. From that point forward, all conversations consist of | |
324 | * a hypervisor request (a call to this function) which directs us to refill | |
1f34c71a | 325 | * the virtqueue with a fresh stats buffer. Since stats collection can sleep, |
fad7b7b2 PM |
326 | * we delegate the job to a freezable workqueue that will do the actual work via |
327 | * stats_handle_request(). | |
9564e138 | 328 | */ |
1f34c71a | 329 | static void stats_request(struct virtqueue *vq) |
9564e138 | 330 | { |
9c378abc | 331 | struct virtio_balloon *vb = vq->vdev->priv; |
9564e138 | 332 | |
fad7b7b2 PM |
333 | spin_lock(&vb->stop_update_lock); |
334 | if (!vb->stop_update) | |
fd0e21c3 | 335 | queue_work(system_freezable_wq, &vb->update_balloon_stats_work); |
fad7b7b2 | 336 | spin_unlock(&vb->stop_update_lock); |
1f34c71a AL |
337 | } |
338 | ||
339 | static void stats_handle_request(struct virtio_balloon *vb) | |
340 | { | |
341 | struct virtqueue *vq; | |
342 | struct scatterlist sg; | |
9646b26e | 343 | unsigned int len, num_stats; |
9564e138 | 344 | |
9646b26e | 345 | num_stats = update_balloon_stats(vb); |
9564e138 | 346 | |
1f34c71a | 347 | vq = vb->stats_vq; |
9c378abc MT |
348 | if (!virtqueue_get_buf(vq, &len)) |
349 | return; | |
9646b26e | 350 | sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); |
4951cc90 | 351 | virtqueue_add_outbuf(vq, &sg, 1, vb, GFP_KERNEL); |
946cfe0e | 352 | virtqueue_kick(vq); |
9564e138 AL |
353 | } |
354 | ||
bdc1681c | 355 | static inline s64 towards_target(struct virtio_balloon *vb) |
6b35e407 | 356 | { |
1a87228f | 357 | s64 target; |
df81b29c | 358 | u32 num_pages; |
1a87228f | 359 | |
df81b29c MT |
360 | virtio_cread(vb->vdev, struct virtio_balloon_config, num_pages, |
361 | &num_pages); | |
855e0c52 | 362 | |
df81b29c MT |
363 | /* Legacy balloon config space is LE, unlike all other devices. */ |
364 | if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1)) | |
365 | num_pages = le32_to_cpu((__force __le32)num_pages); | |
366 | ||
367 | target = num_pages; | |
1a87228f | 368 | return target - vb->num_pages; |
6b35e407 RR |
369 | } |
370 | ||
86a55978 WW |
371 | /* Gives back @num_to_return blocks of free pages to mm. */ |
372 | static unsigned long return_free_pages_to_mm(struct virtio_balloon *vb, | |
373 | unsigned long num_to_return) | |
374 | { | |
375 | struct page *page; | |
376 | unsigned long num_returned; | |
377 | ||
378 | spin_lock_irq(&vb->free_page_list_lock); | |
379 | for (num_returned = 0; num_returned < num_to_return; num_returned++) { | |
380 | page = balloon_page_pop(&vb->free_page_list); | |
381 | if (!page) | |
382 | break; | |
383 | free_pages((unsigned long)page_address(page), | |
2a946fa1 | 384 | VIRTIO_BALLOON_HINT_BLOCK_ORDER); |
86a55978 WW |
385 | } |
386 | vb->num_free_page_blocks -= num_returned; | |
387 | spin_unlock_irq(&vb->free_page_list_lock); | |
388 | ||
389 | return num_returned; | |
390 | } | |
391 | ||
bf4dc0b2 WW |
392 | static void virtio_balloon_queue_free_page_work(struct virtio_balloon *vb) |
393 | { | |
394 | if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) | |
395 | return; | |
396 | ||
397 | /* No need to queue the work if the bit was already set. */ | |
398 | if (test_and_set_bit(VIRTIO_BALLOON_CONFIG_READ_CMD_ID, | |
399 | &vb->config_read_bitmap)) | |
400 | return; | |
401 | ||
402 | queue_work(vb->balloon_wq, &vb->report_free_page_work); | |
403 | } | |
404 | ||
86a55978 WW |
405 | static void virtballoon_changed(struct virtio_device *vdev) |
406 | { | |
407 | struct virtio_balloon *vb = vdev->priv; | |
408 | unsigned long flags; | |
86a55978 | 409 | |
bf4dc0b2 WW |
410 | spin_lock_irqsave(&vb->stop_update_lock, flags); |
411 | if (!vb->stop_update) { | |
412 | queue_work(system_freezable_wq, | |
413 | &vb->update_balloon_size_work); | |
414 | virtio_balloon_queue_free_page_work(vb); | |
86a55978 | 415 | } |
bf4dc0b2 | 416 | spin_unlock_irqrestore(&vb->stop_update_lock, flags); |
86a55978 WW |
417 | } |
418 | ||
6b35e407 RR |
419 | static void update_balloon_size(struct virtio_balloon *vb) |
420 | { | |
df81b29c MT |
421 | u32 actual = vb->num_pages; |
422 | ||
423 | /* Legacy balloon config space is LE, unlike all other devices. */ | |
424 | if (!virtio_has_feature(vb->vdev, VIRTIO_F_VERSION_1)) | |
425 | actual = (__force u32)cpu_to_le32(actual); | |
6b35e407 | 426 | |
3459f11a | 427 | virtio_cwrite(vb->vdev, struct virtio_balloon_config, actual, |
855e0c52 | 428 | &actual); |
6b35e407 RR |
429 | } |
430 | ||
fd0e21c3 | 431 | static void update_balloon_stats_func(struct work_struct *work) |
6b35e407 | 432 | { |
fd0e21c3 | 433 | struct virtio_balloon *vb; |
3d2a3774 | 434 | |
fd0e21c3 PM |
435 | vb = container_of(work, struct virtio_balloon, |
436 | update_balloon_stats_work); | |
437 | stats_handle_request(vb); | |
438 | } | |
1f74ef0f | 439 | |
fd0e21c3 | 440 | static void update_balloon_size_func(struct work_struct *work) |
6b35e407 | 441 | { |
fad7b7b2 PM |
442 | struct virtio_balloon *vb; |
443 | s64 diff; | |
3d2a3774 | 444 | |
fd0e21c3 PM |
445 | vb = container_of(work, struct virtio_balloon, |
446 | update_balloon_size_work); | |
fad7b7b2 | 447 | diff = towards_target(vb); |
1f74ef0f | 448 | |
53e946cb WW |
449 | if (!diff) |
450 | return; | |
451 | ||
fad7b7b2 PM |
452 | if (diff > 0) |
453 | diff -= fill_balloon(vb, diff); | |
53e946cb | 454 | else |
fad7b7b2 PM |
455 | diff += leak_balloon(vb, -diff); |
456 | update_balloon_size(vb); | |
457 | ||
458 | if (diff) | |
459 | queue_work(system_freezable_wq, work); | |
6b35e407 RR |
460 | } |
461 | ||
be91c33d | 462 | static int init_vqs(struct virtio_balloon *vb) |
6b35e407 | 463 | { |
86a55978 WW |
464 | struct virtqueue *vqs[VIRTIO_BALLOON_VQ_MAX]; |
465 | vq_callback_t *callbacks[VIRTIO_BALLOON_VQ_MAX]; | |
466 | const char *names[VIRTIO_BALLOON_VQ_MAX]; | |
467 | int err; | |
6b35e407 | 468 | |
be91c33d | 469 | /* |
86a55978 WW |
470 | * Inflateq and deflateq are used unconditionally. The names[] |
471 | * will be NULL if the related feature is not enabled, which will | |
472 | * cause no allocation for the corresponding virtqueue in find_vqs. | |
be91c33d | 473 | */ |
86a55978 WW |
474 | callbacks[VIRTIO_BALLOON_VQ_INFLATE] = balloon_ack; |
475 | names[VIRTIO_BALLOON_VQ_INFLATE] = "inflate"; | |
476 | callbacks[VIRTIO_BALLOON_VQ_DEFLATE] = balloon_ack; | |
477 | names[VIRTIO_BALLOON_VQ_DEFLATE] = "deflate"; | |
478 | names[VIRTIO_BALLOON_VQ_STATS] = NULL; | |
479 | names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; | |
480 | ||
481 | if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { | |
482 | names[VIRTIO_BALLOON_VQ_STATS] = "stats"; | |
483 | callbacks[VIRTIO_BALLOON_VQ_STATS] = stats_request; | |
484 | } | |
485 | ||
486 | if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { | |
487 | names[VIRTIO_BALLOON_VQ_FREE_PAGE] = "free_page_vq"; | |
488 | callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL; | |
489 | } | |
490 | ||
491 | err = vb->vdev->config->find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, | |
492 | vqs, callbacks, names, NULL, NULL); | |
d2a7ddda | 493 | if (err) |
be91c33d | 494 | return err; |
6b35e407 | 495 | |
86a55978 WW |
496 | vb->inflate_vq = vqs[VIRTIO_BALLOON_VQ_INFLATE]; |
497 | vb->deflate_vq = vqs[VIRTIO_BALLOON_VQ_DEFLATE]; | |
9564e138 AL |
498 | if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) { |
499 | struct scatterlist sg; | |
9646b26e | 500 | unsigned int num_stats; |
86a55978 | 501 | vb->stats_vq = vqs[VIRTIO_BALLOON_VQ_STATS]; |
9564e138 AL |
502 | |
503 | /* | |
504 | * Prime this virtqueue with one buffer so the hypervisor can | |
4951cc90 | 505 | * use it to signal us later (it can't be broken yet!). |
9564e138 | 506 | */ |
9646b26e | 507 | num_stats = update_balloon_stats(vb); |
fc865322 | 508 | |
9646b26e | 509 | sg_init_one(&sg, vb->stats, sizeof(vb->stats[0]) * num_stats); |
74cf5b16 WW |
510 | err = virtqueue_add_outbuf(vb->stats_vq, &sg, 1, vb, |
511 | GFP_KERNEL); | |
512 | if (err) { | |
513 | dev_warn(&vb->vdev->dev, "%s: add stat_vq failed\n", | |
514 | __func__); | |
515 | return err; | |
516 | } | |
946cfe0e | 517 | virtqueue_kick(vb->stats_vq); |
9564e138 | 518 | } |
86a55978 WW |
519 | |
520 | if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) | |
521 | vb->free_page_vq = vqs[VIRTIO_BALLOON_VQ_FREE_PAGE]; | |
522 | ||
523 | return 0; | |
524 | } | |
525 | ||
bf4dc0b2 WW |
526 | static u32 virtio_balloon_cmd_id_received(struct virtio_balloon *vb) |
527 | { | |
528 | if (test_and_clear_bit(VIRTIO_BALLOON_CONFIG_READ_CMD_ID, | |
529 | &vb->config_read_bitmap)) | |
530 | virtio_cread(vb->vdev, struct virtio_balloon_config, | |
531 | free_page_report_cmd_id, | |
532 | &vb->cmd_id_received_cache); | |
533 | ||
534 | return vb->cmd_id_received_cache; | |
535 | } | |
536 | ||
86a55978 WW |
537 | static int send_cmd_id_start(struct virtio_balloon *vb) |
538 | { | |
539 | struct scatterlist sg; | |
540 | struct virtqueue *vq = vb->free_page_vq; | |
541 | int err, unused; | |
542 | ||
543 | /* Detach all the used buffers from the vq */ | |
544 | while (virtqueue_get_buf(vq, &unused)) | |
545 | ; | |
546 | ||
bf4dc0b2 WW |
547 | vb->cmd_id_active = virtio32_to_cpu(vb->vdev, |
548 | virtio_balloon_cmd_id_received(vb)); | |
86a55978 WW |
549 | sg_init_one(&sg, &vb->cmd_id_active, sizeof(vb->cmd_id_active)); |
550 | err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_active, GFP_KERNEL); | |
551 | if (!err) | |
552 | virtqueue_kick(vq); | |
553 | return err; | |
554 | } | |
555 | ||
556 | static int send_cmd_id_stop(struct virtio_balloon *vb) | |
557 | { | |
558 | struct scatterlist sg; | |
559 | struct virtqueue *vq = vb->free_page_vq; | |
560 | int err, unused; | |
561 | ||
562 | /* Detach all the used buffers from the vq */ | |
563 | while (virtqueue_get_buf(vq, &unused)) | |
564 | ; | |
565 | ||
566 | sg_init_one(&sg, &vb->cmd_id_stop, sizeof(vb->cmd_id_stop)); | |
567 | err = virtqueue_add_outbuf(vq, &sg, 1, &vb->cmd_id_stop, GFP_KERNEL); | |
568 | if (!err) | |
569 | virtqueue_kick(vq); | |
570 | return err; | |
571 | } | |
572 | ||
573 | static int get_free_page_and_send(struct virtio_balloon *vb) | |
574 | { | |
575 | struct virtqueue *vq = vb->free_page_vq; | |
576 | struct page *page; | |
577 | struct scatterlist sg; | |
578 | int err, unused; | |
579 | void *p; | |
580 | ||
581 | /* Detach all the used buffers from the vq */ | |
582 | while (virtqueue_get_buf(vq, &unused)) | |
583 | ; | |
584 | ||
585 | page = alloc_pages(VIRTIO_BALLOON_FREE_PAGE_ALLOC_FLAG, | |
2a946fa1 | 586 | VIRTIO_BALLOON_HINT_BLOCK_ORDER); |
86a55978 WW |
587 | /* |
588 | * When the allocation returns NULL, it indicates that we have got all | |
589 | * the possible free pages, so return -EINTR to stop. | |
590 | */ | |
591 | if (!page) | |
592 | return -EINTR; | |
593 | ||
594 | p = page_address(page); | |
2a946fa1 | 595 | sg_init_one(&sg, p, VIRTIO_BALLOON_HINT_BLOCK_BYTES); |
86a55978 WW |
596 | /* There is always 1 entry reserved for the cmd id to use. */ |
597 | if (vq->num_free > 1) { | |
598 | err = virtqueue_add_inbuf(vq, &sg, 1, p, GFP_KERNEL); | |
599 | if (unlikely(err)) { | |
600 | free_pages((unsigned long)p, | |
2a946fa1 | 601 | VIRTIO_BALLOON_HINT_BLOCK_ORDER); |
86a55978 WW |
602 | return err; |
603 | } | |
604 | virtqueue_kick(vq); | |
605 | spin_lock_irq(&vb->free_page_list_lock); | |
606 | balloon_page_push(&vb->free_page_list, page); | |
607 | vb->num_free_page_blocks++; | |
608 | spin_unlock_irq(&vb->free_page_list_lock); | |
609 | } else { | |
610 | /* | |
611 | * The vq has no available entry to add this page block, so | |
612 | * just free it. | |
613 | */ | |
2a946fa1 | 614 | free_pages((unsigned long)p, VIRTIO_BALLOON_HINT_BLOCK_ORDER); |
86a55978 WW |
615 | } |
616 | ||
be91c33d AS |
617 | return 0; |
618 | } | |
619 | ||
86a55978 WW |
620 | static int send_free_pages(struct virtio_balloon *vb) |
621 | { | |
622 | int err; | |
623 | u32 cmd_id_active; | |
624 | ||
625 | while (1) { | |
626 | /* | |
627 | * If a stop id or a new cmd id was just received from host, | |
628 | * stop the reporting. | |
629 | */ | |
630 | cmd_id_active = virtio32_to_cpu(vb->vdev, vb->cmd_id_active); | |
bf4dc0b2 WW |
631 | if (unlikely(cmd_id_active != |
632 | virtio_balloon_cmd_id_received(vb))) | |
86a55978 WW |
633 | break; |
634 | ||
635 | /* | |
636 | * The free page blocks are allocated and sent to host one by | |
637 | * one. | |
638 | */ | |
639 | err = get_free_page_and_send(vb); | |
640 | if (err == -EINTR) | |
641 | break; | |
642 | else if (unlikely(err)) | |
643 | return err; | |
644 | } | |
645 | ||
646 | return 0; | |
647 | } | |
648 | ||
bf4dc0b2 | 649 | static void virtio_balloon_report_free_page(struct virtio_balloon *vb) |
86a55978 WW |
650 | { |
651 | int err; | |
86a55978 WW |
652 | struct device *dev = &vb->vdev->dev; |
653 | ||
654 | /* Start by sending the received cmd id to host with an outbuf. */ | |
655 | err = send_cmd_id_start(vb); | |
656 | if (unlikely(err)) | |
657 | dev_err(dev, "Failed to send a start id, err = %d\n", err); | |
658 | ||
659 | err = send_free_pages(vb); | |
660 | if (unlikely(err)) | |
661 | dev_err(dev, "Failed to send a free page, err = %d\n", err); | |
662 | ||
663 | /* End by sending a stop id to host with an outbuf. */ | |
664 | err = send_cmd_id_stop(vb); | |
665 | if (unlikely(err)) | |
666 | dev_err(dev, "Failed to send a stop id, err = %d\n", err); | |
667 | } | |
668 | ||
bf4dc0b2 WW |
669 | static void report_free_page_func(struct work_struct *work) |
670 | { | |
671 | struct virtio_balloon *vb = container_of(work, struct virtio_balloon, | |
672 | report_free_page_work); | |
673 | u32 cmd_id_received; | |
674 | ||
675 | cmd_id_received = virtio_balloon_cmd_id_received(vb); | |
676 | if (cmd_id_received == VIRTIO_BALLOON_CMD_ID_DONE) { | |
677 | /* Pass ULONG_MAX to give back all the free pages */ | |
678 | return_free_pages_to_mm(vb, ULONG_MAX); | |
679 | } else if (cmd_id_received != VIRTIO_BALLOON_CMD_ID_STOP && | |
680 | cmd_id_received != | |
681 | virtio32_to_cpu(vb->vdev, vb->cmd_id_active)) { | |
682 | virtio_balloon_report_free_page(vb); | |
683 | } | |
684 | } | |
685 | ||
e2250429 RA |
686 | #ifdef CONFIG_BALLOON_COMPACTION |
687 | /* | |
688 | * virtballoon_migratepage - perform the balloon page migration on behalf of | |
689 | * a compation thread. (called under page lock) | |
9d1ba805 | 690 | * @vb_dev_info: the balloon device |
e2250429 RA |
691 | * @newpage: page that will replace the isolated page after migration finishes. |
692 | * @page : the isolated (old) page that is about to be migrated to newpage. | |
693 | * @mode : compaction mode -- not used for balloon page migration. | |
694 | * | |
695 | * After a ballooned page gets isolated by compaction procedures, this is the | |
696 | * function that performs the page migration on behalf of a compaction thread | |
697 | * The page migration for virtio balloon is done in a simple swap fashion which | |
698 | * follows these two macro steps: | |
699 | * 1) insert newpage into vb->pages list and update the host about it; | |
700 | * 2) update the host about the old page removed from vb->pages list; | |
701 | * | |
702 | * This function preforms the balloon page migration task. | |
703 | * Called through balloon_mapping->a_ops->migratepage | |
704 | */ | |
9d1ba805 | 705 | static int virtballoon_migratepage(struct balloon_dev_info *vb_dev_info, |
e2250429 RA |
706 | struct page *newpage, struct page *page, enum migrate_mode mode) |
707 | { | |
9d1ba805 KK |
708 | struct virtio_balloon *vb = container_of(vb_dev_info, |
709 | struct virtio_balloon, vb_dev_info); | |
e2250429 RA |
710 | unsigned long flags; |
711 | ||
e2250429 RA |
712 | /* |
713 | * In order to avoid lock contention while migrating pages concurrently | |
714 | * to leak_balloon() or fill_balloon() we just give up the balloon_lock | |
715 | * this turn, as it is easier to retry the page migration later. | |
716 | * This also prevents fill_balloon() getting stuck into a mutex | |
717 | * recursion in the case it ends up triggering memory compaction | |
718 | * while it is attempting to inflate the ballon. | |
719 | */ | |
720 | if (!mutex_trylock(&vb->balloon_lock)) | |
721 | return -EAGAIN; | |
722 | ||
d6d86c0a KK |
723 | get_page(newpage); /* balloon reference */ |
724 | ||
63341ab0 DH |
725 | /* |
726 | * When we migrate a page to a different zone and adjusted the | |
727 | * managed page count when inflating, we have to fixup the count of | |
728 | * both involved zones. | |
729 | */ | |
730 | if (!virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM) && | |
731 | page_zone(page) != page_zone(newpage)) { | |
732 | adjust_managed_page_count(page, 1); | |
733 | adjust_managed_page_count(newpage, -1); | |
734 | } | |
735 | ||
e2250429 RA |
736 | /* balloon's page migration 1st step -- inflate "newpage" */ |
737 | spin_lock_irqsave(&vb_dev_info->pages_lock, flags); | |
9d1ba805 | 738 | balloon_page_insert(vb_dev_info, newpage); |
e2250429 | 739 | vb_dev_info->isolated_pages--; |
09316c09 | 740 | __count_vm_event(BALLOON_MIGRATE); |
e2250429 RA |
741 | spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); |
742 | vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; | |
87c9403b | 743 | set_page_pfns(vb, vb->pfns, newpage); |
e2250429 RA |
744 | tell_host(vb, vb->inflate_vq); |
745 | ||
d6d86c0a | 746 | /* balloon's page migration 2nd step -- deflate "page" */ |
89da619b | 747 | spin_lock_irqsave(&vb_dev_info->pages_lock, flags); |
e2250429 | 748 | balloon_page_delete(page); |
89da619b | 749 | spin_unlock_irqrestore(&vb_dev_info->pages_lock, flags); |
e2250429 | 750 | vb->num_pfns = VIRTIO_BALLOON_PAGES_PER_PAGE; |
87c9403b | 751 | set_page_pfns(vb, vb->pfns, page); |
e2250429 RA |
752 | tell_host(vb, vb->deflate_vq); |
753 | ||
754 | mutex_unlock(&vb->balloon_lock); | |
755 | ||
d6d86c0a KK |
756 | put_page(page); /* balloon reference */ |
757 | ||
dd4123f3 | 758 | return MIGRATEPAGE_SUCCESS; |
e2250429 | 759 | } |
b1123ea6 | 760 | |
99558d20 | 761 | static int balloon_init_fs_context(struct fs_context *fc) |
b1123ea6 | 762 | { |
99558d20 | 763 | return init_pseudo(fc, BALLOON_KVM_MAGIC) ? 0 : -ENOMEM; |
b1123ea6 MK |
764 | } |
765 | ||
766 | static struct file_system_type balloon_fs = { | |
767 | .name = "balloon-kvm", | |
99558d20 | 768 | .init_fs_context = balloon_init_fs_context, |
b1123ea6 MK |
769 | .kill_sb = kill_anon_super, |
770 | }; | |
771 | ||
e2250429 RA |
772 | #endif /* CONFIG_BALLOON_COMPACTION */ |
773 | ||
86a55978 WW |
774 | static unsigned long shrink_free_pages(struct virtio_balloon *vb, |
775 | unsigned long pages_to_free) | |
71994620 | 776 | { |
86a55978 | 777 | unsigned long blocks_to_free, blocks_freed; |
71994620 | 778 | |
86a55978 | 779 | pages_to_free = round_up(pages_to_free, |
63b9b80e MT |
780 | VIRTIO_BALLOON_HINT_BLOCK_PAGES); |
781 | blocks_to_free = pages_to_free / VIRTIO_BALLOON_HINT_BLOCK_PAGES; | |
86a55978 WW |
782 | blocks_freed = return_free_pages_to_mm(vb, blocks_to_free); |
783 | ||
63b9b80e | 784 | return blocks_freed * VIRTIO_BALLOON_HINT_BLOCK_PAGES; |
86a55978 WW |
785 | } |
786 | ||
60bd04f2 MT |
787 | static unsigned long leak_balloon_pages(struct virtio_balloon *vb, |
788 | unsigned long pages_to_free) | |
789 | { | |
790 | return leak_balloon(vb, pages_to_free * VIRTIO_BALLOON_PAGES_PER_PAGE) / | |
791 | VIRTIO_BALLOON_PAGES_PER_PAGE; | |
792 | } | |
793 | ||
86a55978 WW |
794 | static unsigned long shrink_balloon_pages(struct virtio_balloon *vb, |
795 | unsigned long pages_to_free) | |
796 | { | |
797 | unsigned long pages_freed = 0; | |
71994620 WW |
798 | |
799 | /* | |
800 | * One invocation of leak_balloon can deflate at most | |
801 | * VIRTIO_BALLOON_ARRAY_PFNS_MAX balloon pages, so we call it | |
802 | * multiple times to deflate pages till reaching pages_to_free. | |
803 | */ | |
60bd04f2 MT |
804 | while (vb->num_pages && pages_freed < pages_to_free) |
805 | pages_freed += leak_balloon_pages(vb, | |
806 | pages_to_free - pages_freed); | |
807 | ||
71994620 WW |
808 | update_balloon_size(vb); |
809 | ||
86a55978 WW |
810 | return pages_freed; |
811 | } | |
812 | ||
813 | static unsigned long virtio_balloon_shrinker_scan(struct shrinker *shrinker, | |
814 | struct shrink_control *sc) | |
815 | { | |
816 | unsigned long pages_to_free, pages_freed = 0; | |
817 | struct virtio_balloon *vb = container_of(shrinker, | |
818 | struct virtio_balloon, shrinker); | |
819 | ||
60bd04f2 | 820 | pages_to_free = sc->nr_to_scan; |
86a55978 WW |
821 | |
822 | if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) | |
823 | pages_freed = shrink_free_pages(vb, pages_to_free); | |
824 | ||
825 | if (pages_freed >= pages_to_free) | |
826 | return pages_freed; | |
827 | ||
828 | pages_freed += shrink_balloon_pages(vb, pages_to_free - pages_freed); | |
829 | ||
830 | return pages_freed; | |
71994620 WW |
831 | } |
832 | ||
833 | static unsigned long virtio_balloon_shrinker_count(struct shrinker *shrinker, | |
834 | struct shrink_control *sc) | |
835 | { | |
836 | struct virtio_balloon *vb = container_of(shrinker, | |
837 | struct virtio_balloon, shrinker); | |
86a55978 | 838 | unsigned long count; |
71994620 | 839 | |
86a55978 | 840 | count = vb->num_pages / VIRTIO_BALLOON_PAGES_PER_PAGE; |
63b9b80e | 841 | count += vb->num_free_page_blocks * VIRTIO_BALLOON_HINT_BLOCK_PAGES; |
86a55978 WW |
842 | |
843 | return count; | |
71994620 WW |
844 | } |
845 | ||
846 | static void virtio_balloon_unregister_shrinker(struct virtio_balloon *vb) | |
847 | { | |
848 | unregister_shrinker(&vb->shrinker); | |
849 | } | |
850 | ||
851 | static int virtio_balloon_register_shrinker(struct virtio_balloon *vb) | |
852 | { | |
853 | vb->shrinker.scan_objects = virtio_balloon_shrinker_scan; | |
854 | vb->shrinker.count_objects = virtio_balloon_shrinker_count; | |
855 | vb->shrinker.seeks = DEFAULT_SEEKS; | |
856 | ||
857 | return register_shrinker(&vb->shrinker); | |
858 | } | |
859 | ||
be91c33d AS |
860 | static int virtballoon_probe(struct virtio_device *vdev) |
861 | { | |
862 | struct virtio_balloon *vb; | |
2e991629 | 863 | __u32 poison_val; |
be91c33d AS |
864 | int err; |
865 | ||
2d9becc1 MT |
866 | if (!vdev->config->get) { |
867 | dev_err(&vdev->dev, "%s failure: config access disabled\n", | |
868 | __func__); | |
869 | return -EINVAL; | |
870 | } | |
871 | ||
c51d8fca | 872 | vdev->priv = vb = kzalloc(sizeof(*vb), GFP_KERNEL); |
be91c33d AS |
873 | if (!vb) { |
874 | err = -ENOMEM; | |
875 | goto out; | |
876 | } | |
877 | ||
fd0e21c3 PM |
878 | INIT_WORK(&vb->update_balloon_stats_work, update_balloon_stats_func); |
879 | INIT_WORK(&vb->update_balloon_size_work, update_balloon_size_func); | |
fad7b7b2 | 880 | spin_lock_init(&vb->stop_update_lock); |
e2250429 | 881 | mutex_init(&vb->balloon_lock); |
9c378abc | 882 | init_waitqueue_head(&vb->acked); |
be91c33d | 883 | vb->vdev = vdev; |
be91c33d | 884 | |
9d1ba805 | 885 | balloon_devinfo_init(&vb->vb_dev_info); |
e2250429 | 886 | |
be91c33d AS |
887 | err = init_vqs(vb); |
888 | if (err) | |
9d1ba805 | 889 | goto out_free_vb; |
6b35e407 | 890 | |
b1123ea6 MK |
891 | #ifdef CONFIG_BALLOON_COMPACTION |
892 | balloon_mnt = kern_mount(&balloon_fs); | |
893 | if (IS_ERR(balloon_mnt)) { | |
894 | err = PTR_ERR(balloon_mnt); | |
b1123ea6 MK |
895 | goto out_del_vqs; |
896 | } | |
897 | ||
898 | vb->vb_dev_info.migratepage = virtballoon_migratepage; | |
899 | vb->vb_dev_info.inode = alloc_anon_inode(balloon_mnt->mnt_sb); | |
900 | if (IS_ERR(vb->vb_dev_info.inode)) { | |
901 | err = PTR_ERR(vb->vb_dev_info.inode); | |
902 | kern_unmount(balloon_mnt); | |
b1123ea6 MK |
903 | goto out_del_vqs; |
904 | } | |
905 | vb->vb_dev_info.inode->i_mapping->a_ops = &balloon_aops; | |
906 | #endif | |
86a55978 WW |
907 | if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { |
908 | /* | |
909 | * There is always one entry reserved for cmd id, so the ring | |
910 | * size needs to be at least two to report free page hints. | |
911 | */ | |
912 | if (virtqueue_get_vring_size(vb->free_page_vq) < 2) { | |
913 | err = -ENOSPC; | |
914 | goto out_del_vqs; | |
915 | } | |
916 | vb->balloon_wq = alloc_workqueue("balloon-wq", | |
917 | WQ_FREEZABLE | WQ_CPU_INTENSIVE, 0); | |
918 | if (!vb->balloon_wq) { | |
919 | err = -ENOMEM; | |
920 | goto out_del_vqs; | |
921 | } | |
922 | INIT_WORK(&vb->report_free_page_work, report_free_page_func); | |
bf4dc0b2 | 923 | vb->cmd_id_received_cache = VIRTIO_BALLOON_CMD_ID_STOP; |
86a55978 WW |
924 | vb->cmd_id_active = cpu_to_virtio32(vb->vdev, |
925 | VIRTIO_BALLOON_CMD_ID_STOP); | |
926 | vb->cmd_id_stop = cpu_to_virtio32(vb->vdev, | |
927 | VIRTIO_BALLOON_CMD_ID_STOP); | |
86a55978 WW |
928 | spin_lock_init(&vb->free_page_list_lock); |
929 | INIT_LIST_HEAD(&vb->free_page_list); | |
2e991629 WW |
930 | if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) { |
931 | memset(&poison_val, PAGE_POISON, sizeof(poison_val)); | |
932 | virtio_cwrite(vb->vdev, struct virtio_balloon_config, | |
933 | poison_val, &poison_val); | |
934 | } | |
86a55978 | 935 | } |
71994620 WW |
936 | /* |
937 | * We continue to use VIRTIO_BALLOON_F_DEFLATE_ON_OOM to decide if a | |
938 | * shrinker needs to be registered to relieve memory pressure. | |
939 | */ | |
940 | if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) { | |
941 | err = virtio_balloon_register_shrinker(vb); | |
942 | if (err) | |
86a55978 | 943 | goto out_del_balloon_wq; |
71994620 | 944 | } |
88660f7f MT |
945 | virtio_device_ready(vdev); |
946 | ||
8424af53 KN |
947 | if (towards_target(vb)) |
948 | virtballoon_changed(vdev); | |
6b35e407 RR |
949 | return 0; |
950 | ||
86a55978 WW |
951 | out_del_balloon_wq: |
952 | if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) | |
953 | destroy_workqueue(vb->balloon_wq); | |
b1123ea6 | 954 | out_del_vqs: |
d2a7ddda | 955 | vdev->config->del_vqs(vdev); |
6b35e407 RR |
956 | out_free_vb: |
957 | kfree(vb); | |
958 | out: | |
959 | return err; | |
960 | } | |
961 | ||
c877bab5 | 962 | static void remove_common(struct virtio_balloon *vb) |
6b35e407 | 963 | { |
6b35e407 RR |
964 | /* There might be pages left in the balloon: free them. */ |
965 | while (vb->num_pages) | |
966 | leak_balloon(vb, vb->num_pages); | |
b8ae0eb3 | 967 | update_balloon_size(vb); |
6b35e407 RR |
968 | |
969 | /* Now we reset the device so we can clean up the queues. */ | |
c877bab5 | 970 | vb->vdev->config->reset(vb->vdev); |
6b35e407 | 971 | |
c877bab5 AS |
972 | vb->vdev->config->del_vqs(vb->vdev); |
973 | } | |
974 | ||
8590dbc7 | 975 | static void virtballoon_remove(struct virtio_device *vdev) |
c877bab5 AS |
976 | { |
977 | struct virtio_balloon *vb = vdev->priv; | |
978 | ||
71994620 WW |
979 | if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) |
980 | virtio_balloon_unregister_shrinker(vb); | |
fad7b7b2 PM |
981 | spin_lock_irq(&vb->stop_update_lock); |
982 | vb->stop_update = true; | |
983 | spin_unlock_irq(&vb->stop_update_lock); | |
fd0e21c3 PM |
984 | cancel_work_sync(&vb->update_balloon_size_work); |
985 | cancel_work_sync(&vb->update_balloon_stats_work); | |
fad7b7b2 | 986 | |
86a55978 WW |
987 | if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { |
988 | cancel_work_sync(&vb->report_free_page_work); | |
989 | destroy_workqueue(vb->balloon_wq); | |
990 | } | |
991 | ||
c877bab5 | 992 | remove_common(vb); |
9c57b580 | 993 | #ifdef CONFIG_BALLOON_COMPACTION |
b1123ea6 MK |
994 | if (vb->vb_dev_info.inode) |
995 | iput(vb->vb_dev_info.inode); | |
9c57b580 YX |
996 | |
997 | kern_unmount(balloon_mnt); | |
998 | #endif | |
6b35e407 RR |
999 | kfree(vb); |
1000 | } | |
1001 | ||
89107000 | 1002 | #ifdef CONFIG_PM_SLEEP |
e562966d AS |
1003 | static int virtballoon_freeze(struct virtio_device *vdev) |
1004 | { | |
4eb05d56 AS |
1005 | struct virtio_balloon *vb = vdev->priv; |
1006 | ||
e562966d | 1007 | /* |
fad7b7b2 | 1008 | * The workqueue is already frozen by the PM core before this |
e562966d AS |
1009 | * function is called. |
1010 | */ | |
c877bab5 | 1011 | remove_common(vb); |
e562966d AS |
1012 | return 0; |
1013 | } | |
1014 | ||
c45b4166 | 1015 | static int virtballoon_restore(struct virtio_device *vdev) |
4eb05d56 AS |
1016 | { |
1017 | struct virtio_balloon *vb = vdev->priv; | |
1018 | int ret; | |
1019 | ||
1020 | ret = init_vqs(vdev->priv); | |
1021 | if (ret) | |
1022 | return ret; | |
1023 | ||
486d2e63 MT |
1024 | virtio_device_ready(vdev); |
1025 | ||
fad7b7b2 PM |
1026 | if (towards_target(vb)) |
1027 | virtballoon_changed(vdev); | |
4eb05d56 AS |
1028 | update_balloon_size(vb); |
1029 | return 0; | |
1030 | } | |
e562966d AS |
1031 | #endif |
1032 | ||
e41b1355 MT |
1033 | static int virtballoon_validate(struct virtio_device *vdev) |
1034 | { | |
2e991629 WW |
1035 | if (!page_poisoning_enabled()) |
1036 | __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_PAGE_POISON); | |
1037 | ||
e41b1355 MT |
1038 | __virtio_clear_bit(vdev, VIRTIO_F_IOMMU_PLATFORM); |
1039 | return 0; | |
1040 | } | |
1041 | ||
9564e138 AL |
1042 | static unsigned int features[] = { |
1043 | VIRTIO_BALLOON_F_MUST_TELL_HOST, | |
1044 | VIRTIO_BALLOON_F_STATS_VQ, | |
5a10b7db | 1045 | VIRTIO_BALLOON_F_DEFLATE_ON_OOM, |
86a55978 | 1046 | VIRTIO_BALLOON_F_FREE_PAGE_HINT, |
2e991629 | 1047 | VIRTIO_BALLOON_F_PAGE_POISON, |
9564e138 | 1048 | }; |
c45a6816 | 1049 | |
d817cd52 | 1050 | static struct virtio_driver virtio_balloon_driver = { |
c45a6816 RR |
1051 | .feature_table = features, |
1052 | .feature_table_size = ARRAY_SIZE(features), | |
6b35e407 RR |
1053 | .driver.name = KBUILD_MODNAME, |
1054 | .driver.owner = THIS_MODULE, | |
1055 | .id_table = id_table, | |
e41b1355 | 1056 | .validate = virtballoon_validate, |
6b35e407 | 1057 | .probe = virtballoon_probe, |
8590dbc7 | 1058 | .remove = virtballoon_remove, |
6b35e407 | 1059 | .config_changed = virtballoon_changed, |
89107000 | 1060 | #ifdef CONFIG_PM_SLEEP |
e562966d AS |
1061 | .freeze = virtballoon_freeze, |
1062 | .restore = virtballoon_restore, | |
e562966d | 1063 | #endif |
6b35e407 RR |
1064 | }; |
1065 | ||
b2a17029 | 1066 | module_virtio_driver(virtio_balloon_driver); |
6b35e407 RR |
1067 | MODULE_DEVICE_TABLE(virtio, id_table); |
1068 | MODULE_DESCRIPTION("Virtio balloon driver"); | |
1069 | MODULE_LICENSE("GPL"); |