]>
Commit | Line | Data |
---|---|---|
453dc659 DT |
1 | /* |
2 | * VMware Balloon driver. | |
3 | * | |
48e3d668 | 4 | * Copyright (C) 2000-2014, VMware, Inc. All Rights Reserved. |
453dc659 DT |
5 | * |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License as published by the | |
8 | * Free Software Foundation; version 2 of the License and no later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but | |
11 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
13 | * NON INFRINGEMENT. See the GNU General Public License for more | |
14 | * details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program; if not, write to the Free Software | |
18 | * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
19 | * | |
73b35d07 DT |
20 | * Maintained by: Xavier Deguillard <xdeguillard@vmware.com> |
21 | * Philip Moltmann <moltmann@vmware.com> | |
453dc659 DT |
22 | */ |
23 | ||
24 | /* | |
25 | * This is VMware physical memory management driver for Linux. The driver | |
26 | * acts like a "balloon" that can be inflated to reclaim physical pages by | |
27 | * reserving them in the guest and invalidating them in the monitor, | |
28 | * freeing up the underlying machine pages so they can be allocated to | |
29 | * other guests. The balloon can also be deflated to allow the guest to | |
30 | * use more physical memory. Higher level policies can control the sizes | |
31 | * of balloons in VMs in order to manage physical memory resources. | |
32 | */ | |
33 | ||
34 | //#define DEBUG | |
35 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
36 | ||
37 | #include <linux/types.h> | |
38 | #include <linux/kernel.h> | |
39 | #include <linux/mm.h> | |
f220a80f | 40 | #include <linux/vmalloc.h> |
453dc659 DT |
41 | #include <linux/sched.h> |
42 | #include <linux/module.h> | |
43 | #include <linux/workqueue.h> | |
44 | #include <linux/debugfs.h> | |
45 | #include <linux/seq_file.h> | |
48e3d668 PM |
46 | #include <linux/vmw_vmci_defs.h> |
47 | #include <linux/vmw_vmci_api.h> | |
a10a5698 | 48 | #include <asm/hypervisor.h> |
453dc659 DT |
49 | |
50 | MODULE_AUTHOR("VMware, Inc."); | |
51 | MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver"); | |
48e3d668 | 52 | MODULE_VERSION("1.5.0.0-k"); |
453dc659 DT |
53 | MODULE_ALIAS("dmi:*:svnVMware*:*"); |
54 | MODULE_ALIAS("vmware_vmmemctl"); | |
55 | MODULE_LICENSE("GPL"); | |
56 | ||
57 | /* | |
58 | * Various constants controlling rate of inflaint/deflating balloon, | |
59 | * measured in pages. | |
60 | */ | |
61 | ||
453dc659 DT |
62 | /* |
63 | * Rates of memory allocaton when guest experiences memory pressure | |
64 | * (driver performs sleeping allocations). | |
65 | */ | |
66 | #define VMW_BALLOON_RATE_ALLOC_MIN 512U | |
67 | #define VMW_BALLOON_RATE_ALLOC_MAX 2048U | |
68 | #define VMW_BALLOON_RATE_ALLOC_INC 16U | |
69 | ||
453dc659 DT |
70 | /* |
71 | * When guest is under memory pressure, use a reduced page allocation | |
72 | * rate for next several cycles. | |
73 | */ | |
74 | #define VMW_BALLOON_SLOW_CYCLES 4 | |
75 | ||
76 | /* | |
77 | * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't | |
71baba4b | 78 | * allow wait (__GFP_RECLAIM) for NOSLEEP page allocations. Use |
453dc659 DT |
79 | * __GFP_NOWARN, to suppress page allocation failure warnings. |
80 | */ | |
81 | #define VMW_PAGE_ALLOC_NOSLEEP (__GFP_HIGHMEM|__GFP_NOWARN) | |
82 | ||
83 | /* | |
84 | * Use GFP_HIGHUSER when executing in a separate kernel thread | |
85 | * context and allocation can sleep. This is less stressful to | |
86 | * the guest memory system, since it allows the thread to block | |
87 | * while memory is reclaimed, and won't take pages from emergency | |
88 | * low-memory pools. | |
89 | */ | |
90 | #define VMW_PAGE_ALLOC_CANSLEEP (GFP_HIGHUSER) | |
91 | ||
55adaa49 DT |
92 | /* Maximum number of refused pages we accumulate during inflation cycle */ |
93 | #define VMW_BALLOON_MAX_REFUSED 16 | |
453dc659 DT |
94 | |
95 | /* | |
96 | * Hypervisor communication port definitions. | |
97 | */ | |
98 | #define VMW_BALLOON_HV_PORT 0x5670 | |
99 | #define VMW_BALLOON_HV_MAGIC 0x456c6d6f | |
453dc659 DT |
100 | #define VMW_BALLOON_GUEST_ID 1 /* Linux */ |
101 | ||
eb79100f XD |
102 | enum vmwballoon_capabilities { |
103 | /* | |
104 | * Bit 0 is reserved and not associated to any capability. | |
105 | */ | |
48e3d668 PM |
106 | VMW_BALLOON_BASIC_CMDS = (1 << 1), |
107 | VMW_BALLOON_BATCHED_CMDS = (1 << 2), | |
108 | VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3), | |
109 | VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4), | |
eb79100f XD |
110 | }; |
111 | ||
f220a80f | 112 | #define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_BASIC_CMDS \ |
365bd7ef | 113 | | VMW_BALLOON_BATCHED_CMDS \ |
48e3d668 PM |
114 | | VMW_BALLOON_BATCHED_2M_CMDS \ |
115 | | VMW_BALLOON_SIGNALLED_WAKEUP_CMD) | |
365bd7ef PM |
116 | |
117 | #define VMW_BALLOON_2M_SHIFT (9) | |
118 | #define VMW_BALLOON_NUM_PAGE_SIZES (2) | |
eb79100f | 119 | |
f220a80f XD |
120 | /* |
121 | * Backdoor commands availability: | |
122 | * | |
123 | * START, GET_TARGET and GUEST_ID are always available, | |
124 | * | |
125 | * VMW_BALLOON_BASIC_CMDS: | |
126 | * LOCK and UNLOCK commands, | |
127 | * VMW_BALLOON_BATCHED_CMDS: | |
128 | * BATCHED_LOCK and BATCHED_UNLOCK commands. | |
365bd7ef | 129 | * VMW BALLOON_BATCHED_2M_CMDS: |
48e3d668 PM |
130 | * BATCHED_2M_LOCK and BATCHED_2M_UNLOCK commands, |
131 | * VMW VMW_BALLOON_SIGNALLED_WAKEUP_CMD: | |
132 | * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command. | |
f220a80f | 133 | */ |
365bd7ef PM |
134 | #define VMW_BALLOON_CMD_START 0 |
135 | #define VMW_BALLOON_CMD_GET_TARGET 1 | |
136 | #define VMW_BALLOON_CMD_LOCK 2 | |
137 | #define VMW_BALLOON_CMD_UNLOCK 3 | |
138 | #define VMW_BALLOON_CMD_GUEST_ID 4 | |
139 | #define VMW_BALLOON_CMD_BATCHED_LOCK 6 | |
140 | #define VMW_BALLOON_CMD_BATCHED_UNLOCK 7 | |
141 | #define VMW_BALLOON_CMD_BATCHED_2M_LOCK 8 | |
142 | #define VMW_BALLOON_CMD_BATCHED_2M_UNLOCK 9 | |
48e3d668 | 143 | #define VMW_BALLOON_CMD_VMCI_DOORBELL_SET 10 |
365bd7ef | 144 | |
453dc659 DT |
145 | |
146 | /* error codes */ | |
eb79100f XD |
147 | #define VMW_BALLOON_SUCCESS 0 |
148 | #define VMW_BALLOON_FAILURE -1 | |
149 | #define VMW_BALLOON_ERROR_CMD_INVALID 1 | |
150 | #define VMW_BALLOON_ERROR_PPN_INVALID 2 | |
151 | #define VMW_BALLOON_ERROR_PPN_LOCKED 3 | |
152 | #define VMW_BALLOON_ERROR_PPN_UNLOCKED 4 | |
153 | #define VMW_BALLOON_ERROR_PPN_PINNED 5 | |
154 | #define VMW_BALLOON_ERROR_PPN_NOTNEEDED 6 | |
155 | #define VMW_BALLOON_ERROR_RESET 7 | |
156 | #define VMW_BALLOON_ERROR_BUSY 8 | |
157 | ||
158 | #define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000) | |
159 | ||
f220a80f XD |
160 | /* Batch page description */ |
161 | ||
162 | /* | |
163 | * Layout of a page in the batch page: | |
164 | * | |
165 | * +-------------+----------+--------+ | |
166 | * | | | | | |
167 | * | Page number | Reserved | Status | | |
168 | * | | | | | |
169 | * +-------------+----------+--------+ | |
170 | * 64 PAGE_SHIFT 6 0 | |
171 | * | |
f220a80f XD |
172 | * The reserved field should be set to 0. |
173 | */ | |
174 | #define VMW_BALLOON_BATCH_MAX_PAGES (PAGE_SIZE / sizeof(u64)) | |
175 | #define VMW_BALLOON_BATCH_STATUS_MASK ((1UL << 5) - 1) | |
176 | #define VMW_BALLOON_BATCH_PAGE_MASK (~((1UL << PAGE_SHIFT) - 1)) | |
177 | ||
178 | struct vmballoon_batch_page { | |
179 | u64 pages[VMW_BALLOON_BATCH_MAX_PAGES]; | |
180 | }; | |
181 | ||
182 | static u64 vmballoon_batch_get_pa(struct vmballoon_batch_page *batch, int idx) | |
183 | { | |
184 | return batch->pages[idx] & VMW_BALLOON_BATCH_PAGE_MASK; | |
185 | } | |
186 | ||
187 | static int vmballoon_batch_get_status(struct vmballoon_batch_page *batch, | |
188 | int idx) | |
189 | { | |
190 | return (int)(batch->pages[idx] & VMW_BALLOON_BATCH_STATUS_MASK); | |
191 | } | |
192 | ||
193 | static void vmballoon_batch_set_pa(struct vmballoon_batch_page *batch, int idx, | |
194 | u64 pa) | |
195 | { | |
196 | batch->pages[idx] = pa; | |
197 | } | |
198 | ||
199 | ||
200 | #define VMWARE_BALLOON_CMD(cmd, arg1, arg2, result) \ | |
eb79100f | 201 | ({ \ |
f220a80f | 202 | unsigned long __status, __dummy1, __dummy2, __dummy3; \ |
eb79100f XD |
203 | __asm__ __volatile__ ("inl %%dx" : \ |
204 | "=a"(__status), \ | |
205 | "=c"(__dummy1), \ | |
206 | "=d"(__dummy2), \ | |
f220a80f XD |
207 | "=b"(result), \ |
208 | "=S" (__dummy3) : \ | |
eb79100f XD |
209 | "0"(VMW_BALLOON_HV_MAGIC), \ |
210 | "1"(VMW_BALLOON_CMD_##cmd), \ | |
211 | "2"(VMW_BALLOON_HV_PORT), \ | |
f220a80f XD |
212 | "3"(arg1), \ |
213 | "4" (arg2) : \ | |
eb79100f XD |
214 | "memory"); \ |
215 | if (VMW_BALLOON_CMD_##cmd == VMW_BALLOON_CMD_START) \ | |
216 | result = __dummy1; \ | |
217 | result &= -1UL; \ | |
218 | __status & -1UL; \ | |
453dc659 DT |
219 | }) |
220 | ||
221 | #ifdef CONFIG_DEBUG_FS | |
222 | struct vmballoon_stats { | |
223 | unsigned int timer; | |
48e3d668 | 224 | unsigned int doorbell; |
453dc659 | 225 | |
2ca02df6 | 226 | /* allocation statistics */ |
365bd7ef PM |
227 | unsigned int alloc[VMW_BALLOON_NUM_PAGE_SIZES]; |
228 | unsigned int alloc_fail[VMW_BALLOON_NUM_PAGE_SIZES]; | |
453dc659 DT |
229 | unsigned int sleep_alloc; |
230 | unsigned int sleep_alloc_fail; | |
365bd7ef PM |
231 | unsigned int refused_alloc[VMW_BALLOON_NUM_PAGE_SIZES]; |
232 | unsigned int refused_free[VMW_BALLOON_NUM_PAGE_SIZES]; | |
233 | unsigned int free[VMW_BALLOON_NUM_PAGE_SIZES]; | |
453dc659 DT |
234 | |
235 | /* monitor operations */ | |
365bd7ef PM |
236 | unsigned int lock[VMW_BALLOON_NUM_PAGE_SIZES]; |
237 | unsigned int lock_fail[VMW_BALLOON_NUM_PAGE_SIZES]; | |
238 | unsigned int unlock[VMW_BALLOON_NUM_PAGE_SIZES]; | |
239 | unsigned int unlock_fail[VMW_BALLOON_NUM_PAGE_SIZES]; | |
453dc659 DT |
240 | unsigned int target; |
241 | unsigned int target_fail; | |
242 | unsigned int start; | |
243 | unsigned int start_fail; | |
244 | unsigned int guest_type; | |
245 | unsigned int guest_type_fail; | |
48e3d668 PM |
246 | unsigned int doorbell_set; |
247 | unsigned int doorbell_unset; | |
453dc659 DT |
248 | }; |
249 | ||
250 | #define STATS_INC(stat) (stat)++ | |
251 | #else | |
252 | #define STATS_INC(stat) | |
253 | #endif | |
254 | ||
f220a80f XD |
255 | struct vmballoon; |
256 | ||
257 | struct vmballoon_ops { | |
258 | void (*add_page)(struct vmballoon *b, int idx, struct page *p); | |
4670de4d | 259 | int (*lock)(struct vmballoon *b, unsigned int num_pages, |
365bd7ef | 260 | bool is_2m_pages, unsigned int *target); |
4670de4d | 261 | int (*unlock)(struct vmballoon *b, unsigned int num_pages, |
365bd7ef | 262 | bool is_2m_pages, unsigned int *target); |
f220a80f XD |
263 | }; |
264 | ||
365bd7ef | 265 | struct vmballoon_page_size { |
453dc659 DT |
266 | /* list of reserved physical pages */ |
267 | struct list_head pages; | |
268 | ||
269 | /* transient list of non-balloonable pages */ | |
270 | struct list_head refused_pages; | |
55adaa49 | 271 | unsigned int n_refused_pages; |
365bd7ef PM |
272 | }; |
273 | ||
274 | struct vmballoon { | |
275 | struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES]; | |
276 | ||
277 | /* supported page sizes. 1 == 4k pages only, 2 == 4k and 2m pages */ | |
278 | unsigned supported_page_sizes; | |
453dc659 DT |
279 | |
280 | /* balloon size in pages */ | |
281 | unsigned int size; | |
282 | unsigned int target; | |
283 | ||
284 | /* reset flag */ | |
285 | bool reset_required; | |
286 | ||
287 | /* adjustment rates (pages per second) */ | |
288 | unsigned int rate_alloc; | |
453dc659 DT |
289 | |
290 | /* slowdown page allocations for next few cycles */ | |
291 | unsigned int slow_allocation_cycles; | |
292 | ||
f220a80f XD |
293 | unsigned long capabilities; |
294 | ||
295 | struct vmballoon_batch_page *batch_page; | |
296 | unsigned int batch_max_pages; | |
297 | struct page *page; | |
298 | ||
299 | const struct vmballoon_ops *ops; | |
300 | ||
453dc659 DT |
301 | #ifdef CONFIG_DEBUG_FS |
302 | /* statistics */ | |
303 | struct vmballoon_stats stats; | |
304 | ||
305 | /* debugfs file exporting statistics */ | |
306 | struct dentry *dbg_entry; | |
307 | #endif | |
308 | ||
309 | struct sysinfo sysinfo; | |
310 | ||
311 | struct delayed_work dwork; | |
48e3d668 PM |
312 | |
313 | struct vmci_handle vmci_doorbell; | |
453dc659 DT |
314 | }; |
315 | ||
316 | static struct vmballoon balloon; | |
453dc659 DT |
317 | |
318 | /* | |
319 | * Send "start" command to the host, communicating supported version | |
320 | * of the protocol. | |
321 | */ | |
f220a80f | 322 | static bool vmballoon_send_start(struct vmballoon *b, unsigned long req_caps) |
453dc659 | 323 | { |
f220a80f | 324 | unsigned long status, capabilities, dummy = 0; |
365bd7ef | 325 | bool success; |
453dc659 DT |
326 | |
327 | STATS_INC(b->stats.start); | |
328 | ||
f220a80f XD |
329 | status = VMWARE_BALLOON_CMD(START, req_caps, dummy, capabilities); |
330 | ||
331 | switch (status) { | |
332 | case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES: | |
333 | b->capabilities = capabilities; | |
365bd7ef PM |
334 | success = true; |
335 | break; | |
f220a80f XD |
336 | case VMW_BALLOON_SUCCESS: |
337 | b->capabilities = VMW_BALLOON_BASIC_CMDS; | |
365bd7ef PM |
338 | success = true; |
339 | break; | |
340 | default: | |
341 | success = false; | |
f220a80f | 342 | } |
453dc659 | 343 | |
365bd7ef PM |
344 | if (b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) |
345 | b->supported_page_sizes = 2; | |
346 | else | |
347 | b->supported_page_sizes = 1; | |
348 | ||
349 | if (!success) { | |
350 | pr_debug("%s - failed, hv returns %ld\n", __func__, status); | |
351 | STATS_INC(b->stats.start_fail); | |
352 | } | |
353 | return success; | |
453dc659 DT |
354 | } |
355 | ||
356 | static bool vmballoon_check_status(struct vmballoon *b, unsigned long status) | |
357 | { | |
358 | switch (status) { | |
359 | case VMW_BALLOON_SUCCESS: | |
360 | return true; | |
361 | ||
362 | case VMW_BALLOON_ERROR_RESET: | |
363 | b->reset_required = true; | |
364 | /* fall through */ | |
365 | ||
366 | default: | |
367 | return false; | |
368 | } | |
369 | } | |
370 | ||
371 | /* | |
372 | * Communicate guest type to the host so that it can adjust ballooning | |
373 | * algorithm to the one most appropriate for the guest. This command | |
374 | * is normally issued after sending "start" command and is part of | |
375 | * standard reset sequence. | |
376 | */ | |
377 | static bool vmballoon_send_guest_id(struct vmballoon *b) | |
378 | { | |
f220a80f | 379 | unsigned long status, dummy = 0; |
453dc659 | 380 | |
f220a80f XD |
381 | status = VMWARE_BALLOON_CMD(GUEST_ID, VMW_BALLOON_GUEST_ID, dummy, |
382 | dummy); | |
453dc659 DT |
383 | |
384 | STATS_INC(b->stats.guest_type); | |
385 | ||
386 | if (vmballoon_check_status(b, status)) | |
387 | return true; | |
388 | ||
389 | pr_debug("%s - failed, hv returns %ld\n", __func__, status); | |
390 | STATS_INC(b->stats.guest_type_fail); | |
391 | return false; | |
392 | } | |
393 | ||
365bd7ef PM |
394 | static u16 vmballoon_page_size(bool is_2m_page) |
395 | { | |
396 | if (is_2m_page) | |
397 | return 1 << VMW_BALLOON_2M_SHIFT; | |
398 | ||
399 | return 1; | |
400 | } | |
401 | ||
453dc659 DT |
402 | /* |
403 | * Retrieve desired balloon size from the host. | |
404 | */ | |
405 | static bool vmballoon_send_get_target(struct vmballoon *b, u32 *new_target) | |
406 | { | |
407 | unsigned long status; | |
408 | unsigned long target; | |
409 | unsigned long limit; | |
f220a80f | 410 | unsigned long dummy = 0; |
453dc659 DT |
411 | u32 limit32; |
412 | ||
413 | /* | |
414 | * si_meminfo() is cheap. Moreover, we want to provide dynamic | |
415 | * max balloon size later. So let us call si_meminfo() every | |
416 | * iteration. | |
417 | */ | |
418 | si_meminfo(&b->sysinfo); | |
419 | limit = b->sysinfo.totalram; | |
420 | ||
421 | /* Ensure limit fits in 32-bits */ | |
422 | limit32 = (u32)limit; | |
423 | if (limit != limit32) | |
424 | return false; | |
425 | ||
426 | /* update stats */ | |
427 | STATS_INC(b->stats.target); | |
428 | ||
f220a80f | 429 | status = VMWARE_BALLOON_CMD(GET_TARGET, limit, dummy, target); |
453dc659 DT |
430 | if (vmballoon_check_status(b, status)) { |
431 | *new_target = target; | |
432 | return true; | |
433 | } | |
434 | ||
435 | pr_debug("%s - failed, hv returns %ld\n", __func__, status); | |
436 | STATS_INC(b->stats.target_fail); | |
437 | return false; | |
438 | } | |
439 | ||
440 | /* | |
441 | * Notify the host about allocated page so that host can use it without | |
442 | * fear that guest will need it. Host may reject some pages, we need to | |
443 | * check the return value and maybe submit a different page. | |
444 | */ | |
3e5ba466 | 445 | static int vmballoon_send_lock_page(struct vmballoon *b, unsigned long pfn, |
4670de4d | 446 | unsigned int *hv_status, unsigned int *target) |
453dc659 | 447 | { |
f220a80f | 448 | unsigned long status, dummy = 0; |
453dc659 DT |
449 | u32 pfn32; |
450 | ||
451 | pfn32 = (u32)pfn; | |
452 | if (pfn32 != pfn) | |
3e5ba466 | 453 | return -1; |
453dc659 | 454 | |
365bd7ef | 455 | STATS_INC(b->stats.lock[false]); |
453dc659 | 456 | |
4670de4d | 457 | *hv_status = status = VMWARE_BALLOON_CMD(LOCK, pfn, dummy, *target); |
453dc659 | 458 | if (vmballoon_check_status(b, status)) |
3e5ba466 | 459 | return 0; |
453dc659 DT |
460 | |
461 | pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status); | |
365bd7ef | 462 | STATS_INC(b->stats.lock_fail[false]); |
3e5ba466 | 463 | return 1; |
453dc659 DT |
464 | } |
465 | ||
f220a80f | 466 | static int vmballoon_send_batched_lock(struct vmballoon *b, |
365bd7ef | 467 | unsigned int num_pages, bool is_2m_pages, unsigned int *target) |
f220a80f | 468 | { |
4670de4d | 469 | unsigned long status; |
95fafb7a | 470 | unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page)); |
f220a80f | 471 | |
365bd7ef PM |
472 | STATS_INC(b->stats.lock[is_2m_pages]); |
473 | ||
474 | if (is_2m_pages) | |
475 | status = VMWARE_BALLOON_CMD(BATCHED_2M_LOCK, pfn, num_pages, | |
476 | *target); | |
477 | else | |
478 | status = VMWARE_BALLOON_CMD(BATCHED_LOCK, pfn, num_pages, | |
479 | *target); | |
f220a80f | 480 | |
f220a80f XD |
481 | if (vmballoon_check_status(b, status)) |
482 | return 0; | |
483 | ||
484 | pr_debug("%s - batch ppn %lx, hv returns %ld\n", __func__, pfn, status); | |
365bd7ef | 485 | STATS_INC(b->stats.lock_fail[is_2m_pages]); |
f220a80f XD |
486 | return 1; |
487 | } | |
488 | ||
453dc659 DT |
489 | /* |
490 | * Notify the host that guest intends to release given page back into | |
491 | * the pool of available (to the guest) pages. | |
492 | */ | |
4670de4d XD |
493 | static bool vmballoon_send_unlock_page(struct vmballoon *b, unsigned long pfn, |
494 | unsigned int *target) | |
453dc659 | 495 | { |
f220a80f | 496 | unsigned long status, dummy = 0; |
453dc659 DT |
497 | u32 pfn32; |
498 | ||
499 | pfn32 = (u32)pfn; | |
500 | if (pfn32 != pfn) | |
501 | return false; | |
502 | ||
365bd7ef | 503 | STATS_INC(b->stats.unlock[false]); |
453dc659 | 504 | |
4670de4d | 505 | status = VMWARE_BALLOON_CMD(UNLOCK, pfn, dummy, *target); |
453dc659 DT |
506 | if (vmballoon_check_status(b, status)) |
507 | return true; | |
508 | ||
509 | pr_debug("%s - ppn %lx, hv returns %ld\n", __func__, pfn, status); | |
365bd7ef | 510 | STATS_INC(b->stats.unlock_fail[false]); |
453dc659 DT |
511 | return false; |
512 | } | |
513 | ||
f220a80f | 514 | static bool vmballoon_send_batched_unlock(struct vmballoon *b, |
365bd7ef | 515 | unsigned int num_pages, bool is_2m_pages, unsigned int *target) |
f220a80f | 516 | { |
4670de4d | 517 | unsigned long status; |
95fafb7a | 518 | unsigned long pfn = PHYS_PFN(virt_to_phys(b->batch_page)); |
f220a80f | 519 | |
365bd7ef PM |
520 | STATS_INC(b->stats.unlock[is_2m_pages]); |
521 | ||
522 | if (is_2m_pages) | |
523 | status = VMWARE_BALLOON_CMD(BATCHED_2M_UNLOCK, pfn, num_pages, | |
524 | *target); | |
525 | else | |
526 | status = VMWARE_BALLOON_CMD(BATCHED_UNLOCK, pfn, num_pages, | |
527 | *target); | |
f220a80f | 528 | |
f220a80f XD |
529 | if (vmballoon_check_status(b, status)) |
530 | return true; | |
531 | ||
532 | pr_debug("%s - batch ppn %lx, hv returns %ld\n", __func__, pfn, status); | |
365bd7ef | 533 | STATS_INC(b->stats.unlock_fail[is_2m_pages]); |
f220a80f XD |
534 | return false; |
535 | } | |
536 | ||
365bd7ef PM |
537 | static struct page *vmballoon_alloc_page(gfp_t flags, bool is_2m_page) |
538 | { | |
539 | if (is_2m_page) | |
540 | return alloc_pages(flags, VMW_BALLOON_2M_SHIFT); | |
541 | ||
542 | return alloc_page(flags); | |
543 | } | |
544 | ||
545 | static void vmballoon_free_page(struct page *page, bool is_2m_page) | |
546 | { | |
547 | if (is_2m_page) | |
548 | __free_pages(page, VMW_BALLOON_2M_SHIFT); | |
549 | else | |
550 | __free_page(page); | |
551 | } | |
552 | ||
453dc659 DT |
553 | /* |
554 | * Quickly release all pages allocated for the balloon. This function is | |
555 | * called when host decides to "reset" balloon for one reason or another. | |
556 | * Unlike normal "deflate" we do not (shall not) notify host of the pages | |
557 | * being released. | |
558 | */ | |
559 | static void vmballoon_pop(struct vmballoon *b) | |
560 | { | |
561 | struct page *page, *next; | |
365bd7ef PM |
562 | unsigned is_2m_pages; |
563 | ||
564 | for (is_2m_pages = 0; is_2m_pages < VMW_BALLOON_NUM_PAGE_SIZES; | |
565 | is_2m_pages++) { | |
566 | struct vmballoon_page_size *page_size = | |
567 | &b->page_sizes[is_2m_pages]; | |
568 | u16 size_per_page = vmballoon_page_size(is_2m_pages); | |
569 | ||
570 | list_for_each_entry_safe(page, next, &page_size->pages, lru) { | |
571 | list_del(&page->lru); | |
572 | vmballoon_free_page(page, is_2m_pages); | |
573 | STATS_INC(b->stats.free[is_2m_pages]); | |
574 | b->size -= size_per_page; | |
575 | cond_resched(); | |
576 | } | |
453dc659 | 577 | } |
453dc659 | 578 | |
35968885 GK |
579 | /* Clearing the batch_page unconditionally has no adverse effect */ |
580 | free_page((unsigned long)b->batch_page); | |
581 | b->batch_page = NULL; | |
453dc659 DT |
582 | } |
583 | ||
584 | /* | |
ef0f8f11 XD |
585 | * Notify the host of a ballooned page. If host rejects the page put it on the |
586 | * refuse list, those refused page are then released at the end of the | |
587 | * inflation cycle. | |
453dc659 | 588 | */ |
4670de4d | 589 | static int vmballoon_lock_page(struct vmballoon *b, unsigned int num_pages, |
365bd7ef | 590 | bool is_2m_pages, unsigned int *target) |
453dc659 | 591 | { |
ef0f8f11 | 592 | int locked, hv_status; |
f220a80f | 593 | struct page *page = b->page; |
365bd7ef PM |
594 | struct vmballoon_page_size *page_size = &b->page_sizes[false]; |
595 | ||
596 | /* is_2m_pages can never happen as 2m pages support implies batching */ | |
453dc659 | 597 | |
4670de4d XD |
598 | locked = vmballoon_send_lock_page(b, page_to_pfn(page), &hv_status, |
599 | target); | |
ef0f8f11 | 600 | if (locked > 0) { |
365bd7ef | 601 | STATS_INC(b->stats.refused_alloc[false]); |
453dc659 | 602 | |
ef0f8f11 XD |
603 | if (hv_status == VMW_BALLOON_ERROR_RESET || |
604 | hv_status == VMW_BALLOON_ERROR_PPN_NOTNEEDED) { | |
365bd7ef | 605 | vmballoon_free_page(page, false); |
ef0f8f11 XD |
606 | return -EIO; |
607 | } | |
453dc659 | 608 | |
ef0f8f11 XD |
609 | /* |
610 | * Place page on the list of non-balloonable pages | |
611 | * and retry allocation, unless we already accumulated | |
612 | * too many of them, in which case take a breather. | |
613 | */ | |
365bd7ef PM |
614 | if (page_size->n_refused_pages < VMW_BALLOON_MAX_REFUSED) { |
615 | page_size->n_refused_pages++; | |
616 | list_add(&page->lru, &page_size->refused_pages); | |
ef0f8f11 | 617 | } else { |
365bd7ef | 618 | vmballoon_free_page(page, false); |
453dc659 | 619 | } |
ef0f8f11 XD |
620 | return -EIO; |
621 | } | |
453dc659 DT |
622 | |
623 | /* track allocated page */ | |
365bd7ef | 624 | list_add(&page->lru, &page_size->pages); |
453dc659 DT |
625 | |
626 | /* update balloon size */ | |
627 | b->size++; | |
628 | ||
629 | return 0; | |
630 | } | |
631 | ||
f220a80f | 632 | static int vmballoon_lock_batched_page(struct vmballoon *b, |
365bd7ef | 633 | unsigned int num_pages, bool is_2m_pages, unsigned int *target) |
f220a80f XD |
634 | { |
635 | int locked, i; | |
365bd7ef | 636 | u16 size_per_page = vmballoon_page_size(is_2m_pages); |
f220a80f | 637 | |
365bd7ef PM |
638 | locked = vmballoon_send_batched_lock(b, num_pages, is_2m_pages, |
639 | target); | |
f220a80f XD |
640 | if (locked > 0) { |
641 | for (i = 0; i < num_pages; i++) { | |
642 | u64 pa = vmballoon_batch_get_pa(b->batch_page, i); | |
643 | struct page *p = pfn_to_page(pa >> PAGE_SHIFT); | |
644 | ||
365bd7ef | 645 | vmballoon_free_page(p, is_2m_pages); |
f220a80f XD |
646 | } |
647 | ||
648 | return -EIO; | |
649 | } | |
650 | ||
651 | for (i = 0; i < num_pages; i++) { | |
652 | u64 pa = vmballoon_batch_get_pa(b->batch_page, i); | |
653 | struct page *p = pfn_to_page(pa >> PAGE_SHIFT); | |
365bd7ef PM |
654 | struct vmballoon_page_size *page_size = |
655 | &b->page_sizes[is_2m_pages]; | |
f220a80f XD |
656 | |
657 | locked = vmballoon_batch_get_status(b->batch_page, i); | |
658 | ||
659 | switch (locked) { | |
660 | case VMW_BALLOON_SUCCESS: | |
365bd7ef PM |
661 | list_add(&p->lru, &page_size->pages); |
662 | b->size += size_per_page; | |
f220a80f XD |
663 | break; |
664 | case VMW_BALLOON_ERROR_PPN_PINNED: | |
665 | case VMW_BALLOON_ERROR_PPN_INVALID: | |
365bd7ef PM |
666 | if (page_size->n_refused_pages |
667 | < VMW_BALLOON_MAX_REFUSED) { | |
668 | list_add(&p->lru, &page_size->refused_pages); | |
669 | page_size->n_refused_pages++; | |
f220a80f XD |
670 | break; |
671 | } | |
672 | /* Fallthrough */ | |
673 | case VMW_BALLOON_ERROR_RESET: | |
674 | case VMW_BALLOON_ERROR_PPN_NOTNEEDED: | |
365bd7ef | 675 | vmballoon_free_page(p, is_2m_pages); |
f220a80f XD |
676 | break; |
677 | default: | |
678 | /* This should never happen */ | |
679 | WARN_ON_ONCE(true); | |
680 | } | |
681 | } | |
682 | ||
683 | return 0; | |
684 | } | |
685 | ||
453dc659 DT |
686 | /* |
687 | * Release the page allocated for the balloon. Note that we first notify | |
688 | * the host so it can make sure the page will be available for the guest | |
689 | * to use, if needed. | |
690 | */ | |
4670de4d | 691 | static int vmballoon_unlock_page(struct vmballoon *b, unsigned int num_pages, |
365bd7ef | 692 | bool is_2m_pages, unsigned int *target) |
453dc659 | 693 | { |
f220a80f | 694 | struct page *page = b->page; |
365bd7ef PM |
695 | struct vmballoon_page_size *page_size = &b->page_sizes[false]; |
696 | ||
697 | /* is_2m_pages can never happen as 2m pages support implies batching */ | |
453dc659 | 698 | |
4670de4d | 699 | if (!vmballoon_send_unlock_page(b, page_to_pfn(page), target)) { |
365bd7ef | 700 | list_add(&page->lru, &page_size->pages); |
f220a80f XD |
701 | return -EIO; |
702 | } | |
453dc659 DT |
703 | |
704 | /* deallocate page */ | |
365bd7ef PM |
705 | vmballoon_free_page(page, false); |
706 | STATS_INC(b->stats.free[false]); | |
453dc659 DT |
707 | |
708 | /* update balloon size */ | |
709 | b->size--; | |
710 | ||
711 | return 0; | |
712 | } | |
713 | ||
f220a80f | 714 | static int vmballoon_unlock_batched_page(struct vmballoon *b, |
365bd7ef PM |
715 | unsigned int num_pages, bool is_2m_pages, |
716 | unsigned int *target) | |
f220a80f XD |
717 | { |
718 | int locked, i, ret = 0; | |
719 | bool hv_success; | |
365bd7ef | 720 | u16 size_per_page = vmballoon_page_size(is_2m_pages); |
f220a80f | 721 | |
365bd7ef PM |
722 | hv_success = vmballoon_send_batched_unlock(b, num_pages, is_2m_pages, |
723 | target); | |
f220a80f XD |
724 | if (!hv_success) |
725 | ret = -EIO; | |
726 | ||
727 | for (i = 0; i < num_pages; i++) { | |
728 | u64 pa = vmballoon_batch_get_pa(b->batch_page, i); | |
729 | struct page *p = pfn_to_page(pa >> PAGE_SHIFT); | |
365bd7ef PM |
730 | struct vmballoon_page_size *page_size = |
731 | &b->page_sizes[is_2m_pages]; | |
f220a80f XD |
732 | |
733 | locked = vmballoon_batch_get_status(b->batch_page, i); | |
734 | if (!hv_success || locked != VMW_BALLOON_SUCCESS) { | |
735 | /* | |
736 | * That page wasn't successfully unlocked by the | |
737 | * hypervisor, re-add it to the list of pages owned by | |
738 | * the balloon driver. | |
739 | */ | |
365bd7ef | 740 | list_add(&p->lru, &page_size->pages); |
f220a80f XD |
741 | } else { |
742 | /* deallocate page */ | |
365bd7ef PM |
743 | vmballoon_free_page(p, is_2m_pages); |
744 | STATS_INC(b->stats.free[is_2m_pages]); | |
f220a80f XD |
745 | |
746 | /* update balloon size */ | |
365bd7ef | 747 | b->size -= size_per_page; |
f220a80f XD |
748 | } |
749 | } | |
750 | ||
751 | return ret; | |
752 | } | |
753 | ||
453dc659 DT |
754 | /* |
755 | * Release pages that were allocated while attempting to inflate the | |
756 | * balloon but were refused by the host for one reason or another. | |
757 | */ | |
365bd7ef PM |
758 | static void vmballoon_release_refused_pages(struct vmballoon *b, |
759 | bool is_2m_pages) | |
453dc659 DT |
760 | { |
761 | struct page *page, *next; | |
365bd7ef PM |
762 | struct vmballoon_page_size *page_size = |
763 | &b->page_sizes[is_2m_pages]; | |
453dc659 | 764 | |
365bd7ef | 765 | list_for_each_entry_safe(page, next, &page_size->refused_pages, lru) { |
453dc659 | 766 | list_del(&page->lru); |
365bd7ef PM |
767 | vmballoon_free_page(page, is_2m_pages); |
768 | STATS_INC(b->stats.refused_free[is_2m_pages]); | |
453dc659 | 769 | } |
55adaa49 | 770 | |
365bd7ef | 771 | page_size->n_refused_pages = 0; |
453dc659 DT |
772 | } |
773 | ||
f220a80f XD |
774 | static void vmballoon_add_page(struct vmballoon *b, int idx, struct page *p) |
775 | { | |
776 | b->page = p; | |
777 | } | |
778 | ||
779 | static void vmballoon_add_batched_page(struct vmballoon *b, int idx, | |
780 | struct page *p) | |
781 | { | |
782 | vmballoon_batch_set_pa(b->batch_page, idx, | |
783 | (u64)page_to_pfn(p) << PAGE_SHIFT); | |
784 | } | |
785 | ||
453dc659 DT |
786 | /* |
787 | * Inflate the balloon towards its target size. Note that we try to limit | |
788 | * the rate of allocation to make sure we are not choking the rest of the | |
789 | * system. | |
790 | */ | |
791 | static void vmballoon_inflate(struct vmballoon *b) | |
792 | { | |
33d268ed | 793 | unsigned rate; |
453dc659 | 794 | unsigned int allocations = 0; |
f220a80f | 795 | unsigned int num_pages = 0; |
453dc659 | 796 | int error = 0; |
ef0f8f11 | 797 | gfp_t flags = VMW_PAGE_ALLOC_NOSLEEP; |
365bd7ef | 798 | bool is_2m_pages; |
453dc659 DT |
799 | |
800 | pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target); | |
801 | ||
802 | /* | |
803 | * First try NOSLEEP page allocations to inflate balloon. | |
804 | * | |
805 | * If we do not throttle nosleep allocations, we can drain all | |
806 | * free pages in the guest quickly (if the balloon target is high). | |
807 | * As a side-effect, draining free pages helps to inform (force) | |
808 | * the guest to start swapping if balloon target is not met yet, | |
809 | * which is a desired behavior. However, balloon driver can consume | |
810 | * all available CPU cycles if too many pages are allocated in a | |
811 | * second. Therefore, we throttle nosleep allocations even when | |
812 | * the guest is not under memory pressure. OTOH, if we have already | |
813 | * predicted that the guest is under memory pressure, then we | |
814 | * slowdown page allocations considerably. | |
815 | */ | |
816 | ||
453dc659 DT |
817 | /* |
818 | * Start with no sleep allocation rate which may be higher | |
819 | * than sleeping allocation rate. | |
820 | */ | |
365bd7ef PM |
821 | if (b->slow_allocation_cycles) { |
822 | rate = b->rate_alloc; | |
823 | is_2m_pages = false; | |
824 | } else { | |
825 | rate = UINT_MAX; | |
826 | is_2m_pages = | |
827 | b->supported_page_sizes == VMW_BALLOON_NUM_PAGE_SIZES; | |
828 | } | |
453dc659 | 829 | |
33d268ed | 830 | pr_debug("%s - goal: %d, no-sleep rate: %u, sleep rate: %d\n", |
4670de4d | 831 | __func__, b->target - b->size, rate, b->rate_alloc); |
453dc659 | 832 | |
33d268ed | 833 | while (!b->reset_required && |
365bd7ef PM |
834 | b->size + num_pages * vmballoon_page_size(is_2m_pages) |
835 | < b->target) { | |
4670de4d | 836 | struct page *page; |
453dc659 | 837 | |
ef0f8f11 | 838 | if (flags == VMW_PAGE_ALLOC_NOSLEEP) |
365bd7ef | 839 | STATS_INC(b->stats.alloc[is_2m_pages]); |
ef0f8f11 XD |
840 | else |
841 | STATS_INC(b->stats.sleep_alloc); | |
453dc659 | 842 | |
365bd7ef | 843 | page = vmballoon_alloc_page(flags, is_2m_pages); |
ef0f8f11 | 844 | if (!page) { |
365bd7ef PM |
845 | STATS_INC(b->stats.alloc_fail[is_2m_pages]); |
846 | ||
847 | if (is_2m_pages) { | |
848 | b->ops->lock(b, num_pages, true, &b->target); | |
849 | ||
850 | /* | |
851 | * ignore errors from locking as we now switch | |
852 | * to 4k pages and we might get different | |
853 | * errors. | |
854 | */ | |
855 | ||
856 | num_pages = 0; | |
857 | is_2m_pages = false; | |
858 | continue; | |
859 | } | |
860 | ||
ef0f8f11 | 861 | if (flags == VMW_PAGE_ALLOC_CANSLEEP) { |
453dc659 DT |
862 | /* |
863 | * CANSLEEP page allocation failed, so guest | |
864 | * is under severe memory pressure. Quickly | |
865 | * decrease allocation rate. | |
866 | */ | |
867 | b->rate_alloc = max(b->rate_alloc / 2, | |
868 | VMW_BALLOON_RATE_ALLOC_MIN); | |
ef0f8f11 | 869 | STATS_INC(b->stats.sleep_alloc_fail); |
453dc659 DT |
870 | break; |
871 | } | |
872 | ||
873 | /* | |
874 | * NOSLEEP page allocation failed, so the guest is | |
875 | * under memory pressure. Let us slow down page | |
876 | * allocations for next few cycles so that the guest | |
877 | * gets out of memory pressure. Also, if we already | |
878 | * allocated b->rate_alloc pages, let's pause, | |
879 | * otherwise switch to sleeping allocations. | |
880 | */ | |
881 | b->slow_allocation_cycles = VMW_BALLOON_SLOW_CYCLES; | |
882 | ||
4670de4d | 883 | if (allocations >= b->rate_alloc) |
453dc659 DT |
884 | break; |
885 | ||
ef0f8f11 | 886 | flags = VMW_PAGE_ALLOC_CANSLEEP; |
453dc659 DT |
887 | /* Lower rate for sleeping allocations. */ |
888 | rate = b->rate_alloc; | |
ef0f8f11 | 889 | continue; |
453dc659 DT |
890 | } |
891 | ||
f220a80f XD |
892 | b->ops->add_page(b, num_pages++, page); |
893 | if (num_pages == b->batch_max_pages) { | |
365bd7ef PM |
894 | error = b->ops->lock(b, num_pages, is_2m_pages, |
895 | &b->target); | |
f220a80f XD |
896 | num_pages = 0; |
897 | if (error) | |
898 | break; | |
899 | } | |
ef0f8f11 | 900 | |
33d268ed | 901 | cond_resched(); |
453dc659 | 902 | |
4670de4d | 903 | if (allocations >= rate) { |
453dc659 DT |
904 | /* We allocated enough pages, let's take a break. */ |
905 | break; | |
906 | } | |
907 | } | |
908 | ||
f220a80f | 909 | if (num_pages > 0) |
365bd7ef | 910 | b->ops->lock(b, num_pages, is_2m_pages, &b->target); |
f220a80f | 911 | |
453dc659 DT |
912 | /* |
913 | * We reached our goal without failures so try increasing | |
914 | * allocation rate. | |
915 | */ | |
4670de4d XD |
916 | if (error == 0 && allocations >= b->rate_alloc) { |
917 | unsigned int mult = allocations / b->rate_alloc; | |
453dc659 DT |
918 | |
919 | b->rate_alloc = | |
920 | min(b->rate_alloc + mult * VMW_BALLOON_RATE_ALLOC_INC, | |
921 | VMW_BALLOON_RATE_ALLOC_MAX); | |
922 | } | |
923 | ||
365bd7ef PM |
924 | vmballoon_release_refused_pages(b, true); |
925 | vmballoon_release_refused_pages(b, false); | |
453dc659 DT |
926 | } |
927 | ||
928 | /* | |
929 | * Decrease the size of the balloon allowing guest to use more memory. | |
930 | */ | |
931 | static void vmballoon_deflate(struct vmballoon *b) | |
932 | { | |
365bd7ef | 933 | unsigned is_2m_pages; |
453dc659 | 934 | |
33d268ed | 935 | pr_debug("%s - size: %d, target %d\n", __func__, b->size, b->target); |
453dc659 DT |
936 | |
937 | /* free pages to reach target */ | |
365bd7ef PM |
938 | for (is_2m_pages = 0; is_2m_pages < b->supported_page_sizes; |
939 | is_2m_pages++) { | |
940 | struct page *page, *next; | |
941 | unsigned int num_pages = 0; | |
942 | struct vmballoon_page_size *page_size = | |
943 | &b->page_sizes[is_2m_pages]; | |
944 | ||
945 | list_for_each_entry_safe(page, next, &page_size->pages, lru) { | |
946 | if (b->reset_required || | |
947 | (b->target > 0 && | |
948 | b->size - num_pages | |
949 | * vmballoon_page_size(is_2m_pages) | |
950 | < b->target + vmballoon_page_size(true))) | |
951 | break; | |
f220a80f | 952 | |
365bd7ef PM |
953 | list_del(&page->lru); |
954 | b->ops->add_page(b, num_pages++, page); | |
33d268ed | 955 | |
365bd7ef PM |
956 | if (num_pages == b->batch_max_pages) { |
957 | int error; | |
453dc659 | 958 | |
365bd7ef PM |
959 | error = b->ops->unlock(b, num_pages, |
960 | is_2m_pages, &b->target); | |
961 | num_pages = 0; | |
962 | if (error) | |
963 | return; | |
964 | } | |
33d268ed | 965 | |
365bd7ef PM |
966 | cond_resched(); |
967 | } | |
453dc659 | 968 | |
365bd7ef PM |
969 | if (num_pages > 0) |
970 | b->ops->unlock(b, num_pages, is_2m_pages, &b->target); | |
971 | } | |
f220a80f XD |
972 | } |
973 | ||
974 | static const struct vmballoon_ops vmballoon_basic_ops = { | |
975 | .add_page = vmballoon_add_page, | |
976 | .lock = vmballoon_lock_page, | |
977 | .unlock = vmballoon_unlock_page | |
978 | }; | |
979 | ||
980 | static const struct vmballoon_ops vmballoon_batched_ops = { | |
981 | .add_page = vmballoon_add_batched_page, | |
982 | .lock = vmballoon_lock_batched_page, | |
983 | .unlock = vmballoon_unlock_batched_page | |
984 | }; | |
985 | ||
986 | static bool vmballoon_init_batching(struct vmballoon *b) | |
987 | { | |
35968885 | 988 | struct page *page; |
f220a80f | 989 | |
35968885 GK |
990 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); |
991 | if (!page) | |
f220a80f | 992 | return false; |
f220a80f | 993 | |
35968885 | 994 | b->batch_page = page_address(page); |
f220a80f XD |
995 | return true; |
996 | } | |
997 | ||
48e3d668 PM |
998 | /* |
999 | * Receive notification and resize balloon | |
1000 | */ | |
1001 | static void vmballoon_doorbell(void *client_data) | |
1002 | { | |
1003 | struct vmballoon *b = client_data; | |
1004 | ||
1005 | STATS_INC(b->stats.doorbell); | |
1006 | ||
1007 | mod_delayed_work(system_freezable_wq, &b->dwork, 0); | |
1008 | } | |
1009 | ||
1010 | /* | |
1011 | * Clean up vmci doorbell | |
1012 | */ | |
1013 | static void vmballoon_vmci_cleanup(struct vmballoon *b) | |
1014 | { | |
1015 | int error; | |
1016 | ||
1017 | VMWARE_BALLOON_CMD(VMCI_DOORBELL_SET, VMCI_INVALID_ID, | |
1018 | VMCI_INVALID_ID, error); | |
1019 | STATS_INC(b->stats.doorbell_unset); | |
1020 | ||
1021 | if (!vmci_handle_is_invalid(b->vmci_doorbell)) { | |
1022 | vmci_doorbell_destroy(b->vmci_doorbell); | |
1023 | b->vmci_doorbell = VMCI_INVALID_HANDLE; | |
1024 | } | |
1025 | } | |
1026 | ||
1027 | /* | |
1028 | * Initialize vmci doorbell, to get notified as soon as balloon changes | |
1029 | */ | |
1030 | static int vmballoon_vmci_init(struct vmballoon *b) | |
1031 | { | |
1032 | int error = 0; | |
1033 | ||
1034 | if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) != 0) { | |
1035 | error = vmci_doorbell_create(&b->vmci_doorbell, | |
1036 | VMCI_FLAG_DELAYED_CB, | |
1037 | VMCI_PRIVILEGE_FLAG_RESTRICTED, | |
1038 | vmballoon_doorbell, b); | |
1039 | ||
1040 | if (error == VMCI_SUCCESS) { | |
1041 | VMWARE_BALLOON_CMD(VMCI_DOORBELL_SET, | |
1042 | b->vmci_doorbell.context, | |
1043 | b->vmci_doorbell.resource, error); | |
1044 | STATS_INC(b->stats.doorbell_set); | |
1045 | } | |
1046 | } | |
1047 | ||
1048 | if (error != 0) { | |
1049 | vmballoon_vmci_cleanup(b); | |
1050 | ||
1051 | return -EIO; | |
1052 | } | |
1053 | ||
1054 | return 0; | |
1055 | } | |
1056 | ||
f220a80f XD |
1057 | /* |
1058 | * Perform standard reset sequence by popping the balloon (in case it | |
1059 | * is not empty) and then restarting protocol. This operation normally | |
1060 | * happens when host responds with VMW_BALLOON_ERROR_RESET to a command. | |
1061 | */ | |
1062 | static void vmballoon_reset(struct vmballoon *b) | |
1063 | { | |
48e3d668 PM |
1064 | int error; |
1065 | ||
1066 | vmballoon_vmci_cleanup(b); | |
1067 | ||
f220a80f XD |
1068 | /* free all pages, skipping monitor unlock */ |
1069 | vmballoon_pop(b); | |
1070 | ||
1071 | if (!vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES)) | |
1072 | return; | |
1073 | ||
1074 | if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) { | |
1075 | b->ops = &vmballoon_batched_ops; | |
1076 | b->batch_max_pages = VMW_BALLOON_BATCH_MAX_PAGES; | |
1077 | if (!vmballoon_init_batching(b)) { | |
1078 | /* | |
1079 | * We failed to initialize batching, inform the monitor | |
1080 | * about it by sending a null capability. | |
1081 | * | |
1082 | * The guest will retry in one second. | |
1083 | */ | |
1084 | vmballoon_send_start(b, 0); | |
1085 | return; | |
1086 | } | |
1087 | } else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) { | |
1088 | b->ops = &vmballoon_basic_ops; | |
1089 | b->batch_max_pages = 1; | |
1090 | } | |
1091 | ||
1092 | b->reset_required = false; | |
48e3d668 PM |
1093 | |
1094 | error = vmballoon_vmci_init(b); | |
1095 | if (error) | |
1096 | pr_err("failed to initialize vmci doorbell\n"); | |
1097 | ||
f220a80f XD |
1098 | if (!vmballoon_send_guest_id(b)) |
1099 | pr_err("failed to send guest ID to the host\n"); | |
453dc659 DT |
1100 | } |
1101 | ||
1102 | /* | |
1103 | * Balloon work function: reset protocol, if needed, get the new size and | |
1104 | * adjust balloon as needed. Repeat in 1 sec. | |
1105 | */ | |
1106 | static void vmballoon_work(struct work_struct *work) | |
1107 | { | |
1108 | struct delayed_work *dwork = to_delayed_work(work); | |
1109 | struct vmballoon *b = container_of(dwork, struct vmballoon, dwork); | |
1110 | unsigned int target; | |
1111 | ||
1112 | STATS_INC(b->stats.timer); | |
1113 | ||
1114 | if (b->reset_required) | |
1115 | vmballoon_reset(b); | |
1116 | ||
1117 | if (b->slow_allocation_cycles > 0) | |
1118 | b->slow_allocation_cycles--; | |
1119 | ||
d7568c13 | 1120 | if (!b->reset_required && vmballoon_send_get_target(b, &target)) { |
453dc659 DT |
1121 | /* update target, adjust size */ |
1122 | b->target = target; | |
1123 | ||
1124 | if (b->size < target) | |
1125 | vmballoon_inflate(b); | |
365bd7ef PM |
1126 | else if (target == 0 || |
1127 | b->size > target + vmballoon_page_size(true)) | |
453dc659 DT |
1128 | vmballoon_deflate(b); |
1129 | } | |
1130 | ||
beda94da DT |
1131 | /* |
1132 | * We are using a freezable workqueue so that balloon operations are | |
1133 | * stopped while the system transitions to/from sleep/hibernation. | |
1134 | */ | |
1135 | queue_delayed_work(system_freezable_wq, | |
1136 | dwork, round_jiffies_relative(HZ)); | |
453dc659 DT |
1137 | } |
1138 | ||
1139 | /* | |
1140 | * DEBUGFS Interface | |
1141 | */ | |
1142 | #ifdef CONFIG_DEBUG_FS | |
1143 | ||
1144 | static int vmballoon_debug_show(struct seq_file *f, void *offset) | |
1145 | { | |
1146 | struct vmballoon *b = f->private; | |
1147 | struct vmballoon_stats *stats = &b->stats; | |
1148 | ||
b36e89da PM |
1149 | /* format capabilities info */ |
1150 | seq_printf(f, | |
1151 | "balloon capabilities: %#4x\n" | |
d7568c13 PM |
1152 | "used capabilities: %#4lx\n" |
1153 | "is resetting: %c\n", | |
1154 | VMW_BALLOON_CAPABILITIES, b->capabilities, | |
1155 | b->reset_required ? 'y' : 'n'); | |
b36e89da | 1156 | |
453dc659 DT |
1157 | /* format size info */ |
1158 | seq_printf(f, | |
1159 | "target: %8d pages\n" | |
1160 | "current: %8d pages\n", | |
1161 | b->target, b->size); | |
1162 | ||
1163 | /* format rate info */ | |
1164 | seq_printf(f, | |
33d268ed PM |
1165 | "rateSleepAlloc: %8d pages/sec\n", |
1166 | b->rate_alloc); | |
453dc659 DT |
1167 | |
1168 | seq_printf(f, | |
1169 | "\n" | |
1170 | "timer: %8u\n" | |
48e3d668 | 1171 | "doorbell: %8u\n" |
453dc659 DT |
1172 | "start: %8u (%4u failed)\n" |
1173 | "guestType: %8u (%4u failed)\n" | |
365bd7ef | 1174 | "2m-lock: %8u (%4u failed)\n" |
453dc659 | 1175 | "lock: %8u (%4u failed)\n" |
365bd7ef | 1176 | "2m-unlock: %8u (%4u failed)\n" |
453dc659 DT |
1177 | "unlock: %8u (%4u failed)\n" |
1178 | "target: %8u (%4u failed)\n" | |
365bd7ef | 1179 | "prim2mAlloc: %8u (%4u failed)\n" |
453dc659 DT |
1180 | "primNoSleepAlloc: %8u (%4u failed)\n" |
1181 | "primCanSleepAlloc: %8u (%4u failed)\n" | |
365bd7ef | 1182 | "prim2mFree: %8u\n" |
453dc659 | 1183 | "primFree: %8u\n" |
365bd7ef | 1184 | "err2mAlloc: %8u\n" |
453dc659 | 1185 | "errAlloc: %8u\n" |
365bd7ef | 1186 | "err2mFree: %8u\n" |
48e3d668 PM |
1187 | "errFree: %8u\n" |
1188 | "doorbellSet: %8u\n" | |
1189 | "doorbellUnset: %8u\n", | |
453dc659 | 1190 | stats->timer, |
48e3d668 | 1191 | stats->doorbell, |
453dc659 DT |
1192 | stats->start, stats->start_fail, |
1193 | stats->guest_type, stats->guest_type_fail, | |
365bd7ef PM |
1194 | stats->lock[true], stats->lock_fail[true], |
1195 | stats->lock[false], stats->lock_fail[false], | |
1196 | stats->unlock[true], stats->unlock_fail[true], | |
1197 | stats->unlock[false], stats->unlock_fail[false], | |
453dc659 | 1198 | stats->target, stats->target_fail, |
365bd7ef PM |
1199 | stats->alloc[true], stats->alloc_fail[true], |
1200 | stats->alloc[false], stats->alloc_fail[false], | |
453dc659 | 1201 | stats->sleep_alloc, stats->sleep_alloc_fail, |
365bd7ef PM |
1202 | stats->free[true], |
1203 | stats->free[false], | |
1204 | stats->refused_alloc[true], stats->refused_alloc[false], | |
48e3d668 PM |
1205 | stats->refused_free[true], stats->refused_free[false], |
1206 | stats->doorbell_set, stats->doorbell_unset); | |
453dc659 DT |
1207 | |
1208 | return 0; | |
1209 | } | |
1210 | ||
1211 | static int vmballoon_debug_open(struct inode *inode, struct file *file) | |
1212 | { | |
1213 | return single_open(file, vmballoon_debug_show, inode->i_private); | |
1214 | } | |
1215 | ||
1216 | static const struct file_operations vmballoon_debug_fops = { | |
1217 | .owner = THIS_MODULE, | |
1218 | .open = vmballoon_debug_open, | |
1219 | .read = seq_read, | |
1220 | .llseek = seq_lseek, | |
1221 | .release = single_release, | |
1222 | }; | |
1223 | ||
1224 | static int __init vmballoon_debugfs_init(struct vmballoon *b) | |
1225 | { | |
1226 | int error; | |
1227 | ||
1228 | b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b, | |
1229 | &vmballoon_debug_fops); | |
1230 | if (IS_ERR(b->dbg_entry)) { | |
1231 | error = PTR_ERR(b->dbg_entry); | |
1232 | pr_err("failed to create debugfs entry, error: %d\n", error); | |
1233 | return error; | |
1234 | } | |
1235 | ||
1236 | return 0; | |
1237 | } | |
1238 | ||
1239 | static void __exit vmballoon_debugfs_exit(struct vmballoon *b) | |
1240 | { | |
1241 | debugfs_remove(b->dbg_entry); | |
1242 | } | |
1243 | ||
1244 | #else | |
1245 | ||
1246 | static inline int vmballoon_debugfs_init(struct vmballoon *b) | |
1247 | { | |
1248 | return 0; | |
1249 | } | |
1250 | ||
1251 | static inline void vmballoon_debugfs_exit(struct vmballoon *b) | |
1252 | { | |
1253 | } | |
1254 | ||
1255 | #endif /* CONFIG_DEBUG_FS */ | |
1256 | ||
1257 | static int __init vmballoon_init(void) | |
1258 | { | |
1259 | int error; | |
365bd7ef | 1260 | unsigned is_2m_pages; |
453dc659 DT |
1261 | /* |
1262 | * Check if we are running on VMware's hypervisor and bail out | |
1263 | * if we are not. | |
1264 | */ | |
03b2a320 | 1265 | if (x86_hyper_type != X86_HYPER_VMWARE) |
453dc659 DT |
1266 | return -ENODEV; |
1267 | ||
365bd7ef PM |
1268 | for (is_2m_pages = 0; is_2m_pages < VMW_BALLOON_NUM_PAGE_SIZES; |
1269 | is_2m_pages++) { | |
1270 | INIT_LIST_HEAD(&balloon.page_sizes[is_2m_pages].pages); | |
1271 | INIT_LIST_HEAD(&balloon.page_sizes[is_2m_pages].refused_pages); | |
1272 | } | |
453dc659 DT |
1273 | |
1274 | /* initialize rates */ | |
1275 | balloon.rate_alloc = VMW_BALLOON_RATE_ALLOC_MAX; | |
453dc659 DT |
1276 | |
1277 | INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work); | |
1278 | ||
453dc659 DT |
1279 | error = vmballoon_debugfs_init(&balloon); |
1280 | if (error) | |
beda94da | 1281 | return error; |
453dc659 | 1282 | |
48e3d668 | 1283 | balloon.vmci_doorbell = VMCI_INVALID_HANDLE; |
d7568c13 PM |
1284 | balloon.batch_page = NULL; |
1285 | balloon.page = NULL; | |
1286 | balloon.reset_required = true; | |
1287 | ||
beda94da | 1288 | queue_delayed_work(system_freezable_wq, &balloon.dwork, 0); |
453dc659 DT |
1289 | |
1290 | return 0; | |
453dc659 DT |
1291 | } |
1292 | module_init(vmballoon_init); | |
1293 | ||
1294 | static void __exit vmballoon_exit(void) | |
1295 | { | |
48e3d668 | 1296 | vmballoon_vmci_cleanup(&balloon); |
453dc659 | 1297 | cancel_delayed_work_sync(&balloon.dwork); |
453dc659 DT |
1298 | |
1299 | vmballoon_debugfs_exit(&balloon); | |
1300 | ||
1301 | /* | |
1302 | * Deallocate all reserved memory, and reset connection with monitor. | |
1303 | * Reset connection before deallocating memory to avoid potential for | |
1304 | * additional spurious resets from guest touching deallocated pages. | |
1305 | */ | |
d7568c13 | 1306 | vmballoon_send_start(&balloon, 0); |
453dc659 DT |
1307 | vmballoon_pop(&balloon); |
1308 | } | |
1309 | module_exit(vmballoon_exit); |