]>
Commit | Line | Data |
---|---|---|
910b2576 DH |
1 | /* |
2 | * Virtio MEM device | |
3 | * | |
4 | * Copyright (C) 2020 Red Hat, Inc. | |
5 | * | |
6 | * Authors: | |
7 | * David Hildenbrand <david@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. | |
10 | * See the COPYING file in the top-level directory. | |
11 | */ | |
12 | ||
13 | #include "qemu/osdep.h" | |
910b2576 DH |
14 | #include "qemu/iov.h" |
15 | #include "qemu/cutils.h" | |
16 | #include "qemu/error-report.h" | |
17 | #include "qemu/units.h" | |
18 | #include "sysemu/numa.h" | |
19 | #include "sysemu/sysemu.h" | |
20 | #include "sysemu/reset.h" | |
21 | #include "hw/virtio/virtio.h" | |
22 | #include "hw/virtio/virtio-bus.h" | |
23 | #include "hw/virtio/virtio-access.h" | |
24 | #include "hw/virtio/virtio-mem.h" | |
25 | #include "qapi/error.h" | |
26 | #include "qapi/visitor.h" | |
27 | #include "exec/ram_addr.h" | |
28 | #include "migration/misc.h" | |
29 | #include "hw/boards.h" | |
30 | #include "hw/qdev-properties.h" | |
2becc36a | 31 | #include CONFIG_DEVICES |
43e54950 | 32 | #include "trace.h" |
910b2576 | 33 | |
23ad8dec DH |
34 | /* |
35 | * We only had legacy x86 guests that did not support | |
36 | * VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE. Other targets don't have legacy guests. | |
37 | */ | |
38 | #if defined(TARGET_X86_64) || defined(TARGET_I386) | |
39 | #define VIRTIO_MEM_HAS_LEGACY_GUESTS | |
40 | #endif | |
41 | ||
910b2576 | 42 | /* |
228957fe DH |
43 | * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking |
44 | * bitmap small. | |
910b2576 | 45 | */ |
228957fe DH |
46 | #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)(1 * MiB)) |
47 | ||
1263615e GS |
48 | static uint32_t virtio_mem_default_thp_size(void) |
49 | { | |
50 | uint32_t default_thp_size = VIRTIO_MEM_MIN_BLOCK_SIZE; | |
51 | ||
52 | #if defined(__x86_64__) || defined(__arm__) || defined(__powerpc64__) | |
53 | default_thp_size = 2 * MiB; | |
54 | #elif defined(__aarch64__) | |
8e3b0cbb | 55 | if (qemu_real_host_page_size() == 4 * KiB) { |
1263615e | 56 | default_thp_size = 2 * MiB; |
8e3b0cbb | 57 | } else if (qemu_real_host_page_size() == 16 * KiB) { |
1263615e | 58 | default_thp_size = 32 * MiB; |
8e3b0cbb | 59 | } else if (qemu_real_host_page_size() == 64 * KiB) { |
1263615e GS |
60 | default_thp_size = 512 * MiB; |
61 | } | |
228957fe DH |
62 | #endif |
63 | ||
1263615e GS |
64 | return default_thp_size; |
65 | } | |
66 | ||
228957fe DH |
67 | /* |
68 | * We want to have a reasonable default block size such that | |
69 | * 1. We avoid splitting THPs when unplugging memory, which degrades | |
70 | * performance. | |
71 | * 2. We avoid placing THPs for plugged blocks that also cover unplugged | |
72 | * blocks. | |
73 | * | |
74 | * The actual THP size might differ between Linux kernels, so we try to probe | |
75 | * it. In the future (if we ever run into issues regarding 2.), we might want | |
76 | * to disable THP in case we fail to properly probe the THP size, or if the | |
77 | * block size is configured smaller than the THP size. | |
78 | */ | |
79 | static uint32_t thp_size; | |
80 | ||
81 | #define HPAGE_PMD_SIZE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" | |
82 | static uint32_t virtio_mem_thp_size(void) | |
83 | { | |
84 | gchar *content = NULL; | |
85 | const char *endptr; | |
86 | uint64_t tmp; | |
87 | ||
88 | if (thp_size) { | |
89 | return thp_size; | |
90 | } | |
91 | ||
92 | /* | |
93 | * Try to probe the actual THP size, fallback to (sane but eventually | |
94 | * incorrect) default sizes. | |
95 | */ | |
96 | if (g_file_get_contents(HPAGE_PMD_SIZE_PATH, &content, NULL, NULL) && | |
97 | !qemu_strtou64(content, &endptr, 0, &tmp) && | |
98 | (!endptr || *endptr == '\n')) { | |
1263615e GS |
99 | /* Sanity-check the value and fallback to something reasonable. */ |
100 | if (!tmp || !is_power_of_2(tmp)) { | |
228957fe DH |
101 | warn_report("Read unsupported THP size: %" PRIx64, tmp); |
102 | } else { | |
103 | thp_size = tmp; | |
104 | } | |
105 | } | |
106 | ||
107 | if (!thp_size) { | |
1263615e | 108 | thp_size = virtio_mem_default_thp_size(); |
228957fe DH |
109 | warn_report("Could not detect THP size, falling back to %" PRIx64 |
110 | " MiB.", thp_size / MiB); | |
111 | } | |
112 | ||
113 | g_free(content); | |
114 | return thp_size; | |
115 | } | |
116 | ||
117 | static uint64_t virtio_mem_default_block_size(RAMBlock *rb) | |
118 | { | |
119 | const uint64_t page_size = qemu_ram_pagesize(rb); | |
120 | ||
121 | /* We can have hugetlbfs with a page size smaller than the THP size. */ | |
8e3b0cbb | 122 | if (page_size == qemu_real_host_page_size()) { |
228957fe DH |
123 | return MAX(page_size, virtio_mem_thp_size()); |
124 | } | |
125 | return MAX(page_size, VIRTIO_MEM_MIN_BLOCK_SIZE); | |
126 | } | |
127 | ||
23ad8dec DH |
128 | #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) |
129 | static bool virtio_mem_has_shared_zeropage(RAMBlock *rb) | |
130 | { | |
131 | /* | |
132 | * We only have a guaranteed shared zeropage on ordinary MAP_PRIVATE | |
133 | * anonymous RAM. In any other case, reading unplugged *can* populate a | |
134 | * fresh page, consuming actual memory. | |
135 | */ | |
136 | return !qemu_ram_is_shared(rb) && rb->fd < 0 && | |
8e3b0cbb | 137 | qemu_ram_pagesize(rb) == qemu_real_host_page_size(); |
23ad8dec DH |
138 | } |
139 | #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ | |
140 | ||
910b2576 DH |
141 | /* |
142 | * Size the usable region bigger than the requested size if possible. Esp. | |
143 | * Linux guests will only add (aligned) memory blocks in case they fully | |
144 | * fit into the usable region, but plug+online only a subset of the pages. | |
145 | * The memory block size corresponds mostly to the section size. | |
146 | * | |
147 | * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and | |
b1b87327 | 148 | * a section size of 512MB on arm64 (as long as the start address is properly |
910b2576 DH |
149 | * aligned, similar to ordinary DIMMs). |
150 | * | |
151 | * We can change this at any time and maybe even make it configurable if | |
152 | * necessary (as the section size can change). But it's more likely that the | |
153 | * section size will rather get smaller and not bigger over time. | |
154 | */ | |
155 | #if defined(TARGET_X86_64) || defined(TARGET_I386) | |
156 | #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB)) | |
b1b87327 GS |
157 | #elif defined(TARGET_ARM) |
158 | #define VIRTIO_MEM_USABLE_EXTENT (2 * (512 * MiB)) | |
910b2576 DH |
159 | #else |
160 | #error VIRTIO_MEM_USABLE_EXTENT not defined | |
161 | #endif | |
162 | ||
163 | static bool virtio_mem_is_busy(void) | |
164 | { | |
165 | /* | |
166 | * Postcopy cannot handle concurrent discards and we don't want to migrate | |
167 | * pages on-demand with stale content when plugging new blocks. | |
0bc7806c DH |
168 | * |
169 | * For precopy, we don't want unplugged blocks in our migration stream, and | |
170 | * when plugging new blocks, the page content might differ between source | |
171 | * and destination (observable by the guest when not initializing pages | |
172 | * after plugging them) until we're running on the destination (as we didn't | |
173 | * migrate these blocks when they were unplugged). | |
910b2576 | 174 | */ |
0bc7806c | 175 | return migration_in_incoming_postcopy() || !migration_is_idle(); |
910b2576 DH |
176 | } |
177 | ||
7a9d5d02 DH |
178 | typedef int (*virtio_mem_range_cb)(const VirtIOMEM *vmem, void *arg, |
179 | uint64_t offset, uint64_t size); | |
180 | ||
181 | static int virtio_mem_for_each_unplugged_range(const VirtIOMEM *vmem, void *arg, | |
182 | virtio_mem_range_cb cb) | |
183 | { | |
184 | unsigned long first_zero_bit, last_zero_bit; | |
185 | uint64_t offset, size; | |
186 | int ret = 0; | |
187 | ||
188 | first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); | |
189 | while (first_zero_bit < vmem->bitmap_size) { | |
190 | offset = first_zero_bit * vmem->block_size; | |
191 | last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
192 | first_zero_bit + 1) - 1; | |
193 | size = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; | |
194 | ||
195 | ret = cb(vmem, arg, offset, size); | |
196 | if (ret) { | |
197 | break; | |
198 | } | |
199 | first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
200 | last_zero_bit + 2); | |
201 | } | |
202 | return ret; | |
203 | } | |
204 | ||
2044969f DH |
205 | /* |
206 | * Adjust the memory section to cover the intersection with the given range. | |
207 | * | |
208 | * Returns false if the intersection is empty, otherwise returns true. | |
209 | */ | |
210 | static bool virito_mem_intersect_memory_section(MemoryRegionSection *s, | |
211 | uint64_t offset, uint64_t size) | |
212 | { | |
213 | uint64_t start = MAX(s->offset_within_region, offset); | |
214 | uint64_t end = MIN(s->offset_within_region + int128_get64(s->size), | |
215 | offset + size); | |
216 | ||
217 | if (end <= start) { | |
218 | return false; | |
219 | } | |
220 | ||
221 | s->offset_within_address_space += start - s->offset_within_region; | |
222 | s->offset_within_region = start; | |
223 | s->size = int128_make64(end - start); | |
224 | return true; | |
225 | } | |
226 | ||
227 | typedef int (*virtio_mem_section_cb)(MemoryRegionSection *s, void *arg); | |
228 | ||
229 | static int virtio_mem_for_each_plugged_section(const VirtIOMEM *vmem, | |
230 | MemoryRegionSection *s, | |
231 | void *arg, | |
232 | virtio_mem_section_cb cb) | |
233 | { | |
234 | unsigned long first_bit, last_bit; | |
235 | uint64_t offset, size; | |
236 | int ret = 0; | |
237 | ||
b11cf32e | 238 | first_bit = s->offset_within_region / vmem->block_size; |
2044969f DH |
239 | first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, first_bit); |
240 | while (first_bit < vmem->bitmap_size) { | |
241 | MemoryRegionSection tmp = *s; | |
242 | ||
243 | offset = first_bit * vmem->block_size; | |
244 | last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
245 | first_bit + 1) - 1; | |
246 | size = (last_bit - first_bit + 1) * vmem->block_size; | |
247 | ||
248 | if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { | |
249 | break; | |
250 | } | |
251 | ret = cb(&tmp, arg); | |
252 | if (ret) { | |
253 | break; | |
254 | } | |
255 | first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
256 | last_bit + 2); | |
257 | } | |
258 | return ret; | |
259 | } | |
260 | ||
372aa6fd DH |
261 | static int virtio_mem_for_each_unplugged_section(const VirtIOMEM *vmem, |
262 | MemoryRegionSection *s, | |
263 | void *arg, | |
264 | virtio_mem_section_cb cb) | |
265 | { | |
266 | unsigned long first_bit, last_bit; | |
267 | uint64_t offset, size; | |
268 | int ret = 0; | |
269 | ||
b11cf32e | 270 | first_bit = s->offset_within_region / vmem->block_size; |
372aa6fd DH |
271 | first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit); |
272 | while (first_bit < vmem->bitmap_size) { | |
273 | MemoryRegionSection tmp = *s; | |
274 | ||
275 | offset = first_bit * vmem->block_size; | |
276 | last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
277 | first_bit + 1) - 1; | |
278 | size = (last_bit - first_bit + 1) * vmem->block_size; | |
279 | ||
280 | if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { | |
281 | break; | |
282 | } | |
283 | ret = cb(&tmp, arg); | |
284 | if (ret) { | |
285 | break; | |
286 | } | |
287 | first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
288 | last_bit + 2); | |
289 | } | |
290 | return ret; | |
291 | } | |
292 | ||
2044969f DH |
293 | static int virtio_mem_notify_populate_cb(MemoryRegionSection *s, void *arg) |
294 | { | |
295 | RamDiscardListener *rdl = arg; | |
296 | ||
297 | return rdl->notify_populate(rdl, s); | |
298 | } | |
299 | ||
300 | static int virtio_mem_notify_discard_cb(MemoryRegionSection *s, void *arg) | |
301 | { | |
302 | RamDiscardListener *rdl = arg; | |
303 | ||
304 | rdl->notify_discard(rdl, s); | |
305 | return 0; | |
306 | } | |
307 | ||
308 | static void virtio_mem_notify_unplug(VirtIOMEM *vmem, uint64_t offset, | |
309 | uint64_t size) | |
310 | { | |
311 | RamDiscardListener *rdl; | |
312 | ||
313 | QLIST_FOREACH(rdl, &vmem->rdl_list, next) { | |
314 | MemoryRegionSection tmp = *rdl->section; | |
315 | ||
316 | if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { | |
317 | continue; | |
318 | } | |
319 | rdl->notify_discard(rdl, &tmp); | |
320 | } | |
321 | } | |
322 | ||
323 | static int virtio_mem_notify_plug(VirtIOMEM *vmem, uint64_t offset, | |
324 | uint64_t size) | |
325 | { | |
326 | RamDiscardListener *rdl, *rdl2; | |
327 | int ret = 0; | |
328 | ||
329 | QLIST_FOREACH(rdl, &vmem->rdl_list, next) { | |
330 | MemoryRegionSection tmp = *rdl->section; | |
331 | ||
332 | if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { | |
333 | continue; | |
334 | } | |
335 | ret = rdl->notify_populate(rdl, &tmp); | |
336 | if (ret) { | |
337 | break; | |
338 | } | |
339 | } | |
340 | ||
341 | if (ret) { | |
342 | /* Notify all already-notified listeners. */ | |
343 | QLIST_FOREACH(rdl2, &vmem->rdl_list, next) { | |
344 | MemoryRegionSection tmp = *rdl->section; | |
345 | ||
346 | if (rdl2 == rdl) { | |
347 | break; | |
348 | } | |
349 | if (!virito_mem_intersect_memory_section(&tmp, offset, size)) { | |
350 | continue; | |
351 | } | |
352 | rdl2->notify_discard(rdl2, &tmp); | |
353 | } | |
354 | } | |
355 | return ret; | |
356 | } | |
357 | ||
358 | static void virtio_mem_notify_unplug_all(VirtIOMEM *vmem) | |
359 | { | |
360 | RamDiscardListener *rdl; | |
361 | ||
362 | if (!vmem->size) { | |
363 | return; | |
364 | } | |
365 | ||
366 | QLIST_FOREACH(rdl, &vmem->rdl_list, next) { | |
367 | if (rdl->double_discard_supported) { | |
368 | rdl->notify_discard(rdl, rdl->section); | |
369 | } else { | |
370 | virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, | |
371 | virtio_mem_notify_discard_cb); | |
372 | } | |
373 | } | |
374 | } | |
375 | ||
376 | static bool virtio_mem_test_bitmap(const VirtIOMEM *vmem, uint64_t start_gpa, | |
910b2576 DH |
377 | uint64_t size, bool plugged) |
378 | { | |
379 | const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; | |
380 | const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; | |
381 | unsigned long found_bit; | |
382 | ||
383 | /* We fake a shorter bitmap to avoid searching too far. */ | |
384 | if (plugged) { | |
385 | found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit); | |
386 | } else { | |
387 | found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit); | |
388 | } | |
389 | return found_bit > last_bit; | |
390 | } | |
391 | ||
392 | static void virtio_mem_set_bitmap(VirtIOMEM *vmem, uint64_t start_gpa, | |
393 | uint64_t size, bool plugged) | |
394 | { | |
395 | const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; | |
396 | const unsigned long nbits = size / vmem->block_size; | |
397 | ||
398 | if (plugged) { | |
399 | bitmap_set(vmem->bitmap, bit, nbits); | |
400 | } else { | |
401 | bitmap_clear(vmem->bitmap, bit, nbits); | |
402 | } | |
403 | } | |
404 | ||
405 | static void virtio_mem_send_response(VirtIOMEM *vmem, VirtQueueElement *elem, | |
406 | struct virtio_mem_resp *resp) | |
407 | { | |
408 | VirtIODevice *vdev = VIRTIO_DEVICE(vmem); | |
409 | VirtQueue *vq = vmem->vq; | |
410 | ||
43e54950 | 411 | trace_virtio_mem_send_response(le16_to_cpu(resp->type)); |
910b2576 DH |
412 | iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp)); |
413 | ||
414 | virtqueue_push(vq, elem, sizeof(*resp)); | |
415 | virtio_notify(vdev, vq); | |
416 | } | |
417 | ||
418 | static void virtio_mem_send_response_simple(VirtIOMEM *vmem, | |
419 | VirtQueueElement *elem, | |
420 | uint16_t type) | |
421 | { | |
422 | struct virtio_mem_resp resp = { | |
423 | .type = cpu_to_le16(type), | |
424 | }; | |
425 | ||
426 | virtio_mem_send_response(vmem, elem, &resp); | |
427 | } | |
428 | ||
2044969f DH |
429 | static bool virtio_mem_valid_range(const VirtIOMEM *vmem, uint64_t gpa, |
430 | uint64_t size) | |
910b2576 DH |
431 | { |
432 | if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) { | |
433 | return false; | |
434 | } | |
435 | if (gpa + size < gpa || !size) { | |
436 | return false; | |
437 | } | |
438 | if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) { | |
439 | return false; | |
440 | } | |
441 | if (gpa + size > vmem->addr + vmem->usable_region_size) { | |
442 | return false; | |
443 | } | |
444 | return true; | |
445 | } | |
446 | ||
447 | static int virtio_mem_set_block_state(VirtIOMEM *vmem, uint64_t start_gpa, | |
448 | uint64_t size, bool plug) | |
449 | { | |
450 | const uint64_t offset = start_gpa - vmem->addr; | |
3aca6380 | 451 | RAMBlock *rb = vmem->memdev->mr.ram_block; |
910b2576 DH |
452 | |
453 | if (virtio_mem_is_busy()) { | |
454 | return -EBUSY; | |
455 | } | |
456 | ||
457 | if (!plug) { | |
3aca6380 | 458 | if (ram_block_discard_range(rb, offset, size)) { |
910b2576 DH |
459 | return -EBUSY; |
460 | } | |
2044969f | 461 | virtio_mem_notify_unplug(vmem, offset, size); |
09b3b7e0 DH |
462 | } else { |
463 | int ret = 0; | |
464 | ||
465 | if (vmem->prealloc) { | |
466 | void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; | |
467 | int fd = memory_region_get_fd(&vmem->memdev->mr); | |
468 | Error *local_err = NULL; | |
469 | ||
e04a34e5 | 470 | qemu_prealloc_mem(fd, area, size, 1, NULL, &local_err); |
09b3b7e0 DH |
471 | if (local_err) { |
472 | static bool warned; | |
473 | ||
474 | /* | |
475 | * Warn only once, we don't want to fill the log with these | |
476 | * warnings. | |
477 | */ | |
478 | if (!warned) { | |
479 | warn_report_err(local_err); | |
480 | warned = true; | |
481 | } else { | |
482 | error_free(local_err); | |
483 | } | |
484 | ret = -EBUSY; | |
485 | } | |
486 | } | |
487 | if (!ret) { | |
488 | ret = virtio_mem_notify_plug(vmem, offset, size); | |
489 | } | |
490 | ||
491 | if (ret) { | |
492 | /* Could be preallocation or a notifier populated memory. */ | |
493 | ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size); | |
494 | return -EBUSY; | |
495 | } | |
910b2576 DH |
496 | } |
497 | virtio_mem_set_bitmap(vmem, start_gpa, size, plug); | |
498 | return 0; | |
499 | } | |
500 | ||
501 | static int virtio_mem_state_change_request(VirtIOMEM *vmem, uint64_t gpa, | |
502 | uint16_t nb_blocks, bool plug) | |
503 | { | |
504 | const uint64_t size = nb_blocks * vmem->block_size; | |
505 | int ret; | |
506 | ||
507 | if (!virtio_mem_valid_range(vmem, gpa, size)) { | |
508 | return VIRTIO_MEM_RESP_ERROR; | |
509 | } | |
510 | ||
511 | if (plug && (vmem->size + size > vmem->requested_size)) { | |
512 | return VIRTIO_MEM_RESP_NACK; | |
513 | } | |
514 | ||
515 | /* test if really all blocks are in the opposite state */ | |
516 | if (!virtio_mem_test_bitmap(vmem, gpa, size, !plug)) { | |
517 | return VIRTIO_MEM_RESP_ERROR; | |
518 | } | |
519 | ||
520 | ret = virtio_mem_set_block_state(vmem, gpa, size, plug); | |
521 | if (ret) { | |
522 | return VIRTIO_MEM_RESP_BUSY; | |
523 | } | |
524 | if (plug) { | |
525 | vmem->size += size; | |
526 | } else { | |
527 | vmem->size -= size; | |
528 | } | |
c95b4437 | 529 | notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); |
910b2576 DH |
530 | return VIRTIO_MEM_RESP_ACK; |
531 | } | |
532 | ||
533 | static void virtio_mem_plug_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
534 | struct virtio_mem_req *req) | |
535 | { | |
536 | const uint64_t gpa = le64_to_cpu(req->u.plug.addr); | |
537 | const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks); | |
538 | uint16_t type; | |
539 | ||
43e54950 | 540 | trace_virtio_mem_plug_request(gpa, nb_blocks); |
910b2576 DH |
541 | type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, true); |
542 | virtio_mem_send_response_simple(vmem, elem, type); | |
543 | } | |
544 | ||
545 | static void virtio_mem_unplug_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
546 | struct virtio_mem_req *req) | |
547 | { | |
548 | const uint64_t gpa = le64_to_cpu(req->u.unplug.addr); | |
549 | const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks); | |
550 | uint16_t type; | |
551 | ||
43e54950 | 552 | trace_virtio_mem_unplug_request(gpa, nb_blocks); |
910b2576 DH |
553 | type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, false); |
554 | virtio_mem_send_response_simple(vmem, elem, type); | |
555 | } | |
556 | ||
557 | static void virtio_mem_resize_usable_region(VirtIOMEM *vmem, | |
558 | uint64_t requested_size, | |
559 | bool can_shrink) | |
560 | { | |
561 | uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr), | |
562 | requested_size + VIRTIO_MEM_USABLE_EXTENT); | |
563 | ||
0aed2800 DH |
564 | /* The usable region size always has to be multiples of the block size. */ |
565 | newsize = QEMU_ALIGN_UP(newsize, vmem->block_size); | |
566 | ||
910b2576 DH |
567 | if (!requested_size) { |
568 | newsize = 0; | |
569 | } | |
570 | ||
571 | if (newsize < vmem->usable_region_size && !can_shrink) { | |
572 | return; | |
573 | } | |
574 | ||
43e54950 | 575 | trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize); |
910b2576 DH |
576 | vmem->usable_region_size = newsize; |
577 | } | |
578 | ||
579 | static int virtio_mem_unplug_all(VirtIOMEM *vmem) | |
580 | { | |
581 | RAMBlock *rb = vmem->memdev->mr.ram_block; | |
910b2576 DH |
582 | |
583 | if (virtio_mem_is_busy()) { | |
584 | return -EBUSY; | |
585 | } | |
586 | ||
3aca6380 | 587 | if (ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb))) { |
910b2576 DH |
588 | return -EBUSY; |
589 | } | |
2044969f DH |
590 | virtio_mem_notify_unplug_all(vmem); |
591 | ||
910b2576 | 592 | bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size); |
c95b4437 DH |
593 | if (vmem->size) { |
594 | vmem->size = 0; | |
595 | notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); | |
596 | } | |
43e54950 | 597 | trace_virtio_mem_unplugged_all(); |
910b2576 DH |
598 | virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); |
599 | return 0; | |
600 | } | |
601 | ||
602 | static void virtio_mem_unplug_all_request(VirtIOMEM *vmem, | |
603 | VirtQueueElement *elem) | |
604 | { | |
43e54950 | 605 | trace_virtio_mem_unplug_all_request(); |
910b2576 DH |
606 | if (virtio_mem_unplug_all(vmem)) { |
607 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_BUSY); | |
608 | } else { | |
609 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ACK); | |
610 | } | |
611 | } | |
612 | ||
613 | static void virtio_mem_state_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
614 | struct virtio_mem_req *req) | |
615 | { | |
616 | const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks); | |
617 | const uint64_t gpa = le64_to_cpu(req->u.state.addr); | |
618 | const uint64_t size = nb_blocks * vmem->block_size; | |
619 | struct virtio_mem_resp resp = { | |
620 | .type = cpu_to_le16(VIRTIO_MEM_RESP_ACK), | |
621 | }; | |
622 | ||
43e54950 | 623 | trace_virtio_mem_state_request(gpa, nb_blocks); |
910b2576 DH |
624 | if (!virtio_mem_valid_range(vmem, gpa, size)) { |
625 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ERROR); | |
626 | return; | |
627 | } | |
628 | ||
629 | if (virtio_mem_test_bitmap(vmem, gpa, size, true)) { | |
630 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED); | |
631 | } else if (virtio_mem_test_bitmap(vmem, gpa, size, false)) { | |
632 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED); | |
633 | } else { | |
634 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_MIXED); | |
635 | } | |
43e54950 | 636 | trace_virtio_mem_state_response(le16_to_cpu(resp.u.state.state)); |
910b2576 DH |
637 | virtio_mem_send_response(vmem, elem, &resp); |
638 | } | |
639 | ||
640 | static void virtio_mem_handle_request(VirtIODevice *vdev, VirtQueue *vq) | |
641 | { | |
642 | const int len = sizeof(struct virtio_mem_req); | |
643 | VirtIOMEM *vmem = VIRTIO_MEM(vdev); | |
644 | VirtQueueElement *elem; | |
645 | struct virtio_mem_req req; | |
646 | uint16_t type; | |
647 | ||
648 | while (true) { | |
649 | elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); | |
650 | if (!elem) { | |
651 | return; | |
652 | } | |
653 | ||
654 | if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) { | |
655 | virtio_error(vdev, "virtio-mem protocol violation: invalid request" | |
656 | " size: %d", len); | |
0c404e45 | 657 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
658 | g_free(elem); |
659 | return; | |
660 | } | |
661 | ||
662 | if (iov_size(elem->in_sg, elem->in_num) < | |
663 | sizeof(struct virtio_mem_resp)) { | |
664 | virtio_error(vdev, "virtio-mem protocol violation: not enough space" | |
665 | " for response: %zu", | |
666 | iov_size(elem->in_sg, elem->in_num)); | |
0c404e45 | 667 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
668 | g_free(elem); |
669 | return; | |
670 | } | |
671 | ||
672 | type = le16_to_cpu(req.type); | |
673 | switch (type) { | |
674 | case VIRTIO_MEM_REQ_PLUG: | |
675 | virtio_mem_plug_request(vmem, elem, &req); | |
676 | break; | |
677 | case VIRTIO_MEM_REQ_UNPLUG: | |
678 | virtio_mem_unplug_request(vmem, elem, &req); | |
679 | break; | |
680 | case VIRTIO_MEM_REQ_UNPLUG_ALL: | |
681 | virtio_mem_unplug_all_request(vmem, elem); | |
682 | break; | |
683 | case VIRTIO_MEM_REQ_STATE: | |
684 | virtio_mem_state_request(vmem, elem, &req); | |
685 | break; | |
686 | default: | |
687 | virtio_error(vdev, "virtio-mem protocol violation: unknown request" | |
688 | " type: %d", type); | |
0c404e45 | 689 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
690 | g_free(elem); |
691 | return; | |
692 | } | |
693 | ||
694 | g_free(elem); | |
695 | } | |
696 | } | |
697 | ||
698 | static void virtio_mem_get_config(VirtIODevice *vdev, uint8_t *config_data) | |
699 | { | |
700 | VirtIOMEM *vmem = VIRTIO_MEM(vdev); | |
701 | struct virtio_mem_config *config = (void *) config_data; | |
702 | ||
703 | config->block_size = cpu_to_le64(vmem->block_size); | |
704 | config->node_id = cpu_to_le16(vmem->node); | |
705 | config->requested_size = cpu_to_le64(vmem->requested_size); | |
706 | config->plugged_size = cpu_to_le64(vmem->size); | |
707 | config->addr = cpu_to_le64(vmem->addr); | |
708 | config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr)); | |
709 | config->usable_region_size = cpu_to_le64(vmem->usable_region_size); | |
710 | } | |
711 | ||
712 | static uint64_t virtio_mem_get_features(VirtIODevice *vdev, uint64_t features, | |
713 | Error **errp) | |
714 | { | |
715 | MachineState *ms = MACHINE(qdev_get_machine()); | |
23ad8dec | 716 | VirtIOMEM *vmem = VIRTIO_MEM(vdev); |
910b2576 DH |
717 | |
718 | if (ms->numa_state) { | |
719 | #if defined(CONFIG_ACPI) | |
720 | virtio_add_feature(&features, VIRTIO_MEM_F_ACPI_PXM); | |
721 | #endif | |
722 | } | |
23ad8dec DH |
723 | assert(vmem->unplugged_inaccessible != ON_OFF_AUTO_AUTO); |
724 | if (vmem->unplugged_inaccessible == ON_OFF_AUTO_ON) { | |
725 | virtio_add_feature(&features, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE); | |
726 | } | |
910b2576 DH |
727 | return features; |
728 | } | |
729 | ||
23ad8dec DH |
730 | static int virtio_mem_validate_features(VirtIODevice *vdev) |
731 | { | |
732 | if (virtio_host_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE) && | |
733 | !virtio_vdev_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE)) { | |
734 | return -EFAULT; | |
735 | } | |
736 | return 0; | |
737 | } | |
738 | ||
910b2576 DH |
739 | static void virtio_mem_system_reset(void *opaque) |
740 | { | |
741 | VirtIOMEM *vmem = VIRTIO_MEM(opaque); | |
742 | ||
743 | /* | |
744 | * During usual resets, we will unplug all memory and shrink the usable | |
745 | * region size. This is, however, not possible in all scenarios. Then, | |
746 | * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL). | |
747 | */ | |
748 | virtio_mem_unplug_all(vmem); | |
749 | } | |
750 | ||
751 | static void virtio_mem_device_realize(DeviceState *dev, Error **errp) | |
752 | { | |
753 | MachineState *ms = MACHINE(qdev_get_machine()); | |
754 | int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0; | |
755 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); | |
756 | VirtIOMEM *vmem = VIRTIO_MEM(dev); | |
757 | uint64_t page_size; | |
758 | RAMBlock *rb; | |
759 | int ret; | |
760 | ||
761 | if (!vmem->memdev) { | |
762 | error_setg(errp, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP); | |
763 | return; | |
764 | } else if (host_memory_backend_is_mapped(vmem->memdev)) { | |
910b2576 | 765 | error_setg(errp, "'%s' property specifies a busy memdev: %s", |
7a309cc9 MA |
766 | VIRTIO_MEM_MEMDEV_PROP, |
767 | object_get_canonical_path_component(OBJECT(vmem->memdev))); | |
910b2576 DH |
768 | return; |
769 | } else if (!memory_region_is_ram(&vmem->memdev->mr) || | |
770 | memory_region_is_rom(&vmem->memdev->mr) || | |
771 | !vmem->memdev->mr.ram_block) { | |
772 | error_setg(errp, "'%s' property specifies an unsupported memdev", | |
773 | VIRTIO_MEM_MEMDEV_PROP); | |
774 | return; | |
775 | } | |
776 | ||
777 | if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) || | |
778 | (!nb_numa_nodes && vmem->node)) { | |
779 | error_setg(errp, "'%s' property has value '%" PRIu32 "', which exceeds" | |
780 | "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP, | |
781 | vmem->node, nb_numa_nodes ? nb_numa_nodes : 1); | |
782 | return; | |
783 | } | |
784 | ||
785 | if (enable_mlock) { | |
786 | error_setg(errp, "Incompatible with mlock"); | |
787 | return; | |
788 | } | |
789 | ||
790 | rb = vmem->memdev->mr.ram_block; | |
791 | page_size = qemu_ram_pagesize(rb); | |
792 | ||
23ad8dec DH |
793 | #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) |
794 | switch (vmem->unplugged_inaccessible) { | |
795 | case ON_OFF_AUTO_AUTO: | |
796 | if (virtio_mem_has_shared_zeropage(rb)) { | |
797 | vmem->unplugged_inaccessible = ON_OFF_AUTO_OFF; | |
798 | } else { | |
799 | vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; | |
800 | } | |
801 | break; | |
802 | case ON_OFF_AUTO_OFF: | |
803 | if (!virtio_mem_has_shared_zeropage(rb)) { | |
804 | warn_report("'%s' property set to 'off' with a memdev that does" | |
805 | " not support the shared zeropage.", | |
806 | VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP); | |
807 | } | |
808 | break; | |
809 | default: | |
810 | break; | |
811 | } | |
812 | #else /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ | |
813 | vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; | |
814 | #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ | |
815 | ||
228957fe DH |
816 | /* |
817 | * If the block size wasn't configured by the user, use a sane default. This | |
818 | * allows using hugetlbfs backends of any page size without manual | |
819 | * intervention. | |
820 | */ | |
821 | if (!vmem->block_size) { | |
822 | vmem->block_size = virtio_mem_default_block_size(rb); | |
823 | } | |
824 | ||
910b2576 DH |
825 | if (vmem->block_size < page_size) { |
826 | error_setg(errp, "'%s' property has to be at least the page size (0x%" | |
827 | PRIx64 ")", VIRTIO_MEM_BLOCK_SIZE_PROP, page_size); | |
828 | return; | |
228957fe DH |
829 | } else if (vmem->block_size < virtio_mem_default_block_size(rb)) { |
830 | warn_report("'%s' property is smaller than the default block size (%" | |
831 | PRIx64 " MiB)", VIRTIO_MEM_BLOCK_SIZE_PROP, | |
832 | virtio_mem_default_block_size(rb) / MiB); | |
7656d9ce DH |
833 | } |
834 | if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) { | |
910b2576 DH |
835 | error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 |
836 | ")", VIRTIO_MEM_REQUESTED_SIZE_PROP, | |
837 | VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); | |
838 | return; | |
d31992ae DH |
839 | } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) { |
840 | error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 | |
841 | ")", VIRTIO_MEM_ADDR_PROP, VIRTIO_MEM_BLOCK_SIZE_PROP, | |
842 | vmem->block_size); | |
843 | return; | |
910b2576 DH |
844 | } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr), |
845 | vmem->block_size)) { | |
846 | error_setg(errp, "'%s' property memdev size has to be multiples of" | |
847 | "'%s' (0x%" PRIx64 ")", VIRTIO_MEM_MEMDEV_PROP, | |
848 | VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); | |
849 | return; | |
850 | } | |
851 | ||
bc072ed4 | 852 | if (ram_block_coordinated_discard_require(true)) { |
910b2576 DH |
853 | error_setg(errp, "Discarding RAM is disabled"); |
854 | return; | |
855 | } | |
856 | ||
857 | ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb)); | |
858 | if (ret) { | |
859 | error_setg_errno(errp, -ret, "Unexpected error discarding RAM"); | |
bc072ed4 | 860 | ram_block_coordinated_discard_require(false); |
910b2576 DH |
861 | return; |
862 | } | |
863 | ||
864 | virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); | |
865 | ||
866 | vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) / | |
867 | vmem->block_size; | |
868 | vmem->bitmap = bitmap_new(vmem->bitmap_size); | |
869 | ||
3857cd5c | 870 | virtio_init(vdev, VIRTIO_ID_MEM, sizeof(struct virtio_mem_config)); |
910b2576 DH |
871 | vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request); |
872 | ||
873 | host_memory_backend_set_mapped(vmem->memdev, true); | |
874 | vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem)); | |
875 | qemu_register_reset(virtio_mem_system_reset, vmem); | |
2044969f DH |
876 | |
877 | /* | |
878 | * Set ourselves as RamDiscardManager before the plug handler maps the | |
879 | * memory region and exposes it via an address space. | |
880 | */ | |
881 | memory_region_set_ram_discard_manager(&vmem->memdev->mr, | |
882 | RAM_DISCARD_MANAGER(vmem)); | |
910b2576 DH |
883 | } |
884 | ||
885 | static void virtio_mem_device_unrealize(DeviceState *dev) | |
886 | { | |
887 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); | |
888 | VirtIOMEM *vmem = VIRTIO_MEM(dev); | |
889 | ||
2044969f DH |
890 | /* |
891 | * The unplug handler unmapped the memory region, it cannot be | |
892 | * found via an address space anymore. Unset ourselves. | |
893 | */ | |
894 | memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL); | |
910b2576 DH |
895 | qemu_unregister_reset(virtio_mem_system_reset, vmem); |
896 | vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem)); | |
897 | host_memory_backend_set_mapped(vmem->memdev, false); | |
898 | virtio_del_queue(vdev, 0); | |
899 | virtio_cleanup(vdev); | |
900 | g_free(vmem->bitmap); | |
bc072ed4 | 901 | ram_block_coordinated_discard_require(false); |
910b2576 DH |
902 | } |
903 | ||
7a9d5d02 DH |
904 | static int virtio_mem_discard_range_cb(const VirtIOMEM *vmem, void *arg, |
905 | uint64_t offset, uint64_t size) | |
910b2576 DH |
906 | { |
907 | RAMBlock *rb = vmem->memdev->mr.ram_block; | |
910b2576 | 908 | |
3aca6380 | 909 | return ram_block_discard_range(rb, offset, size) ? -EINVAL : 0; |
910b2576 DH |
910 | } |
911 | ||
7a9d5d02 DH |
912 | static int virtio_mem_restore_unplugged(VirtIOMEM *vmem) |
913 | { | |
914 | /* Make sure all memory is really discarded after migration. */ | |
915 | return virtio_mem_for_each_unplugged_range(vmem, NULL, | |
916 | virtio_mem_discard_range_cb); | |
917 | } | |
918 | ||
910b2576 DH |
919 | static int virtio_mem_post_load(void *opaque, int version_id) |
920 | { | |
2044969f DH |
921 | VirtIOMEM *vmem = VIRTIO_MEM(opaque); |
922 | RamDiscardListener *rdl; | |
923 | int ret; | |
924 | ||
925 | /* | |
926 | * We started out with all memory discarded and our memory region is mapped | |
927 | * into an address space. Replay, now that we updated the bitmap. | |
928 | */ | |
929 | QLIST_FOREACH(rdl, &vmem->rdl_list, next) { | |
930 | ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, | |
931 | virtio_mem_notify_populate_cb); | |
932 | if (ret) { | |
933 | return ret; | |
934 | } | |
935 | } | |
936 | ||
910b2576 DH |
937 | if (migration_in_incoming_postcopy()) { |
938 | return 0; | |
939 | } | |
940 | ||
2044969f | 941 | return virtio_mem_restore_unplugged(vmem); |
910b2576 DH |
942 | } |
943 | ||
383ee445 DH |
944 | typedef struct VirtIOMEMMigSanityChecks { |
945 | VirtIOMEM *parent; | |
946 | uint64_t addr; | |
947 | uint64_t region_size; | |
948 | uint64_t block_size; | |
949 | uint32_t node; | |
950 | } VirtIOMEMMigSanityChecks; | |
951 | ||
952 | static int virtio_mem_mig_sanity_checks_pre_save(void *opaque) | |
953 | { | |
954 | VirtIOMEMMigSanityChecks *tmp = opaque; | |
955 | VirtIOMEM *vmem = tmp->parent; | |
956 | ||
957 | tmp->addr = vmem->addr; | |
958 | tmp->region_size = memory_region_size(&vmem->memdev->mr); | |
959 | tmp->block_size = vmem->block_size; | |
960 | tmp->node = vmem->node; | |
961 | return 0; | |
962 | } | |
963 | ||
964 | static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id) | |
965 | { | |
966 | VirtIOMEMMigSanityChecks *tmp = opaque; | |
967 | VirtIOMEM *vmem = tmp->parent; | |
968 | const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr); | |
969 | ||
970 | if (tmp->addr != vmem->addr) { | |
971 | error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, | |
972 | VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr); | |
973 | return -EINVAL; | |
974 | } | |
975 | /* | |
976 | * Note: Preparation for resizeable memory regions. The maximum size | |
977 | * of the memory region must not change during migration. | |
978 | */ | |
979 | if (tmp->region_size != new_region_size) { | |
980 | error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%" | |
981 | PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size, | |
982 | new_region_size); | |
983 | return -EINVAL; | |
984 | } | |
985 | if (tmp->block_size != vmem->block_size) { | |
986 | error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, | |
987 | VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size, | |
988 | vmem->block_size); | |
989 | return -EINVAL; | |
990 | } | |
991 | if (tmp->node != vmem->node) { | |
992 | error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32, | |
993 | VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node); | |
994 | return -EINVAL; | |
995 | } | |
996 | return 0; | |
997 | } | |
998 | ||
999 | static const VMStateDescription vmstate_virtio_mem_sanity_checks = { | |
1000 | .name = "virtio-mem-device/sanity-checks", | |
1001 | .pre_save = virtio_mem_mig_sanity_checks_pre_save, | |
1002 | .post_load = virtio_mem_mig_sanity_checks_post_load, | |
1003 | .fields = (VMStateField[]) { | |
1004 | VMSTATE_UINT64(addr, VirtIOMEMMigSanityChecks), | |
1005 | VMSTATE_UINT64(region_size, VirtIOMEMMigSanityChecks), | |
1006 | VMSTATE_UINT64(block_size, VirtIOMEMMigSanityChecks), | |
1007 | VMSTATE_UINT32(node, VirtIOMEMMigSanityChecks), | |
1008 | VMSTATE_END_OF_LIST(), | |
1009 | }, | |
1010 | }; | |
1011 | ||
910b2576 DH |
1012 | static const VMStateDescription vmstate_virtio_mem_device = { |
1013 | .name = "virtio-mem-device", | |
1014 | .minimum_version_id = 1, | |
1015 | .version_id = 1, | |
0fd7616e | 1016 | .priority = MIG_PRI_VIRTIO_MEM, |
910b2576 DH |
1017 | .post_load = virtio_mem_post_load, |
1018 | .fields = (VMStateField[]) { | |
383ee445 DH |
1019 | VMSTATE_WITH_TMP(VirtIOMEM, VirtIOMEMMigSanityChecks, |
1020 | vmstate_virtio_mem_sanity_checks), | |
910b2576 DH |
1021 | VMSTATE_UINT64(usable_region_size, VirtIOMEM), |
1022 | VMSTATE_UINT64(size, VirtIOMEM), | |
1023 | VMSTATE_UINT64(requested_size, VirtIOMEM), | |
1024 | VMSTATE_BITMAP(bitmap, VirtIOMEM, 0, bitmap_size), | |
1025 | VMSTATE_END_OF_LIST() | |
1026 | }, | |
1027 | }; | |
1028 | ||
1029 | static const VMStateDescription vmstate_virtio_mem = { | |
1030 | .name = "virtio-mem", | |
1031 | .minimum_version_id = 1, | |
1032 | .version_id = 1, | |
1033 | .fields = (VMStateField[]) { | |
1034 | VMSTATE_VIRTIO_DEVICE, | |
1035 | VMSTATE_END_OF_LIST() | |
1036 | }, | |
1037 | }; | |
1038 | ||
1039 | static void virtio_mem_fill_device_info(const VirtIOMEM *vmem, | |
1040 | VirtioMEMDeviceInfo *vi) | |
1041 | { | |
1042 | vi->memaddr = vmem->addr; | |
1043 | vi->node = vmem->node; | |
1044 | vi->requested_size = vmem->requested_size; | |
1045 | vi->size = vmem->size; | |
1046 | vi->max_size = memory_region_size(&vmem->memdev->mr); | |
1047 | vi->block_size = vmem->block_size; | |
1048 | vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev)); | |
1049 | } | |
1050 | ||
1051 | static MemoryRegion *virtio_mem_get_memory_region(VirtIOMEM *vmem, Error **errp) | |
1052 | { | |
1053 | if (!vmem->memdev) { | |
1054 | error_setg(errp, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP); | |
1055 | return NULL; | |
1056 | } | |
1057 | ||
1058 | return &vmem->memdev->mr; | |
1059 | } | |
1060 | ||
c95b4437 DH |
1061 | static void virtio_mem_add_size_change_notifier(VirtIOMEM *vmem, |
1062 | Notifier *notifier) | |
1063 | { | |
1064 | notifier_list_add(&vmem->size_change_notifiers, notifier); | |
1065 | } | |
1066 | ||
1067 | static void virtio_mem_remove_size_change_notifier(VirtIOMEM *vmem, | |
1068 | Notifier *notifier) | |
1069 | { | |
1070 | notifier_remove(notifier); | |
1071 | } | |
1072 | ||
910b2576 DH |
1073 | static void virtio_mem_get_size(Object *obj, Visitor *v, const char *name, |
1074 | void *opaque, Error **errp) | |
1075 | { | |
1076 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1077 | uint64_t value = vmem->size; | |
1078 | ||
1079 | visit_type_size(v, name, &value, errp); | |
1080 | } | |
1081 | ||
1082 | static void virtio_mem_get_requested_size(Object *obj, Visitor *v, | |
1083 | const char *name, void *opaque, | |
1084 | Error **errp) | |
1085 | { | |
1086 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1087 | uint64_t value = vmem->requested_size; | |
1088 | ||
1089 | visit_type_size(v, name, &value, errp); | |
1090 | } | |
1091 | ||
1092 | static void virtio_mem_set_requested_size(Object *obj, Visitor *v, | |
1093 | const char *name, void *opaque, | |
1094 | Error **errp) | |
1095 | { | |
1096 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
910b2576 DH |
1097 | uint64_t value; |
1098 | ||
d1c81c34 | 1099 | if (!visit_type_size(v, name, &value, errp)) { |
910b2576 DH |
1100 | return; |
1101 | } | |
1102 | ||
1103 | /* | |
1104 | * The block size and memory backend are not fixed until the device was | |
1105 | * realized. realize() will verify these properties then. | |
1106 | */ | |
1107 | if (DEVICE(obj)->realized) { | |
1108 | if (!QEMU_IS_ALIGNED(value, vmem->block_size)) { | |
1109 | error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64 | |
1110 | ")", name, VIRTIO_MEM_BLOCK_SIZE_PROP, | |
1111 | vmem->block_size); | |
1112 | return; | |
1113 | } else if (value > memory_region_size(&vmem->memdev->mr)) { | |
1114 | error_setg(errp, "'%s' cannot exceed the memory backend size" | |
1115 | "(0x%" PRIx64 ")", name, | |
1116 | memory_region_size(&vmem->memdev->mr)); | |
1117 | return; | |
1118 | } | |
1119 | ||
1120 | if (value != vmem->requested_size) { | |
1121 | virtio_mem_resize_usable_region(vmem, value, false); | |
1122 | vmem->requested_size = value; | |
1123 | } | |
1124 | /* | |
1125 | * Trigger a config update so the guest gets notified. We trigger | |
1126 | * even if the size didn't change (especially helpful for debugging). | |
1127 | */ | |
1128 | virtio_notify_config(VIRTIO_DEVICE(vmem)); | |
1129 | } else { | |
1130 | vmem->requested_size = value; | |
1131 | } | |
1132 | } | |
1133 | ||
1134 | static void virtio_mem_get_block_size(Object *obj, Visitor *v, const char *name, | |
1135 | void *opaque, Error **errp) | |
1136 | { | |
1137 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1138 | uint64_t value = vmem->block_size; | |
1139 | ||
228957fe DH |
1140 | /* |
1141 | * If not configured by the user (and we're not realized yet), use the | |
1142 | * default block size we would use with the current memory backend. | |
1143 | */ | |
1144 | if (!value) { | |
1145 | if (vmem->memdev && memory_region_is_ram(&vmem->memdev->mr)) { | |
1146 | value = virtio_mem_default_block_size(vmem->memdev->mr.ram_block); | |
1147 | } else { | |
1148 | value = virtio_mem_thp_size(); | |
1149 | } | |
1150 | } | |
1151 | ||
910b2576 DH |
1152 | visit_type_size(v, name, &value, errp); |
1153 | } | |
1154 | ||
1155 | static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name, | |
1156 | void *opaque, Error **errp) | |
1157 | { | |
1158 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
910b2576 DH |
1159 | uint64_t value; |
1160 | ||
1161 | if (DEVICE(obj)->realized) { | |
1162 | error_setg(errp, "'%s' cannot be changed", name); | |
1163 | return; | |
1164 | } | |
1165 | ||
d1c81c34 | 1166 | if (!visit_type_size(v, name, &value, errp)) { |
910b2576 DH |
1167 | return; |
1168 | } | |
1169 | ||
1170 | if (value < VIRTIO_MEM_MIN_BLOCK_SIZE) { | |
1171 | error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name, | |
1172 | VIRTIO_MEM_MIN_BLOCK_SIZE); | |
1173 | return; | |
1174 | } else if (!is_power_of_2(value)) { | |
1175 | error_setg(errp, "'%s' property has to be a power of two", name); | |
1176 | return; | |
1177 | } | |
1178 | vmem->block_size = value; | |
1179 | } | |
1180 | ||
1181 | static void virtio_mem_instance_init(Object *obj) | |
1182 | { | |
1183 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1184 | ||
c95b4437 | 1185 | notifier_list_init(&vmem->size_change_notifiers); |
2044969f | 1186 | QLIST_INIT(&vmem->rdl_list); |
910b2576 DH |
1187 | |
1188 | object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size, | |
1189 | NULL, NULL, NULL); | |
1190 | object_property_add(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP, "size", | |
1191 | virtio_mem_get_requested_size, | |
1192 | virtio_mem_set_requested_size, NULL, NULL); | |
1193 | object_property_add(obj, VIRTIO_MEM_BLOCK_SIZE_PROP, "size", | |
1194 | virtio_mem_get_block_size, virtio_mem_set_block_size, | |
1195 | NULL, NULL); | |
1196 | } | |
1197 | ||
1198 | static Property virtio_mem_properties[] = { | |
1199 | DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP, VirtIOMEM, addr, 0), | |
1200 | DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP, VirtIOMEM, node, 0), | |
09b3b7e0 | 1201 | DEFINE_PROP_BOOL(VIRTIO_MEM_PREALLOC_PROP, VirtIOMEM, prealloc, false), |
910b2576 DH |
1202 | DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP, VirtIOMEM, memdev, |
1203 | TYPE_MEMORY_BACKEND, HostMemoryBackend *), | |
23ad8dec DH |
1204 | #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) |
1205 | DEFINE_PROP_ON_OFF_AUTO(VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP, VirtIOMEM, | |
60f1f77c | 1206 | unplugged_inaccessible, ON_OFF_AUTO_AUTO), |
23ad8dec | 1207 | #endif |
910b2576 DH |
1208 | DEFINE_PROP_END_OF_LIST(), |
1209 | }; | |
1210 | ||
2044969f DH |
1211 | static uint64_t virtio_mem_rdm_get_min_granularity(const RamDiscardManager *rdm, |
1212 | const MemoryRegion *mr) | |
1213 | { | |
1214 | const VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1215 | ||
1216 | g_assert(mr == &vmem->memdev->mr); | |
1217 | return vmem->block_size; | |
1218 | } | |
1219 | ||
1220 | static bool virtio_mem_rdm_is_populated(const RamDiscardManager *rdm, | |
1221 | const MemoryRegionSection *s) | |
1222 | { | |
1223 | const VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1224 | uint64_t start_gpa = vmem->addr + s->offset_within_region; | |
1225 | uint64_t end_gpa = start_gpa + int128_get64(s->size); | |
1226 | ||
1227 | g_assert(s->mr == &vmem->memdev->mr); | |
1228 | ||
1229 | start_gpa = QEMU_ALIGN_DOWN(start_gpa, vmem->block_size); | |
1230 | end_gpa = QEMU_ALIGN_UP(end_gpa, vmem->block_size); | |
1231 | ||
1232 | if (!virtio_mem_valid_range(vmem, start_gpa, end_gpa - start_gpa)) { | |
1233 | return false; | |
1234 | } | |
1235 | ||
1236 | return virtio_mem_test_bitmap(vmem, start_gpa, end_gpa - start_gpa, true); | |
1237 | } | |
1238 | ||
1239 | struct VirtIOMEMReplayData { | |
1240 | void *fn; | |
1241 | void *opaque; | |
1242 | }; | |
1243 | ||
1244 | static int virtio_mem_rdm_replay_populated_cb(MemoryRegionSection *s, void *arg) | |
1245 | { | |
1246 | struct VirtIOMEMReplayData *data = arg; | |
1247 | ||
1248 | return ((ReplayRamPopulate)data->fn)(s, data->opaque); | |
1249 | } | |
1250 | ||
1251 | static int virtio_mem_rdm_replay_populated(const RamDiscardManager *rdm, | |
1252 | MemoryRegionSection *s, | |
1253 | ReplayRamPopulate replay_fn, | |
1254 | void *opaque) | |
1255 | { | |
1256 | const VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1257 | struct VirtIOMEMReplayData data = { | |
1258 | .fn = replay_fn, | |
1259 | .opaque = opaque, | |
1260 | }; | |
1261 | ||
1262 | g_assert(s->mr == &vmem->memdev->mr); | |
1263 | return virtio_mem_for_each_plugged_section(vmem, s, &data, | |
1264 | virtio_mem_rdm_replay_populated_cb); | |
1265 | } | |
1266 | ||
372aa6fd DH |
1267 | static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection *s, |
1268 | void *arg) | |
1269 | { | |
1270 | struct VirtIOMEMReplayData *data = arg; | |
1271 | ||
1272 | ((ReplayRamDiscard)data->fn)(s, data->opaque); | |
1273 | return 0; | |
1274 | } | |
1275 | ||
1276 | static void virtio_mem_rdm_replay_discarded(const RamDiscardManager *rdm, | |
1277 | MemoryRegionSection *s, | |
1278 | ReplayRamDiscard replay_fn, | |
1279 | void *opaque) | |
1280 | { | |
1281 | const VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1282 | struct VirtIOMEMReplayData data = { | |
1283 | .fn = replay_fn, | |
1284 | .opaque = opaque, | |
1285 | }; | |
1286 | ||
1287 | g_assert(s->mr == &vmem->memdev->mr); | |
1288 | virtio_mem_for_each_unplugged_section(vmem, s, &data, | |
1289 | virtio_mem_rdm_replay_discarded_cb); | |
1290 | } | |
1291 | ||
2044969f DH |
1292 | static void virtio_mem_rdm_register_listener(RamDiscardManager *rdm, |
1293 | RamDiscardListener *rdl, | |
1294 | MemoryRegionSection *s) | |
1295 | { | |
1296 | VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1297 | int ret; | |
1298 | ||
1299 | g_assert(s->mr == &vmem->memdev->mr); | |
1300 | rdl->section = memory_region_section_new_copy(s); | |
1301 | ||
1302 | QLIST_INSERT_HEAD(&vmem->rdl_list, rdl, next); | |
1303 | ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, | |
1304 | virtio_mem_notify_populate_cb); | |
1305 | if (ret) { | |
1306 | error_report("%s: Replaying plugged ranges failed: %s", __func__, | |
1307 | strerror(-ret)); | |
1308 | } | |
1309 | } | |
1310 | ||
1311 | static void virtio_mem_rdm_unregister_listener(RamDiscardManager *rdm, | |
1312 | RamDiscardListener *rdl) | |
1313 | { | |
1314 | VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1315 | ||
1316 | g_assert(rdl->section->mr == &vmem->memdev->mr); | |
1317 | if (vmem->size) { | |
1318 | if (rdl->double_discard_supported) { | |
1319 | rdl->notify_discard(rdl, rdl->section); | |
1320 | } else { | |
1321 | virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, | |
1322 | virtio_mem_notify_discard_cb); | |
1323 | } | |
1324 | } | |
1325 | ||
1326 | memory_region_section_free_copy(rdl->section); | |
1327 | rdl->section = NULL; | |
1328 | QLIST_REMOVE(rdl, next); | |
1329 | } | |
1330 | ||
910b2576 DH |
1331 | static void virtio_mem_class_init(ObjectClass *klass, void *data) |
1332 | { | |
1333 | DeviceClass *dc = DEVICE_CLASS(klass); | |
1334 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); | |
1335 | VirtIOMEMClass *vmc = VIRTIO_MEM_CLASS(klass); | |
2044969f | 1336 | RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass); |
910b2576 DH |
1337 | |
1338 | device_class_set_props(dc, virtio_mem_properties); | |
1339 | dc->vmsd = &vmstate_virtio_mem; | |
1340 | ||
1341 | set_bit(DEVICE_CATEGORY_MISC, dc->categories); | |
1342 | vdc->realize = virtio_mem_device_realize; | |
1343 | vdc->unrealize = virtio_mem_device_unrealize; | |
1344 | vdc->get_config = virtio_mem_get_config; | |
1345 | vdc->get_features = virtio_mem_get_features; | |
23ad8dec | 1346 | vdc->validate_features = virtio_mem_validate_features; |
910b2576 DH |
1347 | vdc->vmsd = &vmstate_virtio_mem_device; |
1348 | ||
1349 | vmc->fill_device_info = virtio_mem_fill_device_info; | |
1350 | vmc->get_memory_region = virtio_mem_get_memory_region; | |
c95b4437 DH |
1351 | vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier; |
1352 | vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier; | |
2044969f DH |
1353 | |
1354 | rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity; | |
1355 | rdmc->is_populated = virtio_mem_rdm_is_populated; | |
1356 | rdmc->replay_populated = virtio_mem_rdm_replay_populated; | |
372aa6fd | 1357 | rdmc->replay_discarded = virtio_mem_rdm_replay_discarded; |
2044969f DH |
1358 | rdmc->register_listener = virtio_mem_rdm_register_listener; |
1359 | rdmc->unregister_listener = virtio_mem_rdm_unregister_listener; | |
910b2576 DH |
1360 | } |
1361 | ||
1362 | static const TypeInfo virtio_mem_info = { | |
1363 | .name = TYPE_VIRTIO_MEM, | |
1364 | .parent = TYPE_VIRTIO_DEVICE, | |
1365 | .instance_size = sizeof(VirtIOMEM), | |
1366 | .instance_init = virtio_mem_instance_init, | |
1367 | .class_init = virtio_mem_class_init, | |
1368 | .class_size = sizeof(VirtIOMEMClass), | |
2044969f DH |
1369 | .interfaces = (InterfaceInfo[]) { |
1370 | { TYPE_RAM_DISCARD_MANAGER }, | |
1371 | { } | |
1372 | }, | |
910b2576 DH |
1373 | }; |
1374 | ||
1375 | static void virtio_register_types(void) | |
1376 | { | |
1377 | type_register_static(&virtio_mem_info); | |
1378 | } | |
1379 | ||
1380 | type_init(virtio_register_types) |