]>
Commit | Line | Data |
---|---|---|
910b2576 DH |
1 | /* |
2 | * Virtio MEM device | |
3 | * | |
4 | * Copyright (C) 2020 Red Hat, Inc. | |
5 | * | |
6 | * Authors: | |
7 | * David Hildenbrand <david@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. | |
10 | * See the COPYING file in the top-level directory. | |
11 | */ | |
12 | ||
13 | #include "qemu/osdep.h" | |
910b2576 DH |
14 | #include "qemu/iov.h" |
15 | #include "qemu/cutils.h" | |
16 | #include "qemu/error-report.h" | |
17 | #include "qemu/units.h" | |
18 | #include "sysemu/numa.h" | |
19 | #include "sysemu/sysemu.h" | |
20 | #include "sysemu/reset.h" | |
b01fd4b6 | 21 | #include "sysemu/runstate.h" |
910b2576 DH |
22 | #include "hw/virtio/virtio.h" |
23 | #include "hw/virtio/virtio-bus.h" | |
910b2576 DH |
24 | #include "hw/virtio/virtio-mem.h" |
25 | #include "qapi/error.h" | |
26 | #include "qapi/visitor.h" | |
27 | #include "exec/ram_addr.h" | |
28 | #include "migration/misc.h" | |
29 | #include "hw/boards.h" | |
30 | #include "hw/qdev-properties.h" | |
2becc36a | 31 | #include CONFIG_DEVICES |
43e54950 | 32 | #include "trace.h" |
910b2576 | 33 | |
3b95a71b DH |
34 | static const VMStateDescription vmstate_virtio_mem_device_early; |
35 | ||
23ad8dec DH |
36 | /* |
37 | * We only had legacy x86 guests that did not support | |
38 | * VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE. Other targets don't have legacy guests. | |
39 | */ | |
40 | #if defined(TARGET_X86_64) || defined(TARGET_I386) | |
41 | #define VIRTIO_MEM_HAS_LEGACY_GUESTS | |
42 | #endif | |
43 | ||
910b2576 | 44 | /* |
228957fe DH |
45 | * Let's not allow blocks smaller than 1 MiB, for example, to keep the tracking |
46 | * bitmap small. | |
910b2576 | 47 | */ |
228957fe DH |
48 | #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)(1 * MiB)) |
49 | ||
1263615e GS |
50 | static uint32_t virtio_mem_default_thp_size(void) |
51 | { | |
52 | uint32_t default_thp_size = VIRTIO_MEM_MIN_BLOCK_SIZE; | |
53 | ||
54 | #if defined(__x86_64__) || defined(__arm__) || defined(__powerpc64__) | |
55 | default_thp_size = 2 * MiB; | |
56 | #elif defined(__aarch64__) | |
8e3b0cbb | 57 | if (qemu_real_host_page_size() == 4 * KiB) { |
1263615e | 58 | default_thp_size = 2 * MiB; |
8e3b0cbb | 59 | } else if (qemu_real_host_page_size() == 16 * KiB) { |
1263615e | 60 | default_thp_size = 32 * MiB; |
8e3b0cbb | 61 | } else if (qemu_real_host_page_size() == 64 * KiB) { |
1263615e GS |
62 | default_thp_size = 512 * MiB; |
63 | } | |
228957fe DH |
64 | #endif |
65 | ||
1263615e GS |
66 | return default_thp_size; |
67 | } | |
68 | ||
177f9b1e DH |
69 | /* |
70 | * The minimum memslot size depends on this setting ("sane default"), the | |
71 | * device block size, and the memory backend page size. The last (or single) | |
72 | * memslot might be smaller than this constant. | |
73 | */ | |
74 | #define VIRTIO_MEM_MIN_MEMSLOT_SIZE (1 * GiB) | |
75 | ||
228957fe DH |
76 | /* |
77 | * We want to have a reasonable default block size such that | |
78 | * 1. We avoid splitting THPs when unplugging memory, which degrades | |
79 | * performance. | |
80 | * 2. We avoid placing THPs for plugged blocks that also cover unplugged | |
81 | * blocks. | |
82 | * | |
83 | * The actual THP size might differ between Linux kernels, so we try to probe | |
84 | * it. In the future (if we ever run into issues regarding 2.), we might want | |
85 | * to disable THP in case we fail to properly probe the THP size, or if the | |
86 | * block size is configured smaller than the THP size. | |
87 | */ | |
88 | static uint32_t thp_size; | |
89 | ||
90 | #define HPAGE_PMD_SIZE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" | |
91 | static uint32_t virtio_mem_thp_size(void) | |
92 | { | |
93 | gchar *content = NULL; | |
94 | const char *endptr; | |
95 | uint64_t tmp; | |
96 | ||
97 | if (thp_size) { | |
98 | return thp_size; | |
99 | } | |
100 | ||
101 | /* | |
102 | * Try to probe the actual THP size, fallback to (sane but eventually | |
103 | * incorrect) default sizes. | |
104 | */ | |
105 | if (g_file_get_contents(HPAGE_PMD_SIZE_PATH, &content, NULL, NULL) && | |
106 | !qemu_strtou64(content, &endptr, 0, &tmp) && | |
107 | (!endptr || *endptr == '\n')) { | |
1263615e GS |
108 | /* Sanity-check the value and fallback to something reasonable. */ |
109 | if (!tmp || !is_power_of_2(tmp)) { | |
228957fe DH |
110 | warn_report("Read unsupported THP size: %" PRIx64, tmp); |
111 | } else { | |
112 | thp_size = tmp; | |
113 | } | |
114 | } | |
115 | ||
116 | if (!thp_size) { | |
1263615e | 117 | thp_size = virtio_mem_default_thp_size(); |
228957fe DH |
118 | warn_report("Could not detect THP size, falling back to %" PRIx64 |
119 | " MiB.", thp_size / MiB); | |
120 | } | |
121 | ||
122 | g_free(content); | |
123 | return thp_size; | |
124 | } | |
125 | ||
126 | static uint64_t virtio_mem_default_block_size(RAMBlock *rb) | |
127 | { | |
128 | const uint64_t page_size = qemu_ram_pagesize(rb); | |
129 | ||
130 | /* We can have hugetlbfs with a page size smaller than the THP size. */ | |
8e3b0cbb | 131 | if (page_size == qemu_real_host_page_size()) { |
228957fe DH |
132 | return MAX(page_size, virtio_mem_thp_size()); |
133 | } | |
134 | return MAX(page_size, VIRTIO_MEM_MIN_BLOCK_SIZE); | |
135 | } | |
136 | ||
23ad8dec DH |
137 | #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) |
138 | static bool virtio_mem_has_shared_zeropage(RAMBlock *rb) | |
139 | { | |
140 | /* | |
141 | * We only have a guaranteed shared zeropage on ordinary MAP_PRIVATE | |
142 | * anonymous RAM. In any other case, reading unplugged *can* populate a | |
143 | * fresh page, consuming actual memory. | |
144 | */ | |
21e64350 | 145 | return !qemu_ram_is_shared(rb) && qemu_ram_get_fd(rb) < 0 && |
8e3b0cbb | 146 | qemu_ram_pagesize(rb) == qemu_real_host_page_size(); |
23ad8dec DH |
147 | } |
148 | #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ | |
149 | ||
910b2576 DH |
150 | /* |
151 | * Size the usable region bigger than the requested size if possible. Esp. | |
152 | * Linux guests will only add (aligned) memory blocks in case they fully | |
153 | * fit into the usable region, but plug+online only a subset of the pages. | |
154 | * The memory block size corresponds mostly to the section size. | |
155 | * | |
156 | * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and | |
b1b87327 | 157 | * a section size of 512MB on arm64 (as long as the start address is properly |
910b2576 DH |
158 | * aligned, similar to ordinary DIMMs). |
159 | * | |
160 | * We can change this at any time and maybe even make it configurable if | |
161 | * necessary (as the section size can change). But it's more likely that the | |
162 | * section size will rather get smaller and not bigger over time. | |
163 | */ | |
164 | #if defined(TARGET_X86_64) || defined(TARGET_I386) | |
165 | #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB)) | |
b1b87327 GS |
166 | #elif defined(TARGET_ARM) |
167 | #define VIRTIO_MEM_USABLE_EXTENT (2 * (512 * MiB)) | |
910b2576 DH |
168 | #else |
169 | #error VIRTIO_MEM_USABLE_EXTENT not defined | |
170 | #endif | |
171 | ||
172 | static bool virtio_mem_is_busy(void) | |
173 | { | |
174 | /* | |
175 | * Postcopy cannot handle concurrent discards and we don't want to migrate | |
176 | * pages on-demand with stale content when plugging new blocks. | |
0bc7806c DH |
177 | * |
178 | * For precopy, we don't want unplugged blocks in our migration stream, and | |
179 | * when plugging new blocks, the page content might differ between source | |
180 | * and destination (observable by the guest when not initializing pages | |
181 | * after plugging them) until we're running on the destination (as we didn't | |
182 | * migrate these blocks when they were unplugged). | |
910b2576 | 183 | */ |
0bc7806c | 184 | return migration_in_incoming_postcopy() || !migration_is_idle(); |
910b2576 DH |
185 | } |
186 | ||
a45171db | 187 | typedef int (*virtio_mem_range_cb)(VirtIOMEM *vmem, void *arg, |
7a9d5d02 DH |
188 | uint64_t offset, uint64_t size); |
189 | ||
a45171db | 190 | static int virtio_mem_for_each_unplugged_range(VirtIOMEM *vmem, void *arg, |
7a9d5d02 DH |
191 | virtio_mem_range_cb cb) |
192 | { | |
193 | unsigned long first_zero_bit, last_zero_bit; | |
194 | uint64_t offset, size; | |
195 | int ret = 0; | |
196 | ||
197 | first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); | |
198 | while (first_zero_bit < vmem->bitmap_size) { | |
199 | offset = first_zero_bit * vmem->block_size; | |
200 | last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
201 | first_zero_bit + 1) - 1; | |
202 | size = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; | |
203 | ||
204 | ret = cb(vmem, arg, offset, size); | |
205 | if (ret) { | |
206 | break; | |
207 | } | |
208 | first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
209 | last_zero_bit + 2); | |
210 | } | |
211 | return ret; | |
212 | } | |
213 | ||
a45171db | 214 | static int virtio_mem_for_each_plugged_range(VirtIOMEM *vmem, void *arg, |
d71920d4 DH |
215 | virtio_mem_range_cb cb) |
216 | { | |
217 | unsigned long first_bit, last_bit; | |
218 | uint64_t offset, size; | |
219 | int ret = 0; | |
220 | ||
221 | first_bit = find_first_bit(vmem->bitmap, vmem->bitmap_size); | |
222 | while (first_bit < vmem->bitmap_size) { | |
223 | offset = first_bit * vmem->block_size; | |
224 | last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
225 | first_bit + 1) - 1; | |
226 | size = (last_bit - first_bit + 1) * vmem->block_size; | |
227 | ||
228 | ret = cb(vmem, arg, offset, size); | |
229 | if (ret) { | |
230 | break; | |
231 | } | |
232 | first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
233 | last_bit + 2); | |
234 | } | |
235 | return ret; | |
236 | } | |
237 | ||
2044969f DH |
238 | /* |
239 | * Adjust the memory section to cover the intersection with the given range. | |
240 | * | |
241 | * Returns false if the intersection is empty, otherwise returns true. | |
242 | */ | |
82ba778e | 243 | static bool virtio_mem_intersect_memory_section(MemoryRegionSection *s, |
2044969f DH |
244 | uint64_t offset, uint64_t size) |
245 | { | |
246 | uint64_t start = MAX(s->offset_within_region, offset); | |
247 | uint64_t end = MIN(s->offset_within_region + int128_get64(s->size), | |
248 | offset + size); | |
249 | ||
250 | if (end <= start) { | |
251 | return false; | |
252 | } | |
253 | ||
254 | s->offset_within_address_space += start - s->offset_within_region; | |
255 | s->offset_within_region = start; | |
256 | s->size = int128_make64(end - start); | |
257 | return true; | |
258 | } | |
259 | ||
260 | typedef int (*virtio_mem_section_cb)(MemoryRegionSection *s, void *arg); | |
261 | ||
262 | static int virtio_mem_for_each_plugged_section(const VirtIOMEM *vmem, | |
263 | MemoryRegionSection *s, | |
264 | void *arg, | |
265 | virtio_mem_section_cb cb) | |
266 | { | |
267 | unsigned long first_bit, last_bit; | |
268 | uint64_t offset, size; | |
269 | int ret = 0; | |
270 | ||
b11cf32e | 271 | first_bit = s->offset_within_region / vmem->block_size; |
2044969f DH |
272 | first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, first_bit); |
273 | while (first_bit < vmem->bitmap_size) { | |
274 | MemoryRegionSection tmp = *s; | |
275 | ||
276 | offset = first_bit * vmem->block_size; | |
277 | last_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
278 | first_bit + 1) - 1; | |
279 | size = (last_bit - first_bit + 1) * vmem->block_size; | |
280 | ||
82ba778e | 281 | if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { |
2044969f DH |
282 | break; |
283 | } | |
284 | ret = cb(&tmp, arg); | |
285 | if (ret) { | |
286 | break; | |
287 | } | |
288 | first_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
289 | last_bit + 2); | |
290 | } | |
291 | return ret; | |
292 | } | |
293 | ||
372aa6fd DH |
294 | static int virtio_mem_for_each_unplugged_section(const VirtIOMEM *vmem, |
295 | MemoryRegionSection *s, | |
296 | void *arg, | |
297 | virtio_mem_section_cb cb) | |
298 | { | |
299 | unsigned long first_bit, last_bit; | |
300 | uint64_t offset, size; | |
301 | int ret = 0; | |
302 | ||
b11cf32e | 303 | first_bit = s->offset_within_region / vmem->block_size; |
372aa6fd DH |
304 | first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, first_bit); |
305 | while (first_bit < vmem->bitmap_size) { | |
306 | MemoryRegionSection tmp = *s; | |
307 | ||
308 | offset = first_bit * vmem->block_size; | |
309 | last_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
310 | first_bit + 1) - 1; | |
311 | size = (last_bit - first_bit + 1) * vmem->block_size; | |
312 | ||
82ba778e | 313 | if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { |
372aa6fd DH |
314 | break; |
315 | } | |
316 | ret = cb(&tmp, arg); | |
317 | if (ret) { | |
318 | break; | |
319 | } | |
320 | first_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
321 | last_bit + 2); | |
322 | } | |
323 | return ret; | |
324 | } | |
325 | ||
2044969f DH |
326 | static int virtio_mem_notify_populate_cb(MemoryRegionSection *s, void *arg) |
327 | { | |
328 | RamDiscardListener *rdl = arg; | |
329 | ||
330 | return rdl->notify_populate(rdl, s); | |
331 | } | |
332 | ||
333 | static int virtio_mem_notify_discard_cb(MemoryRegionSection *s, void *arg) | |
334 | { | |
335 | RamDiscardListener *rdl = arg; | |
336 | ||
337 | rdl->notify_discard(rdl, s); | |
338 | return 0; | |
339 | } | |
340 | ||
341 | static void virtio_mem_notify_unplug(VirtIOMEM *vmem, uint64_t offset, | |
342 | uint64_t size) | |
343 | { | |
344 | RamDiscardListener *rdl; | |
345 | ||
346 | QLIST_FOREACH(rdl, &vmem->rdl_list, next) { | |
347 | MemoryRegionSection tmp = *rdl->section; | |
348 | ||
82ba778e | 349 | if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { |
2044969f DH |
350 | continue; |
351 | } | |
352 | rdl->notify_discard(rdl, &tmp); | |
353 | } | |
354 | } | |
355 | ||
356 | static int virtio_mem_notify_plug(VirtIOMEM *vmem, uint64_t offset, | |
357 | uint64_t size) | |
358 | { | |
359 | RamDiscardListener *rdl, *rdl2; | |
360 | int ret = 0; | |
361 | ||
362 | QLIST_FOREACH(rdl, &vmem->rdl_list, next) { | |
363 | MemoryRegionSection tmp = *rdl->section; | |
364 | ||
82ba778e | 365 | if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { |
2044969f DH |
366 | continue; |
367 | } | |
368 | ret = rdl->notify_populate(rdl, &tmp); | |
369 | if (ret) { | |
370 | break; | |
371 | } | |
372 | } | |
373 | ||
374 | if (ret) { | |
375 | /* Notify all already-notified listeners. */ | |
376 | QLIST_FOREACH(rdl2, &vmem->rdl_list, next) { | |
29f1b328 | 377 | MemoryRegionSection tmp = *rdl2->section; |
2044969f DH |
378 | |
379 | if (rdl2 == rdl) { | |
380 | break; | |
381 | } | |
82ba778e | 382 | if (!virtio_mem_intersect_memory_section(&tmp, offset, size)) { |
2044969f DH |
383 | continue; |
384 | } | |
385 | rdl2->notify_discard(rdl2, &tmp); | |
386 | } | |
387 | } | |
388 | return ret; | |
389 | } | |
390 | ||
391 | static void virtio_mem_notify_unplug_all(VirtIOMEM *vmem) | |
392 | { | |
393 | RamDiscardListener *rdl; | |
394 | ||
395 | if (!vmem->size) { | |
396 | return; | |
397 | } | |
398 | ||
399 | QLIST_FOREACH(rdl, &vmem->rdl_list, next) { | |
400 | if (rdl->double_discard_supported) { | |
401 | rdl->notify_discard(rdl, rdl->section); | |
402 | } else { | |
403 | virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, | |
404 | virtio_mem_notify_discard_cb); | |
405 | } | |
406 | } | |
407 | } | |
408 | ||
25c89303 DH |
409 | static bool virtio_mem_is_range_plugged(const VirtIOMEM *vmem, |
410 | uint64_t start_gpa, uint64_t size) | |
910b2576 DH |
411 | { |
412 | const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; | |
413 | const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; | |
414 | unsigned long found_bit; | |
415 | ||
416 | /* We fake a shorter bitmap to avoid searching too far. */ | |
25c89303 | 417 | found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit); |
910b2576 DH |
418 | return found_bit > last_bit; |
419 | } | |
420 | ||
25c89303 DH |
421 | static bool virtio_mem_is_range_unplugged(const VirtIOMEM *vmem, |
422 | uint64_t start_gpa, uint64_t size) | |
423 | { | |
424 | const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; | |
425 | const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; | |
426 | unsigned long found_bit; | |
427 | ||
428 | /* We fake a shorter bitmap to avoid searching too far. */ | |
429 | found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit); | |
430 | return found_bit > last_bit; | |
431 | } | |
432 | ||
433 | static void virtio_mem_set_range_plugged(VirtIOMEM *vmem, uint64_t start_gpa, | |
434 | uint64_t size) | |
910b2576 DH |
435 | { |
436 | const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; | |
437 | const unsigned long nbits = size / vmem->block_size; | |
438 | ||
25c89303 DH |
439 | bitmap_set(vmem->bitmap, bit, nbits); |
440 | } | |
441 | ||
442 | static void virtio_mem_set_range_unplugged(VirtIOMEM *vmem, uint64_t start_gpa, | |
443 | uint64_t size) | |
444 | { | |
445 | const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; | |
446 | const unsigned long nbits = size / vmem->block_size; | |
447 | ||
448 | bitmap_clear(vmem->bitmap, bit, nbits); | |
910b2576 DH |
449 | } |
450 | ||
451 | static void virtio_mem_send_response(VirtIOMEM *vmem, VirtQueueElement *elem, | |
452 | struct virtio_mem_resp *resp) | |
453 | { | |
454 | VirtIODevice *vdev = VIRTIO_DEVICE(vmem); | |
455 | VirtQueue *vq = vmem->vq; | |
456 | ||
43e54950 | 457 | trace_virtio_mem_send_response(le16_to_cpu(resp->type)); |
910b2576 DH |
458 | iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp)); |
459 | ||
460 | virtqueue_push(vq, elem, sizeof(*resp)); | |
461 | virtio_notify(vdev, vq); | |
462 | } | |
463 | ||
464 | static void virtio_mem_send_response_simple(VirtIOMEM *vmem, | |
465 | VirtQueueElement *elem, | |
466 | uint16_t type) | |
467 | { | |
468 | struct virtio_mem_resp resp = { | |
469 | .type = cpu_to_le16(type), | |
470 | }; | |
471 | ||
472 | virtio_mem_send_response(vmem, elem, &resp); | |
473 | } | |
474 | ||
2044969f DH |
475 | static bool virtio_mem_valid_range(const VirtIOMEM *vmem, uint64_t gpa, |
476 | uint64_t size) | |
910b2576 DH |
477 | { |
478 | if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) { | |
479 | return false; | |
480 | } | |
481 | if (gpa + size < gpa || !size) { | |
482 | return false; | |
483 | } | |
484 | if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) { | |
485 | return false; | |
486 | } | |
487 | if (gpa + size > vmem->addr + vmem->usable_region_size) { | |
488 | return false; | |
489 | } | |
490 | return true; | |
491 | } | |
492 | ||
177f9b1e DH |
493 | static void virtio_mem_activate_memslot(VirtIOMEM *vmem, unsigned int idx) |
494 | { | |
495 | const uint64_t memslot_offset = idx * vmem->memslot_size; | |
496 | ||
497 | assert(vmem->memslots); | |
498 | ||
499 | /* | |
500 | * Instead of enabling/disabling memslots, we add/remove them. This should | |
501 | * make address space updates faster, because we don't have to loop over | |
502 | * many disabled subregions. | |
503 | */ | |
504 | if (memory_region_is_mapped(&vmem->memslots[idx])) { | |
505 | return; | |
506 | } | |
507 | memory_region_add_subregion(vmem->mr, memslot_offset, &vmem->memslots[idx]); | |
508 | } | |
509 | ||
510 | static void virtio_mem_deactivate_memslot(VirtIOMEM *vmem, unsigned int idx) | |
511 | { | |
512 | assert(vmem->memslots); | |
513 | ||
514 | if (!memory_region_is_mapped(&vmem->memslots[idx])) { | |
515 | return; | |
516 | } | |
517 | memory_region_del_subregion(vmem->mr, &vmem->memslots[idx]); | |
518 | } | |
519 | ||
520 | static void virtio_mem_activate_memslots_to_plug(VirtIOMEM *vmem, | |
521 | uint64_t offset, uint64_t size) | |
522 | { | |
523 | const unsigned int start_idx = offset / vmem->memslot_size; | |
524 | const unsigned int end_idx = (offset + size + vmem->memslot_size - 1) / | |
525 | vmem->memslot_size; | |
526 | unsigned int idx; | |
527 | ||
364eff68 | 528 | assert(vmem->dynamic_memslots); |
177f9b1e DH |
529 | |
530 | /* Activate all involved memslots in a single transaction. */ | |
531 | memory_region_transaction_begin(); | |
532 | for (idx = start_idx; idx < end_idx; idx++) { | |
533 | virtio_mem_activate_memslot(vmem, idx); | |
534 | } | |
535 | memory_region_transaction_commit(); | |
536 | } | |
537 | ||
538 | static void virtio_mem_deactivate_unplugged_memslots(VirtIOMEM *vmem, | |
539 | uint64_t offset, | |
540 | uint64_t size) | |
541 | { | |
542 | const uint64_t region_size = memory_region_size(&vmem->memdev->mr); | |
543 | const unsigned int start_idx = offset / vmem->memslot_size; | |
544 | const unsigned int end_idx = (offset + size + vmem->memslot_size - 1) / | |
545 | vmem->memslot_size; | |
546 | unsigned int idx; | |
547 | ||
364eff68 | 548 | assert(vmem->dynamic_memslots); |
177f9b1e DH |
549 | |
550 | /* Deactivate all memslots with unplugged blocks in a single transaction. */ | |
551 | memory_region_transaction_begin(); | |
552 | for (idx = start_idx; idx < end_idx; idx++) { | |
553 | const uint64_t memslot_offset = idx * vmem->memslot_size; | |
554 | uint64_t memslot_size = vmem->memslot_size; | |
555 | ||
556 | /* The size of the last memslot might be smaller. */ | |
557 | if (idx == vmem->nb_memslots - 1) { | |
558 | memslot_size = region_size - memslot_offset; | |
559 | } | |
560 | ||
561 | /* | |
562 | * Partially covered memslots might still have some blocks plugged and | |
563 | * have to remain active if that's the case. | |
564 | */ | |
565 | if (offset > memslot_offset || | |
566 | offset + size < memslot_offset + memslot_size) { | |
567 | const uint64_t gpa = vmem->addr + memslot_offset; | |
568 | ||
569 | if (!virtio_mem_is_range_unplugged(vmem, gpa, memslot_size)) { | |
570 | continue; | |
571 | } | |
572 | } | |
573 | ||
574 | virtio_mem_deactivate_memslot(vmem, idx); | |
575 | } | |
576 | memory_region_transaction_commit(); | |
577 | } | |
578 | ||
910b2576 DH |
579 | static int virtio_mem_set_block_state(VirtIOMEM *vmem, uint64_t start_gpa, |
580 | uint64_t size, bool plug) | |
581 | { | |
582 | const uint64_t offset = start_gpa - vmem->addr; | |
3aca6380 | 583 | RAMBlock *rb = vmem->memdev->mr.ram_block; |
25c89303 | 584 | int ret = 0; |
910b2576 DH |
585 | |
586 | if (virtio_mem_is_busy()) { | |
587 | return -EBUSY; | |
588 | } | |
589 | ||
590 | if (!plug) { | |
3aca6380 | 591 | if (ram_block_discard_range(rb, offset, size)) { |
910b2576 DH |
592 | return -EBUSY; |
593 | } | |
2044969f | 594 | virtio_mem_notify_unplug(vmem, offset, size); |
25c89303 | 595 | virtio_mem_set_range_unplugged(vmem, start_gpa, size); |
177f9b1e | 596 | /* Deactivate completely unplugged memslots after updating the state. */ |
364eff68 DH |
597 | if (vmem->dynamic_memslots) { |
598 | virtio_mem_deactivate_unplugged_memslots(vmem, offset, size); | |
599 | } | |
25c89303 DH |
600 | return 0; |
601 | } | |
602 | ||
603 | if (vmem->prealloc) { | |
604 | void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; | |
605 | int fd = memory_region_get_fd(&vmem->memdev->mr); | |
606 | Error *local_err = NULL; | |
607 | ||
608 | qemu_prealloc_mem(fd, area, size, 1, NULL, &local_err); | |
609 | if (local_err) { | |
610 | static bool warned; | |
611 | ||
612 | /* | |
613 | * Warn only once, we don't want to fill the log with these | |
614 | * warnings. | |
615 | */ | |
616 | if (!warned) { | |
617 | warn_report_err(local_err); | |
618 | warned = true; | |
619 | } else { | |
620 | error_free(local_err); | |
09b3b7e0 | 621 | } |
25c89303 | 622 | ret = -EBUSY; |
09b3b7e0 | 623 | } |
25c89303 | 624 | } |
09b3b7e0 | 625 | |
25c89303 | 626 | if (!ret) { |
177f9b1e DH |
627 | /* |
628 | * Activate before notifying and rollback in case of any errors. | |
629 | * | |
630 | * When activating a yet inactive memslot, memory notifiers will get | |
631 | * notified about the added memory region and can register with the | |
632 | * RamDiscardManager; this will traverse all plugged blocks and skip the | |
633 | * blocks we are plugging here. The following notification will inform | |
634 | * registered listeners about the blocks we're plugging. | |
635 | */ | |
364eff68 DH |
636 | if (vmem->dynamic_memslots) { |
637 | virtio_mem_activate_memslots_to_plug(vmem, offset, size); | |
638 | } | |
25c89303 | 639 | ret = virtio_mem_notify_plug(vmem, offset, size); |
364eff68 | 640 | if (ret && vmem->dynamic_memslots) { |
177f9b1e DH |
641 | virtio_mem_deactivate_unplugged_memslots(vmem, offset, size); |
642 | } | |
910b2576 | 643 | } |
25c89303 DH |
644 | if (ret) { |
645 | /* Could be preallocation or a notifier populated memory. */ | |
646 | ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size); | |
647 | return -EBUSY; | |
648 | } | |
649 | ||
650 | virtio_mem_set_range_plugged(vmem, start_gpa, size); | |
910b2576 DH |
651 | return 0; |
652 | } | |
653 | ||
654 | static int virtio_mem_state_change_request(VirtIOMEM *vmem, uint64_t gpa, | |
655 | uint16_t nb_blocks, bool plug) | |
656 | { | |
657 | const uint64_t size = nb_blocks * vmem->block_size; | |
658 | int ret; | |
659 | ||
660 | if (!virtio_mem_valid_range(vmem, gpa, size)) { | |
661 | return VIRTIO_MEM_RESP_ERROR; | |
662 | } | |
663 | ||
664 | if (plug && (vmem->size + size > vmem->requested_size)) { | |
665 | return VIRTIO_MEM_RESP_NACK; | |
666 | } | |
667 | ||
668 | /* test if really all blocks are in the opposite state */ | |
25c89303 DH |
669 | if ((plug && !virtio_mem_is_range_unplugged(vmem, gpa, size)) || |
670 | (!plug && !virtio_mem_is_range_plugged(vmem, gpa, size))) { | |
910b2576 DH |
671 | return VIRTIO_MEM_RESP_ERROR; |
672 | } | |
673 | ||
674 | ret = virtio_mem_set_block_state(vmem, gpa, size, plug); | |
675 | if (ret) { | |
676 | return VIRTIO_MEM_RESP_BUSY; | |
677 | } | |
678 | if (plug) { | |
679 | vmem->size += size; | |
680 | } else { | |
681 | vmem->size -= size; | |
682 | } | |
c95b4437 | 683 | notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); |
910b2576 DH |
684 | return VIRTIO_MEM_RESP_ACK; |
685 | } | |
686 | ||
687 | static void virtio_mem_plug_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
688 | struct virtio_mem_req *req) | |
689 | { | |
690 | const uint64_t gpa = le64_to_cpu(req->u.plug.addr); | |
691 | const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks); | |
692 | uint16_t type; | |
693 | ||
43e54950 | 694 | trace_virtio_mem_plug_request(gpa, nb_blocks); |
910b2576 DH |
695 | type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, true); |
696 | virtio_mem_send_response_simple(vmem, elem, type); | |
697 | } | |
698 | ||
699 | static void virtio_mem_unplug_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
700 | struct virtio_mem_req *req) | |
701 | { | |
702 | const uint64_t gpa = le64_to_cpu(req->u.unplug.addr); | |
703 | const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks); | |
704 | uint16_t type; | |
705 | ||
43e54950 | 706 | trace_virtio_mem_unplug_request(gpa, nb_blocks); |
910b2576 DH |
707 | type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, false); |
708 | virtio_mem_send_response_simple(vmem, elem, type); | |
709 | } | |
710 | ||
711 | static void virtio_mem_resize_usable_region(VirtIOMEM *vmem, | |
712 | uint64_t requested_size, | |
713 | bool can_shrink) | |
714 | { | |
715 | uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr), | |
716 | requested_size + VIRTIO_MEM_USABLE_EXTENT); | |
717 | ||
0aed2800 DH |
718 | /* The usable region size always has to be multiples of the block size. */ |
719 | newsize = QEMU_ALIGN_UP(newsize, vmem->block_size); | |
720 | ||
910b2576 DH |
721 | if (!requested_size) { |
722 | newsize = 0; | |
723 | } | |
724 | ||
725 | if (newsize < vmem->usable_region_size && !can_shrink) { | |
726 | return; | |
727 | } | |
728 | ||
43e54950 | 729 | trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize); |
910b2576 DH |
730 | vmem->usable_region_size = newsize; |
731 | } | |
732 | ||
733 | static int virtio_mem_unplug_all(VirtIOMEM *vmem) | |
734 | { | |
177f9b1e | 735 | const uint64_t region_size = memory_region_size(&vmem->memdev->mr); |
910b2576 | 736 | RAMBlock *rb = vmem->memdev->mr.ram_block; |
910b2576 | 737 | |
c95b4437 | 738 | if (vmem->size) { |
836f657b DH |
739 | if (virtio_mem_is_busy()) { |
740 | return -EBUSY; | |
741 | } | |
742 | if (ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb))) { | |
743 | return -EBUSY; | |
744 | } | |
745 | virtio_mem_notify_unplug_all(vmem); | |
746 | ||
747 | bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size); | |
c95b4437 DH |
748 | vmem->size = 0; |
749 | notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); | |
177f9b1e DH |
750 | |
751 | /* Deactivate all memslots after updating the state. */ | |
364eff68 DH |
752 | if (vmem->dynamic_memslots) { |
753 | virtio_mem_deactivate_unplugged_memslots(vmem, 0, region_size); | |
754 | } | |
c95b4437 | 755 | } |
836f657b | 756 | |
43e54950 | 757 | trace_virtio_mem_unplugged_all(); |
910b2576 DH |
758 | virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); |
759 | return 0; | |
760 | } | |
761 | ||
762 | static void virtio_mem_unplug_all_request(VirtIOMEM *vmem, | |
763 | VirtQueueElement *elem) | |
764 | { | |
43e54950 | 765 | trace_virtio_mem_unplug_all_request(); |
910b2576 DH |
766 | if (virtio_mem_unplug_all(vmem)) { |
767 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_BUSY); | |
768 | } else { | |
769 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ACK); | |
770 | } | |
771 | } | |
772 | ||
773 | static void virtio_mem_state_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
774 | struct virtio_mem_req *req) | |
775 | { | |
776 | const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks); | |
777 | const uint64_t gpa = le64_to_cpu(req->u.state.addr); | |
778 | const uint64_t size = nb_blocks * vmem->block_size; | |
779 | struct virtio_mem_resp resp = { | |
780 | .type = cpu_to_le16(VIRTIO_MEM_RESP_ACK), | |
781 | }; | |
782 | ||
43e54950 | 783 | trace_virtio_mem_state_request(gpa, nb_blocks); |
910b2576 DH |
784 | if (!virtio_mem_valid_range(vmem, gpa, size)) { |
785 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ERROR); | |
786 | return; | |
787 | } | |
788 | ||
25c89303 | 789 | if (virtio_mem_is_range_plugged(vmem, gpa, size)) { |
910b2576 | 790 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED); |
25c89303 | 791 | } else if (virtio_mem_is_range_unplugged(vmem, gpa, size)) { |
910b2576 DH |
792 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED); |
793 | } else { | |
794 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_MIXED); | |
795 | } | |
43e54950 | 796 | trace_virtio_mem_state_response(le16_to_cpu(resp.u.state.state)); |
910b2576 DH |
797 | virtio_mem_send_response(vmem, elem, &resp); |
798 | } | |
799 | ||
800 | static void virtio_mem_handle_request(VirtIODevice *vdev, VirtQueue *vq) | |
801 | { | |
802 | const int len = sizeof(struct virtio_mem_req); | |
803 | VirtIOMEM *vmem = VIRTIO_MEM(vdev); | |
804 | VirtQueueElement *elem; | |
805 | struct virtio_mem_req req; | |
806 | uint16_t type; | |
807 | ||
808 | while (true) { | |
809 | elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); | |
810 | if (!elem) { | |
811 | return; | |
812 | } | |
813 | ||
814 | if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) { | |
815 | virtio_error(vdev, "virtio-mem protocol violation: invalid request" | |
816 | " size: %d", len); | |
0c404e45 | 817 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
818 | g_free(elem); |
819 | return; | |
820 | } | |
821 | ||
822 | if (iov_size(elem->in_sg, elem->in_num) < | |
823 | sizeof(struct virtio_mem_resp)) { | |
824 | virtio_error(vdev, "virtio-mem protocol violation: not enough space" | |
825 | " for response: %zu", | |
826 | iov_size(elem->in_sg, elem->in_num)); | |
0c404e45 | 827 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
828 | g_free(elem); |
829 | return; | |
830 | } | |
831 | ||
832 | type = le16_to_cpu(req.type); | |
833 | switch (type) { | |
834 | case VIRTIO_MEM_REQ_PLUG: | |
835 | virtio_mem_plug_request(vmem, elem, &req); | |
836 | break; | |
837 | case VIRTIO_MEM_REQ_UNPLUG: | |
838 | virtio_mem_unplug_request(vmem, elem, &req); | |
839 | break; | |
840 | case VIRTIO_MEM_REQ_UNPLUG_ALL: | |
841 | virtio_mem_unplug_all_request(vmem, elem); | |
842 | break; | |
843 | case VIRTIO_MEM_REQ_STATE: | |
844 | virtio_mem_state_request(vmem, elem, &req); | |
845 | break; | |
846 | default: | |
847 | virtio_error(vdev, "virtio-mem protocol violation: unknown request" | |
848 | " type: %d", type); | |
0c404e45 | 849 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
850 | g_free(elem); |
851 | return; | |
852 | } | |
853 | ||
854 | g_free(elem); | |
855 | } | |
856 | } | |
857 | ||
858 | static void virtio_mem_get_config(VirtIODevice *vdev, uint8_t *config_data) | |
859 | { | |
860 | VirtIOMEM *vmem = VIRTIO_MEM(vdev); | |
861 | struct virtio_mem_config *config = (void *) config_data; | |
862 | ||
863 | config->block_size = cpu_to_le64(vmem->block_size); | |
864 | config->node_id = cpu_to_le16(vmem->node); | |
865 | config->requested_size = cpu_to_le64(vmem->requested_size); | |
866 | config->plugged_size = cpu_to_le64(vmem->size); | |
867 | config->addr = cpu_to_le64(vmem->addr); | |
868 | config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr)); | |
869 | config->usable_region_size = cpu_to_le64(vmem->usable_region_size); | |
870 | } | |
871 | ||
872 | static uint64_t virtio_mem_get_features(VirtIODevice *vdev, uint64_t features, | |
873 | Error **errp) | |
874 | { | |
875 | MachineState *ms = MACHINE(qdev_get_machine()); | |
23ad8dec | 876 | VirtIOMEM *vmem = VIRTIO_MEM(vdev); |
910b2576 DH |
877 | |
878 | if (ms->numa_state) { | |
879 | #if defined(CONFIG_ACPI) | |
880 | virtio_add_feature(&features, VIRTIO_MEM_F_ACPI_PXM); | |
881 | #endif | |
882 | } | |
23ad8dec DH |
883 | assert(vmem->unplugged_inaccessible != ON_OFF_AUTO_AUTO); |
884 | if (vmem->unplugged_inaccessible == ON_OFF_AUTO_ON) { | |
885 | virtio_add_feature(&features, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE); | |
886 | } | |
910b2576 DH |
887 | return features; |
888 | } | |
889 | ||
23ad8dec DH |
890 | static int virtio_mem_validate_features(VirtIODevice *vdev) |
891 | { | |
892 | if (virtio_host_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE) && | |
893 | !virtio_vdev_has_feature(vdev, VIRTIO_MEM_F_UNPLUGGED_INACCESSIBLE)) { | |
894 | return -EFAULT; | |
895 | } | |
896 | return 0; | |
897 | } | |
898 | ||
910b2576 DH |
899 | static void virtio_mem_system_reset(void *opaque) |
900 | { | |
901 | VirtIOMEM *vmem = VIRTIO_MEM(opaque); | |
902 | ||
903 | /* | |
904 | * During usual resets, we will unplug all memory and shrink the usable | |
905 | * region size. This is, however, not possible in all scenarios. Then, | |
906 | * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL). | |
907 | */ | |
908 | virtio_mem_unplug_all(vmem); | |
909 | } | |
910 | ||
177f9b1e DH |
911 | static void virtio_mem_prepare_mr(VirtIOMEM *vmem) |
912 | { | |
913 | const uint64_t region_size = memory_region_size(&vmem->memdev->mr); | |
914 | ||
915 | assert(!vmem->mr && vmem->dynamic_memslots); | |
916 | vmem->mr = g_new0(MemoryRegion, 1); | |
917 | memory_region_init(vmem->mr, OBJECT(vmem), "virtio-mem", | |
918 | region_size); | |
919 | vmem->mr->align = memory_region_get_alignment(&vmem->memdev->mr); | |
920 | } | |
921 | ||
922 | static void virtio_mem_prepare_memslots(VirtIOMEM *vmem) | |
923 | { | |
924 | const uint64_t region_size = memory_region_size(&vmem->memdev->mr); | |
925 | unsigned int idx; | |
926 | ||
927 | g_assert(!vmem->memslots && vmem->nb_memslots && vmem->dynamic_memslots); | |
928 | vmem->memslots = g_new0(MemoryRegion, vmem->nb_memslots); | |
929 | ||
930 | /* Initialize our memslots, but don't map them yet. */ | |
931 | for (idx = 0; idx < vmem->nb_memslots; idx++) { | |
932 | const uint64_t memslot_offset = idx * vmem->memslot_size; | |
933 | uint64_t memslot_size = vmem->memslot_size; | |
934 | char name[20]; | |
935 | ||
936 | /* The size of the last memslot might be smaller. */ | |
937 | if (idx == vmem->nb_memslots - 1) { | |
938 | memslot_size = region_size - memslot_offset; | |
939 | } | |
940 | ||
941 | snprintf(name, sizeof(name), "memslot-%u", idx); | |
942 | memory_region_init_alias(&vmem->memslots[idx], OBJECT(vmem), name, | |
943 | &vmem->memdev->mr, memslot_offset, | |
944 | memslot_size); | |
ee6398d8 DH |
945 | /* |
946 | * We want to be able to atomically and efficiently activate/deactivate | |
947 | * individual memslots without affecting adjacent memslots in memory | |
948 | * notifiers. | |
949 | */ | |
950 | memory_region_set_unmergeable(&vmem->memslots[idx], true); | |
177f9b1e DH |
951 | } |
952 | } | |
953 | ||
910b2576 DH |
954 | static void virtio_mem_device_realize(DeviceState *dev, Error **errp) |
955 | { | |
956 | MachineState *ms = MACHINE(qdev_get_machine()); | |
957 | int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0; | |
958 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); | |
959 | VirtIOMEM *vmem = VIRTIO_MEM(dev); | |
960 | uint64_t page_size; | |
961 | RAMBlock *rb; | |
962 | int ret; | |
963 | ||
964 | if (!vmem->memdev) { | |
965 | error_setg(errp, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP); | |
966 | return; | |
967 | } else if (host_memory_backend_is_mapped(vmem->memdev)) { | |
910b2576 | 968 | error_setg(errp, "'%s' property specifies a busy memdev: %s", |
7a309cc9 MA |
969 | VIRTIO_MEM_MEMDEV_PROP, |
970 | object_get_canonical_path_component(OBJECT(vmem->memdev))); | |
910b2576 DH |
971 | return; |
972 | } else if (!memory_region_is_ram(&vmem->memdev->mr) || | |
973 | memory_region_is_rom(&vmem->memdev->mr) || | |
974 | !vmem->memdev->mr.ram_block) { | |
975 | error_setg(errp, "'%s' property specifies an unsupported memdev", | |
976 | VIRTIO_MEM_MEMDEV_PROP); | |
977 | return; | |
ce1761f0 DH |
978 | } else if (vmem->memdev->prealloc) { |
979 | error_setg(errp, "'%s' property specifies a memdev with preallocation" | |
980 | " enabled: %s. Instead, specify 'prealloc=on' for the" | |
981 | " virtio-mem device. ", VIRTIO_MEM_MEMDEV_PROP, | |
982 | object_get_canonical_path_component(OBJECT(vmem->memdev))); | |
983 | return; | |
910b2576 DH |
984 | } |
985 | ||
986 | if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) || | |
987 | (!nb_numa_nodes && vmem->node)) { | |
988 | error_setg(errp, "'%s' property has value '%" PRIu32 "', which exceeds" | |
989 | "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP, | |
990 | vmem->node, nb_numa_nodes ? nb_numa_nodes : 1); | |
991 | return; | |
992 | } | |
993 | ||
994 | if (enable_mlock) { | |
995 | error_setg(errp, "Incompatible with mlock"); | |
996 | return; | |
997 | } | |
998 | ||
999 | rb = vmem->memdev->mr.ram_block; | |
1000 | page_size = qemu_ram_pagesize(rb); | |
1001 | ||
23ad8dec DH |
1002 | #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) |
1003 | switch (vmem->unplugged_inaccessible) { | |
1004 | case ON_OFF_AUTO_AUTO: | |
1005 | if (virtio_mem_has_shared_zeropage(rb)) { | |
1006 | vmem->unplugged_inaccessible = ON_OFF_AUTO_OFF; | |
1007 | } else { | |
1008 | vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; | |
1009 | } | |
1010 | break; | |
1011 | case ON_OFF_AUTO_OFF: | |
1012 | if (!virtio_mem_has_shared_zeropage(rb)) { | |
1013 | warn_report("'%s' property set to 'off' with a memdev that does" | |
1014 | " not support the shared zeropage.", | |
1015 | VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP); | |
1016 | } | |
1017 | break; | |
1018 | default: | |
1019 | break; | |
1020 | } | |
1021 | #else /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ | |
1022 | vmem->unplugged_inaccessible = ON_OFF_AUTO_ON; | |
1023 | #endif /* VIRTIO_MEM_HAS_LEGACY_GUESTS */ | |
1024 | ||
177f9b1e DH |
1025 | if (vmem->dynamic_memslots && |
1026 | vmem->unplugged_inaccessible != ON_OFF_AUTO_ON) { | |
1027 | error_setg(errp, "'%s' property set to 'on' requires '%s' to be 'on'", | |
1028 | VIRTIO_MEM_DYNAMIC_MEMSLOTS_PROP, | |
1029 | VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP); | |
1030 | return; | |
1031 | } | |
1032 | ||
228957fe DH |
1033 | /* |
1034 | * If the block size wasn't configured by the user, use a sane default. This | |
1035 | * allows using hugetlbfs backends of any page size without manual | |
1036 | * intervention. | |
1037 | */ | |
1038 | if (!vmem->block_size) { | |
1039 | vmem->block_size = virtio_mem_default_block_size(rb); | |
1040 | } | |
1041 | ||
910b2576 DH |
1042 | if (vmem->block_size < page_size) { |
1043 | error_setg(errp, "'%s' property has to be at least the page size (0x%" | |
1044 | PRIx64 ")", VIRTIO_MEM_BLOCK_SIZE_PROP, page_size); | |
1045 | return; | |
228957fe DH |
1046 | } else if (vmem->block_size < virtio_mem_default_block_size(rb)) { |
1047 | warn_report("'%s' property is smaller than the default block size (%" | |
1048 | PRIx64 " MiB)", VIRTIO_MEM_BLOCK_SIZE_PROP, | |
1049 | virtio_mem_default_block_size(rb) / MiB); | |
7656d9ce DH |
1050 | } |
1051 | if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) { | |
910b2576 DH |
1052 | error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 |
1053 | ")", VIRTIO_MEM_REQUESTED_SIZE_PROP, | |
1054 | VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); | |
1055 | return; | |
d31992ae DH |
1056 | } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) { |
1057 | error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 | |
1058 | ")", VIRTIO_MEM_ADDR_PROP, VIRTIO_MEM_BLOCK_SIZE_PROP, | |
1059 | vmem->block_size); | |
1060 | return; | |
910b2576 DH |
1061 | } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr), |
1062 | vmem->block_size)) { | |
1063 | error_setg(errp, "'%s' property memdev size has to be multiples of" | |
1064 | "'%s' (0x%" PRIx64 ")", VIRTIO_MEM_MEMDEV_PROP, | |
1065 | VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); | |
1066 | return; | |
1067 | } | |
1068 | ||
bc072ed4 | 1069 | if (ram_block_coordinated_discard_require(true)) { |
910b2576 DH |
1070 | error_setg(errp, "Discarding RAM is disabled"); |
1071 | return; | |
1072 | } | |
1073 | ||
b01fd4b6 DH |
1074 | /* |
1075 | * We don't know at this point whether shared RAM is migrated using | |
1076 | * QEMU or migrated using the file content. "x-ignore-shared" will be | |
1077 | * configured after realizing the device. So in case we have an | |
1078 | * incoming migration, simply always skip the discard step. | |
1079 | * | |
1080 | * Otherwise, make sure that we start with a clean slate: either the | |
1081 | * memory backend might get reused or the shared file might still have | |
1082 | * memory allocated. | |
1083 | */ | |
1084 | if (!runstate_check(RUN_STATE_INMIGRATE)) { | |
1085 | ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb)); | |
1086 | if (ret) { | |
1087 | error_setg_errno(errp, -ret, "Unexpected error discarding RAM"); | |
1088 | ram_block_coordinated_discard_require(false); | |
1089 | return; | |
1090 | } | |
910b2576 DH |
1091 | } |
1092 | ||
1093 | virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); | |
1094 | ||
1095 | vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) / | |
1096 | vmem->block_size; | |
1097 | vmem->bitmap = bitmap_new(vmem->bitmap_size); | |
1098 | ||
3857cd5c | 1099 | virtio_init(vdev, VIRTIO_ID_MEM, sizeof(struct virtio_mem_config)); |
910b2576 DH |
1100 | vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request); |
1101 | ||
177f9b1e DH |
1102 | /* |
1103 | * With "dynamic-memslots=off" (old behavior) we always map the whole | |
1104 | * RAM memory region directly. | |
1105 | */ | |
1106 | if (vmem->dynamic_memslots) { | |
1107 | if (!vmem->mr) { | |
1108 | virtio_mem_prepare_mr(vmem); | |
1109 | } | |
1110 | if (vmem->nb_memslots <= 1) { | |
1111 | vmem->nb_memslots = 1; | |
1112 | vmem->memslot_size = memory_region_size(&vmem->memdev->mr); | |
1113 | } | |
1114 | if (!vmem->memslots) { | |
1115 | virtio_mem_prepare_memslots(vmem); | |
1116 | } | |
1117 | } else { | |
1118 | assert(!vmem->mr && !vmem->nb_memslots && !vmem->memslots); | |
1119 | } | |
1120 | ||
910b2576 DH |
1121 | host_memory_backend_set_mapped(vmem->memdev, true); |
1122 | vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem)); | |
3b95a71b | 1123 | if (vmem->early_migration) { |
99b16e8e JQ |
1124 | vmstate_register_any(VMSTATE_IF(vmem), |
1125 | &vmstate_virtio_mem_device_early, vmem); | |
3b95a71b | 1126 | } |
910b2576 | 1127 | qemu_register_reset(virtio_mem_system_reset, vmem); |
2044969f DH |
1128 | |
1129 | /* | |
1130 | * Set ourselves as RamDiscardManager before the plug handler maps the | |
1131 | * memory region and exposes it via an address space. | |
1132 | */ | |
1133 | memory_region_set_ram_discard_manager(&vmem->memdev->mr, | |
1134 | RAM_DISCARD_MANAGER(vmem)); | |
910b2576 DH |
1135 | } |
1136 | ||
1137 | static void virtio_mem_device_unrealize(DeviceState *dev) | |
1138 | { | |
1139 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); | |
1140 | VirtIOMEM *vmem = VIRTIO_MEM(dev); | |
1141 | ||
2044969f DH |
1142 | /* |
1143 | * The unplug handler unmapped the memory region, it cannot be | |
1144 | * found via an address space anymore. Unset ourselves. | |
1145 | */ | |
1146 | memory_region_set_ram_discard_manager(&vmem->memdev->mr, NULL); | |
910b2576 | 1147 | qemu_unregister_reset(virtio_mem_system_reset, vmem); |
3b95a71b DH |
1148 | if (vmem->early_migration) { |
1149 | vmstate_unregister(VMSTATE_IF(vmem), &vmstate_virtio_mem_device_early, | |
1150 | vmem); | |
1151 | } | |
910b2576 DH |
1152 | vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem)); |
1153 | host_memory_backend_set_mapped(vmem->memdev, false); | |
1154 | virtio_del_queue(vdev, 0); | |
1155 | virtio_cleanup(vdev); | |
1156 | g_free(vmem->bitmap); | |
bc072ed4 | 1157 | ram_block_coordinated_discard_require(false); |
910b2576 DH |
1158 | } |
1159 | ||
a45171db | 1160 | static int virtio_mem_discard_range_cb(VirtIOMEM *vmem, void *arg, |
7a9d5d02 | 1161 | uint64_t offset, uint64_t size) |
910b2576 DH |
1162 | { |
1163 | RAMBlock *rb = vmem->memdev->mr.ram_block; | |
910b2576 | 1164 | |
3aca6380 | 1165 | return ram_block_discard_range(rb, offset, size) ? -EINVAL : 0; |
910b2576 DH |
1166 | } |
1167 | ||
7a9d5d02 DH |
1168 | static int virtio_mem_restore_unplugged(VirtIOMEM *vmem) |
1169 | { | |
1170 | /* Make sure all memory is really discarded after migration. */ | |
1171 | return virtio_mem_for_each_unplugged_range(vmem, NULL, | |
1172 | virtio_mem_discard_range_cb); | |
1173 | } | |
1174 | ||
177f9b1e DH |
1175 | static int virtio_mem_activate_memslot_range_cb(VirtIOMEM *vmem, void *arg, |
1176 | uint64_t offset, uint64_t size) | |
1177 | { | |
1178 | virtio_mem_activate_memslots_to_plug(vmem, offset, size); | |
1179 | return 0; | |
1180 | } | |
1181 | ||
884a0c20 | 1182 | static int virtio_mem_post_load_bitmap(VirtIOMEM *vmem) |
910b2576 | 1183 | { |
2044969f DH |
1184 | RamDiscardListener *rdl; |
1185 | int ret; | |
1186 | ||
177f9b1e DH |
1187 | /* |
1188 | * We restored the bitmap and updated the requested size; activate all | |
1189 | * memslots (so listeners register) before notifying about plugged blocks. | |
1190 | */ | |
1191 | if (vmem->dynamic_memslots) { | |
1192 | /* | |
1193 | * We don't expect any active memslots at this point to deactivate: no | |
1194 | * memory was plugged on the migration destination. | |
1195 | */ | |
1196 | virtio_mem_for_each_plugged_range(vmem, NULL, | |
1197 | virtio_mem_activate_memslot_range_cb); | |
1198 | } | |
1199 | ||
2044969f DH |
1200 | /* |
1201 | * We started out with all memory discarded and our memory region is mapped | |
1202 | * into an address space. Replay, now that we updated the bitmap. | |
1203 | */ | |
1204 | QLIST_FOREACH(rdl, &vmem->rdl_list, next) { | |
1205 | ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, | |
1206 | virtio_mem_notify_populate_cb); | |
1207 | if (ret) { | |
1208 | return ret; | |
1209 | } | |
1210 | } | |
884a0c20 DH |
1211 | return 0; |
1212 | } | |
1213 | ||
1214 | static int virtio_mem_post_load(void *opaque, int version_id) | |
1215 | { | |
1216 | VirtIOMEM *vmem = VIRTIO_MEM(opaque); | |
1217 | int ret; | |
1218 | ||
1219 | if (!vmem->early_migration) { | |
1220 | ret = virtio_mem_post_load_bitmap(vmem); | |
1221 | if (ret) { | |
1222 | return ret; | |
1223 | } | |
1224 | } | |
2044969f | 1225 | |
b01fd4b6 DH |
1226 | /* |
1227 | * If shared RAM is migrated using the file content and not using QEMU, | |
1228 | * don't mess with preallocation and postcopy. | |
1229 | */ | |
1230 | if (migrate_ram_is_ignored(vmem->memdev->mr.ram_block)) { | |
1231 | return 0; | |
1232 | } | |
1233 | ||
1234 | if (vmem->prealloc && !vmem->early_migration) { | |
1235 | warn_report("Proper preallocation with migration requires a newer QEMU machine"); | |
1236 | } | |
1237 | ||
910b2576 DH |
1238 | if (migration_in_incoming_postcopy()) { |
1239 | return 0; | |
1240 | } | |
1241 | ||
2044969f | 1242 | return virtio_mem_restore_unplugged(vmem); |
910b2576 DH |
1243 | } |
1244 | ||
a45171db | 1245 | static int virtio_mem_prealloc_range_cb(VirtIOMEM *vmem, void *arg, |
d71920d4 DH |
1246 | uint64_t offset, uint64_t size) |
1247 | { | |
1248 | void *area = memory_region_get_ram_ptr(&vmem->memdev->mr) + offset; | |
1249 | int fd = memory_region_get_fd(&vmem->memdev->mr); | |
1250 | Error *local_err = NULL; | |
1251 | ||
1252 | qemu_prealloc_mem(fd, area, size, 1, NULL, &local_err); | |
1253 | if (local_err) { | |
1254 | error_report_err(local_err); | |
1255 | return -ENOMEM; | |
1256 | } | |
1257 | return 0; | |
1258 | } | |
1259 | ||
1260 | static int virtio_mem_post_load_early(void *opaque, int version_id) | |
1261 | { | |
1262 | VirtIOMEM *vmem = VIRTIO_MEM(opaque); | |
1263 | RAMBlock *rb = vmem->memdev->mr.ram_block; | |
1264 | int ret; | |
1265 | ||
1266 | if (!vmem->prealloc) { | |
884a0c20 | 1267 | goto post_load_bitmap; |
d71920d4 DH |
1268 | } |
1269 | ||
b01fd4b6 DH |
1270 | /* |
1271 | * If shared RAM is migrated using the file content and not using QEMU, | |
1272 | * don't mess with preallocation and postcopy. | |
1273 | */ | |
1274 | if (migrate_ram_is_ignored(rb)) { | |
884a0c20 | 1275 | goto post_load_bitmap; |
b01fd4b6 DH |
1276 | } |
1277 | ||
d71920d4 DH |
1278 | /* |
1279 | * We restored the bitmap and verified that the basic properties | |
1280 | * match on source and destination, so we can go ahead and preallocate | |
1281 | * memory for all plugged memory blocks, before actual RAM migration starts | |
1282 | * touching this memory. | |
1283 | */ | |
1284 | ret = virtio_mem_for_each_plugged_range(vmem, NULL, | |
1285 | virtio_mem_prealloc_range_cb); | |
1286 | if (ret) { | |
1287 | return ret; | |
1288 | } | |
1289 | ||
1290 | /* | |
1291 | * This is tricky: postcopy wants to start with a clean slate. On | |
1292 | * POSTCOPY_INCOMING_ADVISE, postcopy code discards all (ordinarily | |
1293 | * preallocated) RAM such that postcopy will work as expected later. | |
1294 | * | |
1295 | * However, we run after POSTCOPY_INCOMING_ADVISE -- but before actual | |
1296 | * RAM migration. So let's discard all memory again. This looks like an | |
1297 | * expensive NOP, but actually serves a purpose: we made sure that we | |
1298 | * were able to allocate all required backend memory once. We cannot | |
1299 | * guarantee that the backend memory we will free will remain free | |
1300 | * until we need it during postcopy, but at least we can catch the | |
1301 | * obvious setup issues this way. | |
1302 | */ | |
1303 | if (migration_incoming_postcopy_advised()) { | |
1304 | if (ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb))) { | |
1305 | return -EBUSY; | |
1306 | } | |
1307 | } | |
884a0c20 DH |
1308 | |
1309 | post_load_bitmap: | |
1310 | /* Finally, update any other state to be consistent with the new bitmap. */ | |
1311 | return virtio_mem_post_load_bitmap(vmem); | |
d71920d4 DH |
1312 | } |
1313 | ||
383ee445 DH |
1314 | typedef struct VirtIOMEMMigSanityChecks { |
1315 | VirtIOMEM *parent; | |
1316 | uint64_t addr; | |
1317 | uint64_t region_size; | |
1318 | uint64_t block_size; | |
1319 | uint32_t node; | |
1320 | } VirtIOMEMMigSanityChecks; | |
1321 | ||
1322 | static int virtio_mem_mig_sanity_checks_pre_save(void *opaque) | |
1323 | { | |
1324 | VirtIOMEMMigSanityChecks *tmp = opaque; | |
1325 | VirtIOMEM *vmem = tmp->parent; | |
1326 | ||
1327 | tmp->addr = vmem->addr; | |
1328 | tmp->region_size = memory_region_size(&vmem->memdev->mr); | |
1329 | tmp->block_size = vmem->block_size; | |
1330 | tmp->node = vmem->node; | |
1331 | return 0; | |
1332 | } | |
1333 | ||
1334 | static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id) | |
1335 | { | |
1336 | VirtIOMEMMigSanityChecks *tmp = opaque; | |
1337 | VirtIOMEM *vmem = tmp->parent; | |
1338 | const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr); | |
1339 | ||
1340 | if (tmp->addr != vmem->addr) { | |
1341 | error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, | |
1342 | VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr); | |
1343 | return -EINVAL; | |
1344 | } | |
1345 | /* | |
9b4b4e51 | 1346 | * Note: Preparation for resizable memory regions. The maximum size |
383ee445 DH |
1347 | * of the memory region must not change during migration. |
1348 | */ | |
1349 | if (tmp->region_size != new_region_size) { | |
1350 | error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%" | |
1351 | PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size, | |
1352 | new_region_size); | |
1353 | return -EINVAL; | |
1354 | } | |
1355 | if (tmp->block_size != vmem->block_size) { | |
1356 | error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, | |
1357 | VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size, | |
1358 | vmem->block_size); | |
1359 | return -EINVAL; | |
1360 | } | |
1361 | if (tmp->node != vmem->node) { | |
1362 | error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32, | |
1363 | VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node); | |
1364 | return -EINVAL; | |
1365 | } | |
1366 | return 0; | |
1367 | } | |
1368 | ||
1369 | static const VMStateDescription vmstate_virtio_mem_sanity_checks = { | |
1370 | .name = "virtio-mem-device/sanity-checks", | |
1371 | .pre_save = virtio_mem_mig_sanity_checks_pre_save, | |
1372 | .post_load = virtio_mem_mig_sanity_checks_post_load, | |
1373 | .fields = (VMStateField[]) { | |
1374 | VMSTATE_UINT64(addr, VirtIOMEMMigSanityChecks), | |
1375 | VMSTATE_UINT64(region_size, VirtIOMEMMigSanityChecks), | |
1376 | VMSTATE_UINT64(block_size, VirtIOMEMMigSanityChecks), | |
1377 | VMSTATE_UINT32(node, VirtIOMEMMigSanityChecks), | |
1378 | VMSTATE_END_OF_LIST(), | |
1379 | }, | |
1380 | }; | |
1381 | ||
3b95a71b DH |
1382 | static bool virtio_mem_vmstate_field_exists(void *opaque, int version_id) |
1383 | { | |
1384 | const VirtIOMEM *vmem = VIRTIO_MEM(opaque); | |
1385 | ||
1386 | /* With early migration, these fields were already migrated. */ | |
1387 | return !vmem->early_migration; | |
1388 | } | |
1389 | ||
910b2576 DH |
1390 | static const VMStateDescription vmstate_virtio_mem_device = { |
1391 | .name = "virtio-mem-device", | |
1392 | .minimum_version_id = 1, | |
1393 | .version_id = 1, | |
0fd7616e | 1394 | .priority = MIG_PRI_VIRTIO_MEM, |
910b2576 | 1395 | .post_load = virtio_mem_post_load, |
3b95a71b DH |
1396 | .fields = (VMStateField[]) { |
1397 | VMSTATE_WITH_TMP_TEST(VirtIOMEM, virtio_mem_vmstate_field_exists, | |
1398 | VirtIOMEMMigSanityChecks, | |
1399 | vmstate_virtio_mem_sanity_checks), | |
1400 | VMSTATE_UINT64(usable_region_size, VirtIOMEM), | |
1401 | VMSTATE_UINT64_TEST(size, VirtIOMEM, virtio_mem_vmstate_field_exists), | |
1402 | VMSTATE_UINT64(requested_size, VirtIOMEM), | |
1403 | VMSTATE_BITMAP_TEST(bitmap, VirtIOMEM, virtio_mem_vmstate_field_exists, | |
1404 | 0, bitmap_size), | |
1405 | VMSTATE_END_OF_LIST() | |
1406 | }, | |
1407 | }; | |
1408 | ||
1409 | /* | |
1410 | * Transfer properties that are immutable while migration is active early, | |
1411 | * such that we have have this information around before migrating any RAM | |
1412 | * content. | |
1413 | * | |
1414 | * Note that virtio_mem_is_busy() makes sure these properties can no longer | |
1415 | * change on the migration source until migration completed. | |
1416 | * | |
1417 | * With QEMU compat machines, we transmit these properties later, via | |
1418 | * vmstate_virtio_mem_device instead -- see virtio_mem_vmstate_field_exists(). | |
1419 | */ | |
1420 | static const VMStateDescription vmstate_virtio_mem_device_early = { | |
1421 | .name = "virtio-mem-device-early", | |
1422 | .minimum_version_id = 1, | |
1423 | .version_id = 1, | |
1424 | .early_setup = true, | |
d71920d4 | 1425 | .post_load = virtio_mem_post_load_early, |
910b2576 | 1426 | .fields = (VMStateField[]) { |
383ee445 DH |
1427 | VMSTATE_WITH_TMP(VirtIOMEM, VirtIOMEMMigSanityChecks, |
1428 | vmstate_virtio_mem_sanity_checks), | |
910b2576 | 1429 | VMSTATE_UINT64(size, VirtIOMEM), |
910b2576 DH |
1430 | VMSTATE_BITMAP(bitmap, VirtIOMEM, 0, bitmap_size), |
1431 | VMSTATE_END_OF_LIST() | |
1432 | }, | |
1433 | }; | |
1434 | ||
1435 | static const VMStateDescription vmstate_virtio_mem = { | |
1436 | .name = "virtio-mem", | |
1437 | .minimum_version_id = 1, | |
1438 | .version_id = 1, | |
1439 | .fields = (VMStateField[]) { | |
1440 | VMSTATE_VIRTIO_DEVICE, | |
1441 | VMSTATE_END_OF_LIST() | |
1442 | }, | |
1443 | }; | |
1444 | ||
1445 | static void virtio_mem_fill_device_info(const VirtIOMEM *vmem, | |
1446 | VirtioMEMDeviceInfo *vi) | |
1447 | { | |
1448 | vi->memaddr = vmem->addr; | |
1449 | vi->node = vmem->node; | |
1450 | vi->requested_size = vmem->requested_size; | |
1451 | vi->size = vmem->size; | |
1452 | vi->max_size = memory_region_size(&vmem->memdev->mr); | |
1453 | vi->block_size = vmem->block_size; | |
1454 | vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev)); | |
1455 | } | |
1456 | ||
1457 | static MemoryRegion *virtio_mem_get_memory_region(VirtIOMEM *vmem, Error **errp) | |
1458 | { | |
1459 | if (!vmem->memdev) { | |
1460 | error_setg(errp, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP); | |
1461 | return NULL; | |
177f9b1e DH |
1462 | } else if (vmem->dynamic_memslots) { |
1463 | if (!vmem->mr) { | |
1464 | virtio_mem_prepare_mr(vmem); | |
1465 | } | |
1466 | return vmem->mr; | |
910b2576 DH |
1467 | } |
1468 | ||
1469 | return &vmem->memdev->mr; | |
1470 | } | |
1471 | ||
177f9b1e DH |
1472 | static void virtio_mem_decide_memslots(VirtIOMEM *vmem, unsigned int limit) |
1473 | { | |
1474 | uint64_t region_size, memslot_size, min_memslot_size; | |
1475 | unsigned int memslots; | |
1476 | RAMBlock *rb; | |
1477 | ||
1478 | if (!vmem->dynamic_memslots) { | |
1479 | return; | |
1480 | } | |
1481 | ||
1482 | /* We're called exactly once, before realizing the device. */ | |
1483 | assert(!vmem->nb_memslots); | |
1484 | ||
1485 | /* If realizing the device will fail, just assume a single memslot. */ | |
1486 | if (limit <= 1 || !vmem->memdev || !vmem->memdev->mr.ram_block) { | |
1487 | vmem->nb_memslots = 1; | |
1488 | return; | |
1489 | } | |
1490 | ||
1491 | rb = vmem->memdev->mr.ram_block; | |
1492 | region_size = memory_region_size(&vmem->memdev->mr); | |
1493 | ||
1494 | /* | |
1495 | * Determine the default block size now, to determine the minimum memslot | |
1496 | * size. We want the minimum slot size to be at least the device block size. | |
1497 | */ | |
1498 | if (!vmem->block_size) { | |
1499 | vmem->block_size = virtio_mem_default_block_size(rb); | |
1500 | } | |
1501 | /* If realizing the device will fail, just assume a single memslot. */ | |
1502 | if (vmem->block_size < qemu_ram_pagesize(rb) || | |
1503 | !QEMU_IS_ALIGNED(region_size, vmem->block_size)) { | |
1504 | vmem->nb_memslots = 1; | |
1505 | return; | |
1506 | } | |
1507 | ||
1508 | /* | |
1509 | * All memslots except the last one have a reasonable minimum size, and | |
1510 | * and all memslot sizes are aligned to the device block size. | |
1511 | */ | |
1512 | memslot_size = QEMU_ALIGN_UP(region_size / limit, vmem->block_size); | |
1513 | min_memslot_size = MAX(vmem->block_size, VIRTIO_MEM_MIN_MEMSLOT_SIZE); | |
1514 | memslot_size = MAX(memslot_size, min_memslot_size); | |
1515 | ||
1516 | memslots = QEMU_ALIGN_UP(region_size, memslot_size) / memslot_size; | |
1517 | if (memslots != 1) { | |
1518 | vmem->memslot_size = memslot_size; | |
1519 | } | |
1520 | vmem->nb_memslots = memslots; | |
1521 | } | |
1522 | ||
1523 | static unsigned int virtio_mem_get_memslots(VirtIOMEM *vmem) | |
1524 | { | |
1525 | if (!vmem->dynamic_memslots) { | |
1526 | /* Exactly one static RAM memory region. */ | |
1527 | return 1; | |
1528 | } | |
1529 | ||
1530 | /* We're called after instructed to make a decision. */ | |
1531 | g_assert(vmem->nb_memslots); | |
1532 | return vmem->nb_memslots; | |
1533 | } | |
1534 | ||
c95b4437 DH |
1535 | static void virtio_mem_add_size_change_notifier(VirtIOMEM *vmem, |
1536 | Notifier *notifier) | |
1537 | { | |
1538 | notifier_list_add(&vmem->size_change_notifiers, notifier); | |
1539 | } | |
1540 | ||
1541 | static void virtio_mem_remove_size_change_notifier(VirtIOMEM *vmem, | |
1542 | Notifier *notifier) | |
1543 | { | |
1544 | notifier_remove(notifier); | |
1545 | } | |
1546 | ||
910b2576 DH |
1547 | static void virtio_mem_get_size(Object *obj, Visitor *v, const char *name, |
1548 | void *opaque, Error **errp) | |
1549 | { | |
1550 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1551 | uint64_t value = vmem->size; | |
1552 | ||
1553 | visit_type_size(v, name, &value, errp); | |
1554 | } | |
1555 | ||
1556 | static void virtio_mem_get_requested_size(Object *obj, Visitor *v, | |
1557 | const char *name, void *opaque, | |
1558 | Error **errp) | |
1559 | { | |
1560 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1561 | uint64_t value = vmem->requested_size; | |
1562 | ||
1563 | visit_type_size(v, name, &value, errp); | |
1564 | } | |
1565 | ||
1566 | static void virtio_mem_set_requested_size(Object *obj, Visitor *v, | |
1567 | const char *name, void *opaque, | |
1568 | Error **errp) | |
1569 | { | |
1570 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
910b2576 DH |
1571 | uint64_t value; |
1572 | ||
d1c81c34 | 1573 | if (!visit_type_size(v, name, &value, errp)) { |
910b2576 DH |
1574 | return; |
1575 | } | |
1576 | ||
1577 | /* | |
1578 | * The block size and memory backend are not fixed until the device was | |
1579 | * realized. realize() will verify these properties then. | |
1580 | */ | |
1581 | if (DEVICE(obj)->realized) { | |
1582 | if (!QEMU_IS_ALIGNED(value, vmem->block_size)) { | |
1583 | error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64 | |
1584 | ")", name, VIRTIO_MEM_BLOCK_SIZE_PROP, | |
1585 | vmem->block_size); | |
1586 | return; | |
1587 | } else if (value > memory_region_size(&vmem->memdev->mr)) { | |
1588 | error_setg(errp, "'%s' cannot exceed the memory backend size" | |
1589 | "(0x%" PRIx64 ")", name, | |
1590 | memory_region_size(&vmem->memdev->mr)); | |
1591 | return; | |
1592 | } | |
1593 | ||
1594 | if (value != vmem->requested_size) { | |
1595 | virtio_mem_resize_usable_region(vmem, value, false); | |
1596 | vmem->requested_size = value; | |
1597 | } | |
1598 | /* | |
1599 | * Trigger a config update so the guest gets notified. We trigger | |
1600 | * even if the size didn't change (especially helpful for debugging). | |
1601 | */ | |
1602 | virtio_notify_config(VIRTIO_DEVICE(vmem)); | |
1603 | } else { | |
1604 | vmem->requested_size = value; | |
1605 | } | |
1606 | } | |
1607 | ||
1608 | static void virtio_mem_get_block_size(Object *obj, Visitor *v, const char *name, | |
1609 | void *opaque, Error **errp) | |
1610 | { | |
1611 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1612 | uint64_t value = vmem->block_size; | |
1613 | ||
228957fe DH |
1614 | /* |
1615 | * If not configured by the user (and we're not realized yet), use the | |
1616 | * default block size we would use with the current memory backend. | |
1617 | */ | |
1618 | if (!value) { | |
1619 | if (vmem->memdev && memory_region_is_ram(&vmem->memdev->mr)) { | |
1620 | value = virtio_mem_default_block_size(vmem->memdev->mr.ram_block); | |
1621 | } else { | |
1622 | value = virtio_mem_thp_size(); | |
1623 | } | |
1624 | } | |
1625 | ||
910b2576 DH |
1626 | visit_type_size(v, name, &value, errp); |
1627 | } | |
1628 | ||
1629 | static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name, | |
1630 | void *opaque, Error **errp) | |
1631 | { | |
1632 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
910b2576 DH |
1633 | uint64_t value; |
1634 | ||
1635 | if (DEVICE(obj)->realized) { | |
1636 | error_setg(errp, "'%s' cannot be changed", name); | |
1637 | return; | |
1638 | } | |
1639 | ||
d1c81c34 | 1640 | if (!visit_type_size(v, name, &value, errp)) { |
910b2576 DH |
1641 | return; |
1642 | } | |
1643 | ||
1644 | if (value < VIRTIO_MEM_MIN_BLOCK_SIZE) { | |
1645 | error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name, | |
1646 | VIRTIO_MEM_MIN_BLOCK_SIZE); | |
1647 | return; | |
1648 | } else if (!is_power_of_2(value)) { | |
1649 | error_setg(errp, "'%s' property has to be a power of two", name); | |
1650 | return; | |
1651 | } | |
1652 | vmem->block_size = value; | |
1653 | } | |
1654 | ||
1655 | static void virtio_mem_instance_init(Object *obj) | |
1656 | { | |
1657 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1658 | ||
c95b4437 | 1659 | notifier_list_init(&vmem->size_change_notifiers); |
2044969f | 1660 | QLIST_INIT(&vmem->rdl_list); |
910b2576 DH |
1661 | |
1662 | object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size, | |
1663 | NULL, NULL, NULL); | |
1664 | object_property_add(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP, "size", | |
1665 | virtio_mem_get_requested_size, | |
1666 | virtio_mem_set_requested_size, NULL, NULL); | |
1667 | object_property_add(obj, VIRTIO_MEM_BLOCK_SIZE_PROP, "size", | |
1668 | virtio_mem_get_block_size, virtio_mem_set_block_size, | |
1669 | NULL, NULL); | |
1670 | } | |
1671 | ||
177f9b1e DH |
1672 | static void virtio_mem_instance_finalize(Object *obj) |
1673 | { | |
1674 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
1675 | ||
1676 | /* | |
1677 | * Note: the core already dropped the references on all memory regions | |
1678 | * (it's passed as the owner to memory_region_init_*()) and finalized | |
1679 | * these objects. We can simply free the memory. | |
1680 | */ | |
1681 | g_free(vmem->memslots); | |
1682 | vmem->memslots = NULL; | |
1683 | g_free(vmem->mr); | |
1684 | vmem->mr = NULL; | |
1685 | } | |
1686 | ||
910b2576 DH |
1687 | static Property virtio_mem_properties[] = { |
1688 | DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP, VirtIOMEM, addr, 0), | |
1689 | DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP, VirtIOMEM, node, 0), | |
09b3b7e0 | 1690 | DEFINE_PROP_BOOL(VIRTIO_MEM_PREALLOC_PROP, VirtIOMEM, prealloc, false), |
910b2576 DH |
1691 | DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP, VirtIOMEM, memdev, |
1692 | TYPE_MEMORY_BACKEND, HostMemoryBackend *), | |
23ad8dec DH |
1693 | #if defined(VIRTIO_MEM_HAS_LEGACY_GUESTS) |
1694 | DEFINE_PROP_ON_OFF_AUTO(VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP, VirtIOMEM, | |
d5cef025 | 1695 | unplugged_inaccessible, ON_OFF_AUTO_ON), |
23ad8dec | 1696 | #endif |
3b95a71b DH |
1697 | DEFINE_PROP_BOOL(VIRTIO_MEM_EARLY_MIGRATION_PROP, VirtIOMEM, |
1698 | early_migration, true), | |
177f9b1e DH |
1699 | DEFINE_PROP_BOOL(VIRTIO_MEM_DYNAMIC_MEMSLOTS_PROP, VirtIOMEM, |
1700 | dynamic_memslots, false), | |
910b2576 DH |
1701 | DEFINE_PROP_END_OF_LIST(), |
1702 | }; | |
1703 | ||
2044969f DH |
1704 | static uint64_t virtio_mem_rdm_get_min_granularity(const RamDiscardManager *rdm, |
1705 | const MemoryRegion *mr) | |
1706 | { | |
1707 | const VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1708 | ||
1709 | g_assert(mr == &vmem->memdev->mr); | |
1710 | return vmem->block_size; | |
1711 | } | |
1712 | ||
1713 | static bool virtio_mem_rdm_is_populated(const RamDiscardManager *rdm, | |
1714 | const MemoryRegionSection *s) | |
1715 | { | |
1716 | const VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1717 | uint64_t start_gpa = vmem->addr + s->offset_within_region; | |
1718 | uint64_t end_gpa = start_gpa + int128_get64(s->size); | |
1719 | ||
1720 | g_assert(s->mr == &vmem->memdev->mr); | |
1721 | ||
1722 | start_gpa = QEMU_ALIGN_DOWN(start_gpa, vmem->block_size); | |
1723 | end_gpa = QEMU_ALIGN_UP(end_gpa, vmem->block_size); | |
1724 | ||
1725 | if (!virtio_mem_valid_range(vmem, start_gpa, end_gpa - start_gpa)) { | |
1726 | return false; | |
1727 | } | |
1728 | ||
25c89303 | 1729 | return virtio_mem_is_range_plugged(vmem, start_gpa, end_gpa - start_gpa); |
2044969f DH |
1730 | } |
1731 | ||
1732 | struct VirtIOMEMReplayData { | |
1733 | void *fn; | |
1734 | void *opaque; | |
1735 | }; | |
1736 | ||
1737 | static int virtio_mem_rdm_replay_populated_cb(MemoryRegionSection *s, void *arg) | |
1738 | { | |
1739 | struct VirtIOMEMReplayData *data = arg; | |
1740 | ||
1741 | return ((ReplayRamPopulate)data->fn)(s, data->opaque); | |
1742 | } | |
1743 | ||
1744 | static int virtio_mem_rdm_replay_populated(const RamDiscardManager *rdm, | |
1745 | MemoryRegionSection *s, | |
1746 | ReplayRamPopulate replay_fn, | |
1747 | void *opaque) | |
1748 | { | |
1749 | const VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1750 | struct VirtIOMEMReplayData data = { | |
1751 | .fn = replay_fn, | |
1752 | .opaque = opaque, | |
1753 | }; | |
1754 | ||
1755 | g_assert(s->mr == &vmem->memdev->mr); | |
1756 | return virtio_mem_for_each_plugged_section(vmem, s, &data, | |
1757 | virtio_mem_rdm_replay_populated_cb); | |
1758 | } | |
1759 | ||
372aa6fd DH |
1760 | static int virtio_mem_rdm_replay_discarded_cb(MemoryRegionSection *s, |
1761 | void *arg) | |
1762 | { | |
1763 | struct VirtIOMEMReplayData *data = arg; | |
1764 | ||
1765 | ((ReplayRamDiscard)data->fn)(s, data->opaque); | |
1766 | return 0; | |
1767 | } | |
1768 | ||
1769 | static void virtio_mem_rdm_replay_discarded(const RamDiscardManager *rdm, | |
1770 | MemoryRegionSection *s, | |
1771 | ReplayRamDiscard replay_fn, | |
1772 | void *opaque) | |
1773 | { | |
1774 | const VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1775 | struct VirtIOMEMReplayData data = { | |
1776 | .fn = replay_fn, | |
1777 | .opaque = opaque, | |
1778 | }; | |
1779 | ||
1780 | g_assert(s->mr == &vmem->memdev->mr); | |
1781 | virtio_mem_for_each_unplugged_section(vmem, s, &data, | |
1782 | virtio_mem_rdm_replay_discarded_cb); | |
1783 | } | |
1784 | ||
2044969f DH |
1785 | static void virtio_mem_rdm_register_listener(RamDiscardManager *rdm, |
1786 | RamDiscardListener *rdl, | |
1787 | MemoryRegionSection *s) | |
1788 | { | |
1789 | VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1790 | int ret; | |
1791 | ||
1792 | g_assert(s->mr == &vmem->memdev->mr); | |
1793 | rdl->section = memory_region_section_new_copy(s); | |
1794 | ||
1795 | QLIST_INSERT_HEAD(&vmem->rdl_list, rdl, next); | |
1796 | ret = virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, | |
1797 | virtio_mem_notify_populate_cb); | |
1798 | if (ret) { | |
1799 | error_report("%s: Replaying plugged ranges failed: %s", __func__, | |
1800 | strerror(-ret)); | |
1801 | } | |
1802 | } | |
1803 | ||
1804 | static void virtio_mem_rdm_unregister_listener(RamDiscardManager *rdm, | |
1805 | RamDiscardListener *rdl) | |
1806 | { | |
1807 | VirtIOMEM *vmem = VIRTIO_MEM(rdm); | |
1808 | ||
1809 | g_assert(rdl->section->mr == &vmem->memdev->mr); | |
1810 | if (vmem->size) { | |
1811 | if (rdl->double_discard_supported) { | |
1812 | rdl->notify_discard(rdl, rdl->section); | |
1813 | } else { | |
1814 | virtio_mem_for_each_plugged_section(vmem, rdl->section, rdl, | |
1815 | virtio_mem_notify_discard_cb); | |
1816 | } | |
1817 | } | |
1818 | ||
1819 | memory_region_section_free_copy(rdl->section); | |
1820 | rdl->section = NULL; | |
1821 | QLIST_REMOVE(rdl, next); | |
1822 | } | |
1823 | ||
92a8ee1b DH |
1824 | static void virtio_mem_unplug_request_check(VirtIOMEM *vmem, Error **errp) |
1825 | { | |
1826 | if (vmem->unplugged_inaccessible == ON_OFF_AUTO_OFF) { | |
1827 | /* | |
1828 | * We could allow it with a usable region size of 0, but let's just | |
1829 | * not care about that legacy setting. | |
1830 | */ | |
1831 | error_setg(errp, "virtio-mem device cannot get unplugged while" | |
1832 | " '" VIRTIO_MEM_UNPLUGGED_INACCESSIBLE_PROP "' != 'on'"); | |
1833 | return; | |
1834 | } | |
1835 | ||
1836 | if (vmem->size) { | |
1837 | error_setg(errp, "virtio-mem device cannot get unplugged while" | |
1838 | " '" VIRTIO_MEM_SIZE_PROP "' != '0'"); | |
1839 | return; | |
1840 | } | |
1841 | if (vmem->requested_size) { | |
1842 | error_setg(errp, "virtio-mem device cannot get unplugged while" | |
1843 | " '" VIRTIO_MEM_REQUESTED_SIZE_PROP "' != '0'"); | |
1844 | return; | |
1845 | } | |
1846 | } | |
1847 | ||
910b2576 DH |
1848 | static void virtio_mem_class_init(ObjectClass *klass, void *data) |
1849 | { | |
1850 | DeviceClass *dc = DEVICE_CLASS(klass); | |
1851 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); | |
1852 | VirtIOMEMClass *vmc = VIRTIO_MEM_CLASS(klass); | |
2044969f | 1853 | RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass); |
910b2576 DH |
1854 | |
1855 | device_class_set_props(dc, virtio_mem_properties); | |
1856 | dc->vmsd = &vmstate_virtio_mem; | |
1857 | ||
1858 | set_bit(DEVICE_CATEGORY_MISC, dc->categories); | |
1859 | vdc->realize = virtio_mem_device_realize; | |
1860 | vdc->unrealize = virtio_mem_device_unrealize; | |
1861 | vdc->get_config = virtio_mem_get_config; | |
1862 | vdc->get_features = virtio_mem_get_features; | |
23ad8dec | 1863 | vdc->validate_features = virtio_mem_validate_features; |
910b2576 DH |
1864 | vdc->vmsd = &vmstate_virtio_mem_device; |
1865 | ||
1866 | vmc->fill_device_info = virtio_mem_fill_device_info; | |
1867 | vmc->get_memory_region = virtio_mem_get_memory_region; | |
177f9b1e DH |
1868 | vmc->decide_memslots = virtio_mem_decide_memslots; |
1869 | vmc->get_memslots = virtio_mem_get_memslots; | |
c95b4437 DH |
1870 | vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier; |
1871 | vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier; | |
92a8ee1b | 1872 | vmc->unplug_request_check = virtio_mem_unplug_request_check; |
2044969f DH |
1873 | |
1874 | rdmc->get_min_granularity = virtio_mem_rdm_get_min_granularity; | |
1875 | rdmc->is_populated = virtio_mem_rdm_is_populated; | |
1876 | rdmc->replay_populated = virtio_mem_rdm_replay_populated; | |
372aa6fd | 1877 | rdmc->replay_discarded = virtio_mem_rdm_replay_discarded; |
2044969f DH |
1878 | rdmc->register_listener = virtio_mem_rdm_register_listener; |
1879 | rdmc->unregister_listener = virtio_mem_rdm_unregister_listener; | |
910b2576 DH |
1880 | } |
1881 | ||
1882 | static const TypeInfo virtio_mem_info = { | |
1883 | .name = TYPE_VIRTIO_MEM, | |
1884 | .parent = TYPE_VIRTIO_DEVICE, | |
1885 | .instance_size = sizeof(VirtIOMEM), | |
1886 | .instance_init = virtio_mem_instance_init, | |
177f9b1e | 1887 | .instance_finalize = virtio_mem_instance_finalize, |
910b2576 DH |
1888 | .class_init = virtio_mem_class_init, |
1889 | .class_size = sizeof(VirtIOMEMClass), | |
2044969f DH |
1890 | .interfaces = (InterfaceInfo[]) { |
1891 | { TYPE_RAM_DISCARD_MANAGER }, | |
1892 | { } | |
1893 | }, | |
910b2576 DH |
1894 | }; |
1895 | ||
1896 | static void virtio_register_types(void) | |
1897 | { | |
1898 | type_register_static(&virtio_mem_info); | |
1899 | } | |
1900 | ||
1901 | type_init(virtio_register_types) |