]>
Commit | Line | Data |
---|---|---|
910b2576 DH |
1 | /* |
2 | * Virtio MEM device | |
3 | * | |
4 | * Copyright (C) 2020 Red Hat, Inc. | |
5 | * | |
6 | * Authors: | |
7 | * David Hildenbrand <david@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. | |
10 | * See the COPYING file in the top-level directory. | |
11 | */ | |
12 | ||
13 | #include "qemu/osdep.h" | |
14 | #include "qemu-common.h" | |
15 | #include "qemu/iov.h" | |
16 | #include "qemu/cutils.h" | |
17 | #include "qemu/error-report.h" | |
18 | #include "qemu/units.h" | |
19 | #include "sysemu/numa.h" | |
20 | #include "sysemu/sysemu.h" | |
21 | #include "sysemu/reset.h" | |
22 | #include "hw/virtio/virtio.h" | |
23 | #include "hw/virtio/virtio-bus.h" | |
24 | #include "hw/virtio/virtio-access.h" | |
25 | #include "hw/virtio/virtio-mem.h" | |
26 | #include "qapi/error.h" | |
27 | #include "qapi/visitor.h" | |
28 | #include "exec/ram_addr.h" | |
29 | #include "migration/misc.h" | |
30 | #include "hw/boards.h" | |
31 | #include "hw/qdev-properties.h" | |
2becc36a | 32 | #include CONFIG_DEVICES |
43e54950 | 33 | #include "trace.h" |
910b2576 DH |
34 | |
35 | /* | |
36 | * Use QEMU_VMALLOC_ALIGN, so no THP will have to be split when unplugging | |
37 | * memory (e.g., 2MB on x86_64). | |
38 | */ | |
facc6851 | 39 | #define VIRTIO_MEM_MIN_BLOCK_SIZE ((uint32_t)QEMU_VMALLOC_ALIGN) |
910b2576 DH |
40 | /* |
41 | * Size the usable region bigger than the requested size if possible. Esp. | |
42 | * Linux guests will only add (aligned) memory blocks in case they fully | |
43 | * fit into the usable region, but plug+online only a subset of the pages. | |
44 | * The memory block size corresponds mostly to the section size. | |
45 | * | |
46 | * This allows e.g., to add 20MB with a section size of 128MB on x86_64, and | |
47 | * a section size of 1GB on arm64 (as long as the start address is properly | |
48 | * aligned, similar to ordinary DIMMs). | |
49 | * | |
50 | * We can change this at any time and maybe even make it configurable if | |
51 | * necessary (as the section size can change). But it's more likely that the | |
52 | * section size will rather get smaller and not bigger over time. | |
53 | */ | |
54 | #if defined(TARGET_X86_64) || defined(TARGET_I386) | |
55 | #define VIRTIO_MEM_USABLE_EXTENT (2 * (128 * MiB)) | |
56 | #else | |
57 | #error VIRTIO_MEM_USABLE_EXTENT not defined | |
58 | #endif | |
59 | ||
60 | static bool virtio_mem_is_busy(void) | |
61 | { | |
62 | /* | |
63 | * Postcopy cannot handle concurrent discards and we don't want to migrate | |
64 | * pages on-demand with stale content when plugging new blocks. | |
0bc7806c DH |
65 | * |
66 | * For precopy, we don't want unplugged blocks in our migration stream, and | |
67 | * when plugging new blocks, the page content might differ between source | |
68 | * and destination (observable by the guest when not initializing pages | |
69 | * after plugging them) until we're running on the destination (as we didn't | |
70 | * migrate these blocks when they were unplugged). | |
910b2576 | 71 | */ |
0bc7806c | 72 | return migration_in_incoming_postcopy() || !migration_is_idle(); |
910b2576 DH |
73 | } |
74 | ||
75 | static bool virtio_mem_test_bitmap(VirtIOMEM *vmem, uint64_t start_gpa, | |
76 | uint64_t size, bool plugged) | |
77 | { | |
78 | const unsigned long first_bit = (start_gpa - vmem->addr) / vmem->block_size; | |
79 | const unsigned long last_bit = first_bit + (size / vmem->block_size) - 1; | |
80 | unsigned long found_bit; | |
81 | ||
82 | /* We fake a shorter bitmap to avoid searching too far. */ | |
83 | if (plugged) { | |
84 | found_bit = find_next_zero_bit(vmem->bitmap, last_bit + 1, first_bit); | |
85 | } else { | |
86 | found_bit = find_next_bit(vmem->bitmap, last_bit + 1, first_bit); | |
87 | } | |
88 | return found_bit > last_bit; | |
89 | } | |
90 | ||
91 | static void virtio_mem_set_bitmap(VirtIOMEM *vmem, uint64_t start_gpa, | |
92 | uint64_t size, bool plugged) | |
93 | { | |
94 | const unsigned long bit = (start_gpa - vmem->addr) / vmem->block_size; | |
95 | const unsigned long nbits = size / vmem->block_size; | |
96 | ||
97 | if (plugged) { | |
98 | bitmap_set(vmem->bitmap, bit, nbits); | |
99 | } else { | |
100 | bitmap_clear(vmem->bitmap, bit, nbits); | |
101 | } | |
102 | } | |
103 | ||
104 | static void virtio_mem_send_response(VirtIOMEM *vmem, VirtQueueElement *elem, | |
105 | struct virtio_mem_resp *resp) | |
106 | { | |
107 | VirtIODevice *vdev = VIRTIO_DEVICE(vmem); | |
108 | VirtQueue *vq = vmem->vq; | |
109 | ||
43e54950 | 110 | trace_virtio_mem_send_response(le16_to_cpu(resp->type)); |
910b2576 DH |
111 | iov_from_buf(elem->in_sg, elem->in_num, 0, resp, sizeof(*resp)); |
112 | ||
113 | virtqueue_push(vq, elem, sizeof(*resp)); | |
114 | virtio_notify(vdev, vq); | |
115 | } | |
116 | ||
117 | static void virtio_mem_send_response_simple(VirtIOMEM *vmem, | |
118 | VirtQueueElement *elem, | |
119 | uint16_t type) | |
120 | { | |
121 | struct virtio_mem_resp resp = { | |
122 | .type = cpu_to_le16(type), | |
123 | }; | |
124 | ||
125 | virtio_mem_send_response(vmem, elem, &resp); | |
126 | } | |
127 | ||
128 | static bool virtio_mem_valid_range(VirtIOMEM *vmem, uint64_t gpa, uint64_t size) | |
129 | { | |
130 | if (!QEMU_IS_ALIGNED(gpa, vmem->block_size)) { | |
131 | return false; | |
132 | } | |
133 | if (gpa + size < gpa || !size) { | |
134 | return false; | |
135 | } | |
136 | if (gpa < vmem->addr || gpa >= vmem->addr + vmem->usable_region_size) { | |
137 | return false; | |
138 | } | |
139 | if (gpa + size > vmem->addr + vmem->usable_region_size) { | |
140 | return false; | |
141 | } | |
142 | return true; | |
143 | } | |
144 | ||
145 | static int virtio_mem_set_block_state(VirtIOMEM *vmem, uint64_t start_gpa, | |
146 | uint64_t size, bool plug) | |
147 | { | |
148 | const uint64_t offset = start_gpa - vmem->addr; | |
149 | int ret; | |
150 | ||
151 | if (virtio_mem_is_busy()) { | |
152 | return -EBUSY; | |
153 | } | |
154 | ||
155 | if (!plug) { | |
156 | ret = ram_block_discard_range(vmem->memdev->mr.ram_block, offset, size); | |
157 | if (ret) { | |
158 | error_report("Unexpected error discarding RAM: %s", | |
159 | strerror(-ret)); | |
160 | return -EBUSY; | |
161 | } | |
162 | } | |
163 | virtio_mem_set_bitmap(vmem, start_gpa, size, plug); | |
164 | return 0; | |
165 | } | |
166 | ||
167 | static int virtio_mem_state_change_request(VirtIOMEM *vmem, uint64_t gpa, | |
168 | uint16_t nb_blocks, bool plug) | |
169 | { | |
170 | const uint64_t size = nb_blocks * vmem->block_size; | |
171 | int ret; | |
172 | ||
173 | if (!virtio_mem_valid_range(vmem, gpa, size)) { | |
174 | return VIRTIO_MEM_RESP_ERROR; | |
175 | } | |
176 | ||
177 | if (plug && (vmem->size + size > vmem->requested_size)) { | |
178 | return VIRTIO_MEM_RESP_NACK; | |
179 | } | |
180 | ||
181 | /* test if really all blocks are in the opposite state */ | |
182 | if (!virtio_mem_test_bitmap(vmem, gpa, size, !plug)) { | |
183 | return VIRTIO_MEM_RESP_ERROR; | |
184 | } | |
185 | ||
186 | ret = virtio_mem_set_block_state(vmem, gpa, size, plug); | |
187 | if (ret) { | |
188 | return VIRTIO_MEM_RESP_BUSY; | |
189 | } | |
190 | if (plug) { | |
191 | vmem->size += size; | |
192 | } else { | |
193 | vmem->size -= size; | |
194 | } | |
c95b4437 | 195 | notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); |
910b2576 DH |
196 | return VIRTIO_MEM_RESP_ACK; |
197 | } | |
198 | ||
199 | static void virtio_mem_plug_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
200 | struct virtio_mem_req *req) | |
201 | { | |
202 | const uint64_t gpa = le64_to_cpu(req->u.plug.addr); | |
203 | const uint16_t nb_blocks = le16_to_cpu(req->u.plug.nb_blocks); | |
204 | uint16_t type; | |
205 | ||
43e54950 | 206 | trace_virtio_mem_plug_request(gpa, nb_blocks); |
910b2576 DH |
207 | type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, true); |
208 | virtio_mem_send_response_simple(vmem, elem, type); | |
209 | } | |
210 | ||
211 | static void virtio_mem_unplug_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
212 | struct virtio_mem_req *req) | |
213 | { | |
214 | const uint64_t gpa = le64_to_cpu(req->u.unplug.addr); | |
215 | const uint16_t nb_blocks = le16_to_cpu(req->u.unplug.nb_blocks); | |
216 | uint16_t type; | |
217 | ||
43e54950 | 218 | trace_virtio_mem_unplug_request(gpa, nb_blocks); |
910b2576 DH |
219 | type = virtio_mem_state_change_request(vmem, gpa, nb_blocks, false); |
220 | virtio_mem_send_response_simple(vmem, elem, type); | |
221 | } | |
222 | ||
223 | static void virtio_mem_resize_usable_region(VirtIOMEM *vmem, | |
224 | uint64_t requested_size, | |
225 | bool can_shrink) | |
226 | { | |
227 | uint64_t newsize = MIN(memory_region_size(&vmem->memdev->mr), | |
228 | requested_size + VIRTIO_MEM_USABLE_EXTENT); | |
229 | ||
230 | if (!requested_size) { | |
231 | newsize = 0; | |
232 | } | |
233 | ||
234 | if (newsize < vmem->usable_region_size && !can_shrink) { | |
235 | return; | |
236 | } | |
237 | ||
43e54950 | 238 | trace_virtio_mem_resized_usable_region(vmem->usable_region_size, newsize); |
910b2576 DH |
239 | vmem->usable_region_size = newsize; |
240 | } | |
241 | ||
242 | static int virtio_mem_unplug_all(VirtIOMEM *vmem) | |
243 | { | |
244 | RAMBlock *rb = vmem->memdev->mr.ram_block; | |
245 | int ret; | |
246 | ||
247 | if (virtio_mem_is_busy()) { | |
248 | return -EBUSY; | |
249 | } | |
250 | ||
251 | ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb)); | |
252 | if (ret) { | |
253 | error_report("Unexpected error discarding RAM: %s", strerror(-ret)); | |
254 | return -EBUSY; | |
255 | } | |
256 | bitmap_clear(vmem->bitmap, 0, vmem->bitmap_size); | |
c95b4437 DH |
257 | if (vmem->size) { |
258 | vmem->size = 0; | |
259 | notifier_list_notify(&vmem->size_change_notifiers, &vmem->size); | |
260 | } | |
43e54950 | 261 | trace_virtio_mem_unplugged_all(); |
910b2576 DH |
262 | virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); |
263 | return 0; | |
264 | } | |
265 | ||
266 | static void virtio_mem_unplug_all_request(VirtIOMEM *vmem, | |
267 | VirtQueueElement *elem) | |
268 | { | |
43e54950 | 269 | trace_virtio_mem_unplug_all_request(); |
910b2576 DH |
270 | if (virtio_mem_unplug_all(vmem)) { |
271 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_BUSY); | |
272 | } else { | |
273 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ACK); | |
274 | } | |
275 | } | |
276 | ||
277 | static void virtio_mem_state_request(VirtIOMEM *vmem, VirtQueueElement *elem, | |
278 | struct virtio_mem_req *req) | |
279 | { | |
280 | const uint16_t nb_blocks = le16_to_cpu(req->u.state.nb_blocks); | |
281 | const uint64_t gpa = le64_to_cpu(req->u.state.addr); | |
282 | const uint64_t size = nb_blocks * vmem->block_size; | |
283 | struct virtio_mem_resp resp = { | |
284 | .type = cpu_to_le16(VIRTIO_MEM_RESP_ACK), | |
285 | }; | |
286 | ||
43e54950 | 287 | trace_virtio_mem_state_request(gpa, nb_blocks); |
910b2576 DH |
288 | if (!virtio_mem_valid_range(vmem, gpa, size)) { |
289 | virtio_mem_send_response_simple(vmem, elem, VIRTIO_MEM_RESP_ERROR); | |
290 | return; | |
291 | } | |
292 | ||
293 | if (virtio_mem_test_bitmap(vmem, gpa, size, true)) { | |
294 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_PLUGGED); | |
295 | } else if (virtio_mem_test_bitmap(vmem, gpa, size, false)) { | |
296 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_UNPLUGGED); | |
297 | } else { | |
298 | resp.u.state.state = cpu_to_le16(VIRTIO_MEM_STATE_MIXED); | |
299 | } | |
43e54950 | 300 | trace_virtio_mem_state_response(le16_to_cpu(resp.u.state.state)); |
910b2576 DH |
301 | virtio_mem_send_response(vmem, elem, &resp); |
302 | } | |
303 | ||
304 | static void virtio_mem_handle_request(VirtIODevice *vdev, VirtQueue *vq) | |
305 | { | |
306 | const int len = sizeof(struct virtio_mem_req); | |
307 | VirtIOMEM *vmem = VIRTIO_MEM(vdev); | |
308 | VirtQueueElement *elem; | |
309 | struct virtio_mem_req req; | |
310 | uint16_t type; | |
311 | ||
312 | while (true) { | |
313 | elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); | |
314 | if (!elem) { | |
315 | return; | |
316 | } | |
317 | ||
318 | if (iov_to_buf(elem->out_sg, elem->out_num, 0, &req, len) < len) { | |
319 | virtio_error(vdev, "virtio-mem protocol violation: invalid request" | |
320 | " size: %d", len); | |
0c404e45 | 321 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
322 | g_free(elem); |
323 | return; | |
324 | } | |
325 | ||
326 | if (iov_size(elem->in_sg, elem->in_num) < | |
327 | sizeof(struct virtio_mem_resp)) { | |
328 | virtio_error(vdev, "virtio-mem protocol violation: not enough space" | |
329 | " for response: %zu", | |
330 | iov_size(elem->in_sg, elem->in_num)); | |
0c404e45 | 331 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
332 | g_free(elem); |
333 | return; | |
334 | } | |
335 | ||
336 | type = le16_to_cpu(req.type); | |
337 | switch (type) { | |
338 | case VIRTIO_MEM_REQ_PLUG: | |
339 | virtio_mem_plug_request(vmem, elem, &req); | |
340 | break; | |
341 | case VIRTIO_MEM_REQ_UNPLUG: | |
342 | virtio_mem_unplug_request(vmem, elem, &req); | |
343 | break; | |
344 | case VIRTIO_MEM_REQ_UNPLUG_ALL: | |
345 | virtio_mem_unplug_all_request(vmem, elem); | |
346 | break; | |
347 | case VIRTIO_MEM_REQ_STATE: | |
348 | virtio_mem_state_request(vmem, elem, &req); | |
349 | break; | |
350 | default: | |
351 | virtio_error(vdev, "virtio-mem protocol violation: unknown request" | |
352 | " type: %d", type); | |
0c404e45 | 353 | virtqueue_detach_element(vq, elem, 0); |
910b2576 DH |
354 | g_free(elem); |
355 | return; | |
356 | } | |
357 | ||
358 | g_free(elem); | |
359 | } | |
360 | } | |
361 | ||
362 | static void virtio_mem_get_config(VirtIODevice *vdev, uint8_t *config_data) | |
363 | { | |
364 | VirtIOMEM *vmem = VIRTIO_MEM(vdev); | |
365 | struct virtio_mem_config *config = (void *) config_data; | |
366 | ||
367 | config->block_size = cpu_to_le64(vmem->block_size); | |
368 | config->node_id = cpu_to_le16(vmem->node); | |
369 | config->requested_size = cpu_to_le64(vmem->requested_size); | |
370 | config->plugged_size = cpu_to_le64(vmem->size); | |
371 | config->addr = cpu_to_le64(vmem->addr); | |
372 | config->region_size = cpu_to_le64(memory_region_size(&vmem->memdev->mr)); | |
373 | config->usable_region_size = cpu_to_le64(vmem->usable_region_size); | |
374 | } | |
375 | ||
376 | static uint64_t virtio_mem_get_features(VirtIODevice *vdev, uint64_t features, | |
377 | Error **errp) | |
378 | { | |
379 | MachineState *ms = MACHINE(qdev_get_machine()); | |
380 | ||
381 | if (ms->numa_state) { | |
382 | #if defined(CONFIG_ACPI) | |
383 | virtio_add_feature(&features, VIRTIO_MEM_F_ACPI_PXM); | |
384 | #endif | |
385 | } | |
386 | return features; | |
387 | } | |
388 | ||
389 | static void virtio_mem_system_reset(void *opaque) | |
390 | { | |
391 | VirtIOMEM *vmem = VIRTIO_MEM(opaque); | |
392 | ||
393 | /* | |
394 | * During usual resets, we will unplug all memory and shrink the usable | |
395 | * region size. This is, however, not possible in all scenarios. Then, | |
396 | * the guest has to deal with this manually (VIRTIO_MEM_REQ_UNPLUG_ALL). | |
397 | */ | |
398 | virtio_mem_unplug_all(vmem); | |
399 | } | |
400 | ||
401 | static void virtio_mem_device_realize(DeviceState *dev, Error **errp) | |
402 | { | |
403 | MachineState *ms = MACHINE(qdev_get_machine()); | |
404 | int nb_numa_nodes = ms->numa_state ? ms->numa_state->num_nodes : 0; | |
405 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); | |
406 | VirtIOMEM *vmem = VIRTIO_MEM(dev); | |
407 | uint64_t page_size; | |
408 | RAMBlock *rb; | |
409 | int ret; | |
410 | ||
411 | if (!vmem->memdev) { | |
412 | error_setg(errp, "'%s' property is not set", VIRTIO_MEM_MEMDEV_PROP); | |
413 | return; | |
414 | } else if (host_memory_backend_is_mapped(vmem->memdev)) { | |
910b2576 | 415 | error_setg(errp, "'%s' property specifies a busy memdev: %s", |
7a309cc9 MA |
416 | VIRTIO_MEM_MEMDEV_PROP, |
417 | object_get_canonical_path_component(OBJECT(vmem->memdev))); | |
910b2576 DH |
418 | return; |
419 | } else if (!memory_region_is_ram(&vmem->memdev->mr) || | |
420 | memory_region_is_rom(&vmem->memdev->mr) || | |
421 | !vmem->memdev->mr.ram_block) { | |
422 | error_setg(errp, "'%s' property specifies an unsupported memdev", | |
423 | VIRTIO_MEM_MEMDEV_PROP); | |
424 | return; | |
425 | } | |
426 | ||
427 | if ((nb_numa_nodes && vmem->node >= nb_numa_nodes) || | |
428 | (!nb_numa_nodes && vmem->node)) { | |
429 | error_setg(errp, "'%s' property has value '%" PRIu32 "', which exceeds" | |
430 | "the number of numa nodes: %d", VIRTIO_MEM_NODE_PROP, | |
431 | vmem->node, nb_numa_nodes ? nb_numa_nodes : 1); | |
432 | return; | |
433 | } | |
434 | ||
435 | if (enable_mlock) { | |
436 | error_setg(errp, "Incompatible with mlock"); | |
437 | return; | |
438 | } | |
439 | ||
440 | rb = vmem->memdev->mr.ram_block; | |
441 | page_size = qemu_ram_pagesize(rb); | |
442 | ||
443 | if (vmem->block_size < page_size) { | |
444 | error_setg(errp, "'%s' property has to be at least the page size (0x%" | |
445 | PRIx64 ")", VIRTIO_MEM_BLOCK_SIZE_PROP, page_size); | |
446 | return; | |
447 | } else if (!QEMU_IS_ALIGNED(vmem->requested_size, vmem->block_size)) { | |
448 | error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 | |
449 | ")", VIRTIO_MEM_REQUESTED_SIZE_PROP, | |
450 | VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); | |
451 | return; | |
d31992ae DH |
452 | } else if (!QEMU_IS_ALIGNED(vmem->addr, vmem->block_size)) { |
453 | error_setg(errp, "'%s' property has to be multiples of '%s' (0x%" PRIx64 | |
454 | ")", VIRTIO_MEM_ADDR_PROP, VIRTIO_MEM_BLOCK_SIZE_PROP, | |
455 | vmem->block_size); | |
456 | return; | |
910b2576 DH |
457 | } else if (!QEMU_IS_ALIGNED(memory_region_size(&vmem->memdev->mr), |
458 | vmem->block_size)) { | |
459 | error_setg(errp, "'%s' property memdev size has to be multiples of" | |
460 | "'%s' (0x%" PRIx64 ")", VIRTIO_MEM_MEMDEV_PROP, | |
461 | VIRTIO_MEM_BLOCK_SIZE_PROP, vmem->block_size); | |
462 | return; | |
463 | } | |
464 | ||
465 | if (ram_block_discard_require(true)) { | |
466 | error_setg(errp, "Discarding RAM is disabled"); | |
467 | return; | |
468 | } | |
469 | ||
470 | ret = ram_block_discard_range(rb, 0, qemu_ram_get_used_length(rb)); | |
471 | if (ret) { | |
472 | error_setg_errno(errp, -ret, "Unexpected error discarding RAM"); | |
473 | ram_block_discard_require(false); | |
474 | return; | |
475 | } | |
476 | ||
477 | virtio_mem_resize_usable_region(vmem, vmem->requested_size, true); | |
478 | ||
479 | vmem->bitmap_size = memory_region_size(&vmem->memdev->mr) / | |
480 | vmem->block_size; | |
481 | vmem->bitmap = bitmap_new(vmem->bitmap_size); | |
482 | ||
483 | virtio_init(vdev, TYPE_VIRTIO_MEM, VIRTIO_ID_MEM, | |
484 | sizeof(struct virtio_mem_config)); | |
485 | vmem->vq = virtio_add_queue(vdev, 128, virtio_mem_handle_request); | |
486 | ||
487 | host_memory_backend_set_mapped(vmem->memdev, true); | |
488 | vmstate_register_ram(&vmem->memdev->mr, DEVICE(vmem)); | |
489 | qemu_register_reset(virtio_mem_system_reset, vmem); | |
0bc7806c | 490 | precopy_add_notifier(&vmem->precopy_notifier); |
910b2576 DH |
491 | } |
492 | ||
493 | static void virtio_mem_device_unrealize(DeviceState *dev) | |
494 | { | |
495 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); | |
496 | VirtIOMEM *vmem = VIRTIO_MEM(dev); | |
497 | ||
0bc7806c | 498 | precopy_remove_notifier(&vmem->precopy_notifier); |
910b2576 DH |
499 | qemu_unregister_reset(virtio_mem_system_reset, vmem); |
500 | vmstate_unregister_ram(&vmem->memdev->mr, DEVICE(vmem)); | |
501 | host_memory_backend_set_mapped(vmem->memdev, false); | |
502 | virtio_del_queue(vdev, 0); | |
503 | virtio_cleanup(vdev); | |
504 | g_free(vmem->bitmap); | |
505 | ram_block_discard_require(false); | |
506 | } | |
507 | ||
508 | static int virtio_mem_restore_unplugged(VirtIOMEM *vmem) | |
509 | { | |
510 | RAMBlock *rb = vmem->memdev->mr.ram_block; | |
511 | unsigned long first_zero_bit, last_zero_bit; | |
512 | uint64_t offset, length; | |
513 | int ret; | |
514 | ||
515 | /* Find consecutive unplugged blocks and discard the consecutive range. */ | |
516 | first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); | |
517 | while (first_zero_bit < vmem->bitmap_size) { | |
518 | offset = first_zero_bit * vmem->block_size; | |
519 | last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
520 | first_zero_bit + 1) - 1; | |
521 | length = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; | |
522 | ||
523 | ret = ram_block_discard_range(rb, offset, length); | |
524 | if (ret) { | |
525 | error_report("Unexpected error discarding RAM: %s", | |
526 | strerror(-ret)); | |
527 | return -EINVAL; | |
528 | } | |
529 | first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
530 | last_zero_bit + 2); | |
531 | } | |
532 | return 0; | |
533 | } | |
534 | ||
535 | static int virtio_mem_post_load(void *opaque, int version_id) | |
536 | { | |
537 | if (migration_in_incoming_postcopy()) { | |
538 | return 0; | |
539 | } | |
540 | ||
541 | return virtio_mem_restore_unplugged(VIRTIO_MEM(opaque)); | |
542 | } | |
543 | ||
383ee445 DH |
544 | typedef struct VirtIOMEMMigSanityChecks { |
545 | VirtIOMEM *parent; | |
546 | uint64_t addr; | |
547 | uint64_t region_size; | |
548 | uint64_t block_size; | |
549 | uint32_t node; | |
550 | } VirtIOMEMMigSanityChecks; | |
551 | ||
552 | static int virtio_mem_mig_sanity_checks_pre_save(void *opaque) | |
553 | { | |
554 | VirtIOMEMMigSanityChecks *tmp = opaque; | |
555 | VirtIOMEM *vmem = tmp->parent; | |
556 | ||
557 | tmp->addr = vmem->addr; | |
558 | tmp->region_size = memory_region_size(&vmem->memdev->mr); | |
559 | tmp->block_size = vmem->block_size; | |
560 | tmp->node = vmem->node; | |
561 | return 0; | |
562 | } | |
563 | ||
564 | static int virtio_mem_mig_sanity_checks_post_load(void *opaque, int version_id) | |
565 | { | |
566 | VirtIOMEMMigSanityChecks *tmp = opaque; | |
567 | VirtIOMEM *vmem = tmp->parent; | |
568 | const uint64_t new_region_size = memory_region_size(&vmem->memdev->mr); | |
569 | ||
570 | if (tmp->addr != vmem->addr) { | |
571 | error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, | |
572 | VIRTIO_MEM_ADDR_PROP, tmp->addr, vmem->addr); | |
573 | return -EINVAL; | |
574 | } | |
575 | /* | |
576 | * Note: Preparation for resizeable memory regions. The maximum size | |
577 | * of the memory region must not change during migration. | |
578 | */ | |
579 | if (tmp->region_size != new_region_size) { | |
580 | error_report("Property '%s' size changed from 0x%" PRIx64 " to 0x%" | |
581 | PRIx64, VIRTIO_MEM_MEMDEV_PROP, tmp->region_size, | |
582 | new_region_size); | |
583 | return -EINVAL; | |
584 | } | |
585 | if (tmp->block_size != vmem->block_size) { | |
586 | error_report("Property '%s' changed from 0x%" PRIx64 " to 0x%" PRIx64, | |
587 | VIRTIO_MEM_BLOCK_SIZE_PROP, tmp->block_size, | |
588 | vmem->block_size); | |
589 | return -EINVAL; | |
590 | } | |
591 | if (tmp->node != vmem->node) { | |
592 | error_report("Property '%s' changed from %" PRIu32 " to %" PRIu32, | |
593 | VIRTIO_MEM_NODE_PROP, tmp->node, vmem->node); | |
594 | return -EINVAL; | |
595 | } | |
596 | return 0; | |
597 | } | |
598 | ||
599 | static const VMStateDescription vmstate_virtio_mem_sanity_checks = { | |
600 | .name = "virtio-mem-device/sanity-checks", | |
601 | .pre_save = virtio_mem_mig_sanity_checks_pre_save, | |
602 | .post_load = virtio_mem_mig_sanity_checks_post_load, | |
603 | .fields = (VMStateField[]) { | |
604 | VMSTATE_UINT64(addr, VirtIOMEMMigSanityChecks), | |
605 | VMSTATE_UINT64(region_size, VirtIOMEMMigSanityChecks), | |
606 | VMSTATE_UINT64(block_size, VirtIOMEMMigSanityChecks), | |
607 | VMSTATE_UINT32(node, VirtIOMEMMigSanityChecks), | |
608 | VMSTATE_END_OF_LIST(), | |
609 | }, | |
610 | }; | |
611 | ||
910b2576 DH |
612 | static const VMStateDescription vmstate_virtio_mem_device = { |
613 | .name = "virtio-mem-device", | |
614 | .minimum_version_id = 1, | |
615 | .version_id = 1, | |
616 | .post_load = virtio_mem_post_load, | |
617 | .fields = (VMStateField[]) { | |
383ee445 DH |
618 | VMSTATE_WITH_TMP(VirtIOMEM, VirtIOMEMMigSanityChecks, |
619 | vmstate_virtio_mem_sanity_checks), | |
910b2576 DH |
620 | VMSTATE_UINT64(usable_region_size, VirtIOMEM), |
621 | VMSTATE_UINT64(size, VirtIOMEM), | |
622 | VMSTATE_UINT64(requested_size, VirtIOMEM), | |
623 | VMSTATE_BITMAP(bitmap, VirtIOMEM, 0, bitmap_size), | |
624 | VMSTATE_END_OF_LIST() | |
625 | }, | |
626 | }; | |
627 | ||
628 | static const VMStateDescription vmstate_virtio_mem = { | |
629 | .name = "virtio-mem", | |
630 | .minimum_version_id = 1, | |
631 | .version_id = 1, | |
632 | .fields = (VMStateField[]) { | |
633 | VMSTATE_VIRTIO_DEVICE, | |
634 | VMSTATE_END_OF_LIST() | |
635 | }, | |
636 | }; | |
637 | ||
638 | static void virtio_mem_fill_device_info(const VirtIOMEM *vmem, | |
639 | VirtioMEMDeviceInfo *vi) | |
640 | { | |
641 | vi->memaddr = vmem->addr; | |
642 | vi->node = vmem->node; | |
643 | vi->requested_size = vmem->requested_size; | |
644 | vi->size = vmem->size; | |
645 | vi->max_size = memory_region_size(&vmem->memdev->mr); | |
646 | vi->block_size = vmem->block_size; | |
647 | vi->memdev = object_get_canonical_path(OBJECT(vmem->memdev)); | |
648 | } | |
649 | ||
650 | static MemoryRegion *virtio_mem_get_memory_region(VirtIOMEM *vmem, Error **errp) | |
651 | { | |
652 | if (!vmem->memdev) { | |
653 | error_setg(errp, "'%s' property must be set", VIRTIO_MEM_MEMDEV_PROP); | |
654 | return NULL; | |
655 | } | |
656 | ||
657 | return &vmem->memdev->mr; | |
658 | } | |
659 | ||
c95b4437 DH |
660 | static void virtio_mem_add_size_change_notifier(VirtIOMEM *vmem, |
661 | Notifier *notifier) | |
662 | { | |
663 | notifier_list_add(&vmem->size_change_notifiers, notifier); | |
664 | } | |
665 | ||
666 | static void virtio_mem_remove_size_change_notifier(VirtIOMEM *vmem, | |
667 | Notifier *notifier) | |
668 | { | |
669 | notifier_remove(notifier); | |
670 | } | |
671 | ||
910b2576 DH |
672 | static void virtio_mem_get_size(Object *obj, Visitor *v, const char *name, |
673 | void *opaque, Error **errp) | |
674 | { | |
675 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
676 | uint64_t value = vmem->size; | |
677 | ||
678 | visit_type_size(v, name, &value, errp); | |
679 | } | |
680 | ||
681 | static void virtio_mem_get_requested_size(Object *obj, Visitor *v, | |
682 | const char *name, void *opaque, | |
683 | Error **errp) | |
684 | { | |
685 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
686 | uint64_t value = vmem->requested_size; | |
687 | ||
688 | visit_type_size(v, name, &value, errp); | |
689 | } | |
690 | ||
691 | static void virtio_mem_set_requested_size(Object *obj, Visitor *v, | |
692 | const char *name, void *opaque, | |
693 | Error **errp) | |
694 | { | |
695 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
696 | Error *err = NULL; | |
697 | uint64_t value; | |
698 | ||
699 | visit_type_size(v, name, &value, &err); | |
700 | if (err) { | |
701 | error_propagate(errp, err); | |
702 | return; | |
703 | } | |
704 | ||
705 | /* | |
706 | * The block size and memory backend are not fixed until the device was | |
707 | * realized. realize() will verify these properties then. | |
708 | */ | |
709 | if (DEVICE(obj)->realized) { | |
710 | if (!QEMU_IS_ALIGNED(value, vmem->block_size)) { | |
711 | error_setg(errp, "'%s' has to be multiples of '%s' (0x%" PRIx64 | |
712 | ")", name, VIRTIO_MEM_BLOCK_SIZE_PROP, | |
713 | vmem->block_size); | |
714 | return; | |
715 | } else if (value > memory_region_size(&vmem->memdev->mr)) { | |
716 | error_setg(errp, "'%s' cannot exceed the memory backend size" | |
717 | "(0x%" PRIx64 ")", name, | |
718 | memory_region_size(&vmem->memdev->mr)); | |
719 | return; | |
720 | } | |
721 | ||
722 | if (value != vmem->requested_size) { | |
723 | virtio_mem_resize_usable_region(vmem, value, false); | |
724 | vmem->requested_size = value; | |
725 | } | |
726 | /* | |
727 | * Trigger a config update so the guest gets notified. We trigger | |
728 | * even if the size didn't change (especially helpful for debugging). | |
729 | */ | |
730 | virtio_notify_config(VIRTIO_DEVICE(vmem)); | |
731 | } else { | |
732 | vmem->requested_size = value; | |
733 | } | |
734 | } | |
735 | ||
736 | static void virtio_mem_get_block_size(Object *obj, Visitor *v, const char *name, | |
737 | void *opaque, Error **errp) | |
738 | { | |
739 | const VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
740 | uint64_t value = vmem->block_size; | |
741 | ||
742 | visit_type_size(v, name, &value, errp); | |
743 | } | |
744 | ||
745 | static void virtio_mem_set_block_size(Object *obj, Visitor *v, const char *name, | |
746 | void *opaque, Error **errp) | |
747 | { | |
748 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
749 | Error *err = NULL; | |
750 | uint64_t value; | |
751 | ||
752 | if (DEVICE(obj)->realized) { | |
753 | error_setg(errp, "'%s' cannot be changed", name); | |
754 | return; | |
755 | } | |
756 | ||
757 | visit_type_size(v, name, &value, &err); | |
758 | if (err) { | |
759 | error_propagate(errp, err); | |
760 | return; | |
761 | } | |
762 | ||
763 | if (value < VIRTIO_MEM_MIN_BLOCK_SIZE) { | |
764 | error_setg(errp, "'%s' property has to be at least 0x%" PRIx32, name, | |
765 | VIRTIO_MEM_MIN_BLOCK_SIZE); | |
766 | return; | |
767 | } else if (!is_power_of_2(value)) { | |
768 | error_setg(errp, "'%s' property has to be a power of two", name); | |
769 | return; | |
770 | } | |
771 | vmem->block_size = value; | |
772 | } | |
773 | ||
0bc7806c DH |
774 | static void virtio_mem_precopy_exclude_unplugged(VirtIOMEM *vmem) |
775 | { | |
776 | void * const host = qemu_ram_get_host_addr(vmem->memdev->mr.ram_block); | |
777 | unsigned long first_zero_bit, last_zero_bit; | |
778 | uint64_t offset, length; | |
779 | ||
780 | /* | |
781 | * Find consecutive unplugged blocks and exclude them from migration. | |
782 | * | |
783 | * Note: Blocks cannot get (un)plugged during precopy, no locking needed. | |
784 | */ | |
785 | first_zero_bit = find_first_zero_bit(vmem->bitmap, vmem->bitmap_size); | |
786 | while (first_zero_bit < vmem->bitmap_size) { | |
787 | offset = first_zero_bit * vmem->block_size; | |
788 | last_zero_bit = find_next_bit(vmem->bitmap, vmem->bitmap_size, | |
789 | first_zero_bit + 1) - 1; | |
790 | length = (last_zero_bit - first_zero_bit + 1) * vmem->block_size; | |
791 | ||
792 | qemu_guest_free_page_hint(host + offset, length); | |
793 | first_zero_bit = find_next_zero_bit(vmem->bitmap, vmem->bitmap_size, | |
794 | last_zero_bit + 2); | |
795 | } | |
796 | } | |
797 | ||
798 | static int virtio_mem_precopy_notify(NotifierWithReturn *n, void *data) | |
799 | { | |
800 | VirtIOMEM *vmem = container_of(n, VirtIOMEM, precopy_notifier); | |
801 | PrecopyNotifyData *pnd = data; | |
802 | ||
803 | switch (pnd->reason) { | |
804 | case PRECOPY_NOTIFY_SETUP: | |
805 | precopy_enable_free_page_optimization(); | |
806 | break; | |
807 | case PRECOPY_NOTIFY_AFTER_BITMAP_SYNC: | |
808 | virtio_mem_precopy_exclude_unplugged(vmem); | |
809 | break; | |
810 | default: | |
811 | break; | |
812 | } | |
813 | ||
814 | return 0; | |
815 | } | |
816 | ||
910b2576 DH |
817 | static void virtio_mem_instance_init(Object *obj) |
818 | { | |
819 | VirtIOMEM *vmem = VIRTIO_MEM(obj); | |
820 | ||
821 | vmem->block_size = VIRTIO_MEM_MIN_BLOCK_SIZE; | |
c95b4437 | 822 | notifier_list_init(&vmem->size_change_notifiers); |
0bc7806c | 823 | vmem->precopy_notifier.notify = virtio_mem_precopy_notify; |
910b2576 DH |
824 | |
825 | object_property_add(obj, VIRTIO_MEM_SIZE_PROP, "size", virtio_mem_get_size, | |
826 | NULL, NULL, NULL); | |
827 | object_property_add(obj, VIRTIO_MEM_REQUESTED_SIZE_PROP, "size", | |
828 | virtio_mem_get_requested_size, | |
829 | virtio_mem_set_requested_size, NULL, NULL); | |
830 | object_property_add(obj, VIRTIO_MEM_BLOCK_SIZE_PROP, "size", | |
831 | virtio_mem_get_block_size, virtio_mem_set_block_size, | |
832 | NULL, NULL); | |
833 | } | |
834 | ||
835 | static Property virtio_mem_properties[] = { | |
836 | DEFINE_PROP_UINT64(VIRTIO_MEM_ADDR_PROP, VirtIOMEM, addr, 0), | |
837 | DEFINE_PROP_UINT32(VIRTIO_MEM_NODE_PROP, VirtIOMEM, node, 0), | |
838 | DEFINE_PROP_LINK(VIRTIO_MEM_MEMDEV_PROP, VirtIOMEM, memdev, | |
839 | TYPE_MEMORY_BACKEND, HostMemoryBackend *), | |
840 | DEFINE_PROP_END_OF_LIST(), | |
841 | }; | |
842 | ||
843 | static void virtio_mem_class_init(ObjectClass *klass, void *data) | |
844 | { | |
845 | DeviceClass *dc = DEVICE_CLASS(klass); | |
846 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); | |
847 | VirtIOMEMClass *vmc = VIRTIO_MEM_CLASS(klass); | |
848 | ||
849 | device_class_set_props(dc, virtio_mem_properties); | |
850 | dc->vmsd = &vmstate_virtio_mem; | |
851 | ||
852 | set_bit(DEVICE_CATEGORY_MISC, dc->categories); | |
853 | vdc->realize = virtio_mem_device_realize; | |
854 | vdc->unrealize = virtio_mem_device_unrealize; | |
855 | vdc->get_config = virtio_mem_get_config; | |
856 | vdc->get_features = virtio_mem_get_features; | |
857 | vdc->vmsd = &vmstate_virtio_mem_device; | |
858 | ||
859 | vmc->fill_device_info = virtio_mem_fill_device_info; | |
860 | vmc->get_memory_region = virtio_mem_get_memory_region; | |
c95b4437 DH |
861 | vmc->add_size_change_notifier = virtio_mem_add_size_change_notifier; |
862 | vmc->remove_size_change_notifier = virtio_mem_remove_size_change_notifier; | |
910b2576 DH |
863 | } |
864 | ||
865 | static const TypeInfo virtio_mem_info = { | |
866 | .name = TYPE_VIRTIO_MEM, | |
867 | .parent = TYPE_VIRTIO_DEVICE, | |
868 | .instance_size = sizeof(VirtIOMEM), | |
869 | .instance_init = virtio_mem_instance_init, | |
870 | .class_init = virtio_mem_class_init, | |
871 | .class_size = sizeof(VirtIOMEMClass), | |
872 | }; | |
873 | ||
874 | static void virtio_register_types(void) | |
875 | { | |
876 | type_register_static(&virtio_mem_info); | |
877 | } | |
878 | ||
879 | type_init(virtio_register_types) |