]>
Commit | Line | Data |
---|---|---|
eb59db53 DDAG |
1 | /* |
2 | * Postcopy migration for RAM | |
3 | * | |
4 | * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates | |
5 | * | |
6 | * Authors: | |
7 | * Dave Gilbert <dgilbert@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
10 | * See the COPYING file in the top-level directory. | |
11 | * | |
12 | */ | |
13 | ||
14 | /* | |
15 | * Postcopy is a migration technique where the execution flips from the | |
16 | * source to the destination before all the data has been copied. | |
17 | */ | |
18 | ||
1393a485 | 19 | #include "qemu/osdep.h" |
51180423 | 20 | #include "exec/target_page.h" |
6666c96a | 21 | #include "migration.h" |
08a0aee1 | 22 | #include "qemu-file.h" |
20a519a0 | 23 | #include "savevm.h" |
be07b0ac | 24 | #include "postcopy-ram.h" |
7b1e1a22 | 25 | #include "ram.h" |
1693c64c DDAG |
26 | #include "qapi/error.h" |
27 | #include "qemu/notify.h" | |
eb59db53 | 28 | #include "sysemu/sysemu.h" |
371ff5a3 | 29 | #include "sysemu/balloon.h" |
eb59db53 DDAG |
30 | #include "qemu/error-report.h" |
31 | #include "trace.h" | |
32 | ||
e0b266f0 DDAG |
33 | /* Arbitrary limit on size of each discard command, |
34 | * keeps them around ~200 bytes | |
35 | */ | |
36 | #define MAX_DISCARDS_PER_COMMAND 12 | |
37 | ||
38 | struct PostcopyDiscardState { | |
39 | const char *ramblock_name; | |
e0b266f0 DDAG |
40 | uint16_t cur_entry; |
41 | /* | |
42 | * Start and length of a discard range (bytes) | |
43 | */ | |
44 | uint64_t start_list[MAX_DISCARDS_PER_COMMAND]; | |
45 | uint64_t length_list[MAX_DISCARDS_PER_COMMAND]; | |
46 | unsigned int nsentwords; | |
47 | unsigned int nsentcmds; | |
48 | }; | |
49 | ||
1693c64c DDAG |
50 | static NotifierWithReturnList postcopy_notifier_list; |
51 | ||
52 | void postcopy_infrastructure_init(void) | |
53 | { | |
54 | notifier_with_return_list_init(&postcopy_notifier_list); | |
55 | } | |
56 | ||
57 | void postcopy_add_notifier(NotifierWithReturn *nn) | |
58 | { | |
59 | notifier_with_return_list_add(&postcopy_notifier_list, nn); | |
60 | } | |
61 | ||
62 | void postcopy_remove_notifier(NotifierWithReturn *n) | |
63 | { | |
64 | notifier_with_return_remove(n); | |
65 | } | |
66 | ||
67 | int postcopy_notify(enum PostcopyNotifyReason reason, Error **errp) | |
68 | { | |
69 | struct PostcopyNotifyData pnd; | |
70 | pnd.reason = reason; | |
71 | pnd.errp = errp; | |
72 | ||
73 | return notifier_with_return_list_notify(&postcopy_notifier_list, | |
74 | &pnd); | |
75 | } | |
76 | ||
eb59db53 DDAG |
77 | /* Postcopy needs to detect accesses to pages that haven't yet been copied |
78 | * across, and efficiently map new pages in, the techniques for doing this | |
79 | * are target OS specific. | |
80 | */ | |
81 | #if defined(__linux__) | |
82 | ||
c4faeed2 | 83 | #include <poll.h> |
eb59db53 DDAG |
84 | #include <sys/ioctl.h> |
85 | #include <sys/syscall.h> | |
eb59db53 DDAG |
86 | #include <asm/types.h> /* for __u64 */ |
87 | #endif | |
88 | ||
d8b9d771 MF |
89 | #if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD) |
90 | #include <sys/eventfd.h> | |
eb59db53 DDAG |
91 | #include <linux/userfaultfd.h> |
92 | ||
2a4c42f1 AP |
93 | typedef struct PostcopyBlocktimeContext { |
94 | /* time when page fault initiated per vCPU */ | |
95 | uint32_t *page_fault_vcpu_time; | |
96 | /* page address per vCPU */ | |
97 | uintptr_t *vcpu_addr; | |
98 | uint32_t total_blocktime; | |
99 | /* blocktime per vCPU */ | |
100 | uint32_t *vcpu_blocktime; | |
101 | /* point in time when last page fault was initiated */ | |
102 | uint32_t last_begin; | |
103 | /* number of vCPU are suspended */ | |
104 | int smp_cpus_down; | |
105 | uint64_t start_time; | |
106 | ||
107 | /* | |
108 | * Handler for exit event, necessary for | |
109 | * releasing whole blocktime_ctx | |
110 | */ | |
111 | Notifier exit_notifier; | |
112 | } PostcopyBlocktimeContext; | |
113 | ||
114 | static void destroy_blocktime_context(struct PostcopyBlocktimeContext *ctx) | |
115 | { | |
116 | g_free(ctx->page_fault_vcpu_time); | |
117 | g_free(ctx->vcpu_addr); | |
118 | g_free(ctx->vcpu_blocktime); | |
119 | g_free(ctx); | |
120 | } | |
121 | ||
122 | static void migration_exit_cb(Notifier *n, void *data) | |
123 | { | |
124 | PostcopyBlocktimeContext *ctx = container_of(n, PostcopyBlocktimeContext, | |
125 | exit_notifier); | |
126 | destroy_blocktime_context(ctx); | |
127 | } | |
128 | ||
129 | static struct PostcopyBlocktimeContext *blocktime_context_new(void) | |
130 | { | |
131 | PostcopyBlocktimeContext *ctx = g_new0(PostcopyBlocktimeContext, 1); | |
132 | ctx->page_fault_vcpu_time = g_new0(uint32_t, smp_cpus); | |
133 | ctx->vcpu_addr = g_new0(uintptr_t, smp_cpus); | |
134 | ctx->vcpu_blocktime = g_new0(uint32_t, smp_cpus); | |
135 | ||
136 | ctx->exit_notifier.notify = migration_exit_cb; | |
137 | ctx->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); | |
138 | qemu_add_exit_notifier(&ctx->exit_notifier); | |
139 | return ctx; | |
140 | } | |
ca6011c2 | 141 | |
65ace060 AP |
142 | static uint32List *get_vcpu_blocktime_list(PostcopyBlocktimeContext *ctx) |
143 | { | |
144 | uint32List *list = NULL, *entry = NULL; | |
145 | int i; | |
146 | ||
147 | for (i = smp_cpus - 1; i >= 0; i--) { | |
148 | entry = g_new0(uint32List, 1); | |
149 | entry->value = ctx->vcpu_blocktime[i]; | |
150 | entry->next = list; | |
151 | list = entry; | |
152 | } | |
153 | ||
154 | return list; | |
155 | } | |
156 | ||
157 | /* | |
158 | * This function just populates MigrationInfo from postcopy's | |
159 | * blocktime context. It will not populate MigrationInfo, | |
160 | * unless postcopy-blocktime capability was set. | |
161 | * | |
162 | * @info: pointer to MigrationInfo to populate | |
163 | */ | |
164 | void fill_destination_postcopy_migration_info(MigrationInfo *info) | |
165 | { | |
166 | MigrationIncomingState *mis = migration_incoming_get_current(); | |
167 | PostcopyBlocktimeContext *bc = mis->blocktime_ctx; | |
168 | ||
169 | if (!bc) { | |
170 | return; | |
171 | } | |
172 | ||
173 | info->has_postcopy_blocktime = true; | |
174 | info->postcopy_blocktime = bc->total_blocktime; | |
175 | info->has_postcopy_vcpu_blocktime = true; | |
176 | info->postcopy_vcpu_blocktime = get_vcpu_blocktime_list(bc); | |
177 | } | |
178 | ||
179 | static uint32_t get_postcopy_total_blocktime(void) | |
180 | { | |
181 | MigrationIncomingState *mis = migration_incoming_get_current(); | |
182 | PostcopyBlocktimeContext *bc = mis->blocktime_ctx; | |
183 | ||
184 | if (!bc) { | |
185 | return 0; | |
186 | } | |
187 | ||
188 | return bc->total_blocktime; | |
189 | } | |
190 | ||
54ae0886 AP |
191 | /** |
192 | * receive_ufd_features: check userfault fd features, to request only supported | |
193 | * features in the future. | |
194 | * | |
195 | * Returns: true on success | |
196 | * | |
197 | * __NR_userfaultfd - should be checked before | |
198 | * @features: out parameter will contain uffdio_api.features provided by kernel | |
199 | * in case of success | |
200 | */ | |
201 | static bool receive_ufd_features(uint64_t *features) | |
eb59db53 | 202 | { |
54ae0886 AP |
203 | struct uffdio_api api_struct = {0}; |
204 | int ufd; | |
205 | bool ret = true; | |
206 | ||
207 | /* if we are here __NR_userfaultfd should exists */ | |
208 | ufd = syscall(__NR_userfaultfd, O_CLOEXEC); | |
209 | if (ufd == -1) { | |
210 | error_report("%s: syscall __NR_userfaultfd failed: %s", __func__, | |
211 | strerror(errno)); | |
212 | return false; | |
213 | } | |
eb59db53 | 214 | |
54ae0886 | 215 | /* ask features */ |
eb59db53 DDAG |
216 | api_struct.api = UFFD_API; |
217 | api_struct.features = 0; | |
218 | if (ioctl(ufd, UFFDIO_API, &api_struct)) { | |
5553499f | 219 | error_report("%s: UFFDIO_API failed: %s", __func__, |
eb59db53 | 220 | strerror(errno)); |
54ae0886 AP |
221 | ret = false; |
222 | goto release_ufd; | |
223 | } | |
224 | ||
225 | *features = api_struct.features; | |
226 | ||
227 | release_ufd: | |
228 | close(ufd); | |
229 | return ret; | |
230 | } | |
231 | ||
232 | /** | |
233 | * request_ufd_features: this function should be called only once on a newly | |
234 | * opened ufd, subsequent calls will lead to error. | |
235 | * | |
236 | * Returns: true on succes | |
237 | * | |
238 | * @ufd: fd obtained from userfaultfd syscall | |
239 | * @features: bit mask see UFFD_API_FEATURES | |
240 | */ | |
241 | static bool request_ufd_features(int ufd, uint64_t features) | |
242 | { | |
243 | struct uffdio_api api_struct = {0}; | |
244 | uint64_t ioctl_mask; | |
245 | ||
246 | api_struct.api = UFFD_API; | |
247 | api_struct.features = features; | |
248 | if (ioctl(ufd, UFFDIO_API, &api_struct)) { | |
249 | error_report("%s failed: UFFDIO_API failed: %s", __func__, | |
250 | strerror(errno)); | |
eb59db53 DDAG |
251 | return false; |
252 | } | |
253 | ||
254 | ioctl_mask = (__u64)1 << _UFFDIO_REGISTER | | |
255 | (__u64)1 << _UFFDIO_UNREGISTER; | |
256 | if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) { | |
257 | error_report("Missing userfault features: %" PRIx64, | |
258 | (uint64_t)(~api_struct.ioctls & ioctl_mask)); | |
259 | return false; | |
260 | } | |
261 | ||
54ae0886 AP |
262 | return true; |
263 | } | |
264 | ||
265 | static bool ufd_check_and_apply(int ufd, MigrationIncomingState *mis) | |
266 | { | |
267 | uint64_t asked_features = 0; | |
268 | static uint64_t supported_features; | |
269 | ||
270 | /* | |
271 | * it's not possible to | |
272 | * request UFFD_API twice per one fd | |
273 | * userfault fd features is persistent | |
274 | */ | |
275 | if (!supported_features) { | |
276 | if (!receive_ufd_features(&supported_features)) { | |
277 | error_report("%s failed", __func__); | |
278 | return false; | |
279 | } | |
280 | } | |
281 | ||
2a4c42f1 AP |
282 | #ifdef UFFD_FEATURE_THREAD_ID |
283 | if (migrate_postcopy_blocktime() && mis && | |
284 | UFFD_FEATURE_THREAD_ID & supported_features) { | |
285 | /* kernel supports that feature */ | |
286 | /* don't create blocktime_context if it exists */ | |
287 | if (!mis->blocktime_ctx) { | |
288 | mis->blocktime_ctx = blocktime_context_new(); | |
289 | } | |
290 | ||
291 | asked_features |= UFFD_FEATURE_THREAD_ID; | |
292 | } | |
293 | #endif | |
294 | ||
54ae0886 AP |
295 | /* |
296 | * request features, even if asked_features is 0, due to | |
297 | * kernel expects UFFD_API before UFFDIO_REGISTER, per | |
298 | * userfault file descriptor | |
299 | */ | |
300 | if (!request_ufd_features(ufd, asked_features)) { | |
301 | error_report("%s failed: features %" PRIu64, __func__, | |
302 | asked_features); | |
303 | return false; | |
304 | } | |
305 | ||
7e8cafb7 DDAG |
306 | if (getpagesize() != ram_pagesize_summary()) { |
307 | bool have_hp = false; | |
308 | /* We've got a huge page */ | |
309 | #ifdef UFFD_FEATURE_MISSING_HUGETLBFS | |
54ae0886 | 310 | have_hp = supported_features & UFFD_FEATURE_MISSING_HUGETLBFS; |
7e8cafb7 DDAG |
311 | #endif |
312 | if (!have_hp) { | |
313 | error_report("Userfault on this host does not support huge pages"); | |
314 | return false; | |
315 | } | |
316 | } | |
eb59db53 DDAG |
317 | return true; |
318 | } | |
319 | ||
8679638b DDAG |
320 | /* Callback from postcopy_ram_supported_by_host block iterator. |
321 | */ | |
754cb9c0 | 322 | static int test_ramblock_postcopiable(RAMBlock *rb, void *opaque) |
8679638b | 323 | { |
754cb9c0 YK |
324 | const char *block_name = qemu_ram_get_idstr(rb); |
325 | ram_addr_t length = qemu_ram_get_used_length(rb); | |
5d214a92 DDAG |
326 | size_t pagesize = qemu_ram_pagesize(rb); |
327 | ||
5d214a92 DDAG |
328 | if (length % pagesize) { |
329 | error_report("Postcopy requires RAM blocks to be a page size multiple," | |
330 | " block %s is 0x" RAM_ADDR_FMT " bytes with a " | |
331 | "page size of 0x%zx", block_name, length, pagesize); | |
332 | return 1; | |
333 | } | |
8679638b DDAG |
334 | return 0; |
335 | } | |
336 | ||
58b7c17e DDAG |
337 | /* |
338 | * Note: This has the side effect of munlock'ing all of RAM, that's | |
339 | * normally fine since if the postcopy succeeds it gets turned back on at the | |
340 | * end. | |
341 | */ | |
d7651f15 | 342 | bool postcopy_ram_supported_by_host(MigrationIncomingState *mis) |
eb59db53 DDAG |
343 | { |
344 | long pagesize = getpagesize(); | |
345 | int ufd = -1; | |
346 | bool ret = false; /* Error unless we change it */ | |
347 | void *testarea = NULL; | |
348 | struct uffdio_register reg_struct; | |
349 | struct uffdio_range range_struct; | |
350 | uint64_t feature_mask; | |
1693c64c | 351 | Error *local_err = NULL; |
eb59db53 | 352 | |
20afaed9 | 353 | if (qemu_target_page_size() > pagesize) { |
eb59db53 DDAG |
354 | error_report("Target page size bigger than host page size"); |
355 | goto out; | |
356 | } | |
357 | ||
358 | ufd = syscall(__NR_userfaultfd, O_CLOEXEC); | |
359 | if (ufd == -1) { | |
360 | error_report("%s: userfaultfd not available: %s", __func__, | |
361 | strerror(errno)); | |
362 | goto out; | |
363 | } | |
364 | ||
1693c64c DDAG |
365 | /* Give devices a chance to object */ |
366 | if (postcopy_notify(POSTCOPY_NOTIFY_PROBE, &local_err)) { | |
367 | error_report_err(local_err); | |
368 | goto out; | |
369 | } | |
370 | ||
eb59db53 | 371 | /* Version and features check */ |
54ae0886 | 372 | if (!ufd_check_and_apply(ufd, mis)) { |
eb59db53 DDAG |
373 | goto out; |
374 | } | |
375 | ||
8679638b | 376 | /* We don't support postcopy with shared RAM yet */ |
fbd162e6 | 377 | if (foreach_not_ignored_block(test_ramblock_postcopiable, NULL)) { |
8679638b DDAG |
378 | goto out; |
379 | } | |
380 | ||
58b7c17e DDAG |
381 | /* |
382 | * userfault and mlock don't go together; we'll put it back later if | |
383 | * it was enabled. | |
384 | */ | |
385 | if (munlockall()) { | |
386 | error_report("%s: munlockall: %s", __func__, strerror(errno)); | |
387 | return -1; | |
388 | } | |
389 | ||
eb59db53 DDAG |
390 | /* |
391 | * We need to check that the ops we need are supported on anon memory | |
392 | * To do that we need to register a chunk and see the flags that | |
393 | * are returned. | |
394 | */ | |
395 | testarea = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE | | |
396 | MAP_ANONYMOUS, -1, 0); | |
397 | if (testarea == MAP_FAILED) { | |
398 | error_report("%s: Failed to map test area: %s", __func__, | |
399 | strerror(errno)); | |
400 | goto out; | |
401 | } | |
402 | g_assert(((size_t)testarea & (pagesize-1)) == 0); | |
403 | ||
404 | reg_struct.range.start = (uintptr_t)testarea; | |
405 | reg_struct.range.len = pagesize; | |
406 | reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; | |
407 | ||
408 | if (ioctl(ufd, UFFDIO_REGISTER, ®_struct)) { | |
409 | error_report("%s userfault register: %s", __func__, strerror(errno)); | |
410 | goto out; | |
411 | } | |
412 | ||
413 | range_struct.start = (uintptr_t)testarea; | |
414 | range_struct.len = pagesize; | |
415 | if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) { | |
416 | error_report("%s userfault unregister: %s", __func__, strerror(errno)); | |
417 | goto out; | |
418 | } | |
419 | ||
420 | feature_mask = (__u64)1 << _UFFDIO_WAKE | | |
421 | (__u64)1 << _UFFDIO_COPY | | |
422 | (__u64)1 << _UFFDIO_ZEROPAGE; | |
423 | if ((reg_struct.ioctls & feature_mask) != feature_mask) { | |
424 | error_report("Missing userfault map features: %" PRIx64, | |
425 | (uint64_t)(~reg_struct.ioctls & feature_mask)); | |
426 | goto out; | |
427 | } | |
428 | ||
429 | /* Success! */ | |
430 | ret = true; | |
431 | out: | |
432 | if (testarea) { | |
433 | munmap(testarea, pagesize); | |
434 | } | |
435 | if (ufd != -1) { | |
436 | close(ufd); | |
437 | } | |
438 | return ret; | |
439 | } | |
440 | ||
1caddf8a DDAG |
441 | /* |
442 | * Setup an area of RAM so that it *can* be used for postcopy later; this | |
443 | * must be done right at the start prior to pre-copy. | |
444 | * opaque should be the MIS. | |
445 | */ | |
754cb9c0 | 446 | static int init_range(RAMBlock *rb, void *opaque) |
1caddf8a | 447 | { |
754cb9c0 YK |
448 | const char *block_name = qemu_ram_get_idstr(rb); |
449 | void *host_addr = qemu_ram_get_host_addr(rb); | |
450 | ram_addr_t offset = qemu_ram_get_offset(rb); | |
451 | ram_addr_t length = qemu_ram_get_used_length(rb); | |
1caddf8a DDAG |
452 | trace_postcopy_init_range(block_name, host_addr, offset, length); |
453 | ||
454 | /* | |
455 | * We need the whole of RAM to be truly empty for postcopy, so things | |
456 | * like ROMs and any data tables built during init must be zero'd | |
457 | * - we're going to get the copy from the source anyway. | |
458 | * (Precopy will just overwrite this data, so doesn't need the discard) | |
459 | */ | |
aaa2064c | 460 | if (ram_discard_range(block_name, 0, length)) { |
1caddf8a DDAG |
461 | return -1; |
462 | } | |
463 | ||
464 | return 0; | |
465 | } | |
466 | ||
467 | /* | |
468 | * At the end of migration, undo the effects of init_range | |
469 | * opaque should be the MIS. | |
470 | */ | |
754cb9c0 | 471 | static int cleanup_range(RAMBlock *rb, void *opaque) |
1caddf8a | 472 | { |
754cb9c0 YK |
473 | const char *block_name = qemu_ram_get_idstr(rb); |
474 | void *host_addr = qemu_ram_get_host_addr(rb); | |
475 | ram_addr_t offset = qemu_ram_get_offset(rb); | |
476 | ram_addr_t length = qemu_ram_get_used_length(rb); | |
1caddf8a DDAG |
477 | MigrationIncomingState *mis = opaque; |
478 | struct uffdio_range range_struct; | |
479 | trace_postcopy_cleanup_range(block_name, host_addr, offset, length); | |
480 | ||
481 | /* | |
482 | * We turned off hugepage for the precopy stage with postcopy enabled | |
483 | * we can turn it back on now. | |
484 | */ | |
1d741439 | 485 | qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE); |
1caddf8a DDAG |
486 | |
487 | /* | |
488 | * We can also turn off userfault now since we should have all the | |
489 | * pages. It can be useful to leave it on to debug postcopy | |
490 | * if you're not sure it's always getting every page. | |
491 | */ | |
492 | range_struct.start = (uintptr_t)host_addr; | |
493 | range_struct.len = length; | |
494 | ||
495 | if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) { | |
496 | error_report("%s: userfault unregister %s", __func__, strerror(errno)); | |
497 | ||
498 | return -1; | |
499 | } | |
500 | ||
501 | return 0; | |
502 | } | |
503 | ||
504 | /* | |
505 | * Initialise postcopy-ram, setting the RAM to a state where we can go into | |
506 | * postcopy later; must be called prior to any precopy. | |
507 | * called from arch_init's similarly named ram_postcopy_incoming_init | |
508 | */ | |
c136180c | 509 | int postcopy_ram_incoming_init(MigrationIncomingState *mis) |
1caddf8a | 510 | { |
fbd162e6 | 511 | if (foreach_not_ignored_block(init_range, NULL)) { |
1caddf8a DDAG |
512 | return -1; |
513 | } | |
514 | ||
515 | return 0; | |
516 | } | |
517 | ||
154304cd AW |
518 | /* |
519 | * Manage a single vote to the QEMU balloon inhibitor for all postcopy usage, | |
520 | * last caller wins. | |
521 | */ | |
522 | static void postcopy_balloon_inhibit(bool state) | |
523 | { | |
524 | static bool cur_state = false; | |
525 | ||
526 | if (state != cur_state) { | |
527 | qemu_balloon_inhibit(state); | |
528 | cur_state = state; | |
529 | } | |
530 | } | |
531 | ||
1caddf8a DDAG |
532 | /* |
533 | * At the end of a migration where postcopy_ram_incoming_init was called. | |
534 | */ | |
535 | int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) | |
536 | { | |
c4faeed2 DDAG |
537 | trace_postcopy_ram_incoming_cleanup_entry(); |
538 | ||
539 | if (mis->have_fault_thread) { | |
46343570 DDAG |
540 | Error *local_err = NULL; |
541 | ||
55d0fe82 IM |
542 | /* Let the fault thread quit */ |
543 | atomic_set(&mis->fault_thread_quit, 1); | |
544 | postcopy_fault_thread_notify(mis); | |
545 | trace_postcopy_ram_incoming_cleanup_join(); | |
546 | qemu_thread_join(&mis->fault_thread); | |
547 | ||
46343570 DDAG |
548 | if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_END, &local_err)) { |
549 | error_report_err(local_err); | |
550 | return -1; | |
551 | } | |
552 | ||
fbd162e6 | 553 | if (foreach_not_ignored_block(cleanup_range, mis)) { |
c4faeed2 DDAG |
554 | return -1; |
555 | } | |
9ab7ef9b | 556 | |
c4faeed2 DDAG |
557 | trace_postcopy_ram_incoming_cleanup_closeuf(); |
558 | close(mis->userfault_fd); | |
64f615fe | 559 | close(mis->userfault_event_fd); |
c4faeed2 | 560 | mis->have_fault_thread = false; |
1caddf8a DDAG |
561 | } |
562 | ||
154304cd | 563 | postcopy_balloon_inhibit(false); |
371ff5a3 | 564 | |
58b7c17e DDAG |
565 | if (enable_mlock) { |
566 | if (os_mlock() < 0) { | |
567 | error_report("mlock: %s", strerror(errno)); | |
568 | /* | |
569 | * It doesn't feel right to fail at this point, we have a valid | |
570 | * VM state. | |
571 | */ | |
572 | } | |
573 | } | |
574 | ||
c4faeed2 | 575 | postcopy_state_set(POSTCOPY_INCOMING_END); |
c4faeed2 | 576 | |
696ed9a9 | 577 | if (mis->postcopy_tmp_page) { |
df9ff5e1 | 578 | munmap(mis->postcopy_tmp_page, mis->largest_page_size); |
696ed9a9 DDAG |
579 | mis->postcopy_tmp_page = NULL; |
580 | } | |
41d84210 DDAG |
581 | if (mis->postcopy_tmp_zero_page) { |
582 | munmap(mis->postcopy_tmp_zero_page, mis->largest_page_size); | |
583 | mis->postcopy_tmp_zero_page = NULL; | |
584 | } | |
65ace060 AP |
585 | trace_postcopy_ram_incoming_cleanup_blocktime( |
586 | get_postcopy_total_blocktime()); | |
587 | ||
c4faeed2 | 588 | trace_postcopy_ram_incoming_cleanup_exit(); |
1caddf8a DDAG |
589 | return 0; |
590 | } | |
591 | ||
f9527107 DDAG |
592 | /* |
593 | * Disable huge pages on an area | |
594 | */ | |
754cb9c0 | 595 | static int nhp_range(RAMBlock *rb, void *opaque) |
f9527107 | 596 | { |
754cb9c0 YK |
597 | const char *block_name = qemu_ram_get_idstr(rb); |
598 | void *host_addr = qemu_ram_get_host_addr(rb); | |
599 | ram_addr_t offset = qemu_ram_get_offset(rb); | |
600 | ram_addr_t length = qemu_ram_get_used_length(rb); | |
f9527107 DDAG |
601 | trace_postcopy_nhp_range(block_name, host_addr, offset, length); |
602 | ||
603 | /* | |
604 | * Before we do discards we need to ensure those discards really | |
605 | * do delete areas of the page, even if THP thinks a hugepage would | |
606 | * be a good idea, so force hugepages off. | |
607 | */ | |
1d741439 | 608 | qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE); |
f9527107 DDAG |
609 | |
610 | return 0; | |
611 | } | |
612 | ||
613 | /* | |
614 | * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard | |
615 | * however leaving it until after precopy means that most of the precopy | |
616 | * data is still THPd | |
617 | */ | |
618 | int postcopy_ram_prepare_discard(MigrationIncomingState *mis) | |
619 | { | |
fbd162e6 | 620 | if (foreach_not_ignored_block(nhp_range, mis)) { |
f9527107 DDAG |
621 | return -1; |
622 | } | |
623 | ||
624 | postcopy_state_set(POSTCOPY_INCOMING_DISCARD); | |
625 | ||
626 | return 0; | |
627 | } | |
628 | ||
f0a227ad DDAG |
629 | /* |
630 | * Mark the given area of RAM as requiring notification to unwritten areas | |
fbd162e6 | 631 | * Used as a callback on foreach_not_ignored_block. |
f0a227ad DDAG |
632 | * host_addr: Base of area to mark |
633 | * offset: Offset in the whole ram arena | |
634 | * length: Length of the section | |
635 | * opaque: MigrationIncomingState pointer | |
636 | * Returns 0 on success | |
637 | */ | |
754cb9c0 | 638 | static int ram_block_enable_notify(RAMBlock *rb, void *opaque) |
f0a227ad DDAG |
639 | { |
640 | MigrationIncomingState *mis = opaque; | |
641 | struct uffdio_register reg_struct; | |
642 | ||
754cb9c0 YK |
643 | reg_struct.range.start = (uintptr_t)qemu_ram_get_host_addr(rb); |
644 | reg_struct.range.len = qemu_ram_get_used_length(rb); | |
f0a227ad DDAG |
645 | reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; |
646 | ||
647 | /* Now tell our userfault_fd that it's responsible for this area */ | |
648 | if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, ®_struct)) { | |
649 | error_report("%s userfault register: %s", __func__, strerror(errno)); | |
650 | return -1; | |
651 | } | |
665414ad DDAG |
652 | if (!(reg_struct.ioctls & ((__u64)1 << _UFFDIO_COPY))) { |
653 | error_report("%s userfault: Region doesn't support COPY", __func__); | |
654 | return -1; | |
655 | } | |
2ce16640 | 656 | if (reg_struct.ioctls & ((__u64)1 << _UFFDIO_ZEROPAGE)) { |
2ce16640 DDAG |
657 | qemu_ram_set_uf_zeroable(rb); |
658 | } | |
f0a227ad DDAG |
659 | |
660 | return 0; | |
661 | } | |
662 | ||
5efc3564 DDAG |
663 | int postcopy_wake_shared(struct PostCopyFD *pcfd, |
664 | uint64_t client_addr, | |
665 | RAMBlock *rb) | |
666 | { | |
667 | size_t pagesize = qemu_ram_pagesize(rb); | |
668 | struct uffdio_range range; | |
669 | int ret; | |
670 | trace_postcopy_wake_shared(client_addr, qemu_ram_get_idstr(rb)); | |
671 | range.start = client_addr & ~(pagesize - 1); | |
672 | range.len = pagesize; | |
673 | ret = ioctl(pcfd->fd, UFFDIO_WAKE, &range); | |
674 | if (ret) { | |
675 | error_report("%s: Failed to wake: %zx in %s (%s)", | |
676 | __func__, (size_t)client_addr, qemu_ram_get_idstr(rb), | |
677 | strerror(errno)); | |
678 | } | |
679 | return ret; | |
680 | } | |
681 | ||
096bf4c8 DDAG |
682 | /* |
683 | * Callback from shared fault handlers to ask for a page, | |
684 | * the page must be specified by a RAMBlock and an offset in that rb | |
685 | * Note: Only for use by shared fault handlers (in fault thread) | |
686 | */ | |
687 | int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, | |
688 | uint64_t client_addr, uint64_t rb_offset) | |
689 | { | |
690 | size_t pagesize = qemu_ram_pagesize(rb); | |
691 | uint64_t aligned_rbo = rb_offset & ~(pagesize - 1); | |
692 | MigrationIncomingState *mis = migration_incoming_get_current(); | |
693 | ||
694 | trace_postcopy_request_shared_page(pcfd->idstr, qemu_ram_get_idstr(rb), | |
695 | rb_offset); | |
dedfb4b2 DDAG |
696 | if (ramblock_recv_bitmap_test_byte_offset(rb, aligned_rbo)) { |
697 | trace_postcopy_request_shared_page_present(pcfd->idstr, | |
698 | qemu_ram_get_idstr(rb), rb_offset); | |
699 | return postcopy_wake_shared(pcfd, client_addr, rb); | |
700 | } | |
096bf4c8 DDAG |
701 | if (rb != mis->last_rb) { |
702 | mis->last_rb = rb; | |
703 | migrate_send_rp_req_pages(mis, qemu_ram_get_idstr(rb), | |
704 | aligned_rbo, pagesize); | |
705 | } else { | |
706 | /* Save some space */ | |
707 | migrate_send_rp_req_pages(mis, NULL, aligned_rbo, pagesize); | |
708 | } | |
709 | return 0; | |
710 | } | |
711 | ||
575b0b33 AP |
712 | static int get_mem_fault_cpu_index(uint32_t pid) |
713 | { | |
714 | CPUState *cpu_iter; | |
715 | ||
716 | CPU_FOREACH(cpu_iter) { | |
717 | if (cpu_iter->thread_id == pid) { | |
718 | trace_get_mem_fault_cpu_index(cpu_iter->cpu_index, pid); | |
719 | return cpu_iter->cpu_index; | |
720 | } | |
721 | } | |
722 | trace_get_mem_fault_cpu_index(-1, pid); | |
723 | return -1; | |
724 | } | |
725 | ||
726 | static uint32_t get_low_time_offset(PostcopyBlocktimeContext *dc) | |
727 | { | |
728 | int64_t start_time_offset = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - | |
729 | dc->start_time; | |
730 | return start_time_offset < 1 ? 1 : start_time_offset & UINT32_MAX; | |
731 | } | |
732 | ||
733 | /* | |
734 | * This function is being called when pagefault occurs. It | |
735 | * tracks down vCPU blocking time. | |
736 | * | |
737 | * @addr: faulted host virtual address | |
738 | * @ptid: faulted process thread id | |
739 | * @rb: ramblock appropriate to addr | |
740 | */ | |
741 | static void mark_postcopy_blocktime_begin(uintptr_t addr, uint32_t ptid, | |
742 | RAMBlock *rb) | |
743 | { | |
744 | int cpu, already_received; | |
745 | MigrationIncomingState *mis = migration_incoming_get_current(); | |
746 | PostcopyBlocktimeContext *dc = mis->blocktime_ctx; | |
747 | uint32_t low_time_offset; | |
748 | ||
749 | if (!dc || ptid == 0) { | |
750 | return; | |
751 | } | |
752 | cpu = get_mem_fault_cpu_index(ptid); | |
753 | if (cpu < 0) { | |
754 | return; | |
755 | } | |
756 | ||
757 | low_time_offset = get_low_time_offset(dc); | |
758 | if (dc->vcpu_addr[cpu] == 0) { | |
759 | atomic_inc(&dc->smp_cpus_down); | |
760 | } | |
761 | ||
762 | atomic_xchg(&dc->last_begin, low_time_offset); | |
763 | atomic_xchg(&dc->page_fault_vcpu_time[cpu], low_time_offset); | |
764 | atomic_xchg(&dc->vcpu_addr[cpu], addr); | |
765 | ||
766 | /* check it here, not at the begining of the function, | |
767 | * due to, check could accur early than bitmap_set in | |
768 | * qemu_ufd_copy_ioctl */ | |
769 | already_received = ramblock_recv_bitmap_test(rb, (void *)addr); | |
770 | if (already_received) { | |
771 | atomic_xchg(&dc->vcpu_addr[cpu], 0); | |
772 | atomic_xchg(&dc->page_fault_vcpu_time[cpu], 0); | |
773 | atomic_dec(&dc->smp_cpus_down); | |
774 | } | |
775 | trace_mark_postcopy_blocktime_begin(addr, dc, dc->page_fault_vcpu_time[cpu], | |
776 | cpu, already_received); | |
777 | } | |
778 | ||
779 | /* | |
780 | * This function just provide calculated blocktime per cpu and trace it. | |
781 | * Total blocktime is calculated in mark_postcopy_blocktime_end. | |
782 | * | |
783 | * | |
784 | * Assume we have 3 CPU | |
785 | * | |
786 | * S1 E1 S1 E1 | |
787 | * -----***********------------xxx***************------------------------> CPU1 | |
788 | * | |
789 | * S2 E2 | |
790 | * ------------****************xxx---------------------------------------> CPU2 | |
791 | * | |
792 | * S3 E3 | |
793 | * ------------------------****xxx********-------------------------------> CPU3 | |
794 | * | |
795 | * We have sequence S1,S2,E1,S3,S1,E2,E3,E1 | |
796 | * S2,E1 - doesn't match condition due to sequence S1,S2,E1 doesn't include CPU3 | |
797 | * S3,S1,E2 - sequence includes all CPUs, in this case overlap will be S1,E2 - | |
798 | * it's a part of total blocktime. | |
799 | * S1 - here is last_begin | |
800 | * Legend of the picture is following: | |
801 | * * - means blocktime per vCPU | |
802 | * x - means overlapped blocktime (total blocktime) | |
803 | * | |
804 | * @addr: host virtual address | |
805 | */ | |
806 | static void mark_postcopy_blocktime_end(uintptr_t addr) | |
807 | { | |
808 | MigrationIncomingState *mis = migration_incoming_get_current(); | |
809 | PostcopyBlocktimeContext *dc = mis->blocktime_ctx; | |
810 | int i, affected_cpu = 0; | |
811 | bool vcpu_total_blocktime = false; | |
812 | uint32_t read_vcpu_time, low_time_offset; | |
813 | ||
814 | if (!dc) { | |
815 | return; | |
816 | } | |
817 | ||
818 | low_time_offset = get_low_time_offset(dc); | |
819 | /* lookup cpu, to clear it, | |
820 | * that algorithm looks straighforward, but it's not | |
821 | * optimal, more optimal algorithm is keeping tree or hash | |
822 | * where key is address value is a list of */ | |
823 | for (i = 0; i < smp_cpus; i++) { | |
824 | uint32_t vcpu_blocktime = 0; | |
825 | ||
826 | read_vcpu_time = atomic_fetch_add(&dc->page_fault_vcpu_time[i], 0); | |
827 | if (atomic_fetch_add(&dc->vcpu_addr[i], 0) != addr || | |
828 | read_vcpu_time == 0) { | |
829 | continue; | |
830 | } | |
831 | atomic_xchg(&dc->vcpu_addr[i], 0); | |
832 | vcpu_blocktime = low_time_offset - read_vcpu_time; | |
833 | affected_cpu += 1; | |
834 | /* we need to know is that mark_postcopy_end was due to | |
835 | * faulted page, another possible case it's prefetched | |
836 | * page and in that case we shouldn't be here */ | |
837 | if (!vcpu_total_blocktime && | |
838 | atomic_fetch_add(&dc->smp_cpus_down, 0) == smp_cpus) { | |
839 | vcpu_total_blocktime = true; | |
840 | } | |
841 | /* continue cycle, due to one page could affect several vCPUs */ | |
842 | dc->vcpu_blocktime[i] += vcpu_blocktime; | |
843 | } | |
844 | ||
845 | atomic_sub(&dc->smp_cpus_down, affected_cpu); | |
846 | if (vcpu_total_blocktime) { | |
847 | dc->total_blocktime += low_time_offset - atomic_fetch_add( | |
848 | &dc->last_begin, 0); | |
849 | } | |
850 | trace_mark_postcopy_blocktime_end(addr, dc, dc->total_blocktime, | |
851 | affected_cpu); | |
852 | } | |
853 | ||
3a7804c3 PX |
854 | static bool postcopy_pause_fault_thread(MigrationIncomingState *mis) |
855 | { | |
856 | trace_postcopy_pause_fault_thread(); | |
857 | ||
858 | qemu_sem_wait(&mis->postcopy_pause_sem_fault); | |
859 | ||
860 | trace_postcopy_pause_fault_thread_continued(); | |
861 | ||
862 | return true; | |
863 | } | |
864 | ||
f0a227ad DDAG |
865 | /* |
866 | * Handle faults detected by the USERFAULT markings | |
867 | */ | |
868 | static void *postcopy_ram_fault_thread(void *opaque) | |
869 | { | |
870 | MigrationIncomingState *mis = opaque; | |
c4faeed2 DDAG |
871 | struct uffd_msg msg; |
872 | int ret; | |
00fa4fc8 | 873 | size_t index; |
c4faeed2 | 874 | RAMBlock *rb = NULL; |
f0a227ad | 875 | |
c4faeed2 | 876 | trace_postcopy_ram_fault_thread_entry(); |
74637e6f | 877 | rcu_register_thread(); |
096bf4c8 | 878 | mis->last_rb = NULL; /* last RAMBlock we sent part of */ |
f0a227ad | 879 | qemu_sem_post(&mis->fault_thread_sem); |
f0a227ad | 880 | |
00fa4fc8 DDAG |
881 | struct pollfd *pfd; |
882 | size_t pfd_len = 2 + mis->postcopy_remote_fds->len; | |
883 | ||
884 | pfd = g_new0(struct pollfd, pfd_len); | |
885 | ||
886 | pfd[0].fd = mis->userfault_fd; | |
887 | pfd[0].events = POLLIN; | |
888 | pfd[1].fd = mis->userfault_event_fd; | |
889 | pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */ | |
890 | trace_postcopy_ram_fault_thread_fds_core(pfd[0].fd, pfd[1].fd); | |
891 | for (index = 0; index < mis->postcopy_remote_fds->len; index++) { | |
892 | struct PostCopyFD *pcfd = &g_array_index(mis->postcopy_remote_fds, | |
893 | struct PostCopyFD, index); | |
894 | pfd[2 + index].fd = pcfd->fd; | |
895 | pfd[2 + index].events = POLLIN; | |
896 | trace_postcopy_ram_fault_thread_fds_extra(2 + index, pcfd->idstr, | |
897 | pcfd->fd); | |
898 | } | |
899 | ||
c4faeed2 DDAG |
900 | while (true) { |
901 | ram_addr_t rb_offset; | |
00fa4fc8 | 902 | int poll_result; |
c4faeed2 DDAG |
903 | |
904 | /* | |
905 | * We're mainly waiting for the kernel to give us a faulting HVA, | |
906 | * however we can be told to quit via userfault_quit_fd which is | |
907 | * an eventfd | |
908 | */ | |
00fa4fc8 DDAG |
909 | |
910 | poll_result = poll(pfd, pfd_len, -1 /* Wait forever */); | |
911 | if (poll_result == -1) { | |
c4faeed2 DDAG |
912 | error_report("%s: userfault poll: %s", __func__, strerror(errno)); |
913 | break; | |
914 | } | |
915 | ||
3a7804c3 PX |
916 | if (!mis->to_src_file) { |
917 | /* | |
918 | * Possibly someone tells us that the return path is | |
919 | * broken already using the event. We should hold until | |
920 | * the channel is rebuilt. | |
921 | */ | |
922 | if (postcopy_pause_fault_thread(mis)) { | |
923 | mis->last_rb = NULL; | |
924 | /* Continue to read the userfaultfd */ | |
925 | } else { | |
926 | error_report("%s: paused but don't allow to continue", | |
927 | __func__); | |
928 | break; | |
929 | } | |
930 | } | |
931 | ||
c4faeed2 | 932 | if (pfd[1].revents) { |
64f615fe PX |
933 | uint64_t tmp64 = 0; |
934 | ||
935 | /* Consume the signal */ | |
936 | if (read(mis->userfault_event_fd, &tmp64, 8) != 8) { | |
937 | /* Nothing obviously nicer than posting this error. */ | |
938 | error_report("%s: read() failed", __func__); | |
939 | } | |
940 | ||
941 | if (atomic_read(&mis->fault_thread_quit)) { | |
942 | trace_postcopy_ram_fault_thread_quit(); | |
943 | break; | |
944 | } | |
c4faeed2 DDAG |
945 | } |
946 | ||
00fa4fc8 DDAG |
947 | if (pfd[0].revents) { |
948 | poll_result--; | |
949 | ret = read(mis->userfault_fd, &msg, sizeof(msg)); | |
950 | if (ret != sizeof(msg)) { | |
951 | if (errno == EAGAIN) { | |
952 | /* | |
953 | * if a wake up happens on the other thread just after | |
954 | * the poll, there is nothing to read. | |
955 | */ | |
956 | continue; | |
957 | } | |
958 | if (ret < 0) { | |
959 | error_report("%s: Failed to read full userfault " | |
960 | "message: %s", | |
961 | __func__, strerror(errno)); | |
962 | break; | |
963 | } else { | |
964 | error_report("%s: Read %d bytes from userfaultfd " | |
965 | "expected %zd", | |
966 | __func__, ret, sizeof(msg)); | |
967 | break; /* Lost alignment, don't know what we'd read next */ | |
968 | } | |
c4faeed2 | 969 | } |
00fa4fc8 DDAG |
970 | if (msg.event != UFFD_EVENT_PAGEFAULT) { |
971 | error_report("%s: Read unexpected event %ud from userfaultfd", | |
972 | __func__, msg.event); | |
973 | continue; /* It's not a page fault, shouldn't happen */ | |
c4faeed2 | 974 | } |
c4faeed2 | 975 | |
00fa4fc8 DDAG |
976 | rb = qemu_ram_block_from_host( |
977 | (void *)(uintptr_t)msg.arg.pagefault.address, | |
978 | true, &rb_offset); | |
979 | if (!rb) { | |
980 | error_report("postcopy_ram_fault_thread: Fault outside guest: %" | |
981 | PRIx64, (uint64_t)msg.arg.pagefault.address); | |
982 | break; | |
983 | } | |
c4faeed2 | 984 | |
00fa4fc8 DDAG |
985 | rb_offset &= ~(qemu_ram_pagesize(rb) - 1); |
986 | trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address, | |
c4faeed2 | 987 | qemu_ram_get_idstr(rb), |
575b0b33 AP |
988 | rb_offset, |
989 | msg.arg.pagefault.feat.ptid); | |
990 | mark_postcopy_blocktime_begin( | |
991 | (uintptr_t)(msg.arg.pagefault.address), | |
992 | msg.arg.pagefault.feat.ptid, rb); | |
993 | ||
3a7804c3 | 994 | retry: |
00fa4fc8 DDAG |
995 | /* |
996 | * Send the request to the source - we want to request one | |
997 | * of our host page sizes (which is >= TPS) | |
998 | */ | |
096bf4c8 DDAG |
999 | if (rb != mis->last_rb) { |
1000 | mis->last_rb = rb; | |
3a7804c3 PX |
1001 | ret = migrate_send_rp_req_pages(mis, |
1002 | qemu_ram_get_idstr(rb), | |
1003 | rb_offset, | |
1004 | qemu_ram_pagesize(rb)); | |
00fa4fc8 DDAG |
1005 | } else { |
1006 | /* Save some space */ | |
3a7804c3 PX |
1007 | ret = migrate_send_rp_req_pages(mis, |
1008 | NULL, | |
1009 | rb_offset, | |
1010 | qemu_ram_pagesize(rb)); | |
1011 | } | |
1012 | ||
1013 | if (ret) { | |
1014 | /* May be network failure, try to wait for recovery */ | |
1015 | if (ret == -EIO && postcopy_pause_fault_thread(mis)) { | |
1016 | /* We got reconnected somehow, try to continue */ | |
1017 | mis->last_rb = NULL; | |
1018 | goto retry; | |
1019 | } else { | |
1020 | /* This is a unavoidable fault */ | |
1021 | error_report("%s: migrate_send_rp_req_pages() get %d", | |
1022 | __func__, ret); | |
1023 | break; | |
1024 | } | |
00fa4fc8 DDAG |
1025 | } |
1026 | } | |
c4faeed2 | 1027 | |
00fa4fc8 DDAG |
1028 | /* Now handle any requests from external processes on shared memory */ |
1029 | /* TODO: May need to handle devices deregistering during postcopy */ | |
1030 | for (index = 2; index < pfd_len && poll_result; index++) { | |
1031 | if (pfd[index].revents) { | |
1032 | struct PostCopyFD *pcfd = | |
1033 | &g_array_index(mis->postcopy_remote_fds, | |
1034 | struct PostCopyFD, index - 2); | |
1035 | ||
1036 | poll_result--; | |
1037 | if (pfd[index].revents & POLLERR) { | |
1038 | error_report("%s: POLLERR on poll %zd fd=%d", | |
1039 | __func__, index, pcfd->fd); | |
1040 | pfd[index].events = 0; | |
1041 | continue; | |
1042 | } | |
1043 | ||
1044 | ret = read(pcfd->fd, &msg, sizeof(msg)); | |
1045 | if (ret != sizeof(msg)) { | |
1046 | if (errno == EAGAIN) { | |
1047 | /* | |
1048 | * if a wake up happens on the other thread just after | |
1049 | * the poll, there is nothing to read. | |
1050 | */ | |
1051 | continue; | |
1052 | } | |
1053 | if (ret < 0) { | |
1054 | error_report("%s: Failed to read full userfault " | |
1055 | "message: %s (shared) revents=%d", | |
1056 | __func__, strerror(errno), | |
1057 | pfd[index].revents); | |
1058 | /*TODO: Could just disable this sharer */ | |
1059 | break; | |
1060 | } else { | |
1061 | error_report("%s: Read %d bytes from userfaultfd " | |
1062 | "expected %zd (shared)", | |
1063 | __func__, ret, sizeof(msg)); | |
1064 | /*TODO: Could just disable this sharer */ | |
1065 | break; /*Lost alignment,don't know what we'd read next*/ | |
1066 | } | |
1067 | } | |
1068 | if (msg.event != UFFD_EVENT_PAGEFAULT) { | |
1069 | error_report("%s: Read unexpected event %ud " | |
1070 | "from userfaultfd (shared)", | |
1071 | __func__, msg.event); | |
1072 | continue; /* It's not a page fault, shouldn't happen */ | |
1073 | } | |
1074 | /* Call the device handler registered with us */ | |
1075 | ret = pcfd->handler(pcfd, &msg); | |
1076 | if (ret) { | |
1077 | error_report("%s: Failed to resolve shared fault on %zd/%s", | |
1078 | __func__, index, pcfd->idstr); | |
1079 | /* TODO: Fail? Disable this sharer? */ | |
1080 | } | |
1081 | } | |
c4faeed2 DDAG |
1082 | } |
1083 | } | |
74637e6f | 1084 | rcu_unregister_thread(); |
c4faeed2 | 1085 | trace_postcopy_ram_fault_thread_exit(); |
fc6008f3 | 1086 | g_free(pfd); |
f0a227ad DDAG |
1087 | return NULL; |
1088 | } | |
1089 | ||
1090 | int postcopy_ram_enable_notify(MigrationIncomingState *mis) | |
1091 | { | |
c4faeed2 DDAG |
1092 | /* Open the fd for the kernel to give us userfaults */ |
1093 | mis->userfault_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); | |
1094 | if (mis->userfault_fd == -1) { | |
1095 | error_report("%s: Failed to open userfault fd: %s", __func__, | |
1096 | strerror(errno)); | |
1097 | return -1; | |
1098 | } | |
1099 | ||
1100 | /* | |
1101 | * Although the host check already tested the API, we need to | |
1102 | * do the check again as an ABI handshake on the new fd. | |
1103 | */ | |
54ae0886 | 1104 | if (!ufd_check_and_apply(mis->userfault_fd, mis)) { |
c4faeed2 DDAG |
1105 | return -1; |
1106 | } | |
1107 | ||
1108 | /* Now an eventfd we use to tell the fault-thread to quit */ | |
64f615fe PX |
1109 | mis->userfault_event_fd = eventfd(0, EFD_CLOEXEC); |
1110 | if (mis->userfault_event_fd == -1) { | |
1111 | error_report("%s: Opening userfault_event_fd: %s", __func__, | |
c4faeed2 DDAG |
1112 | strerror(errno)); |
1113 | close(mis->userfault_fd); | |
1114 | return -1; | |
1115 | } | |
1116 | ||
f0a227ad DDAG |
1117 | qemu_sem_init(&mis->fault_thread_sem, 0); |
1118 | qemu_thread_create(&mis->fault_thread, "postcopy/fault", | |
1119 | postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE); | |
1120 | qemu_sem_wait(&mis->fault_thread_sem); | |
1121 | qemu_sem_destroy(&mis->fault_thread_sem); | |
c4faeed2 | 1122 | mis->have_fault_thread = true; |
f0a227ad DDAG |
1123 | |
1124 | /* Mark so that we get notified of accesses to unwritten areas */ | |
fbd162e6 | 1125 | if (foreach_not_ignored_block(ram_block_enable_notify, mis)) { |
91b02dc7 | 1126 | error_report("ram_block_enable_notify failed"); |
f0a227ad DDAG |
1127 | return -1; |
1128 | } | |
1129 | ||
371ff5a3 DDAG |
1130 | /* |
1131 | * Ballooning can mark pages as absent while we're postcopying | |
1132 | * that would cause false userfaults. | |
1133 | */ | |
154304cd | 1134 | postcopy_balloon_inhibit(true); |
371ff5a3 | 1135 | |
c4faeed2 DDAG |
1136 | trace_postcopy_ram_enable_notify(); |
1137 | ||
f0a227ad DDAG |
1138 | return 0; |
1139 | } | |
1140 | ||
727b9d7e | 1141 | static int qemu_ufd_copy_ioctl(int userfault_fd, void *host_addr, |
f9494614 | 1142 | void *from_addr, uint64_t pagesize, RAMBlock *rb) |
727b9d7e | 1143 | { |
f9494614 | 1144 | int ret; |
727b9d7e AP |
1145 | if (from_addr) { |
1146 | struct uffdio_copy copy_struct; | |
1147 | copy_struct.dst = (uint64_t)(uintptr_t)host_addr; | |
1148 | copy_struct.src = (uint64_t)(uintptr_t)from_addr; | |
1149 | copy_struct.len = pagesize; | |
1150 | copy_struct.mode = 0; | |
f9494614 | 1151 | ret = ioctl(userfault_fd, UFFDIO_COPY, ©_struct); |
727b9d7e AP |
1152 | } else { |
1153 | struct uffdio_zeropage zero_struct; | |
1154 | zero_struct.range.start = (uint64_t)(uintptr_t)host_addr; | |
1155 | zero_struct.range.len = pagesize; | |
1156 | zero_struct.mode = 0; | |
f9494614 AP |
1157 | ret = ioctl(userfault_fd, UFFDIO_ZEROPAGE, &zero_struct); |
1158 | } | |
1159 | if (!ret) { | |
1160 | ramblock_recv_bitmap_set_range(rb, host_addr, | |
1161 | pagesize / qemu_target_page_size()); | |
575b0b33 AP |
1162 | mark_postcopy_blocktime_end((uintptr_t)host_addr); |
1163 | ||
727b9d7e | 1164 | } |
f9494614 | 1165 | return ret; |
727b9d7e AP |
1166 | } |
1167 | ||
d488b349 DDAG |
1168 | int postcopy_notify_shared_wake(RAMBlock *rb, uint64_t offset) |
1169 | { | |
1170 | int i; | |
1171 | MigrationIncomingState *mis = migration_incoming_get_current(); | |
1172 | GArray *pcrfds = mis->postcopy_remote_fds; | |
1173 | ||
1174 | for (i = 0; i < pcrfds->len; i++) { | |
1175 | struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i); | |
1176 | int ret = cur->waker(cur, rb, offset); | |
1177 | if (ret) { | |
1178 | return ret; | |
1179 | } | |
1180 | } | |
1181 | return 0; | |
1182 | } | |
1183 | ||
696ed9a9 DDAG |
1184 | /* |
1185 | * Place a host page (from) at (host) atomically | |
1186 | * returns 0 on success | |
1187 | */ | |
df9ff5e1 | 1188 | int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, |
8be4620b | 1189 | RAMBlock *rb) |
696ed9a9 | 1190 | { |
8be4620b | 1191 | size_t pagesize = qemu_ram_pagesize(rb); |
696ed9a9 | 1192 | |
696ed9a9 DDAG |
1193 | /* copy also acks to the kernel waking the stalled thread up |
1194 | * TODO: We can inhibit that ack and only do it if it was requested | |
1195 | * which would be slightly cheaper, but we'd have to be careful | |
1196 | * of the order of updating our page state. | |
1197 | */ | |
f9494614 | 1198 | if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, from, pagesize, rb)) { |
696ed9a9 | 1199 | int e = errno; |
df9ff5e1 DDAG |
1200 | error_report("%s: %s copy host: %p from: %p (size: %zd)", |
1201 | __func__, strerror(e), host, from, pagesize); | |
696ed9a9 DDAG |
1202 | |
1203 | return -e; | |
1204 | } | |
1205 | ||
1206 | trace_postcopy_place_page(host); | |
dedfb4b2 DDAG |
1207 | return postcopy_notify_shared_wake(rb, |
1208 | qemu_ram_block_host_offset(rb, host)); | |
696ed9a9 DDAG |
1209 | } |
1210 | ||
1211 | /* | |
1212 | * Place a zero page at (host) atomically | |
1213 | * returns 0 on success | |
1214 | */ | |
df9ff5e1 | 1215 | int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, |
8be4620b | 1216 | RAMBlock *rb) |
696ed9a9 | 1217 | { |
2ce16640 | 1218 | size_t pagesize = qemu_ram_pagesize(rb); |
df9ff5e1 | 1219 | trace_postcopy_place_page_zero(host); |
696ed9a9 | 1220 | |
2ce16640 DDAG |
1221 | /* Normal RAMBlocks can zero a page using UFFDIO_ZEROPAGE |
1222 | * but it's not available for everything (e.g. hugetlbpages) | |
1223 | */ | |
1224 | if (qemu_ram_is_uf_zeroable(rb)) { | |
1225 | if (qemu_ufd_copy_ioctl(mis->userfault_fd, host, NULL, pagesize, rb)) { | |
df9ff5e1 DDAG |
1226 | int e = errno; |
1227 | error_report("%s: %s zero host: %p", | |
1228 | __func__, strerror(e), host); | |
696ed9a9 | 1229 | |
df9ff5e1 DDAG |
1230 | return -e; |
1231 | } | |
dedfb4b2 DDAG |
1232 | return postcopy_notify_shared_wake(rb, |
1233 | qemu_ram_block_host_offset(rb, | |
1234 | host)); | |
df9ff5e1 | 1235 | } else { |
41d84210 DDAG |
1236 | /* The kernel can't use UFFDIO_ZEROPAGE for hugepages */ |
1237 | if (!mis->postcopy_tmp_zero_page) { | |
1238 | mis->postcopy_tmp_zero_page = mmap(NULL, mis->largest_page_size, | |
1239 | PROT_READ | PROT_WRITE, | |
1240 | MAP_PRIVATE | MAP_ANONYMOUS, | |
1241 | -1, 0); | |
1242 | if (mis->postcopy_tmp_zero_page == MAP_FAILED) { | |
1243 | int e = errno; | |
1244 | mis->postcopy_tmp_zero_page = NULL; | |
1245 | error_report("%s: %s mapping large zero page", | |
1246 | __func__, strerror(e)); | |
1247 | return -e; | |
1248 | } | |
1249 | memset(mis->postcopy_tmp_zero_page, '\0', mis->largest_page_size); | |
1250 | } | |
1251 | return postcopy_place_page(mis, host, mis->postcopy_tmp_zero_page, | |
8be4620b | 1252 | rb); |
696ed9a9 | 1253 | } |
696ed9a9 DDAG |
1254 | } |
1255 | ||
1256 | /* | |
1257 | * Returns a target page of memory that can be mapped at a later point in time | |
1258 | * using postcopy_place_page | |
1259 | * The same address is used repeatedly, postcopy_place_page just takes the | |
1260 | * backing page away. | |
1261 | * Returns: Pointer to allocated page | |
1262 | * | |
1263 | */ | |
1264 | void *postcopy_get_tmp_page(MigrationIncomingState *mis) | |
1265 | { | |
1266 | if (!mis->postcopy_tmp_page) { | |
df9ff5e1 | 1267 | mis->postcopy_tmp_page = mmap(NULL, mis->largest_page_size, |
696ed9a9 DDAG |
1268 | PROT_READ | PROT_WRITE, MAP_PRIVATE | |
1269 | MAP_ANONYMOUS, -1, 0); | |
0e8b3cdf EY |
1270 | if (mis->postcopy_tmp_page == MAP_FAILED) { |
1271 | mis->postcopy_tmp_page = NULL; | |
696ed9a9 DDAG |
1272 | error_report("%s: %s", __func__, strerror(errno)); |
1273 | return NULL; | |
1274 | } | |
1275 | } | |
1276 | ||
1277 | return mis->postcopy_tmp_page; | |
1278 | } | |
1279 | ||
eb59db53 DDAG |
1280 | #else |
1281 | /* No target OS support, stubs just fail */ | |
65ace060 AP |
1282 | void fill_destination_postcopy_migration_info(MigrationInfo *info) |
1283 | { | |
1284 | } | |
1285 | ||
d7651f15 | 1286 | bool postcopy_ram_supported_by_host(MigrationIncomingState *mis) |
eb59db53 DDAG |
1287 | { |
1288 | error_report("%s: No OS support", __func__); | |
1289 | return false; | |
1290 | } | |
1291 | ||
c136180c | 1292 | int postcopy_ram_incoming_init(MigrationIncomingState *mis) |
1caddf8a DDAG |
1293 | { |
1294 | error_report("postcopy_ram_incoming_init: No OS support"); | |
1295 | return -1; | |
1296 | } | |
1297 | ||
1298 | int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) | |
1299 | { | |
1300 | assert(0); | |
1301 | return -1; | |
1302 | } | |
1303 | ||
f9527107 DDAG |
1304 | int postcopy_ram_prepare_discard(MigrationIncomingState *mis) |
1305 | { | |
1306 | assert(0); | |
1307 | return -1; | |
1308 | } | |
1309 | ||
c188c539 MT |
1310 | int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, |
1311 | uint64_t client_addr, uint64_t rb_offset) | |
1312 | { | |
1313 | assert(0); | |
1314 | return -1; | |
1315 | } | |
1316 | ||
f0a227ad DDAG |
1317 | int postcopy_ram_enable_notify(MigrationIncomingState *mis) |
1318 | { | |
1319 | assert(0); | |
1320 | return -1; | |
1321 | } | |
696ed9a9 | 1322 | |
df9ff5e1 | 1323 | int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, |
8be4620b | 1324 | RAMBlock *rb) |
696ed9a9 DDAG |
1325 | { |
1326 | assert(0); | |
1327 | return -1; | |
1328 | } | |
1329 | ||
df9ff5e1 | 1330 | int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, |
8be4620b | 1331 | RAMBlock *rb) |
696ed9a9 DDAG |
1332 | { |
1333 | assert(0); | |
1334 | return -1; | |
1335 | } | |
1336 | ||
1337 | void *postcopy_get_tmp_page(MigrationIncomingState *mis) | |
1338 | { | |
1339 | assert(0); | |
1340 | return NULL; | |
1341 | } | |
1342 | ||
5efc3564 DDAG |
1343 | int postcopy_wake_shared(struct PostCopyFD *pcfd, |
1344 | uint64_t client_addr, | |
1345 | RAMBlock *rb) | |
1346 | { | |
1347 | assert(0); | |
1348 | return -1; | |
1349 | } | |
eb59db53 DDAG |
1350 | #endif |
1351 | ||
e0b266f0 DDAG |
1352 | /* ------------------------------------------------------------------------- */ |
1353 | ||
9ab7ef9b PX |
1354 | void postcopy_fault_thread_notify(MigrationIncomingState *mis) |
1355 | { | |
1356 | uint64_t tmp64 = 1; | |
1357 | ||
1358 | /* | |
1359 | * Wakeup the fault_thread. It's an eventfd that should currently | |
1360 | * be at 0, we're going to increment it to 1 | |
1361 | */ | |
1362 | if (write(mis->userfault_event_fd, &tmp64, 8) != 8) { | |
1363 | /* Not much we can do here, but may as well report it */ | |
1364 | error_report("%s: incrementing failed: %s", __func__, | |
1365 | strerror(errno)); | |
1366 | } | |
1367 | } | |
1368 | ||
e0b266f0 DDAG |
1369 | /** |
1370 | * postcopy_discard_send_init: Called at the start of each RAMBlock before | |
1371 | * asking to discard individual ranges. | |
1372 | * | |
1373 | * @ms: The current migration state. | |
1374 | * @offset: the bitmap offset of the named RAMBlock in the migration | |
1375 | * bitmap. | |
1376 | * @name: RAMBlock that discards will operate on. | |
1377 | * | |
1378 | * returns: a new PDS. | |
1379 | */ | |
1380 | PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms, | |
e0b266f0 DDAG |
1381 | const char *name) |
1382 | { | |
1383 | PostcopyDiscardState *res = g_malloc0(sizeof(PostcopyDiscardState)); | |
1384 | ||
1385 | if (res) { | |
1386 | res->ramblock_name = name; | |
e0b266f0 DDAG |
1387 | } |
1388 | ||
1389 | return res; | |
1390 | } | |
1391 | ||
1392 | /** | |
1393 | * postcopy_discard_send_range: Called by the bitmap code for each chunk to | |
1394 | * discard. May send a discard message, may just leave it queued to | |
1395 | * be sent later. | |
1396 | * | |
1397 | * @ms: Current migration state. | |
1398 | * @pds: Structure initialised by postcopy_discard_send_init(). | |
1399 | * @start,@length: a range of pages in the migration bitmap in the | |
1400 | * RAM block passed to postcopy_discard_send_init() (length=1 is one page) | |
1401 | */ | |
1402 | void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds, | |
1403 | unsigned long start, unsigned long length) | |
1404 | { | |
20afaed9 | 1405 | size_t tp_size = qemu_target_page_size(); |
e0b266f0 | 1406 | /* Convert to byte offsets within the RAM block */ |
6b6712ef | 1407 | pds->start_list[pds->cur_entry] = start * tp_size; |
20afaed9 | 1408 | pds->length_list[pds->cur_entry] = length * tp_size; |
e0b266f0 DDAG |
1409 | trace_postcopy_discard_send_range(pds->ramblock_name, start, length); |
1410 | pds->cur_entry++; | |
1411 | pds->nsentwords++; | |
1412 | ||
1413 | if (pds->cur_entry == MAX_DISCARDS_PER_COMMAND) { | |
1414 | /* Full set, ship it! */ | |
89a02a9f HZ |
1415 | qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, |
1416 | pds->ramblock_name, | |
e0b266f0 DDAG |
1417 | pds->cur_entry, |
1418 | pds->start_list, | |
1419 | pds->length_list); | |
1420 | pds->nsentcmds++; | |
1421 | pds->cur_entry = 0; | |
1422 | } | |
1423 | } | |
1424 | ||
1425 | /** | |
1426 | * postcopy_discard_send_finish: Called at the end of each RAMBlock by the | |
1427 | * bitmap code. Sends any outstanding discard messages, frees the PDS | |
1428 | * | |
1429 | * @ms: Current migration state. | |
1430 | * @pds: Structure initialised by postcopy_discard_send_init(). | |
1431 | */ | |
1432 | void postcopy_discard_send_finish(MigrationState *ms, PostcopyDiscardState *pds) | |
1433 | { | |
1434 | /* Anything unsent? */ | |
1435 | if (pds->cur_entry) { | |
89a02a9f HZ |
1436 | qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, |
1437 | pds->ramblock_name, | |
e0b266f0 DDAG |
1438 | pds->cur_entry, |
1439 | pds->start_list, | |
1440 | pds->length_list); | |
1441 | pds->nsentcmds++; | |
1442 | } | |
1443 | ||
1444 | trace_postcopy_discard_send_finish(pds->ramblock_name, pds->nsentwords, | |
1445 | pds->nsentcmds); | |
1446 | ||
1447 | g_free(pds); | |
1448 | } | |
bac3b212 JQ |
1449 | |
1450 | /* | |
1451 | * Current state of incoming postcopy; note this is not part of | |
1452 | * MigrationIncomingState since it's state is used during cleanup | |
1453 | * at the end as MIS is being freed. | |
1454 | */ | |
1455 | static PostcopyState incoming_postcopy_state; | |
1456 | ||
1457 | PostcopyState postcopy_state_get(void) | |
1458 | { | |
1459 | return atomic_mb_read(&incoming_postcopy_state); | |
1460 | } | |
1461 | ||
1462 | /* Set the state and return the old state */ | |
1463 | PostcopyState postcopy_state_set(PostcopyState new_state) | |
1464 | { | |
1465 | return atomic_xchg(&incoming_postcopy_state, new_state); | |
1466 | } | |
00fa4fc8 DDAG |
1467 | |
1468 | /* Register a handler for external shared memory postcopy | |
1469 | * called on the destination. | |
1470 | */ | |
1471 | void postcopy_register_shared_ufd(struct PostCopyFD *pcfd) | |
1472 | { | |
1473 | MigrationIncomingState *mis = migration_incoming_get_current(); | |
1474 | ||
1475 | mis->postcopy_remote_fds = g_array_append_val(mis->postcopy_remote_fds, | |
1476 | *pcfd); | |
1477 | } | |
1478 | ||
1479 | /* Unregister a handler for external shared memory postcopy | |
1480 | */ | |
1481 | void postcopy_unregister_shared_ufd(struct PostCopyFD *pcfd) | |
1482 | { | |
1483 | guint i; | |
1484 | MigrationIncomingState *mis = migration_incoming_get_current(); | |
1485 | GArray *pcrfds = mis->postcopy_remote_fds; | |
1486 | ||
1487 | for (i = 0; i < pcrfds->len; i++) { | |
1488 | struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i); | |
1489 | if (cur->fd == pcfd->fd) { | |
1490 | mis->postcopy_remote_fds = g_array_remove_index(pcrfds, i); | |
1491 | return; | |
1492 | } | |
1493 | } | |
1494 | } |