]>
Commit | Line | Data |
---|---|---|
a76bab49 AL |
1 | /* |
2 | * QEMU aio implementation | |
3 | * | |
4 | * Copyright IBM, Corp. 2008 | |
5 | * | |
6 | * Authors: | |
7 | * Anthony Liguori <aliguori@us.ibm.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
10 | * the COPYING file in the top-level directory. | |
11 | * | |
6b620ca3 PB |
12 | * Contributions after 2012-01-13 are licensed under the terms of the |
13 | * GNU GPL, version 2 or (at your option) any later version. | |
a76bab49 AL |
14 | */ |
15 | ||
d38ea87a | 16 | #include "qemu/osdep.h" |
a76bab49 | 17 | #include "qemu-common.h" |
737e150e | 18 | #include "block/block.h" |
1de7afc9 PB |
19 | #include "qemu/queue.h" |
20 | #include "qemu/sockets.h" | |
4a1cba38 SH |
21 | #include "qemu/cutils.h" |
22 | #include "trace.h" | |
147dfab7 | 23 | #ifdef CONFIG_EPOLL_CREATE1 |
fbe3fc5c FZ |
24 | #include <sys/epoll.h> |
25 | #endif | |
a76bab49 | 26 | |
a76bab49 AL |
27 | struct AioHandler |
28 | { | |
cd9ba1eb | 29 | GPollFD pfd; |
a76bab49 AL |
30 | IOHandler *io_read; |
31 | IOHandler *io_write; | |
4a1cba38 | 32 | AioPollFn *io_poll; |
684e508c SH |
33 | IOHandler *io_poll_begin; |
34 | IOHandler *io_poll_end; | |
a76bab49 AL |
35 | int deleted; |
36 | void *opaque; | |
dca21ef2 | 37 | bool is_external; |
72cf2d4f | 38 | QLIST_ENTRY(AioHandler) node; |
a76bab49 AL |
39 | }; |
40 | ||
147dfab7 | 41 | #ifdef CONFIG_EPOLL_CREATE1 |
fbe3fc5c FZ |
42 | |
43 | /* The fd number threashold to switch to epoll */ | |
44 | #define EPOLL_ENABLE_THRESHOLD 64 | |
45 | ||
46 | static void aio_epoll_disable(AioContext *ctx) | |
47 | { | |
48 | ctx->epoll_available = false; | |
49 | if (!ctx->epoll_enabled) { | |
50 | return; | |
51 | } | |
52 | ctx->epoll_enabled = false; | |
53 | close(ctx->epollfd); | |
54 | } | |
55 | ||
56 | static inline int epoll_events_from_pfd(int pfd_events) | |
57 | { | |
58 | return (pfd_events & G_IO_IN ? EPOLLIN : 0) | | |
59 | (pfd_events & G_IO_OUT ? EPOLLOUT : 0) | | |
60 | (pfd_events & G_IO_HUP ? EPOLLHUP : 0) | | |
61 | (pfd_events & G_IO_ERR ? EPOLLERR : 0); | |
62 | } | |
63 | ||
64 | static bool aio_epoll_try_enable(AioContext *ctx) | |
65 | { | |
66 | AioHandler *node; | |
67 | struct epoll_event event; | |
68 | ||
69 | QLIST_FOREACH(node, &ctx->aio_handlers, node) { | |
70 | int r; | |
71 | if (node->deleted || !node->pfd.events) { | |
72 | continue; | |
73 | } | |
74 | event.events = epoll_events_from_pfd(node->pfd.events); | |
75 | event.data.ptr = node; | |
76 | r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event); | |
77 | if (r) { | |
78 | return false; | |
79 | } | |
80 | } | |
81 | ctx->epoll_enabled = true; | |
82 | return true; | |
83 | } | |
84 | ||
85 | static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new) | |
86 | { | |
87 | struct epoll_event event; | |
88 | int r; | |
35dd66e2 | 89 | int ctl; |
fbe3fc5c FZ |
90 | |
91 | if (!ctx->epoll_enabled) { | |
92 | return; | |
93 | } | |
94 | if (!node->pfd.events) { | |
35dd66e2 | 95 | ctl = EPOLL_CTL_DEL; |
fbe3fc5c FZ |
96 | } else { |
97 | event.data.ptr = node; | |
98 | event.events = epoll_events_from_pfd(node->pfd.events); | |
35dd66e2 PB |
99 | ctl = is_new ? EPOLL_CTL_ADD : EPOLL_CTL_MOD; |
100 | } | |
101 | ||
102 | r = epoll_ctl(ctx->epollfd, ctl, node->pfd.fd, &event); | |
103 | if (r) { | |
104 | aio_epoll_disable(ctx); | |
fbe3fc5c FZ |
105 | } |
106 | } | |
107 | ||
108 | static int aio_epoll(AioContext *ctx, GPollFD *pfds, | |
109 | unsigned npfd, int64_t timeout) | |
110 | { | |
111 | AioHandler *node; | |
112 | int i, ret = 0; | |
113 | struct epoll_event events[128]; | |
114 | ||
115 | assert(npfd == 1); | |
116 | assert(pfds[0].fd == ctx->epollfd); | |
117 | if (timeout > 0) { | |
118 | ret = qemu_poll_ns(pfds, npfd, timeout); | |
119 | } | |
120 | if (timeout <= 0 || ret > 0) { | |
121 | ret = epoll_wait(ctx->epollfd, events, | |
122 | sizeof(events) / sizeof(events[0]), | |
123 | timeout); | |
124 | if (ret <= 0) { | |
125 | goto out; | |
126 | } | |
127 | for (i = 0; i < ret; i++) { | |
128 | int ev = events[i].events; | |
129 | node = events[i].data.ptr; | |
130 | node->pfd.revents = (ev & EPOLLIN ? G_IO_IN : 0) | | |
131 | (ev & EPOLLOUT ? G_IO_OUT : 0) | | |
132 | (ev & EPOLLHUP ? G_IO_HUP : 0) | | |
133 | (ev & EPOLLERR ? G_IO_ERR : 0); | |
134 | } | |
135 | } | |
136 | out: | |
137 | return ret; | |
138 | } | |
139 | ||
140 | static bool aio_epoll_enabled(AioContext *ctx) | |
141 | { | |
142 | /* Fall back to ppoll when external clients are disabled. */ | |
143 | return !aio_external_disabled(ctx) && ctx->epoll_enabled; | |
144 | } | |
145 | ||
146 | static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds, | |
147 | unsigned npfd, int64_t timeout) | |
148 | { | |
149 | if (!ctx->epoll_available) { | |
150 | return false; | |
151 | } | |
152 | if (aio_epoll_enabled(ctx)) { | |
153 | return true; | |
154 | } | |
155 | if (npfd >= EPOLL_ENABLE_THRESHOLD) { | |
156 | if (aio_epoll_try_enable(ctx)) { | |
157 | return true; | |
158 | } else { | |
159 | aio_epoll_disable(ctx); | |
160 | } | |
161 | } | |
162 | return false; | |
163 | } | |
164 | ||
165 | #else | |
166 | ||
167 | static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new) | |
168 | { | |
169 | } | |
170 | ||
171 | static int aio_epoll(AioContext *ctx, GPollFD *pfds, | |
172 | unsigned npfd, int64_t timeout) | |
173 | { | |
174 | assert(false); | |
175 | } | |
176 | ||
177 | static bool aio_epoll_enabled(AioContext *ctx) | |
178 | { | |
179 | return false; | |
180 | } | |
181 | ||
182 | static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds, | |
183 | unsigned npfd, int64_t timeout) | |
184 | { | |
185 | return false; | |
186 | } | |
187 | ||
188 | #endif | |
189 | ||
a915f4bc | 190 | static AioHandler *find_aio_handler(AioContext *ctx, int fd) |
a76bab49 AL |
191 | { |
192 | AioHandler *node; | |
193 | ||
a915f4bc | 194 | QLIST_FOREACH(node, &ctx->aio_handlers, node) { |
cd9ba1eb | 195 | if (node->pfd.fd == fd) |
79d5ca56 AG |
196 | if (!node->deleted) |
197 | return node; | |
a76bab49 AL |
198 | } |
199 | ||
200 | return NULL; | |
201 | } | |
202 | ||
a915f4bc PB |
203 | void aio_set_fd_handler(AioContext *ctx, |
204 | int fd, | |
dca21ef2 | 205 | bool is_external, |
a915f4bc PB |
206 | IOHandler *io_read, |
207 | IOHandler *io_write, | |
f6a51c84 | 208 | AioPollFn *io_poll, |
a915f4bc | 209 | void *opaque) |
a76bab49 AL |
210 | { |
211 | AioHandler *node; | |
fbe3fc5c | 212 | bool is_new = false; |
0ed39f3d | 213 | bool deleted = false; |
a76bab49 | 214 | |
a915f4bc | 215 | node = find_aio_handler(ctx, fd); |
a76bab49 AL |
216 | |
217 | /* Are we deleting the fd handler? */ | |
4a1cba38 | 218 | if (!io_read && !io_write && !io_poll) { |
36173ec5 PB |
219 | if (node == NULL) { |
220 | return; | |
221 | } | |
222 | ||
223 | g_source_remove_poll(&ctx->source, &node->pfd); | |
224 | ||
225 | /* If the lock is held, just mark the node as deleted */ | |
226 | if (ctx->walking_handlers) { | |
227 | node->deleted = 1; | |
228 | node->pfd.revents = 0; | |
229 | } else { | |
230 | /* Otherwise, delete it for real. We can't just mark it as | |
231 | * deleted because deleted nodes are only cleaned up after | |
232 | * releasing the walking_handlers lock. | |
233 | */ | |
234 | QLIST_REMOVE(node, node); | |
235 | deleted = true; | |
a76bab49 | 236 | } |
4a1cba38 SH |
237 | |
238 | if (!node->io_poll) { | |
239 | ctx->poll_disable_cnt--; | |
240 | } | |
a76bab49 AL |
241 | } else { |
242 | if (node == NULL) { | |
243 | /* Alloc and insert if it's not already there */ | |
3ba235a0 | 244 | node = g_new0(AioHandler, 1); |
cd9ba1eb | 245 | node->pfd.fd = fd; |
a915f4bc | 246 | QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node); |
e3713e00 PB |
247 | |
248 | g_source_add_poll(&ctx->source, &node->pfd); | |
fbe3fc5c | 249 | is_new = true; |
4a1cba38 SH |
250 | |
251 | ctx->poll_disable_cnt += !io_poll; | |
252 | } else { | |
253 | ctx->poll_disable_cnt += !io_poll - !node->io_poll; | |
a76bab49 | 254 | } |
4a1cba38 | 255 | |
a76bab49 AL |
256 | /* Update handler with latest information */ |
257 | node->io_read = io_read; | |
258 | node->io_write = io_write; | |
4a1cba38 | 259 | node->io_poll = io_poll; |
a76bab49 | 260 | node->opaque = opaque; |
dca21ef2 | 261 | node->is_external = is_external; |
cd9ba1eb | 262 | |
b5a01a70 SH |
263 | node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0); |
264 | node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0); | |
a76bab49 | 265 | } |
7ed2b24c | 266 | |
fbe3fc5c | 267 | aio_epoll_update(ctx, node, is_new); |
7ed2b24c | 268 | aio_notify(ctx); |
4a1cba38 | 269 | |
0ed39f3d FZ |
270 | if (deleted) { |
271 | g_free(node); | |
272 | } | |
9958c351 PB |
273 | } |
274 | ||
684e508c SH |
275 | void aio_set_fd_poll(AioContext *ctx, int fd, |
276 | IOHandler *io_poll_begin, | |
277 | IOHandler *io_poll_end) | |
278 | { | |
279 | AioHandler *node = find_aio_handler(ctx, fd); | |
280 | ||
281 | if (!node) { | |
282 | return; | |
283 | } | |
284 | ||
285 | node->io_poll_begin = io_poll_begin; | |
286 | node->io_poll_end = io_poll_end; | |
287 | } | |
288 | ||
a915f4bc PB |
289 | void aio_set_event_notifier(AioContext *ctx, |
290 | EventNotifier *notifier, | |
dca21ef2 | 291 | bool is_external, |
f6a51c84 SH |
292 | EventNotifierHandler *io_read, |
293 | AioPollFn *io_poll) | |
a76bab49 | 294 | { |
f6a51c84 SH |
295 | aio_set_fd_handler(ctx, event_notifier_get_fd(notifier), is_external, |
296 | (IOHandler *)io_read, NULL, io_poll, notifier); | |
a76bab49 AL |
297 | } |
298 | ||
684e508c SH |
299 | void aio_set_event_notifier_poll(AioContext *ctx, |
300 | EventNotifier *notifier, | |
301 | EventNotifierHandler *io_poll_begin, | |
302 | EventNotifierHandler *io_poll_end) | |
303 | { | |
304 | aio_set_fd_poll(ctx, event_notifier_get_fd(notifier), | |
305 | (IOHandler *)io_poll_begin, | |
306 | (IOHandler *)io_poll_end); | |
307 | } | |
308 | ||
309 | static void poll_set_started(AioContext *ctx, bool started) | |
310 | { | |
311 | AioHandler *node; | |
312 | ||
313 | if (started == ctx->poll_started) { | |
314 | return; | |
315 | } | |
316 | ||
317 | ctx->poll_started = started; | |
318 | ||
319 | ctx->walking_handlers++; | |
320 | QLIST_FOREACH(node, &ctx->aio_handlers, node) { | |
321 | IOHandler *fn; | |
322 | ||
323 | if (node->deleted) { | |
324 | continue; | |
325 | } | |
326 | ||
327 | if (started) { | |
328 | fn = node->io_poll_begin; | |
329 | } else { | |
330 | fn = node->io_poll_end; | |
331 | } | |
332 | ||
333 | if (fn) { | |
334 | fn(node->opaque); | |
335 | } | |
336 | } | |
337 | ctx->walking_handlers--; | |
338 | } | |
339 | ||
340 | ||
a3462c65 PB |
341 | bool aio_prepare(AioContext *ctx) |
342 | { | |
684e508c SH |
343 | /* Poll mode cannot be used with glib's event loop, disable it. */ |
344 | poll_set_started(ctx, false); | |
345 | ||
a3462c65 PB |
346 | return false; |
347 | } | |
348 | ||
cd9ba1eb PB |
349 | bool aio_pending(AioContext *ctx) |
350 | { | |
351 | AioHandler *node; | |
352 | ||
353 | QLIST_FOREACH(node, &ctx->aio_handlers, node) { | |
354 | int revents; | |
355 | ||
cd9ba1eb | 356 | revents = node->pfd.revents & node->pfd.events; |
37989ced FZ |
357 | if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read && |
358 | aio_node_check(ctx, node->is_external)) { | |
cd9ba1eb PB |
359 | return true; |
360 | } | |
37989ced FZ |
361 | if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write && |
362 | aio_node_check(ctx, node->is_external)) { | |
cd9ba1eb PB |
363 | return true; |
364 | } | |
365 | } | |
366 | ||
367 | return false; | |
368 | } | |
369 | ||
56d2c3c6 | 370 | static bool aio_dispatch_handlers(AioContext *ctx) |
a76bab49 | 371 | { |
56d2c3c6 | 372 | AioHandler *node; |
d0c8d2c0 | 373 | bool progress = false; |
7c0628b2 | 374 | |
cd9ba1eb | 375 | /* |
87f68d31 | 376 | * We have to walk very carefully in case aio_set_fd_handler is |
cd9ba1eb PB |
377 | * called while we're walking. |
378 | */ | |
56d2c3c6 | 379 | node = QLIST_FIRST(&ctx->aio_handlers); |
cd9ba1eb PB |
380 | while (node) { |
381 | AioHandler *tmp; | |
382 | int revents; | |
383 | ||
384 | ctx->walking_handlers++; | |
385 | ||
386 | revents = node->pfd.revents & node->pfd.events; | |
387 | node->pfd.revents = 0; | |
388 | ||
d0c8d2c0 SH |
389 | if (!node->deleted && |
390 | (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) && | |
37989ced | 391 | aio_node_check(ctx, node->is_external) && |
d0c8d2c0 | 392 | node->io_read) { |
cd9ba1eb | 393 | node->io_read(node->opaque); |
164a101f SH |
394 | |
395 | /* aio_notify() does not count as progress */ | |
396 | if (node->opaque != &ctx->notifier) { | |
397 | progress = true; | |
398 | } | |
cd9ba1eb | 399 | } |
d0c8d2c0 SH |
400 | if (!node->deleted && |
401 | (revents & (G_IO_OUT | G_IO_ERR)) && | |
37989ced | 402 | aio_node_check(ctx, node->is_external) && |
d0c8d2c0 | 403 | node->io_write) { |
cd9ba1eb PB |
404 | node->io_write(node->opaque); |
405 | progress = true; | |
406 | } | |
407 | ||
408 | tmp = node; | |
409 | node = QLIST_NEXT(node, node); | |
410 | ||
411 | ctx->walking_handlers--; | |
412 | ||
413 | if (!ctx->walking_handlers && tmp->deleted) { | |
414 | QLIST_REMOVE(tmp, node); | |
415 | g_free(tmp); | |
416 | } | |
417 | } | |
438e1f47 | 418 | |
56d2c3c6 PB |
419 | return progress; |
420 | } | |
421 | ||
422 | /* | |
423 | * Note that dispatch_fds == false has the side-effect of post-poning the | |
424 | * freeing of deleted handlers. | |
425 | */ | |
426 | bool aio_dispatch(AioContext *ctx, bool dispatch_fds) | |
427 | { | |
428 | bool progress; | |
429 | ||
430 | /* | |
431 | * If there are callbacks left that have been queued, we need to call them. | |
432 | * Do not call select in this case, because it is possible that the caller | |
433 | * does not need a complete flush (as is the case for aio_poll loops). | |
434 | */ | |
435 | progress = aio_bh_poll(ctx); | |
436 | ||
437 | if (dispatch_fds) { | |
438 | progress |= aio_dispatch_handlers(ctx); | |
439 | } | |
440 | ||
438e1f47 AB |
441 | /* Run our timers */ |
442 | progress |= timerlistgroup_run_timers(&ctx->tlg); | |
443 | ||
d0c8d2c0 SH |
444 | return progress; |
445 | } | |
446 | ||
e98ab097 PB |
447 | /* These thread-local variables are used only in a small part of aio_poll |
448 | * around the call to the poll() system call. In particular they are not | |
449 | * used while aio_poll is performing callbacks, which makes it much easier | |
450 | * to think about reentrancy! | |
451 | * | |
452 | * Stack-allocated arrays would be perfect but they have size limitations; | |
453 | * heap allocation is expensive enough that we want to reuse arrays across | |
454 | * calls to aio_poll(). And because poll() has to be called without holding | |
455 | * any lock, the arrays cannot be stored in AioContext. Thread-local data | |
456 | * has none of the disadvantages of these three options. | |
457 | */ | |
458 | static __thread GPollFD *pollfds; | |
459 | static __thread AioHandler **nodes; | |
460 | static __thread unsigned npfd, nalloc; | |
461 | static __thread Notifier pollfds_cleanup_notifier; | |
462 | ||
463 | static void pollfds_cleanup(Notifier *n, void *unused) | |
464 | { | |
465 | g_assert(npfd == 0); | |
466 | g_free(pollfds); | |
467 | g_free(nodes); | |
468 | nalloc = 0; | |
469 | } | |
470 | ||
471 | static void add_pollfd(AioHandler *node) | |
472 | { | |
473 | if (npfd == nalloc) { | |
474 | if (nalloc == 0) { | |
475 | pollfds_cleanup_notifier.notify = pollfds_cleanup; | |
476 | qemu_thread_atexit_add(&pollfds_cleanup_notifier); | |
477 | nalloc = 8; | |
478 | } else { | |
479 | g_assert(nalloc <= INT_MAX); | |
480 | nalloc *= 2; | |
481 | } | |
482 | pollfds = g_renew(GPollFD, pollfds, nalloc); | |
483 | nodes = g_renew(AioHandler *, nodes, nalloc); | |
484 | } | |
485 | nodes[npfd] = node; | |
486 | pollfds[npfd] = (GPollFD) { | |
487 | .fd = node->pfd.fd, | |
488 | .events = node->pfd.events, | |
489 | }; | |
490 | npfd++; | |
491 | } | |
492 | ||
684e508c SH |
493 | static bool run_poll_handlers_once(AioContext *ctx) |
494 | { | |
495 | bool progress = false; | |
496 | AioHandler *node; | |
497 | ||
498 | QLIST_FOREACH(node, &ctx->aio_handlers, node) { | |
499 | if (!node->deleted && node->io_poll && | |
500 | node->io_poll(node->opaque)) { | |
501 | progress = true; | |
502 | } | |
503 | ||
504 | /* Caller handles freeing deleted nodes. Don't do it here. */ | |
505 | } | |
506 | ||
507 | return progress; | |
508 | } | |
509 | ||
4a1cba38 SH |
510 | /* run_poll_handlers: |
511 | * @ctx: the AioContext | |
512 | * @max_ns: maximum time to poll for, in nanoseconds | |
513 | * | |
514 | * Polls for a given time. | |
515 | * | |
516 | * Note that ctx->notify_me must be non-zero so this function can detect | |
517 | * aio_notify(). | |
518 | * | |
519 | * Note that the caller must have incremented ctx->walking_handlers. | |
520 | * | |
521 | * Returns: true if progress was made, false otherwise | |
522 | */ | |
523 | static bool run_poll_handlers(AioContext *ctx, int64_t max_ns) | |
524 | { | |
684e508c | 525 | bool progress; |
4a1cba38 SH |
526 | int64_t end_time; |
527 | ||
528 | assert(ctx->notify_me); | |
529 | assert(ctx->walking_handlers > 0); | |
530 | assert(ctx->poll_disable_cnt == 0); | |
531 | ||
532 | trace_run_poll_handlers_begin(ctx, max_ns); | |
533 | ||
534 | end_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + max_ns; | |
535 | ||
536 | do { | |
684e508c | 537 | progress = run_poll_handlers_once(ctx); |
4a1cba38 SH |
538 | } while (!progress && qemu_clock_get_ns(QEMU_CLOCK_REALTIME) < end_time); |
539 | ||
540 | trace_run_poll_handlers_end(ctx, progress); | |
541 | ||
542 | return progress; | |
543 | } | |
544 | ||
545 | /* try_poll_mode: | |
546 | * @ctx: the AioContext | |
684e508c | 547 | * @blocking: busy polling is only attempted when blocking is true |
4a1cba38 | 548 | * |
684e508c | 549 | * ctx->notify_me must be non-zero so this function can detect aio_notify(). |
4a1cba38 SH |
550 | * |
551 | * Note that the caller must have incremented ctx->walking_handlers. | |
552 | * | |
553 | * Returns: true if progress was made, false otherwise | |
554 | */ | |
555 | static bool try_poll_mode(AioContext *ctx, bool blocking) | |
556 | { | |
557 | if (blocking && ctx->poll_max_ns && ctx->poll_disable_cnt == 0) { | |
558 | /* See qemu_soonest_timeout() uint64_t hack */ | |
559 | int64_t max_ns = MIN((uint64_t)aio_compute_timeout(ctx), | |
82a41186 | 560 | (uint64_t)ctx->poll_ns); |
4a1cba38 SH |
561 | |
562 | if (max_ns) { | |
684e508c SH |
563 | poll_set_started(ctx, true); |
564 | ||
4a1cba38 SH |
565 | if (run_poll_handlers(ctx, max_ns)) { |
566 | return true; | |
567 | } | |
568 | } | |
569 | } | |
570 | ||
684e508c SH |
571 | poll_set_started(ctx, false); |
572 | ||
573 | /* Even if we don't run busy polling, try polling once in case it can make | |
574 | * progress and the caller will be able to avoid ppoll(2)/epoll_wait(2). | |
575 | */ | |
576 | return run_poll_handlers_once(ctx); | |
4a1cba38 SH |
577 | } |
578 | ||
d0c8d2c0 SH |
579 | bool aio_poll(AioContext *ctx, bool blocking) |
580 | { | |
d0c8d2c0 | 581 | AioHandler *node; |
4a1cba38 SH |
582 | int i; |
583 | int ret = 0; | |
164a101f | 584 | bool progress; |
e98ab097 | 585 | int64_t timeout; |
82a41186 | 586 | int64_t start = 0; |
d0c8d2c0 | 587 | |
49110174 | 588 | aio_context_acquire(ctx); |
d0c8d2c0 SH |
589 | progress = false; |
590 | ||
0ceb849b PB |
591 | /* aio_notify can avoid the expensive event_notifier_set if |
592 | * everything (file descriptors, bottom halves, timers) will | |
e4c7e2d1 PB |
593 | * be re-evaluated before the next blocking poll(). This is |
594 | * already true when aio_poll is called with blocking == false; | |
eabc9779 PB |
595 | * if blocking == true, it is only true after poll() returns, |
596 | * so disable the optimization now. | |
0ceb849b | 597 | */ |
eabc9779 PB |
598 | if (blocking) { |
599 | atomic_add(&ctx->notify_me, 2); | |
600 | } | |
0ceb849b | 601 | |
a915f4bc | 602 | ctx->walking_handlers++; |
a76bab49 | 603 | |
82a41186 SH |
604 | if (ctx->poll_max_ns) { |
605 | start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); | |
606 | } | |
607 | ||
4a1cba38 SH |
608 | if (try_poll_mode(ctx, blocking)) { |
609 | progress = true; | |
610 | } else { | |
611 | assert(npfd == 0); | |
a76bab49 | 612 | |
4a1cba38 | 613 | /* fill pollfds */ |
6b942468 | 614 | |
4a1cba38 SH |
615 | if (!aio_epoll_enabled(ctx)) { |
616 | QLIST_FOREACH(node, &ctx->aio_handlers, node) { | |
617 | if (!node->deleted && node->pfd.events | |
618 | && aio_node_check(ctx, node->is_external)) { | |
619 | add_pollfd(node); | |
620 | } | |
6b942468 | 621 | } |
9eb0bfca | 622 | } |
a76bab49 | 623 | |
4a1cba38 | 624 | timeout = blocking ? aio_compute_timeout(ctx) : 0; |
a76bab49 | 625 | |
4a1cba38 SH |
626 | /* wait until next event */ |
627 | if (timeout) { | |
628 | aio_context_release(ctx); | |
629 | } | |
630 | if (aio_epoll_check_poll(ctx, pollfds, npfd, timeout)) { | |
631 | AioHandler epoll_handler; | |
632 | ||
633 | epoll_handler.pfd.fd = ctx->epollfd; | |
634 | epoll_handler.pfd.events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR; | |
635 | npfd = 0; | |
636 | add_pollfd(&epoll_handler); | |
637 | ret = aio_epoll(ctx, pollfds, npfd, timeout); | |
638 | } else { | |
639 | ret = qemu_poll_ns(pollfds, npfd, timeout); | |
640 | } | |
641 | if (timeout) { | |
642 | aio_context_acquire(ctx); | |
643 | } | |
fbe3fc5c | 644 | } |
4a1cba38 | 645 | |
eabc9779 PB |
646 | if (blocking) { |
647 | atomic_sub(&ctx->notify_me, 2); | |
648 | } | |
9eb0bfca | 649 | |
82a41186 SH |
650 | /* Adjust polling time */ |
651 | if (ctx->poll_max_ns) { | |
652 | int64_t block_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start; | |
653 | ||
654 | if (block_ns <= ctx->poll_ns) { | |
655 | /* This is the sweet spot, no adjustment needed */ | |
656 | } else if (block_ns > ctx->poll_max_ns) { | |
657 | /* We'd have to poll for too long, poll less */ | |
658 | int64_t old = ctx->poll_ns; | |
659 | ||
660 | if (ctx->poll_shrink) { | |
661 | ctx->poll_ns /= ctx->poll_shrink; | |
662 | } else { | |
663 | ctx->poll_ns = 0; | |
664 | } | |
665 | ||
666 | trace_poll_shrink(ctx, old, ctx->poll_ns); | |
667 | } else if (ctx->poll_ns < ctx->poll_max_ns && | |
668 | block_ns < ctx->poll_max_ns) { | |
669 | /* There is room to grow, poll longer */ | |
670 | int64_t old = ctx->poll_ns; | |
671 | int64_t grow = ctx->poll_grow; | |
672 | ||
673 | if (grow == 0) { | |
674 | grow = 2; | |
675 | } | |
676 | ||
677 | if (ctx->poll_ns) { | |
678 | ctx->poll_ns *= grow; | |
679 | } else { | |
680 | ctx->poll_ns = 4000; /* start polling at 4 microseconds */ | |
681 | } | |
682 | ||
683 | if (ctx->poll_ns > ctx->poll_max_ns) { | |
684 | ctx->poll_ns = ctx->poll_max_ns; | |
685 | } | |
686 | ||
687 | trace_poll_grow(ctx, old, ctx->poll_ns); | |
688 | } | |
689 | } | |
690 | ||
05e514b1 | 691 | aio_notify_accept(ctx); |
21a03d17 | 692 | |
9eb0bfca PB |
693 | /* if we have any readable fds, dispatch event */ |
694 | if (ret > 0) { | |
e98ab097 PB |
695 | for (i = 0; i < npfd; i++) { |
696 | nodes[i]->pfd.revents = pollfds[i].revents; | |
a76bab49 | 697 | } |
438e1f47 AB |
698 | } |
699 | ||
e98ab097 PB |
700 | npfd = 0; |
701 | ctx->walking_handlers--; | |
702 | ||
438e1f47 | 703 | /* Run dispatch even if there were no readable fds to run timers */ |
721671ad | 704 | if (aio_dispatch(ctx, ret > 0)) { |
438e1f47 | 705 | progress = true; |
9eb0bfca | 706 | } |
bcdc1857 | 707 | |
49110174 PB |
708 | aio_context_release(ctx); |
709 | ||
164a101f | 710 | return progress; |
a76bab49 | 711 | } |
37fcee5d | 712 | |
7e003465 | 713 | void aio_context_setup(AioContext *ctx) |
37fcee5d | 714 | { |
4a1cba38 SH |
715 | /* TODO remove this in final patch submission */ |
716 | if (getenv("QEMU_AIO_POLL_MAX_NS")) { | |
717 | fprintf(stderr, "The QEMU_AIO_POLL_MAX_NS environment variable has " | |
718 | "been replaced with -object iothread,poll-max-ns=NUM\n"); | |
719 | exit(1); | |
720 | } | |
721 | ||
147dfab7 | 722 | #ifdef CONFIG_EPOLL_CREATE1 |
fbe3fc5c FZ |
723 | assert(!ctx->epollfd); |
724 | ctx->epollfd = epoll_create1(EPOLL_CLOEXEC); | |
725 | if (ctx->epollfd == -1) { | |
7e003465 | 726 | fprintf(stderr, "Failed to create epoll instance: %s", strerror(errno)); |
fbe3fc5c FZ |
727 | ctx->epoll_available = false; |
728 | } else { | |
729 | ctx->epoll_available = true; | |
730 | } | |
731 | #endif | |
37fcee5d | 732 | } |
4a1cba38 | 733 | |
82a41186 SH |
734 | void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns, |
735 | int64_t grow, int64_t shrink, Error **errp) | |
4a1cba38 | 736 | { |
82a41186 SH |
737 | /* No thread synchronization here, it doesn't matter if an incorrect value |
738 | * is used once. | |
4a1cba38 SH |
739 | */ |
740 | ctx->poll_max_ns = max_ns; | |
82a41186 SH |
741 | ctx->poll_ns = 0; |
742 | ctx->poll_grow = grow; | |
743 | ctx->poll_shrink = shrink; | |
4a1cba38 SH |
744 | |
745 | aio_notify(ctx); | |
746 | } |