]>
Commit | Line | Data |
---|---|---|
a76bab49 AL |
1 | /* |
2 | * QEMU aio implementation | |
3 | * | |
4 | * Copyright IBM, Corp. 2008 | |
5 | * | |
6 | * Authors: | |
7 | * Anthony Liguori <aliguori@us.ibm.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
10 | * the COPYING file in the top-level directory. | |
11 | * | |
6b620ca3 PB |
12 | * Contributions after 2012-01-13 are licensed under the terms of the |
13 | * GNU GPL, version 2 or (at your option) any later version. | |
a76bab49 AL |
14 | */ |
15 | ||
d38ea87a | 16 | #include "qemu/osdep.h" |
a76bab49 | 17 | #include "qemu-common.h" |
737e150e | 18 | #include "block/block.h" |
2bbf11d7 | 19 | #include "qemu/rcu_queue.h" |
1de7afc9 | 20 | #include "qemu/sockets.h" |
4a1cba38 | 21 | #include "qemu/cutils.h" |
c2b38b27 | 22 | #include "trace.h" |
147dfab7 | 23 | #ifdef CONFIG_EPOLL_CREATE1 |
fbe3fc5c FZ |
24 | #include <sys/epoll.h> |
25 | #endif | |
a76bab49 | 26 | |
a76bab49 AL |
27 | struct AioHandler |
28 | { | |
cd9ba1eb | 29 | GPollFD pfd; |
a76bab49 AL |
30 | IOHandler *io_read; |
31 | IOHandler *io_write; | |
4a1cba38 | 32 | AioPollFn *io_poll; |
684e508c SH |
33 | IOHandler *io_poll_begin; |
34 | IOHandler *io_poll_end; | |
a76bab49 AL |
35 | int deleted; |
36 | void *opaque; | |
dca21ef2 | 37 | bool is_external; |
72cf2d4f | 38 | QLIST_ENTRY(AioHandler) node; |
a76bab49 AL |
39 | }; |
40 | ||
147dfab7 | 41 | #ifdef CONFIG_EPOLL_CREATE1 |
fbe3fc5c | 42 | |
82dfee5a | 43 | /* The fd number threshold to switch to epoll */ |
fbe3fc5c FZ |
44 | #define EPOLL_ENABLE_THRESHOLD 64 |
45 | ||
46 | static void aio_epoll_disable(AioContext *ctx) | |
47 | { | |
cd0a6d2b JW |
48 | ctx->epoll_enabled = false; |
49 | if (!ctx->epoll_available) { | |
fbe3fc5c FZ |
50 | return; |
51 | } | |
cd0a6d2b | 52 | ctx->epoll_available = false; |
fbe3fc5c FZ |
53 | close(ctx->epollfd); |
54 | } | |
55 | ||
56 | static inline int epoll_events_from_pfd(int pfd_events) | |
57 | { | |
58 | return (pfd_events & G_IO_IN ? EPOLLIN : 0) | | |
59 | (pfd_events & G_IO_OUT ? EPOLLOUT : 0) | | |
60 | (pfd_events & G_IO_HUP ? EPOLLHUP : 0) | | |
61 | (pfd_events & G_IO_ERR ? EPOLLERR : 0); | |
62 | } | |
63 | ||
64 | static bool aio_epoll_try_enable(AioContext *ctx) | |
65 | { | |
66 | AioHandler *node; | |
67 | struct epoll_event event; | |
68 | ||
2bbf11d7 | 69 | QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) { |
fbe3fc5c FZ |
70 | int r; |
71 | if (node->deleted || !node->pfd.events) { | |
72 | continue; | |
73 | } | |
74 | event.events = epoll_events_from_pfd(node->pfd.events); | |
75 | event.data.ptr = node; | |
76 | r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event); | |
77 | if (r) { | |
78 | return false; | |
79 | } | |
80 | } | |
81 | ctx->epoll_enabled = true; | |
82 | return true; | |
83 | } | |
84 | ||
85 | static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new) | |
86 | { | |
87 | struct epoll_event event; | |
88 | int r; | |
35dd66e2 | 89 | int ctl; |
fbe3fc5c FZ |
90 | |
91 | if (!ctx->epoll_enabled) { | |
92 | return; | |
93 | } | |
94 | if (!node->pfd.events) { | |
35dd66e2 | 95 | ctl = EPOLL_CTL_DEL; |
fbe3fc5c FZ |
96 | } else { |
97 | event.data.ptr = node; | |
98 | event.events = epoll_events_from_pfd(node->pfd.events); | |
35dd66e2 PB |
99 | ctl = is_new ? EPOLL_CTL_ADD : EPOLL_CTL_MOD; |
100 | } | |
101 | ||
102 | r = epoll_ctl(ctx->epollfd, ctl, node->pfd.fd, &event); | |
103 | if (r) { | |
104 | aio_epoll_disable(ctx); | |
fbe3fc5c FZ |
105 | } |
106 | } | |
107 | ||
108 | static int aio_epoll(AioContext *ctx, GPollFD *pfds, | |
109 | unsigned npfd, int64_t timeout) | |
110 | { | |
111 | AioHandler *node; | |
112 | int i, ret = 0; | |
113 | struct epoll_event events[128]; | |
114 | ||
115 | assert(npfd == 1); | |
116 | assert(pfds[0].fd == ctx->epollfd); | |
117 | if (timeout > 0) { | |
118 | ret = qemu_poll_ns(pfds, npfd, timeout); | |
119 | } | |
120 | if (timeout <= 0 || ret > 0) { | |
121 | ret = epoll_wait(ctx->epollfd, events, | |
8f801baf | 122 | ARRAY_SIZE(events), |
fbe3fc5c FZ |
123 | timeout); |
124 | if (ret <= 0) { | |
125 | goto out; | |
126 | } | |
127 | for (i = 0; i < ret; i++) { | |
128 | int ev = events[i].events; | |
129 | node = events[i].data.ptr; | |
130 | node->pfd.revents = (ev & EPOLLIN ? G_IO_IN : 0) | | |
131 | (ev & EPOLLOUT ? G_IO_OUT : 0) | | |
132 | (ev & EPOLLHUP ? G_IO_HUP : 0) | | |
133 | (ev & EPOLLERR ? G_IO_ERR : 0); | |
134 | } | |
135 | } | |
136 | out: | |
137 | return ret; | |
138 | } | |
139 | ||
140 | static bool aio_epoll_enabled(AioContext *ctx) | |
141 | { | |
142 | /* Fall back to ppoll when external clients are disabled. */ | |
143 | return !aio_external_disabled(ctx) && ctx->epoll_enabled; | |
144 | } | |
145 | ||
146 | static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds, | |
147 | unsigned npfd, int64_t timeout) | |
148 | { | |
149 | if (!ctx->epoll_available) { | |
150 | return false; | |
151 | } | |
152 | if (aio_epoll_enabled(ctx)) { | |
153 | return true; | |
154 | } | |
155 | if (npfd >= EPOLL_ENABLE_THRESHOLD) { | |
156 | if (aio_epoll_try_enable(ctx)) { | |
157 | return true; | |
158 | } else { | |
159 | aio_epoll_disable(ctx); | |
160 | } | |
161 | } | |
162 | return false; | |
163 | } | |
164 | ||
165 | #else | |
166 | ||
167 | static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new) | |
168 | { | |
169 | } | |
170 | ||
171 | static int aio_epoll(AioContext *ctx, GPollFD *pfds, | |
172 | unsigned npfd, int64_t timeout) | |
173 | { | |
174 | assert(false); | |
175 | } | |
176 | ||
177 | static bool aio_epoll_enabled(AioContext *ctx) | |
178 | { | |
179 | return false; | |
180 | } | |
181 | ||
182 | static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds, | |
183 | unsigned npfd, int64_t timeout) | |
184 | { | |
185 | return false; | |
186 | } | |
187 | ||
188 | #endif | |
189 | ||
a915f4bc | 190 | static AioHandler *find_aio_handler(AioContext *ctx, int fd) |
a76bab49 AL |
191 | { |
192 | AioHandler *node; | |
193 | ||
a915f4bc | 194 | QLIST_FOREACH(node, &ctx->aio_handlers, node) { |
cd9ba1eb | 195 | if (node->pfd.fd == fd) |
79d5ca56 AG |
196 | if (!node->deleted) |
197 | return node; | |
a76bab49 AL |
198 | } |
199 | ||
200 | return NULL; | |
201 | } | |
202 | ||
a915f4bc PB |
203 | void aio_set_fd_handler(AioContext *ctx, |
204 | int fd, | |
dca21ef2 | 205 | bool is_external, |
a915f4bc PB |
206 | IOHandler *io_read, |
207 | IOHandler *io_write, | |
f6a51c84 | 208 | AioPollFn *io_poll, |
a915f4bc | 209 | void *opaque) |
a76bab49 AL |
210 | { |
211 | AioHandler *node; | |
fbe3fc5c | 212 | bool is_new = false; |
0ed39f3d | 213 | bool deleted = false; |
d7be5dd1 | 214 | int poll_disable_change; |
a76bab49 | 215 | |
2bbf11d7 PB |
216 | qemu_lockcnt_lock(&ctx->list_lock); |
217 | ||
a915f4bc | 218 | node = find_aio_handler(ctx, fd); |
a76bab49 AL |
219 | |
220 | /* Are we deleting the fd handler? */ | |
4a1cba38 | 221 | if (!io_read && !io_write && !io_poll) { |
36173ec5 | 222 | if (node == NULL) { |
2bbf11d7 | 223 | qemu_lockcnt_unlock(&ctx->list_lock); |
36173ec5 PB |
224 | return; |
225 | } | |
226 | ||
f708a5e7 SH |
227 | /* If the GSource is in the process of being destroyed then |
228 | * g_source_remove_poll() causes an assertion failure. Skip | |
229 | * removal in that case, because glib cleans up its state during | |
230 | * destruction anyway. | |
231 | */ | |
232 | if (!g_source_is_destroyed(&ctx->source)) { | |
233 | g_source_remove_poll(&ctx->source, &node->pfd); | |
234 | } | |
36173ec5 | 235 | |
37a81812 | 236 | /* If a read is in progress, just mark the node as deleted */ |
2bbf11d7 | 237 | if (qemu_lockcnt_count(&ctx->list_lock)) { |
36173ec5 PB |
238 | node->deleted = 1; |
239 | node->pfd.revents = 0; | |
240 | } else { | |
241 | /* Otherwise, delete it for real. We can't just mark it as | |
2bbf11d7 PB |
242 | * deleted because deleted nodes are only cleaned up while |
243 | * no one is walking the handlers list. | |
36173ec5 PB |
244 | */ |
245 | QLIST_REMOVE(node, node); | |
246 | deleted = true; | |
a76bab49 | 247 | } |
8821b34a RN |
248 | /* Clean events in order to unregister fd from the ctx epoll. */ |
249 | node->pfd.events = 0; | |
250 | ||
d7be5dd1 | 251 | poll_disable_change = -!node->io_poll; |
a76bab49 | 252 | } else { |
d7be5dd1 | 253 | poll_disable_change = !io_poll - (node && !node->io_poll); |
a76bab49 AL |
254 | if (node == NULL) { |
255 | /* Alloc and insert if it's not already there */ | |
3ba235a0 | 256 | node = g_new0(AioHandler, 1); |
cd9ba1eb | 257 | node->pfd.fd = fd; |
2bbf11d7 | 258 | QLIST_INSERT_HEAD_RCU(&ctx->aio_handlers, node, node); |
e3713e00 PB |
259 | |
260 | g_source_add_poll(&ctx->source, &node->pfd); | |
fbe3fc5c | 261 | is_new = true; |
a76bab49 | 262 | } |
4a1cba38 | 263 | |
a76bab49 AL |
264 | /* Update handler with latest information */ |
265 | node->io_read = io_read; | |
266 | node->io_write = io_write; | |
4a1cba38 | 267 | node->io_poll = io_poll; |
a76bab49 | 268 | node->opaque = opaque; |
dca21ef2 | 269 | node->is_external = is_external; |
cd9ba1eb | 270 | |
b5a01a70 SH |
271 | node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0); |
272 | node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0); | |
a76bab49 | 273 | } |
7ed2b24c | 274 | |
d7be5dd1 PB |
275 | /* No need to order poll_disable_cnt writes against other updates; |
276 | * the counter is only used to avoid wasting time and latency on | |
277 | * iterated polling when the system call will be ultimately necessary. | |
278 | * Changing handlers is a rare event, and a little wasted polling until | |
279 | * the aio_notify below is not an issue. | |
280 | */ | |
281 | atomic_set(&ctx->poll_disable_cnt, | |
282 | atomic_read(&ctx->poll_disable_cnt) + poll_disable_change); | |
283 | ||
fbe3fc5c | 284 | aio_epoll_update(ctx, node, is_new); |
2bbf11d7 | 285 | qemu_lockcnt_unlock(&ctx->list_lock); |
7ed2b24c | 286 | aio_notify(ctx); |
4a1cba38 | 287 | |
0ed39f3d FZ |
288 | if (deleted) { |
289 | g_free(node); | |
290 | } | |
9958c351 PB |
291 | } |
292 | ||
684e508c SH |
293 | void aio_set_fd_poll(AioContext *ctx, int fd, |
294 | IOHandler *io_poll_begin, | |
295 | IOHandler *io_poll_end) | |
296 | { | |
297 | AioHandler *node = find_aio_handler(ctx, fd); | |
298 | ||
299 | if (!node) { | |
300 | return; | |
301 | } | |
302 | ||
303 | node->io_poll_begin = io_poll_begin; | |
304 | node->io_poll_end = io_poll_end; | |
305 | } | |
306 | ||
a915f4bc PB |
307 | void aio_set_event_notifier(AioContext *ctx, |
308 | EventNotifier *notifier, | |
dca21ef2 | 309 | bool is_external, |
f6a51c84 SH |
310 | EventNotifierHandler *io_read, |
311 | AioPollFn *io_poll) | |
a76bab49 | 312 | { |
f6a51c84 SH |
313 | aio_set_fd_handler(ctx, event_notifier_get_fd(notifier), is_external, |
314 | (IOHandler *)io_read, NULL, io_poll, notifier); | |
a76bab49 AL |
315 | } |
316 | ||
684e508c SH |
317 | void aio_set_event_notifier_poll(AioContext *ctx, |
318 | EventNotifier *notifier, | |
319 | EventNotifierHandler *io_poll_begin, | |
320 | EventNotifierHandler *io_poll_end) | |
321 | { | |
322 | aio_set_fd_poll(ctx, event_notifier_get_fd(notifier), | |
323 | (IOHandler *)io_poll_begin, | |
324 | (IOHandler *)io_poll_end); | |
325 | } | |
326 | ||
327 | static void poll_set_started(AioContext *ctx, bool started) | |
328 | { | |
329 | AioHandler *node; | |
330 | ||
331 | if (started == ctx->poll_started) { | |
332 | return; | |
333 | } | |
334 | ||
335 | ctx->poll_started = started; | |
336 | ||
2bbf11d7 PB |
337 | qemu_lockcnt_inc(&ctx->list_lock); |
338 | QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) { | |
684e508c SH |
339 | IOHandler *fn; |
340 | ||
341 | if (node->deleted) { | |
342 | continue; | |
343 | } | |
344 | ||
345 | if (started) { | |
346 | fn = node->io_poll_begin; | |
347 | } else { | |
348 | fn = node->io_poll_end; | |
349 | } | |
350 | ||
351 | if (fn) { | |
352 | fn(node->opaque); | |
353 | } | |
354 | } | |
2bbf11d7 | 355 | qemu_lockcnt_dec(&ctx->list_lock); |
684e508c SH |
356 | } |
357 | ||
358 | ||
a3462c65 PB |
359 | bool aio_prepare(AioContext *ctx) |
360 | { | |
684e508c SH |
361 | /* Poll mode cannot be used with glib's event loop, disable it. */ |
362 | poll_set_started(ctx, false); | |
363 | ||
a3462c65 PB |
364 | return false; |
365 | } | |
366 | ||
cd9ba1eb PB |
367 | bool aio_pending(AioContext *ctx) |
368 | { | |
369 | AioHandler *node; | |
2bbf11d7 | 370 | bool result = false; |
cd9ba1eb | 371 | |
2bbf11d7 PB |
372 | /* |
373 | * We have to walk very carefully in case aio_set_fd_handler is | |
374 | * called while we're walking. | |
375 | */ | |
376 | qemu_lockcnt_inc(&ctx->list_lock); | |
377 | ||
378 | QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) { | |
cd9ba1eb PB |
379 | int revents; |
380 | ||
cd9ba1eb | 381 | revents = node->pfd.revents & node->pfd.events; |
37989ced FZ |
382 | if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read && |
383 | aio_node_check(ctx, node->is_external)) { | |
2bbf11d7 PB |
384 | result = true; |
385 | break; | |
cd9ba1eb | 386 | } |
37989ced FZ |
387 | if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write && |
388 | aio_node_check(ctx, node->is_external)) { | |
2bbf11d7 PB |
389 | result = true; |
390 | break; | |
cd9ba1eb PB |
391 | } |
392 | } | |
2bbf11d7 | 393 | qemu_lockcnt_dec(&ctx->list_lock); |
cd9ba1eb | 394 | |
2bbf11d7 | 395 | return result; |
cd9ba1eb PB |
396 | } |
397 | ||
56d2c3c6 | 398 | static bool aio_dispatch_handlers(AioContext *ctx) |
a76bab49 | 399 | { |
abf90d39 | 400 | AioHandler *node, *tmp; |
d0c8d2c0 | 401 | bool progress = false; |
7c0628b2 | 402 | |
2bbf11d7 | 403 | QLIST_FOREACH_SAFE_RCU(node, &ctx->aio_handlers, node, tmp) { |
abf90d39 | 404 | int revents; |
cd9ba1eb PB |
405 | |
406 | revents = node->pfd.revents & node->pfd.events; | |
407 | node->pfd.revents = 0; | |
408 | ||
d0c8d2c0 SH |
409 | if (!node->deleted && |
410 | (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) && | |
37989ced | 411 | aio_node_check(ctx, node->is_external) && |
d0c8d2c0 | 412 | node->io_read) { |
cd9ba1eb | 413 | node->io_read(node->opaque); |
164a101f SH |
414 | |
415 | /* aio_notify() does not count as progress */ | |
416 | if (node->opaque != &ctx->notifier) { | |
417 | progress = true; | |
418 | } | |
cd9ba1eb | 419 | } |
d0c8d2c0 SH |
420 | if (!node->deleted && |
421 | (revents & (G_IO_OUT | G_IO_ERR)) && | |
37989ced | 422 | aio_node_check(ctx, node->is_external) && |
d0c8d2c0 | 423 | node->io_write) { |
cd9ba1eb PB |
424 | node->io_write(node->opaque); |
425 | progress = true; | |
426 | } | |
427 | ||
abf90d39 | 428 | if (node->deleted) { |
2bbf11d7 | 429 | if (qemu_lockcnt_dec_if_lock(&ctx->list_lock)) { |
abf90d39 PB |
430 | QLIST_REMOVE(node, node); |
431 | g_free(node); | |
2bbf11d7 | 432 | qemu_lockcnt_inc_and_unlock(&ctx->list_lock); |
abf90d39 | 433 | } |
cd9ba1eb PB |
434 | } |
435 | } | |
438e1f47 | 436 | |
56d2c3c6 PB |
437 | return progress; |
438 | } | |
439 | ||
a153bf52 | 440 | void aio_dispatch(AioContext *ctx) |
56d2c3c6 | 441 | { |
a153bf52 | 442 | qemu_lockcnt_inc(&ctx->list_lock); |
bd451435 | 443 | aio_bh_poll(ctx); |
a153bf52 PB |
444 | aio_dispatch_handlers(ctx); |
445 | qemu_lockcnt_dec(&ctx->list_lock); | |
438e1f47 | 446 | |
a153bf52 | 447 | timerlistgroup_run_timers(&ctx->tlg); |
d0c8d2c0 SH |
448 | } |
449 | ||
e98ab097 PB |
450 | /* These thread-local variables are used only in a small part of aio_poll |
451 | * around the call to the poll() system call. In particular they are not | |
452 | * used while aio_poll is performing callbacks, which makes it much easier | |
453 | * to think about reentrancy! | |
454 | * | |
455 | * Stack-allocated arrays would be perfect but they have size limitations; | |
456 | * heap allocation is expensive enough that we want to reuse arrays across | |
457 | * calls to aio_poll(). And because poll() has to be called without holding | |
458 | * any lock, the arrays cannot be stored in AioContext. Thread-local data | |
459 | * has none of the disadvantages of these three options. | |
460 | */ | |
461 | static __thread GPollFD *pollfds; | |
462 | static __thread AioHandler **nodes; | |
463 | static __thread unsigned npfd, nalloc; | |
464 | static __thread Notifier pollfds_cleanup_notifier; | |
465 | ||
466 | static void pollfds_cleanup(Notifier *n, void *unused) | |
467 | { | |
468 | g_assert(npfd == 0); | |
469 | g_free(pollfds); | |
470 | g_free(nodes); | |
471 | nalloc = 0; | |
472 | } | |
473 | ||
474 | static void add_pollfd(AioHandler *node) | |
475 | { | |
476 | if (npfd == nalloc) { | |
477 | if (nalloc == 0) { | |
478 | pollfds_cleanup_notifier.notify = pollfds_cleanup; | |
479 | qemu_thread_atexit_add(&pollfds_cleanup_notifier); | |
480 | nalloc = 8; | |
481 | } else { | |
482 | g_assert(nalloc <= INT_MAX); | |
483 | nalloc *= 2; | |
484 | } | |
485 | pollfds = g_renew(GPollFD, pollfds, nalloc); | |
486 | nodes = g_renew(AioHandler *, nodes, nalloc); | |
487 | } | |
488 | nodes[npfd] = node; | |
489 | pollfds[npfd] = (GPollFD) { | |
490 | .fd = node->pfd.fd, | |
491 | .events = node->pfd.events, | |
492 | }; | |
493 | npfd++; | |
494 | } | |
495 | ||
e30cffa0 | 496 | static bool run_poll_handlers_once(AioContext *ctx, int64_t *timeout) |
684e508c SH |
497 | { |
498 | bool progress = false; | |
499 | AioHandler *node; | |
500 | ||
2bbf11d7 | 501 | QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) { |
684e508c | 502 | if (!node->deleted && node->io_poll && |
59c9f437 | 503 | aio_node_check(ctx, node->is_external) && |
cfeb35d6 | 504 | node->io_poll(node->opaque)) { |
e30cffa0 | 505 | *timeout = 0; |
cfeb35d6 PB |
506 | if (node->opaque != &ctx->notifier) { |
507 | progress = true; | |
508 | } | |
684e508c SH |
509 | } |
510 | ||
511 | /* Caller handles freeing deleted nodes. Don't do it here. */ | |
512 | } | |
513 | ||
514 | return progress; | |
515 | } | |
516 | ||
4a1cba38 SH |
517 | /* run_poll_handlers: |
518 | * @ctx: the AioContext | |
519 | * @max_ns: maximum time to poll for, in nanoseconds | |
520 | * | |
521 | * Polls for a given time. | |
522 | * | |
523 | * Note that ctx->notify_me must be non-zero so this function can detect | |
524 | * aio_notify(). | |
525 | * | |
2bbf11d7 | 526 | * Note that the caller must have incremented ctx->list_lock. |
4a1cba38 SH |
527 | * |
528 | * Returns: true if progress was made, false otherwise | |
529 | */ | |
e30cffa0 | 530 | static bool run_poll_handlers(AioContext *ctx, int64_t max_ns, int64_t *timeout) |
4a1cba38 | 531 | { |
684e508c | 532 | bool progress; |
e30cffa0 | 533 | int64_t start_time, elapsed_time; |
4a1cba38 SH |
534 | |
535 | assert(ctx->notify_me); | |
2bbf11d7 | 536 | assert(qemu_lockcnt_count(&ctx->list_lock) > 0); |
4a1cba38 | 537 | |
e30cffa0 | 538 | trace_run_poll_handlers_begin(ctx, max_ns, *timeout); |
4a1cba38 | 539 | |
e30cffa0 | 540 | start_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); |
4a1cba38 | 541 | do { |
e30cffa0 PB |
542 | progress = run_poll_handlers_once(ctx, timeout); |
543 | elapsed_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start_time; | |
544 | } while (!progress && elapsed_time < max_ns | |
d7be5dd1 | 545 | && !atomic_read(&ctx->poll_disable_cnt)); |
4a1cba38 | 546 | |
e30cffa0 PB |
547 | /* If time has passed with no successful polling, adjust *timeout to |
548 | * keep the same ending time. | |
549 | */ | |
550 | if (*timeout != -1) { | |
551 | *timeout -= MIN(*timeout, elapsed_time); | |
552 | } | |
4a1cba38 | 553 | |
e30cffa0 | 554 | trace_run_poll_handlers_end(ctx, progress, *timeout); |
4a1cba38 SH |
555 | return progress; |
556 | } | |
557 | ||
558 | /* try_poll_mode: | |
559 | * @ctx: the AioContext | |
e30cffa0 PB |
560 | * @timeout: timeout for blocking wait, computed by the caller and updated if |
561 | * polling succeeds. | |
4a1cba38 | 562 | * |
684e508c | 563 | * ctx->notify_me must be non-zero so this function can detect aio_notify(). |
4a1cba38 | 564 | * |
2bbf11d7 | 565 | * Note that the caller must have incremented ctx->list_lock. |
4a1cba38 SH |
566 | * |
567 | * Returns: true if progress was made, false otherwise | |
568 | */ | |
e30cffa0 | 569 | static bool try_poll_mode(AioContext *ctx, int64_t *timeout) |
4a1cba38 | 570 | { |
e30cffa0 PB |
571 | /* See qemu_soonest_timeout() uint64_t hack */ |
572 | int64_t max_ns = MIN((uint64_t)*timeout, (uint64_t)ctx->poll_ns); | |
4a1cba38 | 573 | |
e30cffa0 PB |
574 | if (max_ns && !atomic_read(&ctx->poll_disable_cnt)) { |
575 | poll_set_started(ctx, true); | |
684e508c | 576 | |
e30cffa0 PB |
577 | if (run_poll_handlers(ctx, max_ns, timeout)) { |
578 | return true; | |
4a1cba38 SH |
579 | } |
580 | } | |
581 | ||
684e508c SH |
582 | poll_set_started(ctx, false); |
583 | ||
584 | /* Even if we don't run busy polling, try polling once in case it can make | |
585 | * progress and the caller will be able to avoid ppoll(2)/epoll_wait(2). | |
586 | */ | |
e30cffa0 | 587 | return run_poll_handlers_once(ctx, timeout); |
4a1cba38 SH |
588 | } |
589 | ||
d0c8d2c0 SH |
590 | bool aio_poll(AioContext *ctx, bool blocking) |
591 | { | |
d0c8d2c0 | 592 | AioHandler *node; |
4a1cba38 SH |
593 | int i; |
594 | int ret = 0; | |
164a101f | 595 | bool progress; |
e98ab097 | 596 | int64_t timeout; |
82a41186 | 597 | int64_t start = 0; |
d0c8d2c0 | 598 | |
0ceb849b PB |
599 | /* aio_notify can avoid the expensive event_notifier_set if |
600 | * everything (file descriptors, bottom halves, timers) will | |
e4c7e2d1 PB |
601 | * be re-evaluated before the next blocking poll(). This is |
602 | * already true when aio_poll is called with blocking == false; | |
eabc9779 PB |
603 | * if blocking == true, it is only true after poll() returns, |
604 | * so disable the optimization now. | |
0ceb849b | 605 | */ |
eabc9779 | 606 | if (blocking) { |
b37548fc | 607 | assert(in_aio_context_home_thread(ctx)); |
eabc9779 PB |
608 | atomic_add(&ctx->notify_me, 2); |
609 | } | |
0ceb849b | 610 | |
2bbf11d7 | 611 | qemu_lockcnt_inc(&ctx->list_lock); |
a76bab49 | 612 | |
82a41186 SH |
613 | if (ctx->poll_max_ns) { |
614 | start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); | |
615 | } | |
616 | ||
e30cffa0 PB |
617 | timeout = blocking ? aio_compute_timeout(ctx) : 0; |
618 | progress = try_poll_mode(ctx, &timeout); | |
619 | assert(!(timeout && progress)); | |
620 | ||
621 | /* If polling is allowed, non-blocking aio_poll does not need the | |
622 | * system call---a single round of run_poll_handlers_once suffices. | |
623 | */ | |
624 | if (timeout || atomic_read(&ctx->poll_disable_cnt)) { | |
4a1cba38 | 625 | assert(npfd == 0); |
a76bab49 | 626 | |
4a1cba38 | 627 | /* fill pollfds */ |
6b942468 | 628 | |
4a1cba38 | 629 | if (!aio_epoll_enabled(ctx)) { |
2bbf11d7 | 630 | QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) { |
4a1cba38 SH |
631 | if (!node->deleted && node->pfd.events |
632 | && aio_node_check(ctx, node->is_external)) { | |
633 | add_pollfd(node); | |
634 | } | |
6b942468 | 635 | } |
9eb0bfca | 636 | } |
a76bab49 | 637 | |
4a1cba38 | 638 | /* wait until next event */ |
4a1cba38 SH |
639 | if (aio_epoll_check_poll(ctx, pollfds, npfd, timeout)) { |
640 | AioHandler epoll_handler; | |
641 | ||
642 | epoll_handler.pfd.fd = ctx->epollfd; | |
643 | epoll_handler.pfd.events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR; | |
644 | npfd = 0; | |
645 | add_pollfd(&epoll_handler); | |
646 | ret = aio_epoll(ctx, pollfds, npfd, timeout); | |
647 | } else { | |
648 | ret = qemu_poll_ns(pollfds, npfd, timeout); | |
649 | } | |
fbe3fc5c | 650 | } |
4a1cba38 | 651 | |
eabc9779 PB |
652 | if (blocking) { |
653 | atomic_sub(&ctx->notify_me, 2); | |
b37548fc | 654 | aio_notify_accept(ctx); |
eabc9779 | 655 | } |
9eb0bfca | 656 | |
82a41186 SH |
657 | /* Adjust polling time */ |
658 | if (ctx->poll_max_ns) { | |
659 | int64_t block_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start; | |
660 | ||
661 | if (block_ns <= ctx->poll_ns) { | |
662 | /* This is the sweet spot, no adjustment needed */ | |
663 | } else if (block_ns > ctx->poll_max_ns) { | |
664 | /* We'd have to poll for too long, poll less */ | |
665 | int64_t old = ctx->poll_ns; | |
666 | ||
667 | if (ctx->poll_shrink) { | |
668 | ctx->poll_ns /= ctx->poll_shrink; | |
669 | } else { | |
670 | ctx->poll_ns = 0; | |
671 | } | |
672 | ||
673 | trace_poll_shrink(ctx, old, ctx->poll_ns); | |
674 | } else if (ctx->poll_ns < ctx->poll_max_ns && | |
675 | block_ns < ctx->poll_max_ns) { | |
676 | /* There is room to grow, poll longer */ | |
677 | int64_t old = ctx->poll_ns; | |
678 | int64_t grow = ctx->poll_grow; | |
679 | ||
680 | if (grow == 0) { | |
681 | grow = 2; | |
682 | } | |
683 | ||
684 | if (ctx->poll_ns) { | |
685 | ctx->poll_ns *= grow; | |
686 | } else { | |
687 | ctx->poll_ns = 4000; /* start polling at 4 microseconds */ | |
688 | } | |
689 | ||
690 | if (ctx->poll_ns > ctx->poll_max_ns) { | |
691 | ctx->poll_ns = ctx->poll_max_ns; | |
692 | } | |
693 | ||
694 | trace_poll_grow(ctx, old, ctx->poll_ns); | |
695 | } | |
696 | } | |
697 | ||
9eb0bfca PB |
698 | /* if we have any readable fds, dispatch event */ |
699 | if (ret > 0) { | |
e98ab097 PB |
700 | for (i = 0; i < npfd; i++) { |
701 | nodes[i]->pfd.revents = pollfds[i].revents; | |
a76bab49 | 702 | } |
438e1f47 AB |
703 | } |
704 | ||
e98ab097 | 705 | npfd = 0; |
e98ab097 | 706 | |
a153bf52 PB |
707 | progress |= aio_bh_poll(ctx); |
708 | ||
709 | if (ret > 0) { | |
a153bf52 | 710 | progress |= aio_dispatch_handlers(ctx); |
9eb0bfca | 711 | } |
bcdc1857 | 712 | |
bd451435 PB |
713 | qemu_lockcnt_dec(&ctx->list_lock); |
714 | ||
a153bf52 PB |
715 | progress |= timerlistgroup_run_timers(&ctx->tlg); |
716 | ||
164a101f | 717 | return progress; |
a76bab49 | 718 | } |
37fcee5d | 719 | |
7e003465 | 720 | void aio_context_setup(AioContext *ctx) |
37fcee5d | 721 | { |
147dfab7 | 722 | #ifdef CONFIG_EPOLL_CREATE1 |
fbe3fc5c FZ |
723 | assert(!ctx->epollfd); |
724 | ctx->epollfd = epoll_create1(EPOLL_CLOEXEC); | |
725 | if (ctx->epollfd == -1) { | |
7e003465 | 726 | fprintf(stderr, "Failed to create epoll instance: %s", strerror(errno)); |
fbe3fc5c FZ |
727 | ctx->epoll_available = false; |
728 | } else { | |
729 | ctx->epoll_available = true; | |
730 | } | |
731 | #endif | |
37fcee5d | 732 | } |
4a1cba38 | 733 | |
cd0a6d2b JW |
734 | void aio_context_destroy(AioContext *ctx) |
735 | { | |
736 | #ifdef CONFIG_EPOLL_CREATE1 | |
737 | aio_epoll_disable(ctx); | |
738 | #endif | |
739 | } | |
740 | ||
82a41186 SH |
741 | void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns, |
742 | int64_t grow, int64_t shrink, Error **errp) | |
4a1cba38 | 743 | { |
82a41186 SH |
744 | /* No thread synchronization here, it doesn't matter if an incorrect value |
745 | * is used once. | |
4a1cba38 SH |
746 | */ |
747 | ctx->poll_max_ns = max_ns; | |
82a41186 SH |
748 | ctx->poll_ns = 0; |
749 | ctx->poll_grow = grow; | |
750 | ctx->poll_shrink = shrink; | |
4a1cba38 SH |
751 | |
752 | aio_notify(ctx); | |
753 | } |