]>
Commit | Line | Data |
---|---|---|
a76bab49 AL |
1 | /* |
2 | * QEMU aio implementation | |
3 | * | |
4 | * Copyright IBM, Corp. 2008 | |
5 | * | |
6 | * Authors: | |
7 | * Anthony Liguori <aliguori@us.ibm.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
10 | * the COPYING file in the top-level directory. | |
11 | * | |
12 | */ | |
13 | ||
14 | #ifndef QEMU_AIO_H | |
15 | #define QEMU_AIO_H | |
16 | ||
73fd282e SH |
17 | #ifdef CONFIG_LINUX_IO_URING |
18 | #include <liburing.h> | |
19 | #endif | |
26b0b698 | 20 | #include "qemu/coroutine.h" |
1de7afc9 PB |
21 | #include "qemu/queue.h" |
22 | #include "qemu/event_notifier.h" | |
dcc772e2 | 23 | #include "qemu/thread.h" |
dae21b98 | 24 | #include "qemu/timer.h" |
a76bab49 | 25 | |
7c84b1b8 | 26 | typedef struct BlockAIOCB BlockAIOCB; |
097310b5 | 27 | typedef void BlockCompletionFunc(void *opaque, int ret); |
85e8dab1 | 28 | |
d7331bed | 29 | typedef struct AIOCBInfo { |
7c84b1b8 MA |
30 | void (*cancel_async)(BlockAIOCB *acb); |
31 | AioContext *(*get_aio_context)(BlockAIOCB *acb); | |
8c82e9a4 | 32 | size_t aiocb_size; |
d7331bed | 33 | } AIOCBInfo; |
85e8dab1 | 34 | |
7c84b1b8 | 35 | struct BlockAIOCB { |
d7331bed | 36 | const AIOCBInfo *aiocb_info; |
85e8dab1 | 37 | BlockDriverState *bs; |
097310b5 | 38 | BlockCompletionFunc *cb; |
85e8dab1 | 39 | void *opaque; |
f197fe2b | 40 | int refcnt; |
85e8dab1 PB |
41 | }; |
42 | ||
d7331bed | 43 | void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs, |
097310b5 | 44 | BlockCompletionFunc *cb, void *opaque); |
8007429a | 45 | void qemu_aio_unref(void *p); |
f197fe2b | 46 | void qemu_aio_ref(void *p); |
85e8dab1 | 47 | |
f627aab1 | 48 | typedef struct AioHandler AioHandler; |
4749079c | 49 | typedef QLIST_HEAD(, AioHandler) AioHandlerList; |
f627aab1 | 50 | typedef void QEMUBHFunc(void *opaque); |
f6a51c84 | 51 | typedef bool AioPollFn(void *opaque); |
f627aab1 PB |
52 | typedef void IOHandler(void *opaque); |
53 | ||
0c330a73 | 54 | struct Coroutine; |
0187f5c9 PB |
55 | struct ThreadPool; |
56 | struct LinuxAioState; | |
6663a0a3 | 57 | struct LuringState; |
0187f5c9 | 58 | |
aa38e19f SH |
59 | /* Is polling disabled? */ |
60 | bool aio_poll_disabled(AioContext *ctx); | |
61 | ||
1f050a46 SH |
62 | /* Callbacks for file descriptor monitoring implementations */ |
63 | typedef struct { | |
64 | /* | |
65 | * update: | |
66 | * @ctx: the AioContext | |
b321051c SH |
67 | * @old_node: the existing handler or NULL if this file descriptor is being |
68 | * monitored for the first time | |
69 | * @new_node: the new handler or NULL if this file descriptor is being | |
70 | * removed | |
1f050a46 | 71 | * |
b321051c | 72 | * Add/remove/modify a monitored file descriptor. |
1f050a46 SH |
73 | * |
74 | * Called with ctx->list_lock acquired. | |
75 | */ | |
b321051c | 76 | void (*update)(AioContext *ctx, AioHandler *old_node, AioHandler *new_node); |
1f050a46 SH |
77 | |
78 | /* | |
79 | * wait: | |
80 | * @ctx: the AioContext | |
81 | * @ready_list: list for handlers that become ready | |
82 | * @timeout: maximum duration to wait, in nanoseconds | |
83 | * | |
84 | * Wait for file descriptors to become ready and place them on ready_list. | |
85 | * | |
86 | * Called with ctx->list_lock incremented but not locked. | |
87 | * | |
88 | * Returns: number of ready file descriptors. | |
89 | */ | |
90 | int (*wait)(AioContext *ctx, AioHandlerList *ready_list, int64_t timeout); | |
aa38e19f SH |
91 | |
92 | /* | |
93 | * need_wait: | |
94 | * @ctx: the AioContext | |
95 | * | |
96 | * Tell aio_poll() when to stop userspace polling early because ->wait() | |
97 | * has fds ready. | |
98 | * | |
99 | * File descriptor monitoring implementations that cannot poll fd readiness | |
100 | * from userspace should use aio_poll_disabled() here. This ensures that | |
101 | * file descriptors are not starved by handlers that frequently make | |
102 | * progress via userspace polling. | |
103 | * | |
104 | * Returns: true if ->wait() should be called, false otherwise. | |
105 | */ | |
106 | bool (*need_wait)(AioContext *ctx); | |
1f050a46 SH |
107 | } FDMonOps; |
108 | ||
8c6b0356 SH |
109 | /* |
110 | * Each aio_bh_poll() call carves off a slice of the BH list, so that newly | |
111 | * scheduled BHs are not processed until the next aio_bh_poll() call. All | |
112 | * active aio_bh_poll() calls chain their slices together in a list, so that | |
113 | * nested aio_bh_poll() calls process all scheduled bottom halves. | |
114 | */ | |
115 | typedef QSLIST_HEAD(, QEMUBH) BHList; | |
116 | typedef struct BHListSlice BHListSlice; | |
117 | struct BHListSlice { | |
118 | BHList bh_list; | |
119 | QSIMPLEQ_ENTRY(BHListSlice) next; | |
120 | }; | |
121 | ||
73fd282e SH |
122 | typedef QSLIST_HEAD(, AioHandler) AioHandlerSList; |
123 | ||
6a1751b7 | 124 | struct AioContext { |
e3713e00 PB |
125 | GSource source; |
126 | ||
7c690fd1 | 127 | /* Used by AioContext users to protect from multi-threaded access. */ |
3fe71223 | 128 | QemuRecMutex lock; |
98563fc3 | 129 | |
7c690fd1 | 130 | /* The list of registered AIO handlers. Protected by ctx->list_lock. */ |
4749079c SH |
131 | AioHandlerList aio_handlers; |
132 | ||
133 | /* The list of AIO handlers to be deleted. Protected by ctx->list_lock. */ | |
134 | AioHandlerList deleted_aio_handlers; | |
a915f4bc | 135 | |
eabc9779 | 136 | /* Used to avoid unnecessary event_notifier_set calls in aio_notify; |
3c18a92d PB |
137 | * only written from the AioContext home thread, or under the BQL in |
138 | * the case of the main AioContext. However, it is read from any | |
139 | * thread so it is still accessed with atomic primitives. | |
140 | * | |
141 | * If this field is 0, everything (file descriptors, bottom halves, | |
142 | * timers) will be re-evaluated before the next blocking poll() or | |
143 | * io_uring wait; therefore, the event_notifier_set call can be | |
144 | * skipped. If it is non-zero, you may need to wake up a concurrent | |
145 | * aio_poll or the glib main event loop, making event_notifier_set | |
146 | * necessary. | |
eabc9779 PB |
147 | * |
148 | * Bit 0 is reserved for GSource usage of the AioContext, and is 1 | |
54a16a63 | 149 | * between a call to aio_ctx_prepare and the next call to aio_ctx_check. |
eabc9779 PB |
150 | * Bits 1-31 simply count the number of active calls to aio_poll |
151 | * that are in the prepare or poll phase. | |
152 | * | |
153 | * The GSource and aio_poll must use a different mechanism because | |
154 | * there is no certainty that a call to GSource's prepare callback | |
155 | * (via g_main_context_prepare) is indeed followed by check and | |
156 | * dispatch. It's not clear whether this would be a bug, but let's | |
157 | * play safe and allow it---it will just cause extra calls to | |
158 | * event_notifier_set until the next call to dispatch. | |
159 | * | |
160 | * Instead, the aio_poll calls include both the prepare and the | |
161 | * dispatch phase, hence a simple counter is enough for them. | |
0ceb849b | 162 | */ |
eabc9779 | 163 | uint32_t notify_me; |
0ceb849b | 164 | |
7c690fd1 PB |
165 | /* A lock to protect between QEMUBH and AioHandler adders and deleter, |
166 | * and to ensure that no callbacks are removed while we're walking and | |
167 | * dispatching them. | |
d7c99a12 PB |
168 | */ |
169 | QemuLockCnt list_lock; | |
0ceb849b | 170 | |
8c6b0356 SH |
171 | /* Bottom Halves pending aio_bh_poll() processing */ |
172 | BHList bh_list; | |
173 | ||
174 | /* Chained BH list slices for each nested aio_bh_poll() call */ | |
175 | QSIMPLEQ_HEAD(, BHListSlice) bh_slice_list; | |
f627aab1 | 176 | |
05e514b1 PB |
177 | /* Used by aio_notify. |
178 | * | |
179 | * "notified" is used to avoid expensive event_notifier_test_and_clear | |
180 | * calls. When it is clear, the EventNotifier is clear, or one thread | |
181 | * is going to clear "notified" before processing more events. False | |
182 | * positives are possible, i.e. "notified" could be set even though the | |
183 | * EventNotifier is clear. | |
184 | * | |
185 | * Note that event_notifier_set *cannot* be optimized the same way. For | |
186 | * more information on the problem that would result, see "#ifdef BUG2" | |
187 | * in the docs/aio_notify_accept.promela formal model. | |
188 | */ | |
189 | bool notified; | |
2f4dc3c1 | 190 | EventNotifier notifier; |
6b5f8762 | 191 | |
0c330a73 PB |
192 | QSLIST_HEAD(, Coroutine) scheduled_coroutines; |
193 | QEMUBH *co_schedule_bh; | |
194 | ||
7c690fd1 PB |
195 | /* Thread pool for performing work and receiving completion callbacks. |
196 | * Has its own locking. | |
197 | */ | |
9b34277d | 198 | struct ThreadPool *thread_pool; |
dae21b98 | 199 | |
0187f5c9 | 200 | #ifdef CONFIG_LINUX_AIO |
6663a0a3 AM |
201 | /* |
202 | * State for native Linux AIO. Uses aio_context_acquire/release for | |
0187f5c9 PB |
203 | * locking. |
204 | */ | |
205 | struct LinuxAioState *linux_aio; | |
206 | #endif | |
6663a0a3 AM |
207 | #ifdef CONFIG_LINUX_IO_URING |
208 | /* | |
209 | * State for Linux io_uring. Uses aio_context_acquire/release for | |
210 | * locking. | |
211 | */ | |
212 | struct LuringState *linux_io_uring; | |
73fd282e SH |
213 | |
214 | /* State for file descriptor monitoring using Linux io_uring */ | |
215 | struct io_uring fdmon_io_uring; | |
216 | AioHandlerSList submit_list; | |
6663a0a3 | 217 | #endif |
0187f5c9 | 218 | |
7c690fd1 PB |
219 | /* TimerLists for calling timers - one per clock type. Has its own |
220 | * locking. | |
221 | */ | |
dae21b98 | 222 | QEMUTimerListGroup tlg; |
c1e1e5fa FZ |
223 | |
224 | int external_disable_cnt; | |
fbe3fc5c | 225 | |
4a1cba38 SH |
226 | /* Number of AioHandlers without .io_poll() */ |
227 | int poll_disable_cnt; | |
228 | ||
82a41186 SH |
229 | /* Polling mode parameters */ |
230 | int64_t poll_ns; /* current polling time in nanoseconds */ | |
231 | int64_t poll_max_ns; /* maximum polling time in nanoseconds */ | |
232 | int64_t poll_grow; /* polling time growth factor */ | |
233 | int64_t poll_shrink; /* polling time shrink factor */ | |
4a1cba38 | 234 | |
1793ad02 SG |
235 | /* AIO engine parameters */ |
236 | int64_t aio_max_batch; /* maximum number of requests in a batch */ | |
237 | ||
d37d0e36 SH |
238 | /* |
239 | * List of handlers participating in userspace polling. Protected by | |
240 | * ctx->list_lock. Iterated and modified mostly by the event loop thread | |
241 | * from aio_poll() with ctx->list_lock incremented. aio_set_fd_handler() | |
242 | * only touches the list to delete nodes if ctx->list_lock's count is zero. | |
243 | */ | |
244 | AioHandlerList poll_aio_handlers; | |
245 | ||
684e508c SH |
246 | /* Are we in polling mode or monitoring file descriptors? */ |
247 | bool poll_started; | |
248 | ||
fbe3fc5c FZ |
249 | /* epoll(7) state used when built with CONFIG_EPOLL */ |
250 | int epollfd; | |
1f050a46 SH |
251 | |
252 | const FDMonOps *fdmon_ops; | |
6a1751b7 | 253 | }; |
f627aab1 | 254 | |
f627aab1 PB |
255 | /** |
256 | * aio_context_new: Allocate a new AioContext. | |
257 | * | |
258 | * AioContext provide a mini event-loop that can be waited on synchronously. | |
259 | * They also provide bottom halves, a service to execute a piece of code | |
260 | * as soon as possible. | |
261 | */ | |
2f78e491 | 262 | AioContext *aio_context_new(Error **errp); |
f627aab1 | 263 | |
e3713e00 PB |
264 | /** |
265 | * aio_context_ref: | |
266 | * @ctx: The AioContext to operate on. | |
267 | * | |
268 | * Add a reference to an AioContext. | |
269 | */ | |
270 | void aio_context_ref(AioContext *ctx); | |
271 | ||
272 | /** | |
273 | * aio_context_unref: | |
274 | * @ctx: The AioContext to operate on. | |
275 | * | |
276 | * Drop a reference to an AioContext. | |
277 | */ | |
278 | void aio_context_unref(AioContext *ctx); | |
279 | ||
98563fc3 | 280 | /* Take ownership of the AioContext. If the AioContext will be shared between |
49110174 PB |
281 | * threads, and a thread does not want to be interrupted, it will have to |
282 | * take ownership around calls to aio_poll(). Otherwise, aio_poll() | |
283 | * automatically takes care of calling aio_context_acquire and | |
284 | * aio_context_release. | |
98563fc3 | 285 | * |
7c690fd1 PB |
286 | * Note that this is separate from bdrv_drained_begin/bdrv_drained_end. A |
287 | * thread still has to call those to avoid being interrupted by the guest. | |
288 | * | |
289 | * Bottom halves, timers and callbacks can be created or removed without | |
290 | * acquiring the AioContext. | |
98563fc3 SH |
291 | */ |
292 | void aio_context_acquire(AioContext *ctx); | |
293 | ||
294 | /* Relinquish ownership of the AioContext. */ | |
295 | void aio_context_release(AioContext *ctx); | |
296 | ||
0f08586c SH |
297 | /** |
298 | * aio_bh_schedule_oneshot_full: Allocate a new bottom half structure that will | |
299 | * run only once and as soon as possible. | |
300 | * | |
301 | * @name: A human-readable identifier for debugging purposes. | |
302 | */ | |
303 | void aio_bh_schedule_oneshot_full(AioContext *ctx, QEMUBHFunc *cb, void *opaque, | |
304 | const char *name); | |
305 | ||
5b8bb359 PB |
306 | /** |
307 | * aio_bh_schedule_oneshot: Allocate a new bottom half structure that will run | |
308 | * only once and as soon as possible. | |
0f08586c SH |
309 | * |
310 | * A convenience wrapper for aio_bh_schedule_oneshot_full() that uses cb as the | |
311 | * name string. | |
5b8bb359 | 312 | */ |
0f08586c SH |
313 | #define aio_bh_schedule_oneshot(ctx, cb, opaque) \ |
314 | aio_bh_schedule_oneshot_full((ctx), (cb), (opaque), (stringify(cb))) | |
5b8bb359 | 315 | |
f627aab1 | 316 | /** |
0f08586c | 317 | * aio_bh_new_full: Allocate a new bottom half structure. |
f627aab1 PB |
318 | * |
319 | * Bottom halves are lightweight callbacks whose invocation is guaranteed | |
320 | * to be wait-free, thread-safe and signal-safe. The #QEMUBH structure | |
321 | * is opaque and must be allocated prior to its use. | |
0f08586c SH |
322 | * |
323 | * @name: A human-readable identifier for debugging purposes. | |
324 | */ | |
325 | QEMUBH *aio_bh_new_full(AioContext *ctx, QEMUBHFunc *cb, void *opaque, | |
326 | const char *name); | |
327 | ||
328 | /** | |
329 | * aio_bh_new: Allocate a new bottom half structure | |
330 | * | |
331 | * A convenience wrapper for aio_bh_new_full() that uses the cb as the name | |
332 | * string. | |
f627aab1 | 333 | */ |
0f08586c SH |
334 | #define aio_bh_new(ctx, cb, opaque) \ |
335 | aio_bh_new_full((ctx), (cb), (opaque), (stringify(cb))) | |
f627aab1 | 336 | |
2f4dc3c1 PB |
337 | /** |
338 | * aio_notify: Force processing of pending events. | |
339 | * | |
340 | * Similar to signaling a condition variable, aio_notify forces | |
722f8d90 YB |
341 | * aio_poll to exit, so that the next call will re-examine pending events. |
342 | * The caller of aio_notify will usually call aio_poll again very soon, | |
2f4dc3c1 PB |
343 | * or go through another iteration of the GLib main loop. Hence, aio_notify |
344 | * also has the side effect of recalculating the sets of file descriptors | |
345 | * that the main loop waits for. | |
346 | * | |
347 | * Calling aio_notify is rarely necessary, because for example scheduling | |
348 | * a bottom half calls it already. | |
349 | */ | |
350 | void aio_notify(AioContext *ctx); | |
351 | ||
05e514b1 PB |
352 | /** |
353 | * aio_notify_accept: Acknowledge receiving an aio_notify. | |
354 | * | |
355 | * aio_notify() uses an EventNotifier in order to wake up a sleeping | |
356 | * aio_poll() or g_main_context_iteration(). Calls to aio_notify() are | |
357 | * usually rare, but the AioContext has to clear the EventNotifier on | |
358 | * every aio_poll() or g_main_context_iteration() in order to avoid | |
359 | * busy waiting. This event_notifier_test_and_clear() cannot be done | |
360 | * using the usual aio_context_set_event_notifier(), because it must | |
361 | * be done before processing all events (file descriptors, bottom halves, | |
362 | * timers). | |
363 | * | |
364 | * aio_notify_accept() is an optimized event_notifier_test_and_clear() | |
365 | * that is specific to an AioContext's notifier; it is used internally | |
366 | * to clear the EventNotifier only if aio_notify() had been called. | |
367 | */ | |
368 | void aio_notify_accept(AioContext *ctx); | |
369 | ||
df281b80 PD |
370 | /** |
371 | * aio_bh_call: Executes callback function of the specified BH. | |
372 | */ | |
373 | void aio_bh_call(QEMUBH *bh); | |
374 | ||
f627aab1 PB |
375 | /** |
376 | * aio_bh_poll: Poll bottom halves for an AioContext. | |
377 | * | |
378 | * These are internal functions used by the QEMU main loop. | |
dcc772e2 LPF |
379 | * And notice that multiple occurrences of aio_bh_poll cannot |
380 | * be called concurrently | |
f627aab1 PB |
381 | */ |
382 | int aio_bh_poll(AioContext *ctx); | |
f627aab1 PB |
383 | |
384 | /** | |
385 | * qemu_bh_schedule: Schedule a bottom half. | |
386 | * | |
387 | * Scheduling a bottom half interrupts the main loop and causes the | |
388 | * execution of the callback that was passed to qemu_bh_new. | |
389 | * | |
390 | * Bottom halves that are scheduled from a bottom half handler are instantly | |
391 | * invoked. This can create an infinite loop if a bottom half handler | |
392 | * schedules itself. | |
393 | * | |
394 | * @bh: The bottom half to be scheduled. | |
395 | */ | |
396 | void qemu_bh_schedule(QEMUBH *bh); | |
397 | ||
398 | /** | |
399 | * qemu_bh_cancel: Cancel execution of a bottom half. | |
400 | * | |
401 | * Canceling execution of a bottom half undoes the effect of calls to | |
402 | * qemu_bh_schedule without freeing its resources yet. While cancellation | |
403 | * itself is also wait-free and thread-safe, it can of course race with the | |
404 | * loop that executes bottom halves unless you are holding the iothread | |
405 | * mutex. This makes it mostly useless if you are not holding the mutex. | |
406 | * | |
407 | * @bh: The bottom half to be canceled. | |
408 | */ | |
409 | void qemu_bh_cancel(QEMUBH *bh); | |
410 | ||
411 | /** | |
412 | *qemu_bh_delete: Cancel execution of a bottom half and free its resources. | |
413 | * | |
414 | * Deleting a bottom half frees the memory that was allocated for it by | |
415 | * qemu_bh_new. It also implies canceling the bottom half if it was | |
416 | * scheduled. | |
dcc772e2 LPF |
417 | * This func is async. The bottom half will do the delete action at the finial |
418 | * end. | |
f627aab1 PB |
419 | * |
420 | * @bh: The bottom half to be deleted. | |
421 | */ | |
422 | void qemu_bh_delete(QEMUBH *bh); | |
423 | ||
cd9ba1eb | 424 | /* Return whether there are any pending callbacks from the GSource |
a3462c65 PB |
425 | * attached to the AioContext, before g_poll is invoked. |
426 | * | |
427 | * This is used internally in the implementation of the GSource. | |
428 | */ | |
429 | bool aio_prepare(AioContext *ctx); | |
430 | ||
431 | /* Return whether there are any pending callbacks from the GSource | |
432 | * attached to the AioContext, after g_poll is invoked. | |
cd9ba1eb PB |
433 | * |
434 | * This is used internally in the implementation of the GSource. | |
435 | */ | |
436 | bool aio_pending(AioContext *ctx); | |
437 | ||
e4c7e2d1 PB |
438 | /* Dispatch any pending callbacks from the GSource attached to the AioContext. |
439 | * | |
440 | * This is used internally in the implementation of the GSource. | |
441 | */ | |
a153bf52 | 442 | void aio_dispatch(AioContext *ctx); |
e4c7e2d1 | 443 | |
7c0628b2 PB |
444 | /* Progress in completing AIO work to occur. This can issue new pending |
445 | * aio as a result of executing I/O completion or bh callbacks. | |
bcdc1857 | 446 | * |
acfb23ad PB |
447 | * Return whether any progress was made by executing AIO or bottom half |
448 | * handlers. If @blocking == true, this should always be true except | |
449 | * if someone called aio_notify. | |
7c0628b2 PB |
450 | * |
451 | * If there are no pending bottom halves, but there are pending AIO | |
452 | * operations, it may not be possible to make any progress without | |
453 | * blocking. If @blocking is true, this function will wait until one | |
454 | * or more AIO events have completed, to ensure something has moved | |
455 | * before returning. | |
7c0628b2 PB |
456 | */ |
457 | bool aio_poll(AioContext *ctx, bool blocking); | |
a76bab49 AL |
458 | |
459 | /* Register a file descriptor and associated callbacks. Behaves very similarly | |
6484e422 | 460 | * to qemu_set_fd_handler. Unlike qemu_set_fd_handler, these callbacks will |
87f68d31 | 461 | * be invoked when using aio_poll(). |
a76bab49 AL |
462 | * |
463 | * Code that invokes AIO completion functions should rely on this function | |
464 | * instead of qemu_set_fd_handler[2]. | |
465 | */ | |
a915f4bc PB |
466 | void aio_set_fd_handler(AioContext *ctx, |
467 | int fd, | |
dca21ef2 | 468 | bool is_external, |
a915f4bc PB |
469 | IOHandler *io_read, |
470 | IOHandler *io_write, | |
f6a51c84 | 471 | AioPollFn *io_poll, |
826cc324 | 472 | IOHandler *io_poll_ready, |
a915f4bc | 473 | void *opaque); |
9958c351 | 474 | |
684e508c SH |
475 | /* Set polling begin/end callbacks for a file descriptor that has already been |
476 | * registered with aio_set_fd_handler. Do nothing if the file descriptor is | |
477 | * not registered. | |
478 | */ | |
479 | void aio_set_fd_poll(AioContext *ctx, int fd, | |
480 | IOHandler *io_poll_begin, | |
481 | IOHandler *io_poll_end); | |
482 | ||
9958c351 PB |
483 | /* Register an event notifier and associated callbacks. Behaves very similarly |
484 | * to event_notifier_set_handler. Unlike event_notifier_set_handler, these callbacks | |
87f68d31 | 485 | * will be invoked when using aio_poll(). |
9958c351 PB |
486 | * |
487 | * Code that invokes AIO completion functions should rely on this function | |
488 | * instead of event_notifier_set_handler. | |
489 | */ | |
a915f4bc PB |
490 | void aio_set_event_notifier(AioContext *ctx, |
491 | EventNotifier *notifier, | |
dca21ef2 | 492 | bool is_external, |
f6a51c84 | 493 | EventNotifierHandler *io_read, |
826cc324 SH |
494 | AioPollFn *io_poll, |
495 | EventNotifierHandler *io_poll_ready); | |
a915f4bc | 496 | |
684e508c SH |
497 | /* Set polling begin/end callbacks for an event notifier that has already been |
498 | * registered with aio_set_event_notifier. Do nothing if the event notifier is | |
499 | * not registered. | |
500 | */ | |
501 | void aio_set_event_notifier_poll(AioContext *ctx, | |
502 | EventNotifier *notifier, | |
503 | EventNotifierHandler *io_poll_begin, | |
504 | EventNotifierHandler *io_poll_end); | |
505 | ||
e3713e00 PB |
506 | /* Return a GSource that lets the main loop poll the file descriptors attached |
507 | * to this AioContext. | |
508 | */ | |
509 | GSource *aio_get_g_source(AioContext *ctx); | |
510 | ||
9b34277d SH |
511 | /* Return the ThreadPool bound to this AioContext */ |
512 | struct ThreadPool *aio_get_thread_pool(AioContext *ctx); | |
513 | ||
ed6e2161 NA |
514 | /* Setup the LinuxAioState bound to this AioContext */ |
515 | struct LinuxAioState *aio_setup_linux_aio(AioContext *ctx, Error **errp); | |
516 | ||
0187f5c9 PB |
517 | /* Return the LinuxAioState bound to this AioContext */ |
518 | struct LinuxAioState *aio_get_linux_aio(AioContext *ctx); | |
519 | ||
6663a0a3 AM |
520 | /* Setup the LuringState bound to this AioContext */ |
521 | struct LuringState *aio_setup_linux_io_uring(AioContext *ctx, Error **errp); | |
522 | ||
523 | /* Return the LuringState bound to this AioContext */ | |
524 | struct LuringState *aio_get_linux_io_uring(AioContext *ctx); | |
4e29e831 | 525 | /** |
89a603a0 | 526 | * aio_timer_new_with_attrs: |
4e29e831 AB |
527 | * @ctx: the aio context |
528 | * @type: the clock type | |
529 | * @scale: the scale | |
89a603a0 AP |
530 | * @attributes: 0, or one to multiple OR'ed QEMU_TIMER_ATTR_<id> values |
531 | * to assign | |
4e29e831 AB |
532 | * @cb: the callback to call on timer expiry |
533 | * @opaque: the opaque pointer to pass to the callback | |
534 | * | |
89a603a0 | 535 | * Allocate a new timer (with attributes) attached to the context @ctx. |
4e29e831 AB |
536 | * The function is responsible for memory allocation. |
537 | * | |
89a603a0 AP |
538 | * The preferred interface is aio_timer_init or aio_timer_init_with_attrs. |
539 | * Use that unless you really need dynamic memory allocation. | |
540 | * | |
541 | * Returns: a pointer to the new timer | |
542 | */ | |
543 | static inline QEMUTimer *aio_timer_new_with_attrs(AioContext *ctx, | |
544 | QEMUClockType type, | |
545 | int scale, int attributes, | |
546 | QEMUTimerCB *cb, void *opaque) | |
547 | { | |
548 | return timer_new_full(&ctx->tlg, type, scale, attributes, cb, opaque); | |
549 | } | |
550 | ||
551 | /** | |
552 | * aio_timer_new: | |
553 | * @ctx: the aio context | |
554 | * @type: the clock type | |
555 | * @scale: the scale | |
556 | * @cb: the callback to call on timer expiry | |
557 | * @opaque: the opaque pointer to pass to the callback | |
558 | * | |
559 | * Allocate a new timer attached to the context @ctx. | |
560 | * See aio_timer_new_with_attrs for details. | |
4e29e831 AB |
561 | * |
562 | * Returns: a pointer to the new timer | |
563 | */ | |
564 | static inline QEMUTimer *aio_timer_new(AioContext *ctx, QEMUClockType type, | |
565 | int scale, | |
566 | QEMUTimerCB *cb, void *opaque) | |
567 | { | |
89a603a0 AP |
568 | return timer_new_full(&ctx->tlg, type, scale, 0, cb, opaque); |
569 | } | |
570 | ||
571 | /** | |
572 | * aio_timer_init_with_attrs: | |
573 | * @ctx: the aio context | |
574 | * @ts: the timer | |
575 | * @type: the clock type | |
576 | * @scale: the scale | |
577 | * @attributes: 0, or one to multiple OR'ed QEMU_TIMER_ATTR_<id> values | |
578 | * to assign | |
579 | * @cb: the callback to call on timer expiry | |
580 | * @opaque: the opaque pointer to pass to the callback | |
581 | * | |
582 | * Initialise a new timer (with attributes) attached to the context @ctx. | |
583 | * The caller is responsible for memory allocation. | |
584 | */ | |
585 | static inline void aio_timer_init_with_attrs(AioContext *ctx, | |
586 | QEMUTimer *ts, QEMUClockType type, | |
587 | int scale, int attributes, | |
588 | QEMUTimerCB *cb, void *opaque) | |
589 | { | |
590 | timer_init_full(ts, &ctx->tlg, type, scale, attributes, cb, opaque); | |
4e29e831 AB |
591 | } |
592 | ||
593 | /** | |
594 | * aio_timer_init: | |
595 | * @ctx: the aio context | |
596 | * @ts: the timer | |
597 | * @type: the clock type | |
598 | * @scale: the scale | |
599 | * @cb: the callback to call on timer expiry | |
600 | * @opaque: the opaque pointer to pass to the callback | |
601 | * | |
602 | * Initialise a new timer attached to the context @ctx. | |
89a603a0 | 603 | * See aio_timer_init_with_attrs for details. |
4e29e831 AB |
604 | */ |
605 | static inline void aio_timer_init(AioContext *ctx, | |
606 | QEMUTimer *ts, QEMUClockType type, | |
607 | int scale, | |
608 | QEMUTimerCB *cb, void *opaque) | |
609 | { | |
89a603a0 | 610 | timer_init_full(ts, &ctx->tlg, type, scale, 0, cb, opaque); |
4e29e831 AB |
611 | } |
612 | ||
845ca10d PB |
613 | /** |
614 | * aio_compute_timeout: | |
615 | * @ctx: the aio context | |
616 | * | |
617 | * Compute the timeout that a blocking aio_poll should use. | |
618 | */ | |
619 | int64_t aio_compute_timeout(AioContext *ctx); | |
620 | ||
c1e1e5fa FZ |
621 | /** |
622 | * aio_disable_external: | |
623 | * @ctx: the aio context | |
624 | * | |
625 | * Disable the further processing of external clients. | |
626 | */ | |
627 | static inline void aio_disable_external(AioContext *ctx) | |
628 | { | |
d73415a3 | 629 | qatomic_inc(&ctx->external_disable_cnt); |
c1e1e5fa FZ |
630 | } |
631 | ||
632 | /** | |
633 | * aio_enable_external: | |
634 | * @ctx: the aio context | |
635 | * | |
636 | * Enable the processing of external clients. | |
637 | */ | |
638 | static inline void aio_enable_external(AioContext *ctx) | |
639 | { | |
321d1dba SH |
640 | int old; |
641 | ||
d73415a3 | 642 | old = qatomic_fetch_dec(&ctx->external_disable_cnt); |
321d1dba SH |
643 | assert(old > 0); |
644 | if (old == 1) { | |
645 | /* Kick event loop so it re-arms file descriptors */ | |
646 | aio_notify(ctx); | |
647 | } | |
c1e1e5fa FZ |
648 | } |
649 | ||
5ceb9e39 FZ |
650 | /** |
651 | * aio_external_disabled: | |
652 | * @ctx: the aio context | |
653 | * | |
654 | * Return true if the external clients are disabled. | |
655 | */ | |
656 | static inline bool aio_external_disabled(AioContext *ctx) | |
657 | { | |
d73415a3 | 658 | return qatomic_read(&ctx->external_disable_cnt); |
5ceb9e39 FZ |
659 | } |
660 | ||
c1e1e5fa FZ |
661 | /** |
662 | * aio_node_check: | |
663 | * @ctx: the aio context | |
664 | * @is_external: Whether or not the checked node is an external event source. | |
665 | * | |
666 | * Check if the node's is_external flag is okay to be polled by the ctx at this | |
667 | * moment. True means green light. | |
668 | */ | |
669 | static inline bool aio_node_check(AioContext *ctx, bool is_external) | |
670 | { | |
d73415a3 | 671 | return !is_external || !qatomic_read(&ctx->external_disable_cnt); |
c1e1e5fa FZ |
672 | } |
673 | ||
0c330a73 PB |
674 | /** |
675 | * aio_co_schedule: | |
676 | * @ctx: the aio context | |
677 | * @co: the coroutine | |
678 | * | |
679 | * Start a coroutine on a remote AioContext. | |
680 | * | |
681 | * The coroutine must not be entered by anyone else while aio_co_schedule() | |
682 | * is active. In addition the coroutine must have yielded unless ctx | |
683 | * is the context in which the coroutine is running (i.e. the value of | |
684 | * qemu_get_current_aio_context() from the coroutine itself). | |
685 | */ | |
686 | void aio_co_schedule(AioContext *ctx, struct Coroutine *co); | |
687 | ||
26b0b698 KW |
688 | /** |
689 | * aio_co_reschedule_self: | |
690 | * @new_ctx: the new context | |
691 | * | |
692 | * Move the currently running coroutine to new_ctx. If the coroutine is already | |
693 | * running in new_ctx, do nothing. | |
694 | */ | |
695 | void coroutine_fn aio_co_reschedule_self(AioContext *new_ctx); | |
696 | ||
0c330a73 PB |
697 | /** |
698 | * aio_co_wake: | |
699 | * @co: the coroutine | |
700 | * | |
701 | * Restart a coroutine on the AioContext where it was running last, thus | |
702 | * preventing coroutines from jumping from one context to another when they | |
703 | * go to sleep. | |
704 | * | |
705 | * aio_co_wake may be executed either in coroutine or non-coroutine | |
706 | * context. The coroutine must not be entered by anyone else while | |
707 | * aio_co_wake() is active. | |
708 | */ | |
709 | void aio_co_wake(struct Coroutine *co); | |
710 | ||
8865852e FZ |
711 | /** |
712 | * aio_co_enter: | |
713 | * @ctx: the context to run the coroutine | |
714 | * @co: the coroutine to run | |
715 | * | |
716 | * Enter a coroutine in the specified AioContext. | |
717 | */ | |
718 | void aio_co_enter(AioContext *ctx, struct Coroutine *co); | |
719 | ||
e4370165 PB |
720 | /** |
721 | * Return the AioContext whose event loop runs in the current thread. | |
722 | * | |
723 | * If called from an IOThread this will be the IOThread's AioContext. If | |
5f50be9b PB |
724 | * called from the main thread or with the "big QEMU lock" taken it |
725 | * will be the main loop AioContext. | |
e4370165 PB |
726 | */ |
727 | AioContext *qemu_get_current_aio_context(void); | |
728 | ||
5f50be9b PB |
729 | void qemu_set_current_aio_context(AioContext *ctx); |
730 | ||
37fcee5d FZ |
731 | /** |
732 | * aio_context_setup: | |
733 | * @ctx: the aio context | |
734 | * | |
735 | * Initialize the aio context. | |
736 | */ | |
7e003465 | 737 | void aio_context_setup(AioContext *ctx); |
37fcee5d | 738 | |
cd0a6d2b JW |
739 | /** |
740 | * aio_context_destroy: | |
741 | * @ctx: the aio context | |
742 | * | |
743 | * Destroy the aio context. | |
744 | */ | |
745 | void aio_context_destroy(AioContext *ctx); | |
746 | ||
ba607ca8 SH |
747 | /* Used internally, do not call outside AioContext code */ |
748 | void aio_context_use_g_source(AioContext *ctx); | |
749 | ||
4a1cba38 SH |
750 | /** |
751 | * aio_context_set_poll_params: | |
752 | * @ctx: the aio context | |
753 | * @max_ns: how long to busy poll for, in nanoseconds | |
82a41186 SH |
754 | * @grow: polling time growth factor |
755 | * @shrink: polling time shrink factor | |
4a1cba38 SH |
756 | * |
757 | * Poll mode can be disabled by setting poll_max_ns to 0. | |
758 | */ | |
759 | void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns, | |
82a41186 | 760 | int64_t grow, int64_t shrink, |
4a1cba38 SH |
761 | Error **errp); |
762 | ||
1793ad02 SG |
763 | /** |
764 | * aio_context_set_aio_params: | |
765 | * @ctx: the aio context | |
766 | * @max_batch: maximum number of requests in a batch, 0 means that the | |
767 | * engine will use its default | |
768 | */ | |
769 | void aio_context_set_aio_params(AioContext *ctx, int64_t max_batch, | |
770 | Error **errp); | |
771 | ||
a76bab49 | 772 | #endif |