]>
Commit | Line | Data |
---|---|---|
4f999d05 KW |
1 | /* |
2 | * QEMU System Emulator | |
3 | * | |
4 | * Copyright (c) 2003-2008 Fabrice Bellard | |
5 | * | |
6 | * Permission is hereby granted, free of charge, to any person obtaining a copy | |
7 | * of this software and associated documentation files (the "Software"), to deal | |
8 | * in the Software without restriction, including without limitation the rights | |
9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
10 | * copies of the Software, and to permit persons to whom the Software is | |
11 | * furnished to do so, subject to the following conditions: | |
12 | * | |
13 | * The above copyright notice and this permission notice shall be included in | |
14 | * all copies or substantial portions of the Software. | |
15 | * | |
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN | |
22 | * THE SOFTWARE. | |
23 | */ | |
24 | ||
d38ea87a | 25 | #include "qemu/osdep.h" |
da34e65c | 26 | #include "qapi/error.h" |
4f999d05 | 27 | #include "qemu-common.h" |
737e150e | 28 | #include "block/aio.h" |
9b34277d | 29 | #include "block/thread-pool.h" |
1de7afc9 | 30 | #include "qemu/main-loop.h" |
0ceb849b | 31 | #include "qemu/atomic.h" |
0187f5c9 | 32 | #include "block/raw-aio.h" |
9a1e9481 | 33 | |
4f999d05 KW |
34 | /***********************************************************/ |
35 | /* bottom halves (can be seen as timers which expire ASAP) */ | |
36 | ||
37 | struct QEMUBH { | |
2f4dc3c1 | 38 | AioContext *ctx; |
4f999d05 KW |
39 | QEMUBHFunc *cb; |
40 | void *opaque; | |
4f999d05 | 41 | QEMUBH *next; |
9b47b17e SW |
42 | bool scheduled; |
43 | bool idle; | |
44 | bool deleted; | |
4f999d05 KW |
45 | }; |
46 | ||
f627aab1 | 47 | QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque) |
4f999d05 KW |
48 | { |
49 | QEMUBH *bh; | |
ee82310f PB |
50 | bh = g_new(QEMUBH, 1); |
51 | *bh = (QEMUBH){ | |
52 | .ctx = ctx, | |
53 | .cb = cb, | |
54 | .opaque = opaque, | |
55 | }; | |
dcc772e2 | 56 | qemu_mutex_lock(&ctx->bh_lock); |
f627aab1 | 57 | bh->next = ctx->first_bh; |
dcc772e2 LPF |
58 | /* Make sure that the members are ready before putting bh into list */ |
59 | smp_wmb(); | |
f627aab1 | 60 | ctx->first_bh = bh; |
dcc772e2 | 61 | qemu_mutex_unlock(&ctx->bh_lock); |
4f999d05 KW |
62 | return bh; |
63 | } | |
64 | ||
df281b80 PD |
65 | void aio_bh_call(QEMUBH *bh) |
66 | { | |
67 | bh->cb(bh->opaque); | |
68 | } | |
69 | ||
dcc772e2 | 70 | /* Multiple occurrences of aio_bh_poll cannot be called concurrently */ |
f627aab1 | 71 | int aio_bh_poll(AioContext *ctx) |
4f999d05 | 72 | { |
7887f620 | 73 | QEMUBH *bh, **bhp, *next; |
4f999d05 | 74 | int ret; |
648fb0ea | 75 | |
f627aab1 | 76 | ctx->walking_bh++; |
4f999d05 KW |
77 | |
78 | ret = 0; | |
f627aab1 | 79 | for (bh = ctx->first_bh; bh; bh = next) { |
dcc772e2 LPF |
80 | /* Make sure that fetching bh happens before accessing its members */ |
81 | smp_read_barrier_depends(); | |
7887f620 | 82 | next = bh->next; |
e8d3b1a2 PB |
83 | /* The atomic_xchg is paired with the one in qemu_bh_schedule. The |
84 | * implicit memory barrier ensures that the callback sees all writes | |
85 | * done by the scheduling thread. It also ensures that the scheduling | |
86 | * thread sees the zero before bh->cb has run, and thus will call | |
87 | * aio_notify again if necessary. | |
88 | */ | |
89 | if (!bh->deleted && atomic_xchg(&bh->scheduled, 0)) { | |
ca96ac44 SH |
90 | /* Idle BHs and the notify BH don't count as progress */ |
91 | if (!bh->idle && bh != ctx->notify_dummy_bh) { | |
4f999d05 | 92 | ret = 1; |
ca96ac44 | 93 | } |
4f999d05 | 94 | bh->idle = 0; |
df281b80 | 95 | aio_bh_call(bh); |
4f999d05 KW |
96 | } |
97 | } | |
98 | ||
f627aab1 | 99 | ctx->walking_bh--; |
648fb0ea | 100 | |
4f999d05 | 101 | /* remove deleted bhs */ |
f627aab1 | 102 | if (!ctx->walking_bh) { |
dcc772e2 | 103 | qemu_mutex_lock(&ctx->bh_lock); |
f627aab1 | 104 | bhp = &ctx->first_bh; |
648fb0ea KW |
105 | while (*bhp) { |
106 | bh = *bhp; | |
107 | if (bh->deleted) { | |
108 | *bhp = bh->next; | |
109 | g_free(bh); | |
110 | } else { | |
111 | bhp = &bh->next; | |
112 | } | |
113 | } | |
dcc772e2 | 114 | qemu_mutex_unlock(&ctx->bh_lock); |
4f999d05 KW |
115 | } |
116 | ||
117 | return ret; | |
118 | } | |
119 | ||
120 | void qemu_bh_schedule_idle(QEMUBH *bh) | |
121 | { | |
4f999d05 | 122 | bh->idle = 1; |
dcc772e2 LPF |
123 | /* Make sure that idle & any writes needed by the callback are done |
124 | * before the locations are read in the aio_bh_poll. | |
125 | */ | |
e8d3b1a2 | 126 | atomic_mb_set(&bh->scheduled, 1); |
4f999d05 KW |
127 | } |
128 | ||
129 | void qemu_bh_schedule(QEMUBH *bh) | |
130 | { | |
924fe129 SH |
131 | AioContext *ctx; |
132 | ||
924fe129 | 133 | ctx = bh->ctx; |
4f999d05 | 134 | bh->idle = 0; |
e8d3b1a2 | 135 | /* The memory barrier implicit in atomic_xchg makes sure that: |
924fe129 SH |
136 | * 1. idle & any writes needed by the callback are done before the |
137 | * locations are read in the aio_bh_poll. | |
138 | * 2. ctx is loaded before scheduled is set and the callback has a chance | |
139 | * to execute. | |
dcc772e2 | 140 | */ |
e8d3b1a2 PB |
141 | if (atomic_xchg(&bh->scheduled, 1) == 0) { |
142 | aio_notify(ctx); | |
143 | } | |
4f999d05 KW |
144 | } |
145 | ||
dcc772e2 LPF |
146 | |
147 | /* This func is async. | |
148 | */ | |
4f999d05 KW |
149 | void qemu_bh_cancel(QEMUBH *bh) |
150 | { | |
151 | bh->scheduled = 0; | |
152 | } | |
153 | ||
dcc772e2 LPF |
154 | /* This func is async.The bottom half will do the delete action at the finial |
155 | * end. | |
156 | */ | |
4f999d05 KW |
157 | void qemu_bh_delete(QEMUBH *bh) |
158 | { | |
159 | bh->scheduled = 0; | |
160 | bh->deleted = 1; | |
161 | } | |
162 | ||
845ca10d PB |
163 | int64_t |
164 | aio_compute_timeout(AioContext *ctx) | |
4f999d05 | 165 | { |
845ca10d PB |
166 | int64_t deadline; |
167 | int timeout = -1; | |
4f999d05 KW |
168 | QEMUBH *bh; |
169 | ||
f627aab1 | 170 | for (bh = ctx->first_bh; bh; bh = bh->next) { |
4f999d05 KW |
171 | if (!bh->deleted && bh->scheduled) { |
172 | if (bh->idle) { | |
173 | /* idle bottom halves will be polled at least | |
174 | * every 10ms */ | |
845ca10d | 175 | timeout = 10000000; |
4f999d05 KW |
176 | } else { |
177 | /* non-idle bottom halves will be executed | |
178 | * immediately */ | |
845ca10d | 179 | return 0; |
4f999d05 KW |
180 | } |
181 | } | |
182 | } | |
e3713e00 | 183 | |
845ca10d | 184 | deadline = timerlistgroup_deadline_ns(&ctx->tlg); |
533a8cf3 | 185 | if (deadline == 0) { |
845ca10d | 186 | return 0; |
533a8cf3 | 187 | } else { |
845ca10d | 188 | return qemu_soonest_timeout(timeout, deadline); |
533a8cf3 | 189 | } |
845ca10d | 190 | } |
533a8cf3 | 191 | |
845ca10d PB |
192 | static gboolean |
193 | aio_ctx_prepare(GSource *source, gint *timeout) | |
194 | { | |
195 | AioContext *ctx = (AioContext *) source; | |
196 | ||
eabc9779 PB |
197 | atomic_or(&ctx->notify_me, 1); |
198 | ||
845ca10d PB |
199 | /* We assume there is no timeout already supplied */ |
200 | *timeout = qemu_timeout_ns_to_ms(aio_compute_timeout(ctx)); | |
a3462c65 PB |
201 | |
202 | if (aio_prepare(ctx)) { | |
203 | *timeout = 0; | |
204 | } | |
205 | ||
845ca10d | 206 | return *timeout == 0; |
e3713e00 PB |
207 | } |
208 | ||
209 | static gboolean | |
210 | aio_ctx_check(GSource *source) | |
211 | { | |
212 | AioContext *ctx = (AioContext *) source; | |
213 | QEMUBH *bh; | |
214 | ||
eabc9779 | 215 | atomic_and(&ctx->notify_me, ~1); |
05e514b1 | 216 | aio_notify_accept(ctx); |
21a03d17 | 217 | |
e3713e00 PB |
218 | for (bh = ctx->first_bh; bh; bh = bh->next) { |
219 | if (!bh->deleted && bh->scheduled) { | |
220 | return true; | |
6977d901 | 221 | } |
e3713e00 | 222 | } |
533a8cf3 | 223 | return aio_pending(ctx) || (timerlistgroup_deadline_ns(&ctx->tlg) == 0); |
e3713e00 PB |
224 | } |
225 | ||
226 | static gboolean | |
227 | aio_ctx_dispatch(GSource *source, | |
228 | GSourceFunc callback, | |
229 | gpointer user_data) | |
230 | { | |
231 | AioContext *ctx = (AioContext *) source; | |
232 | ||
233 | assert(callback == NULL); | |
e4c7e2d1 | 234 | aio_dispatch(ctx); |
e3713e00 PB |
235 | return true; |
236 | } | |
237 | ||
2f4dc3c1 PB |
238 | static void |
239 | aio_ctx_finalize(GSource *source) | |
240 | { | |
241 | AioContext *ctx = (AioContext *) source; | |
242 | ||
ca96ac44 | 243 | qemu_bh_delete(ctx->notify_dummy_bh); |
9b34277d | 244 | thread_pool_free(ctx->thread_pool); |
a076972a | 245 | |
0187f5c9 PB |
246 | #ifdef CONFIG_LINUX_AIO |
247 | if (ctx->linux_aio) { | |
248 | laio_detach_aio_context(ctx->linux_aio, ctx); | |
249 | laio_cleanup(ctx->linux_aio); | |
250 | ctx->linux_aio = NULL; | |
251 | } | |
252 | #endif | |
253 | ||
a076972a SH |
254 | qemu_mutex_lock(&ctx->bh_lock); |
255 | while (ctx->first_bh) { | |
256 | QEMUBH *next = ctx->first_bh->next; | |
257 | ||
258 | /* qemu_bh_delete() must have been called on BHs in this AioContext */ | |
259 | assert(ctx->first_bh->deleted); | |
260 | ||
261 | g_free(ctx->first_bh); | |
262 | ctx->first_bh = next; | |
263 | } | |
264 | qemu_mutex_unlock(&ctx->bh_lock); | |
265 | ||
dca21ef2 | 266 | aio_set_event_notifier(ctx, &ctx->notifier, false, NULL); |
2f4dc3c1 | 267 | event_notifier_cleanup(&ctx->notifier); |
98563fc3 | 268 | rfifolock_destroy(&ctx->lock); |
dcc772e2 | 269 | qemu_mutex_destroy(&ctx->bh_lock); |
dae21b98 | 270 | timerlistgroup_deinit(&ctx->tlg); |
2f4dc3c1 PB |
271 | } |
272 | ||
e3713e00 PB |
273 | static GSourceFuncs aio_source_funcs = { |
274 | aio_ctx_prepare, | |
275 | aio_ctx_check, | |
276 | aio_ctx_dispatch, | |
2f4dc3c1 | 277 | aio_ctx_finalize |
e3713e00 PB |
278 | }; |
279 | ||
280 | GSource *aio_get_g_source(AioContext *ctx) | |
281 | { | |
282 | g_source_ref(&ctx->source); | |
283 | return &ctx->source; | |
284 | } | |
a915f4bc | 285 | |
9b34277d SH |
286 | ThreadPool *aio_get_thread_pool(AioContext *ctx) |
287 | { | |
288 | if (!ctx->thread_pool) { | |
289 | ctx->thread_pool = thread_pool_new(ctx); | |
290 | } | |
291 | return ctx->thread_pool; | |
292 | } | |
293 | ||
0187f5c9 PB |
294 | #ifdef CONFIG_LINUX_AIO |
295 | LinuxAioState *aio_get_linux_aio(AioContext *ctx) | |
296 | { | |
297 | if (!ctx->linux_aio) { | |
298 | ctx->linux_aio = laio_init(); | |
299 | laio_attach_aio_context(ctx->linux_aio, ctx); | |
300 | } | |
301 | return ctx->linux_aio; | |
302 | } | |
303 | #endif | |
304 | ||
2f4dc3c1 PB |
305 | void aio_notify(AioContext *ctx) |
306 | { | |
eabc9779 PB |
307 | /* Write e.g. bh->scheduled before reading ctx->notify_me. Pairs |
308 | * with atomic_or in aio_ctx_prepare or atomic_add in aio_poll. | |
309 | */ | |
0ceb849b | 310 | smp_mb(); |
eabc9779 | 311 | if (ctx->notify_me) { |
0ceb849b | 312 | event_notifier_set(&ctx->notifier); |
05e514b1 PB |
313 | atomic_mb_set(&ctx->notified, true); |
314 | } | |
315 | } | |
316 | ||
317 | void aio_notify_accept(AioContext *ctx) | |
318 | { | |
319 | if (atomic_xchg(&ctx->notified, false)) { | |
320 | event_notifier_test_and_clear(&ctx->notifier); | |
0ceb849b | 321 | } |
2f4dc3c1 PB |
322 | } |
323 | ||
d5541d86 AB |
324 | static void aio_timerlist_notify(void *opaque) |
325 | { | |
326 | aio_notify(opaque); | |
327 | } | |
328 | ||
da5e1de9 SH |
329 | static void aio_rfifolock_cb(void *opaque) |
330 | { | |
ca96ac44 SH |
331 | AioContext *ctx = opaque; |
332 | ||
da5e1de9 | 333 | /* Kick owner thread in case they are blocked in aio_poll() */ |
ca96ac44 SH |
334 | qemu_bh_schedule(ctx->notify_dummy_bh); |
335 | } | |
336 | ||
337 | static void notify_dummy_bh(void *opaque) | |
338 | { | |
339 | /* Do nothing, we were invoked just to force the event loop to iterate */ | |
da5e1de9 SH |
340 | } |
341 | ||
21a03d17 PB |
342 | static void event_notifier_dummy_cb(EventNotifier *e) |
343 | { | |
344 | } | |
345 | ||
2f78e491 | 346 | AioContext *aio_context_new(Error **errp) |
f627aab1 | 347 | { |
2f78e491 | 348 | int ret; |
2f4dc3c1 | 349 | AioContext *ctx; |
37fcee5d | 350 | |
2f4dc3c1 | 351 | ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext)); |
7e003465 C |
352 | aio_context_setup(ctx); |
353 | ||
2f78e491 CN |
354 | ret = event_notifier_init(&ctx->notifier, false); |
355 | if (ret < 0) { | |
2f78e491 | 356 | error_setg_errno(errp, -ret, "Failed to initialize event notifier"); |
37fcee5d | 357 | goto fail; |
2f78e491 | 358 | } |
fcf5def1 | 359 | g_source_set_can_recurse(&ctx->source, true); |
2f78e491 | 360 | aio_set_event_notifier(ctx, &ctx->notifier, |
dca21ef2 | 361 | false, |
2f78e491 | 362 | (EventNotifierHandler *) |
21a03d17 | 363 | event_notifier_dummy_cb); |
0187f5c9 PB |
364 | #ifdef CONFIG_LINUX_AIO |
365 | ctx->linux_aio = NULL; | |
366 | #endif | |
9b34277d | 367 | ctx->thread_pool = NULL; |
dcc772e2 | 368 | qemu_mutex_init(&ctx->bh_lock); |
da5e1de9 | 369 | rfifolock_init(&ctx->lock, aio_rfifolock_cb, ctx); |
d5541d86 | 370 | timerlistgroup_init(&ctx->tlg, aio_timerlist_notify, ctx); |
2f4dc3c1 | 371 | |
ca96ac44 SH |
372 | ctx->notify_dummy_bh = aio_bh_new(ctx, notify_dummy_bh, NULL); |
373 | ||
2f4dc3c1 | 374 | return ctx; |
37fcee5d FZ |
375 | fail: |
376 | g_source_destroy(&ctx->source); | |
377 | return NULL; | |
e3713e00 PB |
378 | } |
379 | ||
380 | void aio_context_ref(AioContext *ctx) | |
381 | { | |
382 | g_source_ref(&ctx->source); | |
383 | } | |
384 | ||
385 | void aio_context_unref(AioContext *ctx) | |
386 | { | |
387 | g_source_unref(&ctx->source); | |
f627aab1 | 388 | } |
98563fc3 SH |
389 | |
390 | void aio_context_acquire(AioContext *ctx) | |
391 | { | |
392 | rfifolock_lock(&ctx->lock); | |
393 | } | |
394 | ||
395 | void aio_context_release(AioContext *ctx) | |
396 | { | |
397 | rfifolock_unlock(&ctx->lock); | |
398 | } |