]>
Commit | Line | Data |
---|---|---|
00dccaf1 KW |
1 | /* |
2 | * QEMU coroutines | |
3 | * | |
4 | * Copyright IBM, Corp. 2011 | |
5 | * | |
6 | * Authors: | |
7 | * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com> | |
8 | * Kevin Wolf <kwolf@redhat.com> | |
9 | * | |
10 | * This work is licensed under the terms of the GNU LGPL, version 2 or later. | |
11 | * See the COPYING.LIB file in the top-level directory. | |
12 | * | |
13 | */ | |
14 | ||
aafd7584 | 15 | #include "qemu/osdep.h" |
00dccaf1 | 16 | #include "trace.h" |
b84c4586 | 17 | #include "qemu/thread.h" |
4d68e86b | 18 | #include "qemu/atomic.h" |
10817bf0 | 19 | #include "qemu/coroutine_int.h" |
ac387a08 | 20 | #include "qemu/coroutine-tls.h" |
0c330a73 | 21 | #include "block/aio.h" |
00dccaf1 | 22 | |
9ec7a59b KW |
23 | /** |
24 | * The minimal batch size is always 64, coroutines from the release_pool are | |
25 | * reused as soon as there are 64 coroutines in it. The maximum pool size starts | |
26 | * with 64 and is increased on demand so that coroutines are not deleted even if | |
27 | * they are not immediately reused. | |
28 | */ | |
40239784 | 29 | enum { |
9ec7a59b KW |
30 | POOL_MIN_BATCH_SIZE = 64, |
31 | POOL_INITIAL_MAX_SIZE = 64, | |
40239784 PB |
32 | }; |
33 | ||
34 | /** Free list to speed up creation */ | |
4d68e86b | 35 | static QSLIST_HEAD(, Coroutine) release_pool = QSLIST_HEAD_INITIALIZER(pool); |
9ec7a59b | 36 | static unsigned int pool_max_size = POOL_INITIAL_MAX_SIZE; |
4d68e86b | 37 | static unsigned int release_pool_size; |
ac387a08 SH |
38 | |
39 | typedef QSLIST_HEAD(, Coroutine) CoroutineQSList; | |
40 | QEMU_DEFINE_STATIC_CO_TLS(CoroutineQSList, alloc_pool); | |
41 | QEMU_DEFINE_STATIC_CO_TLS(unsigned int, alloc_pool_size); | |
42 | QEMU_DEFINE_STATIC_CO_TLS(Notifier, coroutine_pool_cleanup_notifier); | |
4d68e86b PB |
43 | |
44 | static void coroutine_pool_cleanup(Notifier *n, void *value) | |
45 | { | |
46 | Coroutine *co; | |
47 | Coroutine *tmp; | |
ac387a08 | 48 | CoroutineQSList *alloc_pool = get_ptr_alloc_pool(); |
4d68e86b | 49 | |
ac387a08 SH |
50 | QSLIST_FOREACH_SAFE(co, alloc_pool, pool_next, tmp) { |
51 | QSLIST_REMOVE_HEAD(alloc_pool, pool_next); | |
4d68e86b PB |
52 | qemu_coroutine_delete(co); |
53 | } | |
54 | } | |
40239784 | 55 | |
0b8b8753 | 56 | Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque) |
00dccaf1 | 57 | { |
70c60c08 | 58 | Coroutine *co = NULL; |
40239784 | 59 | |
70c60c08 | 60 | if (CONFIG_COROUTINE_POOL) { |
ac387a08 SH |
61 | CoroutineQSList *alloc_pool = get_ptr_alloc_pool(); |
62 | ||
63 | co = QSLIST_FIRST(alloc_pool); | |
4d68e86b | 64 | if (!co) { |
9ec7a59b | 65 | if (release_pool_size > POOL_MIN_BATCH_SIZE) { |
4d68e86b | 66 | /* Slow path; a good place to register the destructor, too. */ |
ac387a08 SH |
67 | Notifier *notifier = get_ptr_coroutine_pool_cleanup_notifier(); |
68 | if (!notifier->notify) { | |
69 | notifier->notify = coroutine_pool_cleanup; | |
70 | qemu_thread_atexit_add(notifier); | |
4d68e86b PB |
71 | } |
72 | ||
73 | /* This is not exact; there could be a little skew between | |
74 | * release_pool_size and the actual size of release_pool. But | |
75 | * it is just a heuristic, it does not need to be perfect. | |
76 | */ | |
ac387a08 SH |
77 | set_alloc_pool_size(qatomic_xchg(&release_pool_size, 0)); |
78 | QSLIST_MOVE_ATOMIC(alloc_pool, &release_pool); | |
79 | co = QSLIST_FIRST(alloc_pool); | |
4d68e86b PB |
80 | } |
81 | } | |
70c60c08 | 82 | if (co) { |
ac387a08 SH |
83 | QSLIST_REMOVE_HEAD(alloc_pool, pool_next); |
84 | set_alloc_pool_size(get_alloc_pool_size() - 1); | |
70c60c08 | 85 | } |
b84c4586 | 86 | } |
b84c4586 SH |
87 | |
88 | if (!co) { | |
40239784 PB |
89 | co = qemu_coroutine_new(); |
90 | } | |
91 | ||
00dccaf1 | 92 | co->entry = entry; |
0b8b8753 | 93 | co->entry_arg = opaque; |
7d9c8581 | 94 | QSIMPLEQ_INIT(&co->co_queue_wakeup); |
00dccaf1 KW |
95 | return co; |
96 | } | |
97 | ||
40239784 PB |
98 | static void coroutine_delete(Coroutine *co) |
99 | { | |
4d68e86b PB |
100 | co->caller = NULL; |
101 | ||
70c60c08 | 102 | if (CONFIG_COROUTINE_POOL) { |
9ec7a59b | 103 | if (release_pool_size < qatomic_read(&pool_max_size) * 2) { |
4d68e86b | 104 | QSLIST_INSERT_HEAD_ATOMIC(&release_pool, co, pool_next); |
d73415a3 | 105 | qatomic_inc(&release_pool_size); |
70c60c08 SH |
106 | return; |
107 | } | |
9ec7a59b | 108 | if (get_alloc_pool_size() < qatomic_read(&pool_max_size)) { |
ac387a08 SH |
109 | QSLIST_INSERT_HEAD(get_ptr_alloc_pool(), co, pool_next); |
110 | set_alloc_pool_size(get_alloc_pool_size() + 1); | |
51a2219b PL |
111 | return; |
112 | } | |
40239784 PB |
113 | } |
114 | ||
115 | qemu_coroutine_delete(co); | |
116 | } | |
117 | ||
ba9e75ce | 118 | void qemu_aio_coroutine_enter(AioContext *ctx, Coroutine *co) |
00dccaf1 | 119 | { |
c40a2545 SH |
120 | QSIMPLEQ_HEAD(, Coroutine) pending = QSIMPLEQ_HEAD_INITIALIZER(pending); |
121 | Coroutine *from = qemu_coroutine_self(); | |
6133b39f | 122 | |
c40a2545 | 123 | QSIMPLEQ_INSERT_TAIL(&pending, co, co_queue_next); |
00dccaf1 | 124 | |
c40a2545 SH |
125 | /* Run co and any queued coroutines */ |
126 | while (!QSIMPLEQ_EMPTY(&pending)) { | |
127 | Coroutine *to = QSIMPLEQ_FIRST(&pending); | |
128 | CoroutineAction ret; | |
6133b39f | 129 | |
c40a2545 SH |
130 | /* Cannot rely on the read barrier for to in aio_co_wake(), as there are |
131 | * callers outside of aio_co_wake() */ | |
d73415a3 | 132 | const char *scheduled = qatomic_mb_read(&to->scheduled); |
00dccaf1 | 133 | |
c40a2545 | 134 | QSIMPLEQ_REMOVE_HEAD(&pending, co_queue_next); |
0c330a73 | 135 | |
c40a2545 | 136 | trace_qemu_aio_coroutine_enter(ctx, from, to, to->entry_arg); |
0c330a73 | 137 | |
c40a2545 SH |
138 | /* if the Coroutine has already been scheduled, entering it again will |
139 | * cause us to enter it twice, potentially even after the coroutine has | |
140 | * been deleted */ | |
141 | if (scheduled) { | |
142 | fprintf(stderr, | |
143 | "%s: Co-routine was already scheduled in '%s'\n", | |
144 | __func__, scheduled); | |
145 | abort(); | |
146 | } | |
cd12bb56 | 147 | |
c40a2545 SH |
148 | if (to->caller) { |
149 | fprintf(stderr, "Co-routine re-entered recursively\n"); | |
150 | abort(); | |
151 | } | |
528f449f | 152 | |
c40a2545 SH |
153 | to->caller = from; |
154 | to->ctx = ctx; | |
155 | ||
156 | /* Store to->ctx before anything that stores to. Matches | |
157 | * barrier in aio_co_wake and qemu_co_mutex_wake. | |
158 | */ | |
159 | smp_wmb(); | |
160 | ||
161 | ret = qemu_coroutine_switch(from, to, COROUTINE_ENTER); | |
162 | ||
163 | /* Queued coroutines are run depth-first; previously pending coroutines | |
164 | * run after those queued more recently. | |
165 | */ | |
166 | QSIMPLEQ_PREPEND(&pending, &to->co_queue_wakeup); | |
167 | ||
168 | switch (ret) { | |
169 | case COROUTINE_YIELD: | |
170 | break; | |
171 | case COROUTINE_TERMINATE: | |
172 | assert(!to->locks_held); | |
173 | trace_qemu_coroutine_terminate(to); | |
174 | coroutine_delete(to); | |
175 | break; | |
176 | default: | |
177 | abort(); | |
178 | } | |
cd12bb56 | 179 | } |
00dccaf1 KW |
180 | } |
181 | ||
ba9e75ce FZ |
182 | void qemu_coroutine_enter(Coroutine *co) |
183 | { | |
184 | qemu_aio_coroutine_enter(qemu_get_current_aio_context(), co); | |
185 | } | |
186 | ||
536fca7f KW |
187 | void qemu_coroutine_enter_if_inactive(Coroutine *co) |
188 | { | |
189 | if (!qemu_coroutine_entered(co)) { | |
190 | qemu_coroutine_enter(co); | |
191 | } | |
192 | } | |
193 | ||
00dccaf1 KW |
194 | void coroutine_fn qemu_coroutine_yield(void) |
195 | { | |
196 | Coroutine *self = qemu_coroutine_self(); | |
197 | Coroutine *to = self->caller; | |
198 | ||
199 | trace_qemu_coroutine_yield(self, to); | |
200 | ||
201 | if (!to) { | |
202 | fprintf(stderr, "Co-routine is yielding to no one\n"); | |
203 | abort(); | |
204 | } | |
205 | ||
206 | self->caller = NULL; | |
315a1309 | 207 | qemu_coroutine_switch(self, to, COROUTINE_YIELD); |
00dccaf1 | 208 | } |
f643e469 SH |
209 | |
210 | bool qemu_coroutine_entered(Coroutine *co) | |
211 | { | |
212 | return co->caller; | |
213 | } | |
aa1361d5 | 214 | |
a248b856 | 215 | AioContext *qemu_coroutine_get_aio_context(Coroutine *co) |
aa1361d5 KW |
216 | { |
217 | return co->ctx; | |
218 | } | |
4c41c69e | 219 | |
98e3ab35 | 220 | void qemu_coroutine_inc_pool_size(unsigned int additional_pool_size) |
4c41c69e | 221 | { |
9ec7a59b | 222 | qatomic_add(&pool_max_size, additional_pool_size); |
4c41c69e HN |
223 | } |
224 | ||
98e3ab35 | 225 | void qemu_coroutine_dec_pool_size(unsigned int removing_pool_size) |
4c41c69e | 226 | { |
9ec7a59b | 227 | qatomic_sub(&pool_max_size, removing_pool_size); |
4c41c69e | 228 | } |