]>
Commit | Line | Data |
---|---|---|
e72f66a0 SH |
1 | /* |
2 | * Dedicated thread for virtio-blk I/O processing | |
3 | * | |
4 | * Copyright 2012 IBM, Corp. | |
5 | * Copyright 2012 Red Hat, Inc. and/or its affiliates | |
6 | * | |
7 | * Authors: | |
8 | * Stefan Hajnoczi <stefanha@redhat.com> | |
9 | * | |
10 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
11 | * See the COPYING file in the top-level directory. | |
12 | * | |
13 | */ | |
14 | ||
80c71a24 | 15 | #include "qemu/osdep.h" |
e72f66a0 SH |
16 | #include "trace.h" |
17 | #include "qemu/iov.h" | |
e72f66a0 | 18 | #include "qemu/thread.h" |
b4a42f81 | 19 | #include "qemu/error-report.h" |
b0e5d90e | 20 | #include "hw/virtio/virtio-access.h" |
0d09e41a | 21 | #include "hw/virtio/dataplane/vring.h" |
b0e5d90e | 22 | #include "hw/virtio/dataplane/vring-accessors.h" |
4be74634 | 23 | #include "sysemu/block-backend.h" |
0d09e41a PB |
24 | #include "hw/virtio/virtio-blk.h" |
25 | #include "virtio-blk.h" | |
2c20e711 | 26 | #include "block/aio.h" |
1c819449 | 27 | #include "hw/virtio/virtio-bus.h" |
54bee5c2 | 28 | #include "qom/object_interfaces.h" |
e72f66a0 | 29 | |
e72f66a0 SH |
30 | struct VirtIOBlockDataPlane { |
31 | bool started; | |
8caf907f | 32 | bool starting; |
cd7fdfe5 | 33 | bool stopping; |
2f5f70fa | 34 | bool disabled; |
e72f66a0 | 35 | |
2a30307f | 36 | VirtIOBlkConf *conf; |
e72f66a0 SH |
37 | |
38 | VirtIODevice *vdev; | |
39 | Vring vring; /* virtqueue vring */ | |
40 | EventNotifier *guest_notifier; /* irq */ | |
5b2ffbe4 | 41 | QEMUBH *bh; /* bh for guest notification */ |
e72f66a0 | 42 | |
1b1e0659 HR |
43 | Notifier insert_notifier, remove_notifier; |
44 | ||
2c20e711 PB |
45 | /* Note that these EventNotifiers are assigned by value. This is |
46 | * fine as long as you do not call event_notifier_cleanup on them | |
47 | * (because you don't own the file descriptor or handle; you just | |
48 | * use it). | |
49 | */ | |
48ff2692 | 50 | IOThread *iothread; |
2c20e711 | 51 | AioContext *ctx; |
2c20e711 | 52 | EventNotifier host_notifier; /* doorbell */ |
e72f66a0 | 53 | |
3718d8ab FZ |
54 | /* Operation blocker on BDS */ |
55 | Error *blocker; | |
b002254d FZ |
56 | void (*saved_complete_request)(struct VirtIOBlockReq *req, |
57 | unsigned char status); | |
e72f66a0 SH |
58 | }; |
59 | ||
60 | /* Raise an interrupt to signal guest, if necessary */ | |
61 | static void notify_guest(VirtIOBlockDataPlane *s) | |
62 | { | |
63 | if (!vring_should_notify(s->vdev, &s->vring)) { | |
64 | return; | |
65 | } | |
66 | ||
67 | event_notifier_set(s->guest_notifier); | |
68 | } | |
69 | ||
5b2ffbe4 ML |
70 | static void notify_guest_bh(void *opaque) |
71 | { | |
72 | VirtIOBlockDataPlane *s = opaque; | |
73 | ||
74 | notify_guest(s); | |
75 | } | |
76 | ||
d64c60a7 | 77 | static void complete_request_vring(VirtIOBlockReq *req, unsigned char status) |
e72f66a0 | 78 | { |
5b2ffbe4 | 79 | VirtIOBlockDataPlane *s = req->dev->dataplane; |
ab2e3cd2 | 80 | stb_p(&req->in->status, status); |
e72f66a0 | 81 | |
2a6cdd6d | 82 | vring_push(s->vdev, &req->dev->dataplane->vring, &req->elem, req->in_len); |
5b2ffbe4 ML |
83 | |
84 | /* Suppress notification to guest by BH and its scheduled | |
85 | * flag because requests are completed as a batch after io | |
86 | * plug & unplug is introduced, and the BH can still be | |
87 | * executed in dataplane aio context even after it is | |
88 | * stopped, so needn't worry about notification loss with BH. | |
89 | */ | |
90 | qemu_bh_schedule(s->bh); | |
e72f66a0 SH |
91 | } |
92 | ||
2c20e711 | 93 | static void handle_notify(EventNotifier *e) |
e72f66a0 | 94 | { |
2c20e711 PB |
95 | VirtIOBlockDataPlane *s = container_of(e, VirtIOBlockDataPlane, |
96 | host_notifier); | |
f897bf75 | 97 | VirtIOBlock *vblk = VIRTIO_BLK(s->vdev); |
e72f66a0 | 98 | |
2c20e711 | 99 | event_notifier_test_and_clear(&s->host_notifier); |
4be74634 | 100 | blk_io_plug(s->conf->conf.blk); |
e72f66a0 | 101 | for (;;) { |
95f7142a | 102 | MultiReqBuffer mrb = {}; |
f897bf75 | 103 | |
e72f66a0 SH |
104 | /* Disable guest->host notifies to avoid unnecessary vmexits */ |
105 | vring_disable_notification(s->vdev, &s->vring); | |
106 | ||
107 | for (;;) { | |
51b19ebe PB |
108 | VirtIOBlockReq *req = vring_pop(s->vdev, &s->vring, |
109 | sizeof(VirtIOBlockReq)); | |
f897bf75 | 110 | |
51b19ebe | 111 | if (req == NULL) { |
e72f66a0 SH |
112 | break; /* no more requests */ |
113 | } | |
114 | ||
51b19ebe | 115 | virtio_blk_init_request(vblk, req); |
f897bf75 SH |
116 | trace_virtio_blk_data_plane_process_request(s, req->elem.out_num, |
117 | req->elem.in_num, | |
118 | req->elem.index); | |
e72f66a0 | 119 | |
b002254d | 120 | virtio_blk_handle_request(req, &mrb); |
e72f66a0 SH |
121 | } |
122 | ||
95f7142a PL |
123 | if (mrb.num_reqs) { |
124 | virtio_blk_submit_multireq(s->conf->conf.blk, &mrb); | |
125 | } | |
b002254d | 126 | |
51b19ebe | 127 | if (likely(!vring_more_avail(s->vdev, &s->vring))) { /* vring emptied */ |
e72f66a0 SH |
128 | /* Re-enable guest->host notifies and stop processing the vring. |
129 | * But if the guest has snuck in more descriptors, keep processing. | |
130 | */ | |
8b1fe1ce PB |
131 | vring_enable_notification(s->vdev, &s->vring); |
132 | if (!vring_more_avail(s->vdev, &s->vring)) { | |
e72f66a0 SH |
133 | break; |
134 | } | |
580b6b2a | 135 | } else { /* fatal error */ |
e72f66a0 SH |
136 | break; |
137 | } | |
138 | } | |
4be74634 | 139 | blk_io_unplug(s->conf->conf.blk); |
e72f66a0 SH |
140 | } |
141 | ||
1b1e0659 HR |
142 | static void data_plane_set_up_op_blockers(VirtIOBlockDataPlane *s) |
143 | { | |
144 | assert(!s->blocker); | |
145 | error_setg(&s->blocker, "block device is in use by data plane"); | |
146 | blk_op_block_all(s->conf->conf.blk, s->blocker); | |
147 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_RESIZE, s->blocker); | |
148 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_DRIVE_DEL, s->blocker); | |
149 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_BACKUP_SOURCE, s->blocker); | |
150 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_CHANGE, s->blocker); | |
151 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_COMMIT_SOURCE, s->blocker); | |
152 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_COMMIT_TARGET, s->blocker); | |
153 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_EJECT, s->blocker); | |
154 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT, | |
155 | s->blocker); | |
156 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT, | |
157 | s->blocker); | |
158 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE, | |
159 | s->blocker); | |
160 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_MIRROR_SOURCE, s->blocker); | |
161 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_STREAM, s->blocker); | |
162 | blk_op_unblock(s->conf->conf.blk, BLOCK_OP_TYPE_REPLACE, s->blocker); | |
163 | } | |
164 | ||
165 | static void data_plane_remove_op_blockers(VirtIOBlockDataPlane *s) | |
166 | { | |
167 | if (s->blocker) { | |
168 | blk_op_unblock_all(s->conf->conf.blk, s->blocker); | |
169 | error_free(s->blocker); | |
170 | s->blocker = NULL; | |
171 | } | |
172 | } | |
173 | ||
174 | static void data_plane_blk_insert_notifier(Notifier *n, void *data) | |
175 | { | |
176 | VirtIOBlockDataPlane *s = container_of(n, VirtIOBlockDataPlane, | |
177 | insert_notifier); | |
178 | assert(s->conf->conf.blk == data); | |
179 | data_plane_set_up_op_blockers(s); | |
180 | } | |
181 | ||
182 | static void data_plane_blk_remove_notifier(Notifier *n, void *data) | |
183 | { | |
184 | VirtIOBlockDataPlane *s = container_of(n, VirtIOBlockDataPlane, | |
185 | remove_notifier); | |
186 | assert(s->conf->conf.blk == data); | |
187 | data_plane_remove_op_blockers(s); | |
188 | } | |
189 | ||
48ff2692 | 190 | /* Context: QEMU global mutex held */ |
2a30307f | 191 | void virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, |
3ffeeef7 AF |
192 | VirtIOBlockDataPlane **dataplane, |
193 | Error **errp) | |
e72f66a0 SH |
194 | { |
195 | VirtIOBlockDataPlane *s; | |
a9968c77 CH |
196 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); |
197 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); | |
e72f66a0 SH |
198 | |
199 | *dataplane = NULL; | |
200 | ||
a616fb75 | 201 | if (!conf->iothread) { |
3ffeeef7 | 202 | return; |
e72f66a0 SH |
203 | } |
204 | ||
a9968c77 CH |
205 | /* Don't try if transport does not support notifiers. */ |
206 | if (!k->set_guest_notifiers || !k->set_host_notifier) { | |
207 | error_setg(errp, | |
a616fb75 | 208 | "device is incompatible with dataplane " |
a9968c77 CH |
209 | "(transport does not support notifiers)"); |
210 | return; | |
211 | } | |
212 | ||
b0f2027c SH |
213 | /* If dataplane is (re-)enabled while the guest is running there could be |
214 | * block jobs that can conflict. | |
215 | */ | |
e43bfd9c MA |
216 | if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { |
217 | error_prepend(errp, "cannot start dataplane thread: "); | |
3ffeeef7 | 218 | return; |
b0f2027c SH |
219 | } |
220 | ||
e72f66a0 SH |
221 | s = g_new0(VirtIOBlockDataPlane, 1); |
222 | s->vdev = vdev; | |
2a30307f | 223 | s->conf = conf; |
e72f66a0 | 224 | |
2a30307f MA |
225 | if (conf->iothread) { |
226 | s->iothread = conf->iothread; | |
54bee5c2 | 227 | object_ref(OBJECT(s->iothread)); |
48ff2692 SH |
228 | } |
229 | s->ctx = iothread_get_aio_context(s->iothread); | |
5b2ffbe4 | 230 | s->bh = aio_bh_new(s->ctx, notify_guest_bh, s); |
48ff2692 | 231 | |
1b1e0659 HR |
232 | s->insert_notifier.notify = data_plane_blk_insert_notifier; |
233 | s->remove_notifier.notify = data_plane_blk_remove_notifier; | |
234 | blk_add_insert_bs_notifier(conf->conf.blk, &s->insert_notifier); | |
235 | blk_add_remove_bs_notifier(conf->conf.blk, &s->remove_notifier); | |
236 | ||
237 | data_plane_set_up_op_blockers(s); | |
e72f66a0 | 238 | |
e72f66a0 | 239 | *dataplane = s; |
e72f66a0 SH |
240 | } |
241 | ||
48ff2692 | 242 | /* Context: QEMU global mutex held */ |
e72f66a0 SH |
243 | void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) |
244 | { | |
245 | if (!s) { | |
246 | return; | |
247 | } | |
248 | ||
249 | virtio_blk_data_plane_stop(s); | |
1b1e0659 HR |
250 | data_plane_remove_op_blockers(s); |
251 | notifier_remove(&s->insert_notifier); | |
252 | notifier_remove(&s->remove_notifier); | |
5b2ffbe4 | 253 | qemu_bh_delete(s->bh); |
fed105e2 | 254 | object_unref(OBJECT(s->iothread)); |
e72f66a0 SH |
255 | g_free(s); |
256 | } | |
257 | ||
48ff2692 | 258 | /* Context: QEMU global mutex held */ |
e72f66a0 SH |
259 | void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s) |
260 | { | |
1c819449 FK |
261 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev))); |
262 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); | |
e926d9b8 | 263 | VirtIOBlock *vblk = VIRTIO_BLK(s->vdev); |
e72f66a0 | 264 | VirtQueue *vq; |
267e1a20 | 265 | int r; |
e72f66a0 | 266 | |
2f5f70fa | 267 | if (s->started || s->disabled) { |
e72f66a0 SH |
268 | return; |
269 | } | |
270 | ||
8caf907f CH |
271 | if (s->starting) { |
272 | return; | |
273 | } | |
274 | ||
275 | s->starting = true; | |
276 | ||
e72f66a0 SH |
277 | vq = virtio_get_queue(s->vdev, 0); |
278 | if (!vring_setup(&s->vring, s->vdev, 0)) { | |
f9907ebc | 279 | goto fail_vring; |
e72f66a0 SH |
280 | } |
281 | ||
e72f66a0 | 282 | /* Set up guest notifier (irq) */ |
267e1a20 CH |
283 | r = k->set_guest_notifiers(qbus->parent, 1, true); |
284 | if (r != 0) { | |
285 | fprintf(stderr, "virtio-blk failed to set guest notifier (%d), " | |
286 | "ensure -enable-kvm is set\n", r); | |
f9907ebc | 287 | goto fail_guest_notifiers; |
e72f66a0 SH |
288 | } |
289 | s->guest_notifier = virtio_queue_get_guest_notifier(vq); | |
290 | ||
291 | /* Set up virtqueue notify */ | |
267e1a20 CH |
292 | r = k->set_host_notifier(qbus->parent, 0, true); |
293 | if (r != 0) { | |
294 | fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); | |
f9907ebc | 295 | goto fail_host_notifier; |
e72f66a0 | 296 | } |
2c20e711 | 297 | s->host_notifier = *virtio_queue_get_host_notifier(vq); |
e72f66a0 | 298 | |
e926d9b8 ML |
299 | s->saved_complete_request = vblk->complete_request; |
300 | vblk->complete_request = complete_request_vring; | |
301 | ||
8caf907f | 302 | s->starting = false; |
e72f66a0 SH |
303 | s->started = true; |
304 | trace_virtio_blk_data_plane_start(s); | |
305 | ||
4be74634 | 306 | blk_set_aio_context(s->conf->conf.blk, s->ctx); |
580b6b2a | 307 | |
e72f66a0 SH |
308 | /* Kick right away to begin processing requests already in vring */ |
309 | event_notifier_set(virtio_queue_get_host_notifier(vq)); | |
310 | ||
48ff2692 SH |
311 | /* Get this show started by hooking up our callbacks */ |
312 | aio_context_acquire(s->ctx); | |
3a1e8074 | 313 | aio_set_event_notifier(s->ctx, &s->host_notifier, true, |
dca21ef2 | 314 | handle_notify); |
48ff2692 | 315 | aio_context_release(s->ctx); |
f9907ebc CH |
316 | return; |
317 | ||
318 | fail_host_notifier: | |
319 | k->set_guest_notifiers(qbus->parent, 1, false); | |
320 | fail_guest_notifiers: | |
321 | vring_teardown(&s->vring, s->vdev, 0); | |
2f5f70fa | 322 | s->disabled = true; |
f9907ebc CH |
323 | fail_vring: |
324 | s->starting = false; | |
e72f66a0 SH |
325 | } |
326 | ||
48ff2692 | 327 | /* Context: QEMU global mutex held */ |
e72f66a0 SH |
328 | void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s) |
329 | { | |
1c819449 FK |
330 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev))); |
331 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); | |
b002254d | 332 | VirtIOBlock *vblk = VIRTIO_BLK(s->vdev); |
2f5f70fa CH |
333 | |
334 | ||
335 | /* Better luck next time. */ | |
336 | if (s->disabled) { | |
337 | s->disabled = false; | |
338 | return; | |
339 | } | |
cd7fdfe5 | 340 | if (!s->started || s->stopping) { |
e72f66a0 SH |
341 | return; |
342 | } | |
cd7fdfe5 | 343 | s->stopping = true; |
b002254d | 344 | vblk->complete_request = s->saved_complete_request; |
e72f66a0 SH |
345 | trace_virtio_blk_data_plane_stop(s); |
346 | ||
48ff2692 SH |
347 | aio_context_acquire(s->ctx); |
348 | ||
349 | /* Stop notifications for new requests from guest */ | |
3a1e8074 | 350 | aio_set_event_notifier(s->ctx, &s->host_notifier, true, NULL); |
48ff2692 | 351 | |
580b6b2a | 352 | /* Drain and switch bs back to the QEMU main loop */ |
4be74634 | 353 | blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context()); |
e72f66a0 | 354 | |
48ff2692 | 355 | aio_context_release(s->ctx); |
e72f66a0 | 356 | |
48ff2692 SH |
357 | /* Sync vring state back to virtqueue so that non-dataplane request |
358 | * processing can continue when we disable the host notifier below. | |
359 | */ | |
360 | vring_teardown(&s->vring, s->vdev, 0); | |
361 | ||
48ff2692 | 362 | k->set_host_notifier(qbus->parent, 0, false); |
e72f66a0 SH |
363 | |
364 | /* Clean up guest notifier (irq) */ | |
1c819449 | 365 | k->set_guest_notifiers(qbus->parent, 1, false); |
e72f66a0 | 366 | |
cd7fdfe5 SH |
367 | s->started = false; |
368 | s->stopping = false; | |
e72f66a0 | 369 | } |