]>
Commit | Line | Data |
---|---|---|
e72f66a0 SH |
1 | /* |
2 | * Dedicated thread for virtio-blk I/O processing | |
3 | * | |
4 | * Copyright 2012 IBM, Corp. | |
5 | * Copyright 2012 Red Hat, Inc. and/or its affiliates | |
6 | * | |
7 | * Authors: | |
8 | * Stefan Hajnoczi <stefanha@redhat.com> | |
9 | * | |
10 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
11 | * See the COPYING file in the top-level directory. | |
12 | * | |
13 | */ | |
14 | ||
80c71a24 | 15 | #include "qemu/osdep.h" |
da34e65c | 16 | #include "qapi/error.h" |
e72f66a0 SH |
17 | #include "trace.h" |
18 | #include "qemu/iov.h" | |
db725815 | 19 | #include "qemu/main-loop.h" |
e72f66a0 | 20 | #include "qemu/thread.h" |
b4a42f81 | 21 | #include "qemu/error-report.h" |
0d09e41a PB |
22 | #include "hw/virtio/virtio-blk.h" |
23 | #include "virtio-blk.h" | |
2c20e711 | 24 | #include "block/aio.h" |
1c819449 | 25 | #include "hw/virtio/virtio-bus.h" |
54bee5c2 | 26 | #include "qom/object_interfaces.h" |
e72f66a0 | 27 | |
e72f66a0 | 28 | struct VirtIOBlockDataPlane { |
8caf907f | 29 | bool starting; |
cd7fdfe5 | 30 | bool stopping; |
e72f66a0 | 31 | |
2a30307f | 32 | VirtIOBlkConf *conf; |
e72f66a0 | 33 | VirtIODevice *vdev; |
e72f66a0 | 34 | |
2c20e711 PB |
35 | /* Note that these EventNotifiers are assigned by value. This is |
36 | * fine as long as you do not call event_notifier_cleanup on them | |
37 | * (because you don't own the file descriptor or handle; you just | |
38 | * use it). | |
39 | */ | |
48ff2692 | 40 | IOThread *iothread; |
2c20e711 | 41 | AioContext *ctx; |
e72f66a0 SH |
42 | }; |
43 | ||
44 | /* Raise an interrupt to signal guest, if necessary */ | |
b234cdda | 45 | void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) |
e72f66a0 | 46 | { |
073458da | 47 | virtio_notify_irqfd(s->vdev, vq); |
e72f66a0 SH |
48 | } |
49 | ||
48ff2692 | 50 | /* Context: QEMU global mutex held */ |
9d3b1551 | 51 | bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, |
3ffeeef7 AF |
52 | VirtIOBlockDataPlane **dataplane, |
53 | Error **errp) | |
e72f66a0 SH |
54 | { |
55 | VirtIOBlockDataPlane *s; | |
a9968c77 CH |
56 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); |
57 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); | |
e72f66a0 SH |
58 | |
59 | *dataplane = NULL; | |
60 | ||
9ffe337c PB |
61 | if (conf->iothread) { |
62 | if (!k->set_guest_notifiers || !k->ioeventfd_assign) { | |
63 | error_setg(errp, | |
64 | "device is incompatible with iothread " | |
65 | "(transport does not support notifiers)"); | |
9d3b1551 | 66 | return false; |
9ffe337c PB |
67 | } |
68 | if (!virtio_device_ioeventfd_enabled(vdev)) { | |
69 | error_setg(errp, "ioeventfd is required for iothread"); | |
9d3b1551 | 70 | return false; |
9ffe337c | 71 | } |
e72f66a0 | 72 | |
9ffe337c PB |
73 | /* If dataplane is (re-)enabled while the guest is running there could |
74 | * be block jobs that can conflict. | |
75 | */ | |
76 | if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { | |
77 | error_prepend(errp, "cannot start virtio-blk dataplane: "); | |
9d3b1551 | 78 | return false; |
9ffe337c | 79 | } |
a9968c77 | 80 | } |
9ffe337c PB |
81 | /* Don't try if transport does not support notifiers. */ |
82 | if (!virtio_device_ioeventfd_enabled(vdev)) { | |
9d3b1551 | 83 | return false; |
b0f2027c SH |
84 | } |
85 | ||
e72f66a0 SH |
86 | s = g_new0(VirtIOBlockDataPlane, 1); |
87 | s->vdev = vdev; | |
2a30307f | 88 | s->conf = conf; |
e72f66a0 | 89 | |
9ffe337c PB |
90 | if (conf->iothread) { |
91 | s->iothread = conf->iothread; | |
92 | object_ref(OBJECT(s->iothread)); | |
93 | s->ctx = iothread_get_aio_context(s->iothread); | |
94 | } else { | |
95 | s->ctx = qemu_get_aio_context(); | |
96 | } | |
48ff2692 | 97 | |
e72f66a0 | 98 | *dataplane = s; |
9d3b1551 MZ |
99 | |
100 | return true; | |
e72f66a0 SH |
101 | } |
102 | ||
48ff2692 | 103 | /* Context: QEMU global mutex held */ |
e72f66a0 SH |
104 | void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) |
105 | { | |
9ffe337c PB |
106 | VirtIOBlock *vblk; |
107 | ||
e72f66a0 SH |
108 | if (!s) { |
109 | return; | |
110 | } | |
111 | ||
9ffe337c PB |
112 | vblk = VIRTIO_BLK(s->vdev); |
113 | assert(!vblk->dataplane_started); | |
9ffe337c PB |
114 | if (s->iothread) { |
115 | object_unref(OBJECT(s->iothread)); | |
116 | } | |
e72f66a0 SH |
117 | g_free(s); |
118 | } | |
119 | ||
48ff2692 | 120 | /* Context: QEMU global mutex held */ |
9ffe337c | 121 | int virtio_blk_data_plane_start(VirtIODevice *vdev) |
e72f66a0 | 122 | { |
9ffe337c PB |
123 | VirtIOBlock *vblk = VIRTIO_BLK(vdev); |
124 | VirtIOBlockDataPlane *s = vblk->dataplane; | |
125 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); | |
1c819449 | 126 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
c7040ff6 | 127 | AioContext *old_context; |
51b04ac5 SH |
128 | unsigned i; |
129 | unsigned nvqs = s->conf->num_queues; | |
97896a48 | 130 | Error *local_err = NULL; |
267e1a20 | 131 | int r; |
e72f66a0 | 132 | |
2906cddf | 133 | if (vblk->dataplane_started || s->starting) { |
9ffe337c | 134 | return 0; |
8caf907f CH |
135 | } |
136 | ||
137 | s->starting = true; | |
e72f66a0 | 138 | |
e72f66a0 | 139 | /* Set up guest notifier (irq) */ |
51b04ac5 | 140 | r = k->set_guest_notifiers(qbus->parent, nvqs, true); |
267e1a20 | 141 | if (r != 0) { |
a1d30f28 TH |
142 | error_report("virtio-blk failed to set guest notifier (%d), " |
143 | "ensure -accel kvm is set.", r); | |
f9907ebc | 144 | goto fail_guest_notifiers; |
e72f66a0 | 145 | } |
e72f66a0 | 146 | |
9cf4fd87 GK |
147 | /* |
148 | * Batch all the host notifiers in a single transaction to avoid | |
149 | * quadratic time complexity in address_space_update_ioeventfds(). | |
150 | */ | |
d0267da6 GK |
151 | memory_region_transaction_begin(); |
152 | ||
e72f66a0 | 153 | /* Set up virtqueue notify */ |
51b04ac5 SH |
154 | for (i = 0; i < nvqs; i++) { |
155 | r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true); | |
156 | if (r != 0) { | |
d0267da6 GK |
157 | int j = i; |
158 | ||
51b04ac5 SH |
159 | fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); |
160 | while (i--) { | |
161 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
162 | } |
163 | ||
9cf4fd87 GK |
164 | /* |
165 | * The transaction expects the ioeventfds to be open when it | |
166 | * commits. Do it now, before the cleanup loop. | |
167 | */ | |
d0267da6 GK |
168 | memory_region_transaction_commit(); |
169 | ||
170 | while (j--) { | |
5b807181 | 171 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j); |
51b04ac5 | 172 | } |
570fe439 | 173 | goto fail_host_notifiers; |
51b04ac5 | 174 | } |
e72f66a0 | 175 | } |
e926d9b8 | 176 | |
d0267da6 GK |
177 | memory_region_transaction_commit(); |
178 | ||
e72f66a0 SH |
179 | trace_virtio_blk_data_plane_start(s); |
180 | ||
c7040ff6 SL |
181 | old_context = blk_get_aio_context(s->conf->conf.blk); |
182 | aio_context_acquire(old_context); | |
97896a48 | 183 | r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); |
c7040ff6 | 184 | aio_context_release(old_context); |
97896a48 KW |
185 | if (r < 0) { |
186 | error_report_err(local_err); | |
570fe439 | 187 | goto fail_aio_context; |
97896a48 | 188 | } |
580b6b2a | 189 | |
e72f66a0 | 190 | /* Kick right away to begin processing requests already in vring */ |
51b04ac5 SH |
191 | for (i = 0; i < nvqs; i++) { |
192 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
193 | ||
194 | event_notifier_set(virtio_queue_get_host_notifier(vq)); | |
195 | } | |
e72f66a0 | 196 | |
75dcb4d7 SH |
197 | /* |
198 | * These fields must be visible to the IOThread when it processes the | |
199 | * virtqueue, otherwise it will think dataplane has not started yet. | |
200 | * | |
201 | * Make sure ->dataplane_started is false when blk_set_aio_context() is | |
202 | * called above so that draining does not cause the host notifier to be | |
203 | * detached/attached prematurely. | |
204 | */ | |
205 | s->starting = false; | |
206 | vblk->dataplane_started = true; | |
207 | smp_wmb(); /* paired with aio_notify_accept() on the read side */ | |
208 | ||
48ff2692 | 209 | /* Get this show started by hooking up our callbacks */ |
1665d932 SH |
210 | if (!blk_in_drain(s->conf->conf.blk)) { |
211 | aio_context_acquire(s->ctx); | |
212 | for (i = 0; i < nvqs; i++) { | |
213 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
51b04ac5 | 214 | |
1665d932 SH |
215 | virtio_queue_aio_attach_host_notifier(vq, s->ctx); |
216 | } | |
217 | aio_context_release(s->ctx); | |
51b04ac5 | 218 | } |
9ffe337c | 219 | return 0; |
f9907ebc | 220 | |
570fe439 | 221 | fail_aio_context: |
d0267da6 GK |
222 | memory_region_transaction_begin(); |
223 | ||
570fe439 GK |
224 | for (i = 0; i < nvqs; i++) { |
225 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
226 | } |
227 | ||
228 | memory_region_transaction_commit(); | |
229 | ||
230 | for (i = 0; i < nvqs; i++) { | |
570fe439 GK |
231 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); |
232 | } | |
233 | fail_host_notifiers: | |
234 | k->set_guest_notifiers(qbus->parent, nvqs, false); | |
f9907ebc | 235 | fail_guest_notifiers: |
eb41cf78 | 236 | vblk->dataplane_disabled = true; |
f9907ebc | 237 | s->starting = false; |
9ffe337c | 238 | return -ENOSYS; |
e72f66a0 SH |
239 | } |
240 | ||
1010cadf SH |
241 | /* Stop notifications for new requests from guest. |
242 | * | |
243 | * Context: BH in IOThread | |
244 | */ | |
245 | static void virtio_blk_data_plane_stop_bh(void *opaque) | |
246 | { | |
247 | VirtIOBlockDataPlane *s = opaque; | |
248 | unsigned i; | |
249 | ||
250 | for (i = 0; i < s->conf->num_queues; i++) { | |
251 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
bd58ab40 | 252 | EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); |
1010cadf | 253 | |
db608fb7 | 254 | virtio_queue_aio_detach_host_notifier(vq, s->ctx); |
bd58ab40 SH |
255 | |
256 | /* | |
257 | * Test and clear notifier after disabling event, in case poll callback | |
258 | * didn't have time to run. | |
259 | */ | |
260 | virtio_queue_host_notifier_read(host_notifier); | |
1010cadf SH |
261 | } |
262 | } | |
263 | ||
48ff2692 | 264 | /* Context: QEMU global mutex held */ |
9ffe337c | 265 | void virtio_blk_data_plane_stop(VirtIODevice *vdev) |
e72f66a0 | 266 | { |
9ffe337c PB |
267 | VirtIOBlock *vblk = VIRTIO_BLK(vdev); |
268 | VirtIOBlockDataPlane *s = vblk->dataplane; | |
269 | BusState *qbus = qdev_get_parent_bus(DEVICE(vblk)); | |
1c819449 | 270 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
51b04ac5 SH |
271 | unsigned i; |
272 | unsigned nvqs = s->conf->num_queues; | |
2f5f70fa | 273 | |
2906cddf PB |
274 | if (!vblk->dataplane_started || s->stopping) { |
275 | return; | |
276 | } | |
2f5f70fa CH |
277 | |
278 | /* Better luck next time. */ | |
eb41cf78 PB |
279 | if (vblk->dataplane_disabled) { |
280 | vblk->dataplane_disabled = false; | |
2906cddf | 281 | vblk->dataplane_started = false; |
e72f66a0 SH |
282 | return; |
283 | } | |
cd7fdfe5 | 284 | s->stopping = true; |
e72f66a0 SH |
285 | trace_virtio_blk_data_plane_stop(s); |
286 | ||
1665d932 SH |
287 | if (!blk_in_drain(s->conf->conf.blk)) { |
288 | aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); | |
289 | } | |
48ff2692 | 290 | |
9cf4fd87 GK |
291 | /* |
292 | * Batch all the host notifiers in a single transaction to avoid | |
293 | * quadratic time complexity in address_space_update_ioeventfds(). | |
294 | */ | |
d0267da6 GK |
295 | memory_region_transaction_begin(); |
296 | ||
51b04ac5 SH |
297 | for (i = 0; i < nvqs; i++) { |
298 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
299 | } |
300 | ||
9cf4fd87 GK |
301 | /* |
302 | * The transaction expects the ioeventfds to be open when it | |
303 | * commits. Do it now, before the cleanup loop. | |
304 | */ | |
d0267da6 GK |
305 | memory_region_transaction_commit(); |
306 | ||
307 | for (i = 0; i < nvqs; i++) { | |
76143618 | 308 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); |
51b04ac5 | 309 | } |
e72f66a0 | 310 | |
75dcb4d7 SH |
311 | /* |
312 | * Set ->dataplane_started to false before draining so that host notifiers | |
313 | * are not detached/attached anymore. | |
314 | */ | |
315 | vblk->dataplane_started = false; | |
316 | ||
317 | aio_context_acquire(s->ctx); | |
318 | ||
319 | /* Wait for virtio_blk_dma_restart_bh() and in flight I/O to complete */ | |
320 | blk_drain(s->conf->conf.blk); | |
321 | ||
322 | /* | |
323 | * Try to switch bs back to the QEMU main loop. If other users keep the | |
324 | * BlockBackend in the iothread, that's ok | |
325 | */ | |
326 | blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL); | |
327 | ||
328 | aio_context_release(s->ctx); | |
329 | ||
e72f66a0 | 330 | /* Clean up guest notifier (irq) */ |
51b04ac5 | 331 | k->set_guest_notifiers(qbus->parent, nvqs, false); |
e72f66a0 | 332 | |
cd7fdfe5 | 333 | s->stopping = false; |
e72f66a0 | 334 | } |