]>
Commit | Line | Data |
---|---|---|
e72f66a0 SH |
1 | /* |
2 | * Dedicated thread for virtio-blk I/O processing | |
3 | * | |
4 | * Copyright 2012 IBM, Corp. | |
5 | * Copyright 2012 Red Hat, Inc. and/or its affiliates | |
6 | * | |
7 | * Authors: | |
8 | * Stefan Hajnoczi <stefanha@redhat.com> | |
9 | * | |
10 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
11 | * See the COPYING file in the top-level directory. | |
12 | * | |
13 | */ | |
14 | ||
80c71a24 | 15 | #include "qemu/osdep.h" |
da34e65c | 16 | #include "qapi/error.h" |
e72f66a0 SH |
17 | #include "trace.h" |
18 | #include "qemu/iov.h" | |
db725815 | 19 | #include "qemu/main-loop.h" |
e72f66a0 | 20 | #include "qemu/thread.h" |
b4a42f81 | 21 | #include "qemu/error-report.h" |
0d09e41a PB |
22 | #include "hw/virtio/virtio-blk.h" |
23 | #include "virtio-blk.h" | |
2c20e711 | 24 | #include "block/aio.h" |
1c819449 | 25 | #include "hw/virtio/virtio-bus.h" |
54bee5c2 | 26 | #include "qom/object_interfaces.h" |
e72f66a0 | 27 | |
e72f66a0 | 28 | struct VirtIOBlockDataPlane { |
8caf907f | 29 | bool starting; |
cd7fdfe5 | 30 | bool stopping; |
e72f66a0 | 31 | |
2a30307f | 32 | VirtIOBlkConf *conf; |
e72f66a0 | 33 | VirtIODevice *vdev; |
e72f66a0 | 34 | |
2c20e711 PB |
35 | /* Note that these EventNotifiers are assigned by value. This is |
36 | * fine as long as you do not call event_notifier_cleanup on them | |
37 | * (because you don't own the file descriptor or handle; you just | |
38 | * use it). | |
39 | */ | |
48ff2692 | 40 | IOThread *iothread; |
2c20e711 | 41 | AioContext *ctx; |
e72f66a0 SH |
42 | }; |
43 | ||
44 | /* Raise an interrupt to signal guest, if necessary */ | |
b234cdda | 45 | void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) |
e72f66a0 | 46 | { |
073458da | 47 | virtio_notify_irqfd(s->vdev, vq); |
e72f66a0 SH |
48 | } |
49 | ||
48ff2692 | 50 | /* Context: QEMU global mutex held */ |
9d3b1551 | 51 | bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, |
3ffeeef7 AF |
52 | VirtIOBlockDataPlane **dataplane, |
53 | Error **errp) | |
e72f66a0 SH |
54 | { |
55 | VirtIOBlockDataPlane *s; | |
a9968c77 CH |
56 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); |
57 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); | |
e72f66a0 SH |
58 | |
59 | *dataplane = NULL; | |
60 | ||
9ffe337c PB |
61 | if (conf->iothread) { |
62 | if (!k->set_guest_notifiers || !k->ioeventfd_assign) { | |
63 | error_setg(errp, | |
64 | "device is incompatible with iothread " | |
65 | "(transport does not support notifiers)"); | |
9d3b1551 | 66 | return false; |
9ffe337c PB |
67 | } |
68 | if (!virtio_device_ioeventfd_enabled(vdev)) { | |
69 | error_setg(errp, "ioeventfd is required for iothread"); | |
9d3b1551 | 70 | return false; |
9ffe337c | 71 | } |
e72f66a0 | 72 | |
9ffe337c PB |
73 | /* If dataplane is (re-)enabled while the guest is running there could |
74 | * be block jobs that can conflict. | |
75 | */ | |
76 | if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { | |
77 | error_prepend(errp, "cannot start virtio-blk dataplane: "); | |
9d3b1551 | 78 | return false; |
9ffe337c | 79 | } |
a9968c77 | 80 | } |
9ffe337c PB |
81 | /* Don't try if transport does not support notifiers. */ |
82 | if (!virtio_device_ioeventfd_enabled(vdev)) { | |
9d3b1551 | 83 | return false; |
b0f2027c SH |
84 | } |
85 | ||
e72f66a0 SH |
86 | s = g_new0(VirtIOBlockDataPlane, 1); |
87 | s->vdev = vdev; | |
2a30307f | 88 | s->conf = conf; |
e72f66a0 | 89 | |
9ffe337c PB |
90 | if (conf->iothread) { |
91 | s->iothread = conf->iothread; | |
92 | object_ref(OBJECT(s->iothread)); | |
93 | s->ctx = iothread_get_aio_context(s->iothread); | |
94 | } else { | |
95 | s->ctx = qemu_get_aio_context(); | |
96 | } | |
48ff2692 | 97 | |
e72f66a0 | 98 | *dataplane = s; |
9d3b1551 MZ |
99 | |
100 | return true; | |
e72f66a0 SH |
101 | } |
102 | ||
48ff2692 | 103 | /* Context: QEMU global mutex held */ |
e72f66a0 SH |
104 | void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) |
105 | { | |
9ffe337c PB |
106 | VirtIOBlock *vblk; |
107 | ||
e72f66a0 SH |
108 | if (!s) { |
109 | return; | |
110 | } | |
111 | ||
9ffe337c PB |
112 | vblk = VIRTIO_BLK(s->vdev); |
113 | assert(!vblk->dataplane_started); | |
9ffe337c PB |
114 | if (s->iothread) { |
115 | object_unref(OBJECT(s->iothread)); | |
116 | } | |
e72f66a0 SH |
117 | g_free(s); |
118 | } | |
119 | ||
48ff2692 | 120 | /* Context: QEMU global mutex held */ |
9ffe337c | 121 | int virtio_blk_data_plane_start(VirtIODevice *vdev) |
e72f66a0 | 122 | { |
9ffe337c PB |
123 | VirtIOBlock *vblk = VIRTIO_BLK(vdev); |
124 | VirtIOBlockDataPlane *s = vblk->dataplane; | |
125 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); | |
1c819449 | 126 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
51b04ac5 SH |
127 | unsigned i; |
128 | unsigned nvqs = s->conf->num_queues; | |
97896a48 | 129 | Error *local_err = NULL; |
267e1a20 | 130 | int r; |
e72f66a0 | 131 | |
2906cddf | 132 | if (vblk->dataplane_started || s->starting) { |
9ffe337c | 133 | return 0; |
8caf907f CH |
134 | } |
135 | ||
136 | s->starting = true; | |
e72f66a0 | 137 | |
e72f66a0 | 138 | /* Set up guest notifier (irq) */ |
51b04ac5 | 139 | r = k->set_guest_notifiers(qbus->parent, nvqs, true); |
267e1a20 | 140 | if (r != 0) { |
a1d30f28 TH |
141 | error_report("virtio-blk failed to set guest notifier (%d), " |
142 | "ensure -accel kvm is set.", r); | |
f9907ebc | 143 | goto fail_guest_notifiers; |
e72f66a0 | 144 | } |
e72f66a0 | 145 | |
9cf4fd87 GK |
146 | /* |
147 | * Batch all the host notifiers in a single transaction to avoid | |
148 | * quadratic time complexity in address_space_update_ioeventfds(). | |
149 | */ | |
d0267da6 GK |
150 | memory_region_transaction_begin(); |
151 | ||
e72f66a0 | 152 | /* Set up virtqueue notify */ |
51b04ac5 SH |
153 | for (i = 0; i < nvqs; i++) { |
154 | r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true); | |
155 | if (r != 0) { | |
d0267da6 GK |
156 | int j = i; |
157 | ||
51b04ac5 SH |
158 | fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); |
159 | while (i--) { | |
160 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
161 | } |
162 | ||
9cf4fd87 GK |
163 | /* |
164 | * The transaction expects the ioeventfds to be open when it | |
165 | * commits. Do it now, before the cleanup loop. | |
166 | */ | |
d0267da6 GK |
167 | memory_region_transaction_commit(); |
168 | ||
169 | while (j--) { | |
5b807181 | 170 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j); |
51b04ac5 | 171 | } |
570fe439 | 172 | goto fail_host_notifiers; |
51b04ac5 | 173 | } |
e72f66a0 | 174 | } |
e926d9b8 | 175 | |
d0267da6 GK |
176 | memory_region_transaction_commit(); |
177 | ||
e72f66a0 SH |
178 | trace_virtio_blk_data_plane_start(s); |
179 | ||
97896a48 KW |
180 | r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); |
181 | if (r < 0) { | |
182 | error_report_err(local_err); | |
570fe439 | 183 | goto fail_aio_context; |
97896a48 | 184 | } |
580b6b2a | 185 | |
e72f66a0 | 186 | /* Kick right away to begin processing requests already in vring */ |
51b04ac5 SH |
187 | for (i = 0; i < nvqs; i++) { |
188 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
189 | ||
190 | event_notifier_set(virtio_queue_get_host_notifier(vq)); | |
191 | } | |
e72f66a0 | 192 | |
75dcb4d7 SH |
193 | /* |
194 | * These fields must be visible to the IOThread when it processes the | |
195 | * virtqueue, otherwise it will think dataplane has not started yet. | |
196 | * | |
197 | * Make sure ->dataplane_started is false when blk_set_aio_context() is | |
198 | * called above so that draining does not cause the host notifier to be | |
199 | * detached/attached prematurely. | |
200 | */ | |
201 | s->starting = false; | |
202 | vblk->dataplane_started = true; | |
203 | smp_wmb(); /* paired with aio_notify_accept() on the read side */ | |
204 | ||
48ff2692 | 205 | /* Get this show started by hooking up our callbacks */ |
1665d932 | 206 | if (!blk_in_drain(s->conf->conf.blk)) { |
1665d932 SH |
207 | for (i = 0; i < nvqs; i++) { |
208 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
51b04ac5 | 209 | |
1665d932 SH |
210 | virtio_queue_aio_attach_host_notifier(vq, s->ctx); |
211 | } | |
51b04ac5 | 212 | } |
9ffe337c | 213 | return 0; |
f9907ebc | 214 | |
570fe439 | 215 | fail_aio_context: |
d0267da6 GK |
216 | memory_region_transaction_begin(); |
217 | ||
570fe439 GK |
218 | for (i = 0; i < nvqs; i++) { |
219 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
220 | } |
221 | ||
222 | memory_region_transaction_commit(); | |
223 | ||
224 | for (i = 0; i < nvqs; i++) { | |
570fe439 GK |
225 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); |
226 | } | |
227 | fail_host_notifiers: | |
228 | k->set_guest_notifiers(qbus->parent, nvqs, false); | |
f9907ebc | 229 | fail_guest_notifiers: |
eb41cf78 | 230 | vblk->dataplane_disabled = true; |
f9907ebc | 231 | s->starting = false; |
9ffe337c | 232 | return -ENOSYS; |
e72f66a0 SH |
233 | } |
234 | ||
1010cadf SH |
235 | /* Stop notifications for new requests from guest. |
236 | * | |
237 | * Context: BH in IOThread | |
238 | */ | |
239 | static void virtio_blk_data_plane_stop_bh(void *opaque) | |
240 | { | |
241 | VirtIOBlockDataPlane *s = opaque; | |
242 | unsigned i; | |
243 | ||
244 | for (i = 0; i < s->conf->num_queues; i++) { | |
245 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
bd58ab40 | 246 | EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); |
1010cadf | 247 | |
db608fb7 | 248 | virtio_queue_aio_detach_host_notifier(vq, s->ctx); |
bd58ab40 SH |
249 | |
250 | /* | |
251 | * Test and clear notifier after disabling event, in case poll callback | |
252 | * didn't have time to run. | |
253 | */ | |
254 | virtio_queue_host_notifier_read(host_notifier); | |
1010cadf SH |
255 | } |
256 | } | |
257 | ||
48ff2692 | 258 | /* Context: QEMU global mutex held */ |
9ffe337c | 259 | void virtio_blk_data_plane_stop(VirtIODevice *vdev) |
e72f66a0 | 260 | { |
9ffe337c PB |
261 | VirtIOBlock *vblk = VIRTIO_BLK(vdev); |
262 | VirtIOBlockDataPlane *s = vblk->dataplane; | |
263 | BusState *qbus = qdev_get_parent_bus(DEVICE(vblk)); | |
1c819449 | 264 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
51b04ac5 SH |
265 | unsigned i; |
266 | unsigned nvqs = s->conf->num_queues; | |
2f5f70fa | 267 | |
2906cddf PB |
268 | if (!vblk->dataplane_started || s->stopping) { |
269 | return; | |
270 | } | |
2f5f70fa CH |
271 | |
272 | /* Better luck next time. */ | |
eb41cf78 PB |
273 | if (vblk->dataplane_disabled) { |
274 | vblk->dataplane_disabled = false; | |
2906cddf | 275 | vblk->dataplane_started = false; |
e72f66a0 SH |
276 | return; |
277 | } | |
cd7fdfe5 | 278 | s->stopping = true; |
e72f66a0 SH |
279 | trace_virtio_blk_data_plane_stop(s); |
280 | ||
1665d932 SH |
281 | if (!blk_in_drain(s->conf->conf.blk)) { |
282 | aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); | |
283 | } | |
48ff2692 | 284 | |
9cf4fd87 GK |
285 | /* |
286 | * Batch all the host notifiers in a single transaction to avoid | |
287 | * quadratic time complexity in address_space_update_ioeventfds(). | |
288 | */ | |
d0267da6 GK |
289 | memory_region_transaction_begin(); |
290 | ||
51b04ac5 SH |
291 | for (i = 0; i < nvqs; i++) { |
292 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
293 | } |
294 | ||
9cf4fd87 GK |
295 | /* |
296 | * The transaction expects the ioeventfds to be open when it | |
297 | * commits. Do it now, before the cleanup loop. | |
298 | */ | |
d0267da6 GK |
299 | memory_region_transaction_commit(); |
300 | ||
301 | for (i = 0; i < nvqs; i++) { | |
76143618 | 302 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); |
51b04ac5 | 303 | } |
e72f66a0 | 304 | |
75dcb4d7 SH |
305 | /* |
306 | * Set ->dataplane_started to false before draining so that host notifiers | |
307 | * are not detached/attached anymore. | |
308 | */ | |
309 | vblk->dataplane_started = false; | |
310 | ||
75dcb4d7 SH |
311 | /* Wait for virtio_blk_dma_restart_bh() and in flight I/O to complete */ |
312 | blk_drain(s->conf->conf.blk); | |
313 | ||
314 | /* | |
315 | * Try to switch bs back to the QEMU main loop. If other users keep the | |
316 | * BlockBackend in the iothread, that's ok | |
317 | */ | |
318 | blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL); | |
319 | ||
e72f66a0 | 320 | /* Clean up guest notifier (irq) */ |
51b04ac5 | 321 | k->set_guest_notifiers(qbus->parent, nvqs, false); |
e72f66a0 | 322 | |
cd7fdfe5 | 323 | s->stopping = false; |
e72f66a0 | 324 | } |