]>
Commit | Line | Data |
---|---|---|
e72f66a0 SH |
1 | /* |
2 | * Dedicated thread for virtio-blk I/O processing | |
3 | * | |
4 | * Copyright 2012 IBM, Corp. | |
5 | * Copyright 2012 Red Hat, Inc. and/or its affiliates | |
6 | * | |
7 | * Authors: | |
8 | * Stefan Hajnoczi <stefanha@redhat.com> | |
9 | * | |
10 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
11 | * See the COPYING file in the top-level directory. | |
12 | * | |
13 | */ | |
14 | ||
80c71a24 | 15 | #include "qemu/osdep.h" |
da34e65c | 16 | #include "qapi/error.h" |
e72f66a0 SH |
17 | #include "trace.h" |
18 | #include "qemu/iov.h" | |
db725815 | 19 | #include "qemu/main-loop.h" |
e72f66a0 | 20 | #include "qemu/thread.h" |
b4a42f81 | 21 | #include "qemu/error-report.h" |
b0e5d90e | 22 | #include "hw/virtio/virtio-access.h" |
0d09e41a PB |
23 | #include "hw/virtio/virtio-blk.h" |
24 | #include "virtio-blk.h" | |
2c20e711 | 25 | #include "block/aio.h" |
1c819449 | 26 | #include "hw/virtio/virtio-bus.h" |
54bee5c2 | 27 | #include "qom/object_interfaces.h" |
e72f66a0 | 28 | |
e72f66a0 | 29 | struct VirtIOBlockDataPlane { |
8caf907f | 30 | bool starting; |
cd7fdfe5 | 31 | bool stopping; |
e72f66a0 | 32 | |
2a30307f | 33 | VirtIOBlkConf *conf; |
e72f66a0 | 34 | VirtIODevice *vdev; |
5b2ffbe4 | 35 | QEMUBH *bh; /* bh for guest notification */ |
e21737ab | 36 | unsigned long *batch_notify_vqs; |
12c1c7d7 | 37 | bool batch_notifications; |
e72f66a0 | 38 | |
2c20e711 PB |
39 | /* Note that these EventNotifiers are assigned by value. This is |
40 | * fine as long as you do not call event_notifier_cleanup on them | |
41 | * (because you don't own the file descriptor or handle; you just | |
42 | * use it). | |
43 | */ | |
48ff2692 | 44 | IOThread *iothread; |
2c20e711 | 45 | AioContext *ctx; |
e72f66a0 SH |
46 | }; |
47 | ||
48 | /* Raise an interrupt to signal guest, if necessary */ | |
b234cdda | 49 | void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) |
e72f66a0 | 50 | { |
12c1c7d7 SL |
51 | if (s->batch_notifications) { |
52 | set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs); | |
53 | qemu_bh_schedule(s->bh); | |
54 | } else { | |
55 | virtio_notify_irqfd(s->vdev, vq); | |
56 | } | |
e72f66a0 SH |
57 | } |
58 | ||
5b2ffbe4 ML |
59 | static void notify_guest_bh(void *opaque) |
60 | { | |
61 | VirtIOBlockDataPlane *s = opaque; | |
e21737ab SH |
62 | unsigned nvqs = s->conf->num_queues; |
63 | unsigned long bitmap[BITS_TO_LONGS(nvqs)]; | |
64 | unsigned j; | |
5b2ffbe4 | 65 | |
e21737ab SH |
66 | memcpy(bitmap, s->batch_notify_vqs, sizeof(bitmap)); |
67 | memset(s->batch_notify_vqs, 0, sizeof(bitmap)); | |
68 | ||
69 | for (j = 0; j < nvqs; j += BITS_PER_LONG) { | |
725fe5d1 | 70 | unsigned long bits = bitmap[j / BITS_PER_LONG]; |
03de2f52 | 71 | |
e21737ab SH |
72 | while (bits != 0) { |
73 | unsigned i = j + ctzl(bits); | |
74 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
75 | ||
83d768b5 | 76 | virtio_notify_irqfd(s->vdev, vq); |
e21737ab SH |
77 | |
78 | bits &= bits - 1; /* clear right-most bit */ | |
79 | } | |
80 | } | |
e72f66a0 SH |
81 | } |
82 | ||
48ff2692 | 83 | /* Context: QEMU global mutex held */ |
9d3b1551 | 84 | bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, |
3ffeeef7 AF |
85 | VirtIOBlockDataPlane **dataplane, |
86 | Error **errp) | |
e72f66a0 SH |
87 | { |
88 | VirtIOBlockDataPlane *s; | |
a9968c77 CH |
89 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); |
90 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); | |
e72f66a0 SH |
91 | |
92 | *dataplane = NULL; | |
93 | ||
9ffe337c PB |
94 | if (conf->iothread) { |
95 | if (!k->set_guest_notifiers || !k->ioeventfd_assign) { | |
96 | error_setg(errp, | |
97 | "device is incompatible with iothread " | |
98 | "(transport does not support notifiers)"); | |
9d3b1551 | 99 | return false; |
9ffe337c PB |
100 | } |
101 | if (!virtio_device_ioeventfd_enabled(vdev)) { | |
102 | error_setg(errp, "ioeventfd is required for iothread"); | |
9d3b1551 | 103 | return false; |
9ffe337c | 104 | } |
e72f66a0 | 105 | |
9ffe337c PB |
106 | /* If dataplane is (re-)enabled while the guest is running there could |
107 | * be block jobs that can conflict. | |
108 | */ | |
109 | if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { | |
110 | error_prepend(errp, "cannot start virtio-blk dataplane: "); | |
9d3b1551 | 111 | return false; |
9ffe337c | 112 | } |
a9968c77 | 113 | } |
9ffe337c PB |
114 | /* Don't try if transport does not support notifiers. */ |
115 | if (!virtio_device_ioeventfd_enabled(vdev)) { | |
9d3b1551 | 116 | return false; |
b0f2027c SH |
117 | } |
118 | ||
e72f66a0 SH |
119 | s = g_new0(VirtIOBlockDataPlane, 1); |
120 | s->vdev = vdev; | |
2a30307f | 121 | s->conf = conf; |
e72f66a0 | 122 | |
9ffe337c PB |
123 | if (conf->iothread) { |
124 | s->iothread = conf->iothread; | |
125 | object_ref(OBJECT(s->iothread)); | |
126 | s->ctx = iothread_get_aio_context(s->iothread); | |
127 | } else { | |
128 | s->ctx = qemu_get_aio_context(); | |
129 | } | |
5b2ffbe4 | 130 | s->bh = aio_bh_new(s->ctx, notify_guest_bh, s); |
e21737ab | 131 | s->batch_notify_vqs = bitmap_new(conf->num_queues); |
48ff2692 | 132 | |
e72f66a0 | 133 | *dataplane = s; |
9d3b1551 MZ |
134 | |
135 | return true; | |
e72f66a0 SH |
136 | } |
137 | ||
48ff2692 | 138 | /* Context: QEMU global mutex held */ |
e72f66a0 SH |
139 | void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) |
140 | { | |
9ffe337c PB |
141 | VirtIOBlock *vblk; |
142 | ||
e72f66a0 SH |
143 | if (!s) { |
144 | return; | |
145 | } | |
146 | ||
9ffe337c PB |
147 | vblk = VIRTIO_BLK(s->vdev); |
148 | assert(!vblk->dataplane_started); | |
e21737ab | 149 | g_free(s->batch_notify_vqs); |
5b2ffbe4 | 150 | qemu_bh_delete(s->bh); |
9ffe337c PB |
151 | if (s->iothread) { |
152 | object_unref(OBJECT(s->iothread)); | |
153 | } | |
e72f66a0 SH |
154 | g_free(s); |
155 | } | |
156 | ||
d93d16c0 | 157 | static void virtio_blk_data_plane_handle_output(VirtIODevice *vdev, |
8a2fad57 MT |
158 | VirtQueue *vq) |
159 | { | |
160 | VirtIOBlock *s = (VirtIOBlock *)vdev; | |
161 | ||
162 | assert(s->dataplane); | |
163 | assert(s->dataplane_started); | |
164 | ||
d93d16c0 | 165 | virtio_blk_handle_vq(s, vq); |
8a2fad57 MT |
166 | } |
167 | ||
48ff2692 | 168 | /* Context: QEMU global mutex held */ |
9ffe337c | 169 | int virtio_blk_data_plane_start(VirtIODevice *vdev) |
e72f66a0 | 170 | { |
9ffe337c PB |
171 | VirtIOBlock *vblk = VIRTIO_BLK(vdev); |
172 | VirtIOBlockDataPlane *s = vblk->dataplane; | |
173 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); | |
1c819449 | 174 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
c7040ff6 | 175 | AioContext *old_context; |
51b04ac5 SH |
176 | unsigned i; |
177 | unsigned nvqs = s->conf->num_queues; | |
97896a48 | 178 | Error *local_err = NULL; |
267e1a20 | 179 | int r; |
e72f66a0 | 180 | |
2906cddf | 181 | if (vblk->dataplane_started || s->starting) { |
9ffe337c | 182 | return 0; |
8caf907f CH |
183 | } |
184 | ||
185 | s->starting = true; | |
e72f66a0 | 186 | |
12c1c7d7 SL |
187 | if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { |
188 | s->batch_notifications = true; | |
189 | } else { | |
190 | s->batch_notifications = false; | |
191 | } | |
192 | ||
e72f66a0 | 193 | /* Set up guest notifier (irq) */ |
51b04ac5 | 194 | r = k->set_guest_notifiers(qbus->parent, nvqs, true); |
267e1a20 | 195 | if (r != 0) { |
a1d30f28 TH |
196 | error_report("virtio-blk failed to set guest notifier (%d), " |
197 | "ensure -accel kvm is set.", r); | |
f9907ebc | 198 | goto fail_guest_notifiers; |
e72f66a0 | 199 | } |
e72f66a0 | 200 | |
9cf4fd87 GK |
201 | /* |
202 | * Batch all the host notifiers in a single transaction to avoid | |
203 | * quadratic time complexity in address_space_update_ioeventfds(). | |
204 | */ | |
d0267da6 GK |
205 | memory_region_transaction_begin(); |
206 | ||
e72f66a0 | 207 | /* Set up virtqueue notify */ |
51b04ac5 SH |
208 | for (i = 0; i < nvqs; i++) { |
209 | r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true); | |
210 | if (r != 0) { | |
d0267da6 GK |
211 | int j = i; |
212 | ||
51b04ac5 SH |
213 | fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); |
214 | while (i--) { | |
215 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
216 | } |
217 | ||
9cf4fd87 GK |
218 | /* |
219 | * The transaction expects the ioeventfds to be open when it | |
220 | * commits. Do it now, before the cleanup loop. | |
221 | */ | |
d0267da6 GK |
222 | memory_region_transaction_commit(); |
223 | ||
224 | while (j--) { | |
5b807181 | 225 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j); |
51b04ac5 | 226 | } |
570fe439 | 227 | goto fail_host_notifiers; |
51b04ac5 | 228 | } |
e72f66a0 | 229 | } |
e926d9b8 | 230 | |
d0267da6 GK |
231 | memory_region_transaction_commit(); |
232 | ||
8caf907f | 233 | s->starting = false; |
2906cddf | 234 | vblk->dataplane_started = true; |
e72f66a0 SH |
235 | trace_virtio_blk_data_plane_start(s); |
236 | ||
c7040ff6 SL |
237 | old_context = blk_get_aio_context(s->conf->conf.blk); |
238 | aio_context_acquire(old_context); | |
97896a48 | 239 | r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); |
c7040ff6 | 240 | aio_context_release(old_context); |
97896a48 KW |
241 | if (r < 0) { |
242 | error_report_err(local_err); | |
570fe439 | 243 | goto fail_aio_context; |
97896a48 | 244 | } |
580b6b2a | 245 | |
49b44549 SL |
246 | /* Process queued requests before the ones in vring */ |
247 | virtio_blk_process_queued_requests(vblk, false); | |
248 | ||
e72f66a0 | 249 | /* Kick right away to begin processing requests already in vring */ |
51b04ac5 SH |
250 | for (i = 0; i < nvqs; i++) { |
251 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
252 | ||
253 | event_notifier_set(virtio_queue_get_host_notifier(vq)); | |
254 | } | |
e72f66a0 | 255 | |
48ff2692 SH |
256 | /* Get this show started by hooking up our callbacks */ |
257 | aio_context_acquire(s->ctx); | |
51b04ac5 SH |
258 | for (i = 0; i < nvqs; i++) { |
259 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
260 | ||
261 | virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, | |
262 | virtio_blk_data_plane_handle_output); | |
263 | } | |
48ff2692 | 264 | aio_context_release(s->ctx); |
9ffe337c | 265 | return 0; |
f9907ebc | 266 | |
570fe439 | 267 | fail_aio_context: |
d0267da6 GK |
268 | memory_region_transaction_begin(); |
269 | ||
570fe439 GK |
270 | for (i = 0; i < nvqs; i++) { |
271 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
272 | } |
273 | ||
274 | memory_region_transaction_commit(); | |
275 | ||
276 | for (i = 0; i < nvqs; i++) { | |
570fe439 GK |
277 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); |
278 | } | |
279 | fail_host_notifiers: | |
280 | k->set_guest_notifiers(qbus->parent, nvqs, false); | |
f9907ebc | 281 | fail_guest_notifiers: |
49b44549 SL |
282 | /* |
283 | * If we failed to set up the guest notifiers queued requests will be | |
284 | * processed on the main context. | |
285 | */ | |
286 | virtio_blk_process_queued_requests(vblk, false); | |
eb41cf78 | 287 | vblk->dataplane_disabled = true; |
f9907ebc | 288 | s->starting = false; |
2906cddf | 289 | vblk->dataplane_started = true; |
9ffe337c | 290 | return -ENOSYS; |
e72f66a0 SH |
291 | } |
292 | ||
1010cadf SH |
293 | /* Stop notifications for new requests from guest. |
294 | * | |
295 | * Context: BH in IOThread | |
296 | */ | |
297 | static void virtio_blk_data_plane_stop_bh(void *opaque) | |
298 | { | |
299 | VirtIOBlockDataPlane *s = opaque; | |
300 | unsigned i; | |
301 | ||
302 | for (i = 0; i < s->conf->num_queues; i++) { | |
303 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
304 | ||
305 | virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, NULL); | |
306 | } | |
307 | } | |
308 | ||
48ff2692 | 309 | /* Context: QEMU global mutex held */ |
9ffe337c | 310 | void virtio_blk_data_plane_stop(VirtIODevice *vdev) |
e72f66a0 | 311 | { |
9ffe337c PB |
312 | VirtIOBlock *vblk = VIRTIO_BLK(vdev); |
313 | VirtIOBlockDataPlane *s = vblk->dataplane; | |
314 | BusState *qbus = qdev_get_parent_bus(DEVICE(vblk)); | |
1c819449 | 315 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
51b04ac5 SH |
316 | unsigned i; |
317 | unsigned nvqs = s->conf->num_queues; | |
2f5f70fa | 318 | |
2906cddf PB |
319 | if (!vblk->dataplane_started || s->stopping) { |
320 | return; | |
321 | } | |
2f5f70fa CH |
322 | |
323 | /* Better luck next time. */ | |
eb41cf78 PB |
324 | if (vblk->dataplane_disabled) { |
325 | vblk->dataplane_disabled = false; | |
2906cddf | 326 | vblk->dataplane_started = false; |
e72f66a0 SH |
327 | return; |
328 | } | |
cd7fdfe5 | 329 | s->stopping = true; |
e72f66a0 SH |
330 | trace_virtio_blk_data_plane_stop(s); |
331 | ||
48ff2692 | 332 | aio_context_acquire(s->ctx); |
1010cadf | 333 | aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); |
48ff2692 | 334 | |
97896a48 KW |
335 | /* Drain and try to switch bs back to the QEMU main loop. If other users |
336 | * keep the BlockBackend in the iothread, that's ok */ | |
337 | blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL); | |
e72f66a0 | 338 | |
48ff2692 | 339 | aio_context_release(s->ctx); |
e72f66a0 | 340 | |
9cf4fd87 GK |
341 | /* |
342 | * Batch all the host notifiers in a single transaction to avoid | |
343 | * quadratic time complexity in address_space_update_ioeventfds(). | |
344 | */ | |
d0267da6 GK |
345 | memory_region_transaction_begin(); |
346 | ||
51b04ac5 SH |
347 | for (i = 0; i < nvqs; i++) { |
348 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
d0267da6 GK |
349 | } |
350 | ||
9cf4fd87 GK |
351 | /* |
352 | * The transaction expects the ioeventfds to be open when it | |
353 | * commits. Do it now, before the cleanup loop. | |
354 | */ | |
d0267da6 GK |
355 | memory_region_transaction_commit(); |
356 | ||
357 | for (i = 0; i < nvqs; i++) { | |
76143618 | 358 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); |
51b04ac5 | 359 | } |
e72f66a0 | 360 | |
ebb6ff25 PMD |
361 | qemu_bh_cancel(s->bh); |
362 | notify_guest_bh(s); /* final chance to notify guest */ | |
363 | ||
e72f66a0 | 364 | /* Clean up guest notifier (irq) */ |
51b04ac5 | 365 | k->set_guest_notifiers(qbus->parent, nvqs, false); |
e72f66a0 | 366 | |
2906cddf | 367 | vblk->dataplane_started = false; |
cd7fdfe5 | 368 | s->stopping = false; |
e72f66a0 | 369 | } |