]>
Commit | Line | Data |
---|---|---|
e72f66a0 SH |
1 | /* |
2 | * Dedicated thread for virtio-blk I/O processing | |
3 | * | |
4 | * Copyright 2012 IBM, Corp. | |
5 | * Copyright 2012 Red Hat, Inc. and/or its affiliates | |
6 | * | |
7 | * Authors: | |
8 | * Stefan Hajnoczi <stefanha@redhat.com> | |
9 | * | |
10 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
11 | * See the COPYING file in the top-level directory. | |
12 | * | |
13 | */ | |
14 | ||
80c71a24 | 15 | #include "qemu/osdep.h" |
da34e65c | 16 | #include "qapi/error.h" |
e72f66a0 SH |
17 | #include "trace.h" |
18 | #include "qemu/iov.h" | |
e72f66a0 | 19 | #include "qemu/thread.h" |
b4a42f81 | 20 | #include "qemu/error-report.h" |
b0e5d90e | 21 | #include "hw/virtio/virtio-access.h" |
0d09e41a PB |
22 | #include "hw/virtio/virtio-blk.h" |
23 | #include "virtio-blk.h" | |
2c20e711 | 24 | #include "block/aio.h" |
1c819449 | 25 | #include "hw/virtio/virtio-bus.h" |
54bee5c2 | 26 | #include "qom/object_interfaces.h" |
e72f66a0 | 27 | |
e72f66a0 | 28 | struct VirtIOBlockDataPlane { |
8caf907f | 29 | bool starting; |
cd7fdfe5 | 30 | bool stopping; |
e72f66a0 | 31 | |
2a30307f | 32 | VirtIOBlkConf *conf; |
e72f66a0 | 33 | VirtIODevice *vdev; |
5b2ffbe4 | 34 | QEMUBH *bh; /* bh for guest notification */ |
e21737ab | 35 | unsigned long *batch_notify_vqs; |
12c1c7d7 | 36 | bool batch_notifications; |
e72f66a0 | 37 | |
2c20e711 PB |
38 | /* Note that these EventNotifiers are assigned by value. This is |
39 | * fine as long as you do not call event_notifier_cleanup on them | |
40 | * (because you don't own the file descriptor or handle; you just | |
41 | * use it). | |
42 | */ | |
48ff2692 | 43 | IOThread *iothread; |
2c20e711 | 44 | AioContext *ctx; |
e72f66a0 SH |
45 | }; |
46 | ||
47 | /* Raise an interrupt to signal guest, if necessary */ | |
b234cdda | 48 | void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) |
e72f66a0 | 49 | { |
12c1c7d7 SL |
50 | if (s->batch_notifications) { |
51 | set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs); | |
52 | qemu_bh_schedule(s->bh); | |
53 | } else { | |
54 | virtio_notify_irqfd(s->vdev, vq); | |
55 | } | |
e72f66a0 SH |
56 | } |
57 | ||
5b2ffbe4 ML |
58 | static void notify_guest_bh(void *opaque) |
59 | { | |
60 | VirtIOBlockDataPlane *s = opaque; | |
e21737ab SH |
61 | unsigned nvqs = s->conf->num_queues; |
62 | unsigned long bitmap[BITS_TO_LONGS(nvqs)]; | |
63 | unsigned j; | |
5b2ffbe4 | 64 | |
e21737ab SH |
65 | memcpy(bitmap, s->batch_notify_vqs, sizeof(bitmap)); |
66 | memset(s->batch_notify_vqs, 0, sizeof(bitmap)); | |
67 | ||
68 | for (j = 0; j < nvqs; j += BITS_PER_LONG) { | |
69 | unsigned long bits = bitmap[j]; | |
03de2f52 | 70 | |
e21737ab SH |
71 | while (bits != 0) { |
72 | unsigned i = j + ctzl(bits); | |
73 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
74 | ||
83d768b5 | 75 | virtio_notify_irqfd(s->vdev, vq); |
e21737ab SH |
76 | |
77 | bits &= bits - 1; /* clear right-most bit */ | |
78 | } | |
79 | } | |
e72f66a0 SH |
80 | } |
81 | ||
48ff2692 | 82 | /* Context: QEMU global mutex held */ |
9d3b1551 | 83 | bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, |
3ffeeef7 AF |
84 | VirtIOBlockDataPlane **dataplane, |
85 | Error **errp) | |
e72f66a0 SH |
86 | { |
87 | VirtIOBlockDataPlane *s; | |
a9968c77 CH |
88 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); |
89 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); | |
e72f66a0 SH |
90 | |
91 | *dataplane = NULL; | |
92 | ||
9ffe337c PB |
93 | if (conf->iothread) { |
94 | if (!k->set_guest_notifiers || !k->ioeventfd_assign) { | |
95 | error_setg(errp, | |
96 | "device is incompatible with iothread " | |
97 | "(transport does not support notifiers)"); | |
9d3b1551 | 98 | return false; |
9ffe337c PB |
99 | } |
100 | if (!virtio_device_ioeventfd_enabled(vdev)) { | |
101 | error_setg(errp, "ioeventfd is required for iothread"); | |
9d3b1551 | 102 | return false; |
9ffe337c | 103 | } |
e72f66a0 | 104 | |
9ffe337c PB |
105 | /* If dataplane is (re-)enabled while the guest is running there could |
106 | * be block jobs that can conflict. | |
107 | */ | |
108 | if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { | |
109 | error_prepend(errp, "cannot start virtio-blk dataplane: "); | |
9d3b1551 | 110 | return false; |
9ffe337c | 111 | } |
a9968c77 | 112 | } |
9ffe337c PB |
113 | /* Don't try if transport does not support notifiers. */ |
114 | if (!virtio_device_ioeventfd_enabled(vdev)) { | |
9d3b1551 | 115 | return false; |
b0f2027c SH |
116 | } |
117 | ||
e72f66a0 SH |
118 | s = g_new0(VirtIOBlockDataPlane, 1); |
119 | s->vdev = vdev; | |
2a30307f | 120 | s->conf = conf; |
e72f66a0 | 121 | |
9ffe337c PB |
122 | if (conf->iothread) { |
123 | s->iothread = conf->iothread; | |
124 | object_ref(OBJECT(s->iothread)); | |
125 | s->ctx = iothread_get_aio_context(s->iothread); | |
126 | } else { | |
127 | s->ctx = qemu_get_aio_context(); | |
128 | } | |
5b2ffbe4 | 129 | s->bh = aio_bh_new(s->ctx, notify_guest_bh, s); |
e21737ab | 130 | s->batch_notify_vqs = bitmap_new(conf->num_queues); |
48ff2692 | 131 | |
e72f66a0 | 132 | *dataplane = s; |
9d3b1551 MZ |
133 | |
134 | return true; | |
e72f66a0 SH |
135 | } |
136 | ||
48ff2692 | 137 | /* Context: QEMU global mutex held */ |
e72f66a0 SH |
138 | void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) |
139 | { | |
9ffe337c PB |
140 | VirtIOBlock *vblk; |
141 | ||
e72f66a0 SH |
142 | if (!s) { |
143 | return; | |
144 | } | |
145 | ||
9ffe337c PB |
146 | vblk = VIRTIO_BLK(s->vdev); |
147 | assert(!vblk->dataplane_started); | |
e21737ab | 148 | g_free(s->batch_notify_vqs); |
5b2ffbe4 | 149 | qemu_bh_delete(s->bh); |
9ffe337c PB |
150 | if (s->iothread) { |
151 | object_unref(OBJECT(s->iothread)); | |
152 | } | |
e72f66a0 SH |
153 | g_free(s); |
154 | } | |
155 | ||
07931698 | 156 | static bool virtio_blk_data_plane_handle_output(VirtIODevice *vdev, |
8a2fad57 MT |
157 | VirtQueue *vq) |
158 | { | |
159 | VirtIOBlock *s = (VirtIOBlock *)vdev; | |
160 | ||
161 | assert(s->dataplane); | |
162 | assert(s->dataplane_started); | |
163 | ||
07931698 | 164 | return virtio_blk_handle_vq(s, vq); |
8a2fad57 MT |
165 | } |
166 | ||
48ff2692 | 167 | /* Context: QEMU global mutex held */ |
9ffe337c | 168 | int virtio_blk_data_plane_start(VirtIODevice *vdev) |
e72f66a0 | 169 | { |
9ffe337c PB |
170 | VirtIOBlock *vblk = VIRTIO_BLK(vdev); |
171 | VirtIOBlockDataPlane *s = vblk->dataplane; | |
172 | BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); | |
1c819449 | 173 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
51b04ac5 SH |
174 | unsigned i; |
175 | unsigned nvqs = s->conf->num_queues; | |
267e1a20 | 176 | int r; |
e72f66a0 | 177 | |
2906cddf | 178 | if (vblk->dataplane_started || s->starting) { |
9ffe337c | 179 | return 0; |
8caf907f CH |
180 | } |
181 | ||
182 | s->starting = true; | |
e72f66a0 | 183 | |
12c1c7d7 SL |
184 | if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { |
185 | s->batch_notifications = true; | |
186 | } else { | |
187 | s->batch_notifications = false; | |
188 | } | |
189 | ||
e72f66a0 | 190 | /* Set up guest notifier (irq) */ |
51b04ac5 | 191 | r = k->set_guest_notifiers(qbus->parent, nvqs, true); |
267e1a20 | 192 | if (r != 0) { |
a1d30f28 TH |
193 | error_report("virtio-blk failed to set guest notifier (%d), " |
194 | "ensure -accel kvm is set.", r); | |
f9907ebc | 195 | goto fail_guest_notifiers; |
e72f66a0 | 196 | } |
e72f66a0 SH |
197 | |
198 | /* Set up virtqueue notify */ | |
51b04ac5 SH |
199 | for (i = 0; i < nvqs; i++) { |
200 | r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true); | |
201 | if (r != 0) { | |
202 | fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); | |
203 | while (i--) { | |
204 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
76143618 | 205 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); |
51b04ac5 SH |
206 | } |
207 | goto fail_guest_notifiers; | |
208 | } | |
e72f66a0 | 209 | } |
e926d9b8 | 210 | |
8caf907f | 211 | s->starting = false; |
2906cddf | 212 | vblk->dataplane_started = true; |
e72f66a0 SH |
213 | trace_virtio_blk_data_plane_start(s); |
214 | ||
4be74634 | 215 | blk_set_aio_context(s->conf->conf.blk, s->ctx); |
580b6b2a | 216 | |
e72f66a0 | 217 | /* Kick right away to begin processing requests already in vring */ |
51b04ac5 SH |
218 | for (i = 0; i < nvqs; i++) { |
219 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
220 | ||
221 | event_notifier_set(virtio_queue_get_host_notifier(vq)); | |
222 | } | |
e72f66a0 | 223 | |
48ff2692 SH |
224 | /* Get this show started by hooking up our callbacks */ |
225 | aio_context_acquire(s->ctx); | |
51b04ac5 SH |
226 | for (i = 0; i < nvqs; i++) { |
227 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
228 | ||
229 | virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, | |
230 | virtio_blk_data_plane_handle_output); | |
231 | } | |
48ff2692 | 232 | aio_context_release(s->ctx); |
9ffe337c | 233 | return 0; |
f9907ebc | 234 | |
f9907ebc | 235 | fail_guest_notifiers: |
eb41cf78 | 236 | vblk->dataplane_disabled = true; |
f9907ebc | 237 | s->starting = false; |
2906cddf | 238 | vblk->dataplane_started = true; |
9ffe337c | 239 | return -ENOSYS; |
e72f66a0 SH |
240 | } |
241 | ||
1010cadf SH |
242 | /* Stop notifications for new requests from guest. |
243 | * | |
244 | * Context: BH in IOThread | |
245 | */ | |
246 | static void virtio_blk_data_plane_stop_bh(void *opaque) | |
247 | { | |
248 | VirtIOBlockDataPlane *s = opaque; | |
249 | unsigned i; | |
250 | ||
251 | for (i = 0; i < s->conf->num_queues; i++) { | |
252 | VirtQueue *vq = virtio_get_queue(s->vdev, i); | |
253 | ||
254 | virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, NULL); | |
255 | } | |
256 | } | |
257 | ||
48ff2692 | 258 | /* Context: QEMU global mutex held */ |
9ffe337c | 259 | void virtio_blk_data_plane_stop(VirtIODevice *vdev) |
e72f66a0 | 260 | { |
9ffe337c PB |
261 | VirtIOBlock *vblk = VIRTIO_BLK(vdev); |
262 | VirtIOBlockDataPlane *s = vblk->dataplane; | |
263 | BusState *qbus = qdev_get_parent_bus(DEVICE(vblk)); | |
1c819449 | 264 | VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); |
51b04ac5 SH |
265 | unsigned i; |
266 | unsigned nvqs = s->conf->num_queues; | |
2f5f70fa | 267 | |
2906cddf PB |
268 | if (!vblk->dataplane_started || s->stopping) { |
269 | return; | |
270 | } | |
2f5f70fa CH |
271 | |
272 | /* Better luck next time. */ | |
eb41cf78 PB |
273 | if (vblk->dataplane_disabled) { |
274 | vblk->dataplane_disabled = false; | |
2906cddf | 275 | vblk->dataplane_started = false; |
e72f66a0 SH |
276 | return; |
277 | } | |
cd7fdfe5 | 278 | s->stopping = true; |
e72f66a0 SH |
279 | trace_virtio_blk_data_plane_stop(s); |
280 | ||
48ff2692 | 281 | aio_context_acquire(s->ctx); |
1010cadf | 282 | aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); |
48ff2692 | 283 | |
580b6b2a | 284 | /* Drain and switch bs back to the QEMU main loop */ |
4be74634 | 285 | blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context()); |
e72f66a0 | 286 | |
48ff2692 | 287 | aio_context_release(s->ctx); |
e72f66a0 | 288 | |
51b04ac5 SH |
289 | for (i = 0; i < nvqs; i++) { |
290 | virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); | |
76143618 | 291 | virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); |
51b04ac5 | 292 | } |
e72f66a0 SH |
293 | |
294 | /* Clean up guest notifier (irq) */ | |
51b04ac5 | 295 | k->set_guest_notifiers(qbus->parent, nvqs, false); |
e72f66a0 | 296 | |
2906cddf | 297 | vblk->dataplane_started = false; |
cd7fdfe5 | 298 | s->stopping = false; |
e72f66a0 | 299 | } |