2 * Dedicated thread for virtio-blk I/O processing
4 * Copyright 2012 IBM, Corp.
5 * Copyright 2012 Red Hat, Inc. and/or its affiliates
8 * Stefan Hajnoczi <stefanha@redhat.com>
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
17 #include "qemu/thread.h"
18 #include "qemu/error-report.h"
19 #include "hw/virtio/dataplane/vring.h"
20 #include "sysemu/block-backend.h"
21 #include "hw/virtio/virtio-blk.h"
22 #include "virtio-blk.h"
23 #include "block/aio.h"
24 #include "hw/virtio/virtio-bus.h"
25 #include "qom/object_interfaces.h"
27 struct VirtIOBlockDataPlane
{
36 Vring vring
; /* virtqueue vring */
37 EventNotifier
*guest_notifier
; /* irq */
38 QEMUBH
*bh
; /* bh for guest notification */
40 /* Note that these EventNotifiers are assigned by value. This is
41 * fine as long as you do not call event_notifier_cleanup on them
42 * (because you don't own the file descriptor or handle; you just
46 IOThread internal_iothread_obj
;
48 EventNotifier host_notifier
; /* doorbell */
50 /* Operation blocker on BDS */
52 void (*saved_complete_request
)(struct VirtIOBlockReq
*req
,
53 unsigned char status
);
56 /* Raise an interrupt to signal guest, if necessary */
57 static void notify_guest(VirtIOBlockDataPlane
*s
)
59 if (!vring_should_notify(s
->vdev
, &s
->vring
)) {
63 event_notifier_set(s
->guest_notifier
);
66 static void notify_guest_bh(void *opaque
)
68 VirtIOBlockDataPlane
*s
= opaque
;
73 static void complete_request_vring(VirtIOBlockReq
*req
, unsigned char status
)
75 VirtIOBlockDataPlane
*s
= req
->dev
->dataplane
;
76 stb_p(&req
->in
->status
, status
);
78 vring_push(&req
->dev
->dataplane
->vring
, &req
->elem
,
79 req
->qiov
.size
+ sizeof(*req
->in
));
81 /* Suppress notification to guest by BH and its scheduled
82 * flag because requests are completed as a batch after io
83 * plug & unplug is introduced, and the BH can still be
84 * executed in dataplane aio context even after it is
85 * stopped, so needn't worry about notification loss with BH.
87 qemu_bh_schedule(s
->bh
);
90 static void handle_notify(EventNotifier
*e
)
92 VirtIOBlockDataPlane
*s
= container_of(e
, VirtIOBlockDataPlane
,
94 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
96 event_notifier_test_and_clear(&s
->host_notifier
);
97 blk_io_plug(s
->conf
->conf
.blk
);
99 MultiReqBuffer mrb
= {
104 /* Disable guest->host notifies to avoid unnecessary vmexits */
105 vring_disable_notification(s
->vdev
, &s
->vring
);
108 VirtIOBlockReq
*req
= virtio_blk_alloc_request(vblk
);
110 ret
= vring_pop(s
->vdev
, &s
->vring
, &req
->elem
);
112 virtio_blk_free_request(req
);
113 break; /* no more requests */
116 trace_virtio_blk_data_plane_process_request(s
, req
->elem
.out_num
,
120 virtio_blk_handle_request(req
, &mrb
);
123 virtio_submit_multiwrite(s
->conf
->conf
.blk
, &mrb
);
125 if (likely(ret
== -EAGAIN
)) { /* vring emptied */
126 /* Re-enable guest->host notifies and stop processing the vring.
127 * But if the guest has snuck in more descriptors, keep processing.
129 if (vring_enable_notification(s
->vdev
, &s
->vring
)) {
132 } else { /* fatal error */
136 blk_io_unplug(s
->conf
->conf
.blk
);
139 /* Context: QEMU global mutex held */
140 void virtio_blk_data_plane_create(VirtIODevice
*vdev
, VirtIOBlkConf
*conf
,
141 VirtIOBlockDataPlane
**dataplane
,
144 VirtIOBlockDataPlane
*s
;
145 Error
*local_err
= NULL
;
146 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(vdev
)));
147 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
151 if (!conf
->data_plane
&& !conf
->iothread
) {
155 /* Don't try if transport does not support notifiers. */
156 if (!k
->set_guest_notifiers
|| !k
->set_host_notifier
) {
158 "device is incompatible with x-data-plane "
159 "(transport does not support notifiers)");
163 /* If dataplane is (re-)enabled while the guest is running there could be
164 * block jobs that can conflict.
166 if (blk_op_is_blocked(conf
->conf
.blk
, BLOCK_OP_TYPE_DATAPLANE
,
168 error_setg(errp
, "cannot start dataplane thread: %s",
169 error_get_pretty(local_err
));
170 error_free(local_err
);
174 s
= g_new0(VirtIOBlockDataPlane
, 1);
178 if (conf
->iothread
) {
179 s
->iothread
= conf
->iothread
;
180 object_ref(OBJECT(s
->iothread
));
182 /* Create per-device IOThread if none specified. This is for
183 * x-data-plane option compatibility. If x-data-plane is removed we
186 object_initialize(&s
->internal_iothread_obj
,
187 sizeof(s
->internal_iothread_obj
),
189 user_creatable_complete(OBJECT(&s
->internal_iothread_obj
), &error_abort
);
190 s
->iothread
= &s
->internal_iothread_obj
;
192 s
->ctx
= iothread_get_aio_context(s
->iothread
);
193 s
->bh
= aio_bh_new(s
->ctx
, notify_guest_bh
, s
);
195 error_setg(&s
->blocker
, "block device is in use by data plane");
196 blk_op_block_all(conf
->conf
.blk
, s
->blocker
);
197 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_RESIZE
, s
->blocker
);
198 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_DRIVE_DEL
, s
->blocker
);
199 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_BACKUP_SOURCE
, s
->blocker
);
200 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_COMMIT
, s
->blocker
);
201 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE
,
203 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_MIRROR
, s
->blocker
);
204 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_STREAM
, s
->blocker
);
205 blk_op_unblock(conf
->conf
.blk
, BLOCK_OP_TYPE_REPLACE
, s
->blocker
);
210 /* Context: QEMU global mutex held */
211 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane
*s
)
217 virtio_blk_data_plane_stop(s
);
218 blk_op_unblock_all(s
->conf
->conf
.blk
, s
->blocker
);
219 error_free(s
->blocker
);
220 object_unref(OBJECT(s
->iothread
));
221 qemu_bh_delete(s
->bh
);
225 /* Context: QEMU global mutex held */
226 void virtio_blk_data_plane_start(VirtIOBlockDataPlane
*s
)
228 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
229 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
230 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
234 if (s
->started
|| s
->disabled
) {
244 vq
= virtio_get_queue(s
->vdev
, 0);
245 if (!vring_setup(&s
->vring
, s
->vdev
, 0)) {
249 /* Set up guest notifier (irq) */
250 r
= k
->set_guest_notifiers(qbus
->parent
, 1, true);
252 fprintf(stderr
, "virtio-blk failed to set guest notifier (%d), "
253 "ensure -enable-kvm is set\n", r
);
254 goto fail_guest_notifiers
;
256 s
->guest_notifier
= virtio_queue_get_guest_notifier(vq
);
258 /* Set up virtqueue notify */
259 r
= k
->set_host_notifier(qbus
->parent
, 0, true);
261 fprintf(stderr
, "virtio-blk failed to set host notifier (%d)\n", r
);
262 goto fail_host_notifier
;
264 s
->host_notifier
= *virtio_queue_get_host_notifier(vq
);
266 s
->saved_complete_request
= vblk
->complete_request
;
267 vblk
->complete_request
= complete_request_vring
;
271 trace_virtio_blk_data_plane_start(s
);
273 blk_set_aio_context(s
->conf
->conf
.blk
, s
->ctx
);
275 /* Kick right away to begin processing requests already in vring */
276 event_notifier_set(virtio_queue_get_host_notifier(vq
));
278 /* Get this show started by hooking up our callbacks */
279 aio_context_acquire(s
->ctx
);
280 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, handle_notify
);
281 aio_context_release(s
->ctx
);
285 k
->set_guest_notifiers(qbus
->parent
, 1, false);
286 fail_guest_notifiers
:
287 vring_teardown(&s
->vring
, s
->vdev
, 0);
293 /* Context: QEMU global mutex held */
294 void virtio_blk_data_plane_stop(VirtIOBlockDataPlane
*s
)
296 BusState
*qbus
= BUS(qdev_get_parent_bus(DEVICE(s
->vdev
)));
297 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
298 VirtIOBlock
*vblk
= VIRTIO_BLK(s
->vdev
);
301 /* Better luck next time. */
306 if (!s
->started
|| s
->stopping
) {
310 vblk
->complete_request
= s
->saved_complete_request
;
311 trace_virtio_blk_data_plane_stop(s
);
313 aio_context_acquire(s
->ctx
);
315 /* Stop notifications for new requests from guest */
316 aio_set_event_notifier(s
->ctx
, &s
->host_notifier
, NULL
);
318 /* Drain and switch bs back to the QEMU main loop */
319 blk_set_aio_context(s
->conf
->conf
.blk
, qemu_get_aio_context());
321 aio_context_release(s
->ctx
);
323 /* Sync vring state back to virtqueue so that non-dataplane request
324 * processing can continue when we disable the host notifier below.
326 vring_teardown(&s
->vring
, s
->vdev
, 0);
328 k
->set_host_notifier(qbus
->parent
, 0, false);
330 /* Clean up guest notifier (irq) */
331 k
->set_guest_notifiers(qbus
->parent
, 1, false);