hw/block/dataplane/virtio-blk.c

   1 /*
   2  * Dedicated thread for virtio-blk I/O processing
   3  *
   4  * Copyright 2012 IBM, Corp.
   5  * Copyright 2012 Red Hat, Inc. and/or its affiliates
   6  *
   7  * Authors:
   8  *   Stefan Hajnoczi <stefanha@redhat.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include "qemu/osdep.h"
  16 #include "qapi/error.h"
  17 #include "trace.h"
  18 #include "qemu/iov.h"
  19 #include "qemu/main-loop.h"
  20 #include "qemu/thread.h"
  21 #include "qemu/error-report.h"
  22 #include "hw/virtio/virtio-access.h"
  23 #include "hw/virtio/virtio-blk.h"
  24 #include "virtio-blk.h"
  25 #include "block/aio.h"
  26 #include "hw/virtio/virtio-bus.h"
  27 #include "qom/object_interfaces.h"
  28
  29 struct VirtIOBlockDataPlane {
  30     bool starting;
  31     bool stopping;
  32
  33     VirtIOBlkConf *conf;
  34     VirtIODevice *vdev;
  35     QEMUBH *bh;                     /* bh for guest notification */
  36     unsigned long *batch_notify_vqs;
  37     bool batch_notifications;
  38
  39     /* Note that these EventNotifiers are assigned by value.  This is
  40      * fine as long as you do not call event_notifier_cleanup on them
  41      * (because you don't own the file descriptor or handle; you just
  42      * use it).
  43      */
  44     IOThread *iothread;
  45     AioContext *ctx;
  46 };
  47
  48 /* Raise an interrupt to signal guest, if necessary */
  49 void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq)
  50 {
  51     if (s->batch_notifications) {
  52         set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs);
  53         qemu_bh_schedule(s->bh);
  54     } else {
  55         virtio_notify_irqfd(s->vdev, vq);
  56     }
  57 }
  58
  59 static void notify_guest_bh(void *opaque)
  60 {
  61     VirtIOBlockDataPlane *s = opaque;
  62     unsigned nvqs = s->conf->num_queues;
  63     unsigned long bitmap[BITS_TO_LONGS(nvqs)];
  64     unsigned j;
  65
  66     memcpy(bitmap, s->batch_notify_vqs, sizeof(bitmap));
  67     memset(s->batch_notify_vqs, 0, sizeof(bitmap));
  68
  69     for (j = 0; j < nvqs; j += BITS_PER_LONG) {
  70         unsigned long bits = bitmap[j / BITS_PER_LONG];
  71
  72         while (bits != 0) {
  73             unsigned i = j + ctzl(bits);
  74             VirtQueue *vq = virtio_get_queue(s->vdev, i);
  75
  76             virtio_notify_irqfd(s->vdev, vq);
  77
  78             bits &= bits - 1; /* clear right-most bit */
  79         }
  80     }
  81 }
  82
  83 /* Context: QEMU global mutex held */
  84 bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
  85                                   VirtIOBlockDataPlane **dataplane,
  86                                   Error **errp)
  87 {
  88     VirtIOBlockDataPlane *s;
  89     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
  90     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
  91
  92     *dataplane = NULL;
  93
  94     if (conf->iothread) {
  95         if (!k->set_guest_notifiers || !k->ioeventfd_assign) {
  96             error_setg(errp,
  97                        "device is incompatible with iothread "
  98                        "(transport does not support notifiers)");
  99             return false;
 100         }
 101         if (!virtio_device_ioeventfd_enabled(vdev)) {
 102             error_setg(errp, "ioeventfd is required for iothread");
 103             return false;
 104         }
 105
 106         /* If dataplane is (re-)enabled while the guest is running there could
 107          * be block jobs that can conflict.
 108          */
 109         if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
 110             error_prepend(errp, "cannot start virtio-blk dataplane: ");
 111             return false;
 112         }
 113     }
 114     /* Don't try if transport does not support notifiers. */
 115     if (!virtio_device_ioeventfd_enabled(vdev)) {
 116         return false;
 117     }
 118
 119     s = g_new0(VirtIOBlockDataPlane, 1);
 120     s->vdev = vdev;
 121     s->conf = conf;
 122
 123     if (conf->iothread) {
 124         s->iothread = conf->iothread;
 125         object_ref(OBJECT(s->iothread));
 126         s->ctx = iothread_get_aio_context(s->iothread);
 127     } else {
 128         s->ctx = qemu_get_aio_context();
 129     }
 130     s->bh = aio_bh_new_guarded(s->ctx, notify_guest_bh, s,
 131                                &DEVICE(vdev)->mem_reentrancy_guard);
 132     s->batch_notify_vqs = bitmap_new(conf->num_queues);
 133
 134     *dataplane = s;
 135
 136     return true;
 137 }
 138
 139 /* Context: QEMU global mutex held */
 140 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
 141 {
 142     VirtIOBlock *vblk;
 143
 144     if (!s) {
 145         return;
 146     }
 147
 148     vblk = VIRTIO_BLK(s->vdev);
 149     assert(!vblk->dataplane_started);
 150     g_free(s->batch_notify_vqs);
 151     qemu_bh_delete(s->bh);
 152     if (s->iothread) {
 153         object_unref(OBJECT(s->iothread));
 154     }
 155     g_free(s);
 156 }
 157
 158 /* Context: QEMU global mutex held */
 159 int virtio_blk_data_plane_start(VirtIODevice *vdev)
 160 {
 161     VirtIOBlock *vblk = VIRTIO_BLK(vdev);
 162     VirtIOBlockDataPlane *s = vblk->dataplane;
 163     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk)));
 164     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 165     AioContext *old_context;
 166     unsigned i;
 167     unsigned nvqs = s->conf->num_queues;
 168     Error *local_err = NULL;
 169     int r;
 170
 171     if (vblk->dataplane_started || s->starting) {
 172         return 0;
 173     }
 174
 175     s->starting = true;
 176
 177     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
 178         s->batch_notifications = true;
 179     } else {
 180         s->batch_notifications = false;
 181     }
 182
 183     /* Set up guest notifier (irq) */
 184     r = k->set_guest_notifiers(qbus->parent, nvqs, true);
 185     if (r != 0) {
 186         error_report("virtio-blk failed to set guest notifier (%d), "
 187                      "ensure -accel kvm is set.", r);
 188         goto fail_guest_notifiers;
 189     }
 190
 191     /*
 192      * Batch all the host notifiers in a single transaction to avoid
 193      * quadratic time complexity in address_space_update_ioeventfds().
 194      */
 195     memory_region_transaction_begin();
 196
 197     /* Set up virtqueue notify */
 198     for (i = 0; i < nvqs; i++) {
 199         r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true);
 200         if (r != 0) {
 201             int j = i;
 202
 203             fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
 204             while (i--) {
 205                 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
 206             }
 207
 208             /*
 209              * The transaction expects the ioeventfds to be open when it
 210              * commits. Do it now, before the cleanup loop.
 211              */
 212             memory_region_transaction_commit();
 213
 214             while (j--) {
 215                 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j);
 216             }
 217             goto fail_host_notifiers;
 218         }
 219     }
 220
 221     memory_region_transaction_commit();
 222
 223     /*
 224      * These fields are visible to the IOThread so we rely on implicit barriers
 225      * in aio_context_acquire() on the write side and aio_notify_accept() on
 226      * the read side.
 227      */
 228     s->starting = false;
 229     vblk->dataplane_started = true;
 230     trace_virtio_blk_data_plane_start(s);
 231
 232     old_context = blk_get_aio_context(s->conf->conf.blk);
 233     aio_context_acquire(old_context);
 234     r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err);
 235     aio_context_release(old_context);
 236     if (r < 0) {
 237         error_report_err(local_err);
 238         goto fail_aio_context;
 239     }
 240
 241     /* Kick right away to begin processing requests already in vring */
 242     for (i = 0; i < nvqs; i++) {
 243         VirtQueue *vq = virtio_get_queue(s->vdev, i);
 244
 245         event_notifier_set(virtio_queue_get_host_notifier(vq));
 246     }
 247
 248     /* Get this show started by hooking up our callbacks */
 249     if (!blk_in_drain(s->conf->conf.blk)) {
 250         aio_context_acquire(s->ctx);
 251         for (i = 0; i < nvqs; i++) {
 252             VirtQueue *vq = virtio_get_queue(s->vdev, i);
 253
 254             virtio_queue_aio_attach_host_notifier(vq, s->ctx);
 255         }
 256         aio_context_release(s->ctx);
 257     }
 258     return 0;
 259
 260   fail_aio_context:
 261     memory_region_transaction_begin();
 262
 263     for (i = 0; i < nvqs; i++) {
 264         virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
 265     }
 266
 267     memory_region_transaction_commit();
 268
 269     for (i = 0; i < nvqs; i++) {
 270         virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i);
 271     }
 272   fail_host_notifiers:
 273     k->set_guest_notifiers(qbus->parent, nvqs, false);
 274   fail_guest_notifiers:
 275     vblk->dataplane_disabled = true;
 276     s->starting = false;
 277     vblk->dataplane_started = true;
 278     return -ENOSYS;
 279 }
 280
 281 /* Stop notifications for new requests from guest.
 282  *
 283  * Context: BH in IOThread
 284  */
 285 static void virtio_blk_data_plane_stop_bh(void *opaque)
 286 {
 287     VirtIOBlockDataPlane *s = opaque;
 288     unsigned i;
 289
 290     for (i = 0; i < s->conf->num_queues; i++) {
 291         VirtQueue *vq = virtio_get_queue(s->vdev, i);
 292         EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq);
 293
 294         virtio_queue_aio_detach_host_notifier(vq, s->ctx);
 295
 296         /*
 297          * Test and clear notifier after disabling event, in case poll callback
 298          * didn't have time to run.
 299          */
 300         virtio_queue_host_notifier_read(host_notifier);
 301     }
 302 }
 303
 304 /* Context: QEMU global mutex held */
 305 void virtio_blk_data_plane_stop(VirtIODevice *vdev)
 306 {
 307     VirtIOBlock *vblk = VIRTIO_BLK(vdev);
 308     VirtIOBlockDataPlane *s = vblk->dataplane;
 309     BusState *qbus = qdev_get_parent_bus(DEVICE(vblk));
 310     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 311     unsigned i;
 312     unsigned nvqs = s->conf->num_queues;
 313
 314     if (!vblk->dataplane_started || s->stopping) {
 315         return;
 316     }
 317
 318     /* Better luck next time. */
 319     if (vblk->dataplane_disabled) {
 320         vblk->dataplane_disabled = false;
 321         vblk->dataplane_started = false;
 322         return;
 323     }
 324     s->stopping = true;
 325     trace_virtio_blk_data_plane_stop(s);
 326
 327     if (!blk_in_drain(s->conf->conf.blk)) {
 328         aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s);
 329     }
 330
 331     aio_context_acquire(s->ctx);
 332
 333     /* Wait for virtio_blk_dma_restart_bh() and in flight I/O to complete */
 334     blk_drain(s->conf->conf.blk);
 335
 336     /*
 337      * Try to switch bs back to the QEMU main loop. If other users keep the
 338      * BlockBackend in the iothread, that's ok
 339      */
 340     blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL);
 341
 342     aio_context_release(s->ctx);
 343
 344     /*
 345      * Batch all the host notifiers in a single transaction to avoid
 346      * quadratic time complexity in address_space_update_ioeventfds().
 347      */
 348     memory_region_transaction_begin();
 349
 350     for (i = 0; i < nvqs; i++) {
 351         virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
 352     }
 353
 354     /*
 355      * The transaction expects the ioeventfds to be open when it
 356      * commits. Do it now, before the cleanup loop.
 357      */
 358     memory_region_transaction_commit();
 359
 360     for (i = 0; i < nvqs; i++) {
 361         virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i);
 362     }
 363
 364     qemu_bh_cancel(s->bh);
 365     notify_guest_bh(s); /* final chance to notify guest */
 366
 367     /* Clean up guest notifier (irq) */
 368     k->set_guest_notifiers(qbus->parent, nvqs, false);
 369
 370     vblk->dataplane_started = false;
 371     s->stopping = false;
 372 }