hw/block/dataplane/virtio-blk.c

   1 /*
   2  * Dedicated thread for virtio-blk I/O processing
   3  *
   4  * Copyright 2012 IBM, Corp.
   5  * Copyright 2012 Red Hat, Inc. and/or its affiliates
   6  *
   7  * Authors:
   8  *   Stefan Hajnoczi <stefanha@redhat.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include "qemu/osdep.h"
  16 #include "qapi/error.h"
  17 #include "trace.h"
  18 #include "qemu/iov.h"
  19 #include "qemu/main-loop.h"
  20 #include "qemu/thread.h"
  21 #include "qemu/error-report.h"
  22 #include "hw/virtio/virtio-blk.h"
  23 #include "virtio-blk.h"
  24 #include "block/aio.h"
  25 #include "hw/virtio/virtio-bus.h"
  26 #include "qom/object_interfaces.h"
  27
  28 struct VirtIOBlockDataPlane {
  29     bool starting;
  30     bool stopping;
  31
  32     VirtIOBlkConf *conf;
  33     VirtIODevice *vdev;
  34
  35     /*
  36      * The AioContext for each virtqueue. The BlockDriverState will use the
  37      * first element as its AioContext.
  38      */
  39     AioContext **vq_aio_context;
  40 };
  41
  42 /* Raise an interrupt to signal guest, if necessary */
  43 void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq)
  44 {
  45     virtio_notify_irqfd(s->vdev, vq);
  46 }
  47
  48 /* Generate vq:AioContext mappings from a validated iothread-vq-mapping list */
  49 static void
  50 apply_vq_mapping(IOThreadVirtQueueMappingList *iothread_vq_mapping_list,
  51                  AioContext **vq_aio_context, uint16_t num_queues)
  52 {
  53     IOThreadVirtQueueMappingList *node;
  54     size_t num_iothreads = 0;
  55     size_t cur_iothread = 0;
  56
  57     for (node = iothread_vq_mapping_list; node; node = node->next) {
  58         num_iothreads++;
  59     }
  60
  61     for (node = iothread_vq_mapping_list; node; node = node->next) {
  62         IOThread *iothread = iothread_by_id(node->value->iothread);
  63         AioContext *ctx = iothread_get_aio_context(iothread);
  64
  65         /* Released in virtio_blk_data_plane_destroy() */
  66         object_ref(OBJECT(iothread));
  67
  68         if (node->value->vqs) {
  69             uint16List *vq;
  70
  71             /* Explicit vq:IOThread assignment */
  72             for (vq = node->value->vqs; vq; vq = vq->next) {
  73                 vq_aio_context[vq->value] = ctx;
  74             }
  75         } else {
  76             /* Round-robin vq:IOThread assignment */
  77             for (unsigned i = cur_iothread; i < num_queues;
  78                  i += num_iothreads) {
  79                 vq_aio_context[i] = ctx;
  80             }
  81         }
  82
  83         cur_iothread++;
  84     }
  85 }
  86
  87 /* Context: BQL held */
  88 bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf,
  89                                   VirtIOBlockDataPlane **dataplane,
  90                                   Error **errp)
  91 {
  92     VirtIOBlockDataPlane *s;
  93     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
  94     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
  95
  96     *dataplane = NULL;
  97
  98     if (conf->iothread || conf->iothread_vq_mapping_list) {
  99         if (!k->set_guest_notifiers || !k->ioeventfd_assign) {
 100             error_setg(errp,
 101                        "device is incompatible with iothread "
 102                        "(transport does not support notifiers)");
 103             return false;
 104         }
 105         if (!virtio_device_ioeventfd_enabled(vdev)) {
 106             error_setg(errp, "ioeventfd is required for iothread");
 107             return false;
 108         }
 109
 110         /* If dataplane is (re-)enabled while the guest is running there could
 111          * be block jobs that can conflict.
 112          */
 113         if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
 114             error_prepend(errp, "cannot start virtio-blk dataplane: ");
 115             return false;
 116         }
 117     }
 118     /* Don't try if transport does not support notifiers. */
 119     if (!virtio_device_ioeventfd_enabled(vdev)) {
 120         return false;
 121     }
 122
 123     s = g_new0(VirtIOBlockDataPlane, 1);
 124     s->vdev = vdev;
 125     s->conf = conf;
 126     s->vq_aio_context = g_new(AioContext *, conf->num_queues);
 127
 128     if (conf->iothread_vq_mapping_list) {
 129         apply_vq_mapping(conf->iothread_vq_mapping_list, s->vq_aio_context,
 130                          conf->num_queues);
 131     } else if (conf->iothread) {
 132         AioContext *ctx = iothread_get_aio_context(conf->iothread);
 133         for (unsigned i = 0; i < conf->num_queues; i++) {
 134             s->vq_aio_context[i] = ctx;
 135         }
 136
 137         /* Released in virtio_blk_data_plane_destroy() */
 138         object_ref(OBJECT(conf->iothread));
 139     } else {
 140         AioContext *ctx = qemu_get_aio_context();
 141         for (unsigned i = 0; i < conf->num_queues; i++) {
 142             s->vq_aio_context[i] = ctx;
 143         }
 144     }
 145
 146     *dataplane = s;
 147
 148     return true;
 149 }
 150
 151 /* Context: BQL held */
 152 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s)
 153 {
 154     VirtIOBlock *vblk;
 155     VirtIOBlkConf *conf;
 156
 157     if (!s) {
 158         return;
 159     }
 160
 161     vblk = VIRTIO_BLK(s->vdev);
 162     assert(!vblk->dataplane_started);
 163     conf = s->conf;
 164
 165     if (conf->iothread_vq_mapping_list) {
 166         IOThreadVirtQueueMappingList *node;
 167
 168         for (node = conf->iothread_vq_mapping_list; node; node = node->next) {
 169             IOThread *iothread = iothread_by_id(node->value->iothread);
 170             object_unref(OBJECT(iothread));
 171         }
 172     }
 173
 174     if (conf->iothread) {
 175         object_unref(OBJECT(conf->iothread));
 176     }
 177
 178     g_free(s->vq_aio_context);
 179     g_free(s);
 180 }
 181
 182 /* Context: BQL held */
 183 int virtio_blk_data_plane_start(VirtIODevice *vdev)
 184 {
 185     VirtIOBlock *vblk = VIRTIO_BLK(vdev);
 186     VirtIOBlockDataPlane *s = vblk->dataplane;
 187     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk)));
 188     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 189     unsigned i;
 190     unsigned nvqs = s->conf->num_queues;
 191     Error *local_err = NULL;
 192     int r;
 193
 194     if (vblk->dataplane_started || s->starting) {
 195         return 0;
 196     }
 197
 198     s->starting = true;
 199
 200     /* Set up guest notifier (irq) */
 201     r = k->set_guest_notifiers(qbus->parent, nvqs, true);
 202     if (r != 0) {
 203         error_report("virtio-blk failed to set guest notifier (%d), "
 204                      "ensure -accel kvm is set.", r);
 205         goto fail_guest_notifiers;
 206     }
 207
 208     /*
 209      * Batch all the host notifiers in a single transaction to avoid
 210      * quadratic time complexity in address_space_update_ioeventfds().
 211      */
 212     memory_region_transaction_begin();
 213
 214     /* Set up virtqueue notify */
 215     for (i = 0; i < nvqs; i++) {
 216         r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true);
 217         if (r != 0) {
 218             int j = i;
 219
 220             fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
 221             while (i--) {
 222                 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
 223             }
 224
 225             /*
 226              * The transaction expects the ioeventfds to be open when it
 227              * commits. Do it now, before the cleanup loop.
 228              */
 229             memory_region_transaction_commit();
 230
 231             while (j--) {
 232                 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j);
 233             }
 234             goto fail_host_notifiers;
 235         }
 236     }
 237
 238     memory_region_transaction_commit();
 239
 240     trace_virtio_blk_data_plane_start(s);
 241
 242     r = blk_set_aio_context(s->conf->conf.blk, s->vq_aio_context[0],
 243                             &local_err);
 244     if (r < 0) {
 245         error_report_err(local_err);
 246         goto fail_aio_context;
 247     }
 248
 249     /*
 250      * These fields must be visible to the IOThread when it processes the
 251      * virtqueue, otherwise it will think dataplane has not started yet.
 252      *
 253      * Make sure ->dataplane_started is false when blk_set_aio_context() is
 254      * called above so that draining does not cause the host notifier to be
 255      * detached/attached prematurely.
 256      */
 257     s->starting = false;
 258     vblk->dataplane_started = true;
 259     smp_wmb(); /* paired with aio_notify_accept() on the read side */
 260
 261     /* Get this show started by hooking up our callbacks */
 262     if (!blk_in_drain(s->conf->conf.blk)) {
 263         for (i = 0; i < nvqs; i++) {
 264             VirtQueue *vq = virtio_get_queue(s->vdev, i);
 265             AioContext *ctx = s->vq_aio_context[i];
 266
 267             /* Kick right away to begin processing requests already in vring */
 268             event_notifier_set(virtio_queue_get_host_notifier(vq));
 269
 270             virtio_queue_aio_attach_host_notifier(vq, ctx);
 271         }
 272     }
 273     return 0;
 274
 275   fail_aio_context:
 276     memory_region_transaction_begin();
 277
 278     for (i = 0; i < nvqs; i++) {
 279         virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
 280     }
 281
 282     memory_region_transaction_commit();
 283
 284     for (i = 0; i < nvqs; i++) {
 285         virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i);
 286     }
 287   fail_host_notifiers:
 288     k->set_guest_notifiers(qbus->parent, nvqs, false);
 289   fail_guest_notifiers:
 290     vblk->dataplane_disabled = true;
 291     s->starting = false;
 292     return -ENOSYS;
 293 }
 294
 295 /* Stop notifications for new requests from guest.
 296  *
 297  * Context: BH in IOThread
 298  */
 299 static void virtio_blk_data_plane_stop_vq_bh(void *opaque)
 300 {
 301     VirtQueue *vq = opaque;
 302     EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq);
 303
 304     virtio_queue_aio_detach_host_notifier(vq, qemu_get_current_aio_context());
 305
 306     /*
 307      * Test and clear notifier after disabling event, in case poll callback
 308      * didn't have time to run.
 309      */
 310     virtio_queue_host_notifier_read(host_notifier);
 311 }
 312
 313 /* Context: BQL held */
 314 void virtio_blk_data_plane_stop(VirtIODevice *vdev)
 315 {
 316     VirtIOBlock *vblk = VIRTIO_BLK(vdev);
 317     VirtIOBlockDataPlane *s = vblk->dataplane;
 318     BusState *qbus = qdev_get_parent_bus(DEVICE(vblk));
 319     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 320     unsigned i;
 321     unsigned nvqs = s->conf->num_queues;
 322
 323     if (!vblk->dataplane_started || s->stopping) {
 324         return;
 325     }
 326
 327     /* Better luck next time. */
 328     if (vblk->dataplane_disabled) {
 329         vblk->dataplane_disabled = false;
 330         vblk->dataplane_started = false;
 331         return;
 332     }
 333     s->stopping = true;
 334     trace_virtio_blk_data_plane_stop(s);
 335
 336     if (!blk_in_drain(s->conf->conf.blk)) {
 337         for (i = 0; i < nvqs; i++) {
 338             VirtQueue *vq = virtio_get_queue(s->vdev, i);
 339             AioContext *ctx = s->vq_aio_context[i];
 340
 341             aio_wait_bh_oneshot(ctx, virtio_blk_data_plane_stop_vq_bh, vq);
 342         }
 343     }
 344
 345     /*
 346      * Batch all the host notifiers in a single transaction to avoid
 347      * quadratic time complexity in address_space_update_ioeventfds().
 348      */
 349     memory_region_transaction_begin();
 350
 351     for (i = 0; i < nvqs; i++) {
 352         virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
 353     }
 354
 355     /*
 356      * The transaction expects the ioeventfds to be open when it
 357      * commits. Do it now, before the cleanup loop.
 358      */
 359     memory_region_transaction_commit();
 360
 361     for (i = 0; i < nvqs; i++) {
 362         virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i);
 363     }
 364
 365     /*
 366      * Set ->dataplane_started to false before draining so that host notifiers
 367      * are not detached/attached anymore.
 368      */
 369     vblk->dataplane_started = false;
 370
 371     /* Wait for virtio_blk_dma_restart_bh() and in flight I/O to complete */
 372     blk_drain(s->conf->conf.blk);
 373
 374     /*
 375      * Try to switch bs back to the QEMU main loop. If other users keep the
 376      * BlockBackend in the iothread, that's ok
 377      */
 378     blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL);
 379
 380     /* Clean up guest notifier (irq) */
 381     k->set_guest_notifiers(qbus->parent, nvqs, false);
 382
 383     s->stopping = false;
 384 }
 385
 386 void virtio_blk_data_plane_detach(VirtIOBlockDataPlane *s)
 387 {
 388     VirtIODevice *vdev = VIRTIO_DEVICE(s->vdev);
 389
 390     for (uint16_t i = 0; i < s->conf->num_queues; i++) {
 391         VirtQueue *vq = virtio_get_queue(vdev, i);
 392         virtio_queue_aio_detach_host_notifier(vq, s->vq_aio_context[i]);
 393     }
 394 }
 395
 396 void virtio_blk_data_plane_attach(VirtIOBlockDataPlane *s)
 397 {
 398     VirtIODevice *vdev = VIRTIO_DEVICE(s->vdev);
 399
 400     for (uint16_t i = 0; i < s->conf->num_queues; i++) {
 401         VirtQueue *vq = virtio_get_queue(vdev, i);
 402         virtio_queue_aio_attach_host_notifier(vq, s->vq_aio_context[i]);
 403     }
 404 }