contrib/vhost-user-scsi/vhost-user-scsi.c

   1 /*
   2  * vhost-user-scsi sample application
   3  *
   4  * Copyright (c) 2016 Nutanix Inc. All rights reserved.
   5  *
   6  * Author:
   7  *  Felipe Franciosi <felipe@nutanix.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 only.
  10  * See the COPYING file in the top-level directory.
  11  */
  12
  13 #include "qemu/osdep.h"
  14 #include "contrib/libvhost-user/libvhost-user.h"
  15 #include "standard-headers/linux/virtio_scsi.h"
  16 #include "iscsi/iscsi.h"
  17 #include "iscsi/scsi-lowlevel.h"
  18
  19 #include <glib.h>
  20
  21 /* #define VUS_DEBUG 1 */
  22
  23 /** Log helpers **/
  24
  25 #define PPRE                                                          \
  26     struct timespec ts;                                               \
  27     char   timebuf[64];                                               \
  28     struct tm tm;                                                     \
  29     (void)clock_gettime(CLOCK_REALTIME, &ts);                         \
  30     (void)strftime(timebuf, 64, "%Y%m%d %T", gmtime_r(&ts.tv_sec, &tm))
  31
  32 #define PEXT(lvl, msg, ...) do {                                      \
  33     PPRE;                                                             \
  34     fprintf(stderr, "%s.%06ld " lvl ": %s:%s():%d: " msg "\n",        \
  35             timebuf, ts.tv_nsec / 1000,                               \
  36             __FILE__, __func__, __LINE__, ## __VA_ARGS__);            \
  37 } while (0)
  38
  39 #define PNOR(lvl, msg, ...) do {                                      \
  40     PPRE;                                                             \
  41     fprintf(stderr, "%s.%06ld " lvl ": " msg "\n",                    \
  42             timebuf, ts.tv_nsec / 1000, ## __VA_ARGS__);              \
  43 } while (0)
  44
  45 #ifdef VUS_DEBUG
  46 #define PDBG(msg, ...) PEXT("DBG", msg, ## __VA_ARGS__)
  47 #define PERR(msg, ...) PEXT("ERR", msg, ## __VA_ARGS__)
  48 #define PLOG(msg, ...) PEXT("LOG", msg, ## __VA_ARGS__)
  49 #else
  50 #define PDBG(msg, ...) { }
  51 #define PERR(msg, ...) PNOR("ERR", msg, ## __VA_ARGS__)
  52 #define PLOG(msg, ...) PNOR("LOG", msg, ## __VA_ARGS__)
  53 #endif
  54
  55 /** vhost-user-scsi specific definitions **/
  56
  57 #define VUS_ISCSI_INITIATOR "iqn.2016-11.com.nutanix:vhost-user-scsi"
  58
  59 typedef struct VusIscsiLun {
  60     struct iscsi_context *iscsi_ctx;
  61     int iscsi_lun;
  62 } VusIscsiLun;
  63
  64 typedef struct VusDev {
  65     VuDev vu_dev;
  66     int server_sock;
  67     GMainLoop *loop;
  68     GTree *fdmap;   /* fd -> gsource context id */
  69     VusIscsiLun lun;
  70 } VusDev;
  71
  72 /** glib event loop integration for libvhost-user and misc callbacks **/
  73
  74 QEMU_BUILD_BUG_ON((int)G_IO_IN != (int)VU_WATCH_IN);
  75 QEMU_BUILD_BUG_ON((int)G_IO_OUT != (int)VU_WATCH_OUT);
  76 QEMU_BUILD_BUG_ON((int)G_IO_PRI != (int)VU_WATCH_PRI);
  77 QEMU_BUILD_BUG_ON((int)G_IO_ERR != (int)VU_WATCH_ERR);
  78 QEMU_BUILD_BUG_ON((int)G_IO_HUP != (int)VU_WATCH_HUP);
  79
  80 typedef struct vus_gsrc {
  81     GSource parent;
  82     VusDev *vdev_scsi;
  83     GPollFD gfd;
  84 } vus_gsrc_t;
  85
  86 static gint vus_fdmap_compare(gconstpointer a, gconstpointer b)
  87 {
  88     return (b > a) - (b < a);
  89 }
  90
  91 static gboolean vus_gsrc_prepare(GSource *src, gint *timeout)
  92 {
  93     assert(timeout);
  94
  95     *timeout = -1;
  96     return FALSE;
  97 }
  98
  99 static gboolean vus_gsrc_check(GSource *src)
 100 {
 101     vus_gsrc_t *vus_src = (vus_gsrc_t *)src;
 102
 103     assert(vus_src);
 104
 105     return vus_src->gfd.revents & vus_src->gfd.events;
 106 }
 107
 108 static gboolean vus_gsrc_dispatch(GSource *src, GSourceFunc cb, gpointer data)
 109 {
 110     VusDev *vdev_scsi;
 111     vus_gsrc_t *vus_src = (vus_gsrc_t *)src;
 112
 113     assert(vus_src);
 114
 115     vdev_scsi = vus_src->vdev_scsi;
 116
 117     assert(vdev_scsi);
 118
 119     ((vu_watch_cb)cb)(&vdev_scsi->vu_dev, vus_src->gfd.revents, data);
 120
 121     return G_SOURCE_CONTINUE;
 122 }
 123
 124 static GSourceFuncs vus_gsrc_funcs = {
 125     vus_gsrc_prepare,
 126     vus_gsrc_check,
 127     vus_gsrc_dispatch,
 128     NULL
 129 };
 130
 131 static void vus_gsrc_new(VusDev *vdev_scsi, int fd, GIOCondition cond,
 132                          vu_watch_cb vu_cb, GSourceFunc gsrc_cb, gpointer data)
 133 {
 134     GSource *vus_gsrc;
 135     vus_gsrc_t *vus_src;
 136     guint id;
 137
 138     assert(vdev_scsi);
 139     assert(fd >= 0);
 140     assert(vu_cb || gsrc_cb);
 141     assert(!(vu_cb && gsrc_cb));
 142
 143     vus_gsrc = g_source_new(&vus_gsrc_funcs, sizeof(vus_gsrc_t));
 144     g_source_set_callback(vus_gsrc, (GSourceFunc) vu_cb, data, NULL);
 145     vus_src = (vus_gsrc_t *)vus_gsrc;
 146
 147     vus_src->vdev_scsi = vdev_scsi;
 148     vus_src->gfd.fd = fd;
 149     vus_src->gfd.events = cond;
 150
 151     g_source_add_poll(vus_gsrc, &vus_src->gfd);
 152     g_source_set_callback(vus_gsrc, gsrc_cb, data, NULL);
 153     id = g_source_attach(vus_gsrc, NULL);
 154     assert(id);
 155     g_source_unref(vus_gsrc);
 156
 157     g_tree_insert(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd,
 158                                     (gpointer)(uintptr_t)id);
 159 }
 160
 161 /** libiscsi integration **/
 162
 163 typedef struct virtio_scsi_cmd_req VirtIOSCSICmdReq;
 164 typedef struct virtio_scsi_cmd_resp VirtIOSCSICmdResp;
 165
 166 static int vus_iscsi_add_lun(VusIscsiLun *lun, char *iscsi_uri)
 167 {
 168     struct iscsi_url *iscsi_url;
 169     struct iscsi_context *iscsi_ctx;
 170     int ret = 0;
 171
 172     assert(lun);
 173     assert(iscsi_uri);
 174     assert(!lun->iscsi_ctx);
 175
 176     iscsi_ctx = iscsi_create_context(VUS_ISCSI_INITIATOR);
 177     if (!iscsi_ctx) {
 178         PERR("Unable to create iSCSI context");
 179         return -1;
 180     }
 181
 182     iscsi_url = iscsi_parse_full_url(iscsi_ctx, iscsi_uri);
 183     if (!iscsi_url) {
 184         PERR("Unable to parse iSCSI URL: %s", iscsi_get_error(iscsi_ctx));
 185         goto fail;
 186     }
 187
 188     iscsi_set_session_type(iscsi_ctx, ISCSI_SESSION_NORMAL);
 189     iscsi_set_header_digest(iscsi_ctx, ISCSI_HEADER_DIGEST_NONE_CRC32C);
 190     if (iscsi_full_connect_sync(iscsi_ctx, iscsi_url->portal, iscsi_url->lun)) {
 191         PERR("Unable to login to iSCSI portal: %s", iscsi_get_error(iscsi_ctx));
 192         goto fail;
 193     }
 194
 195     lun->iscsi_ctx = iscsi_ctx;
 196     lun->iscsi_lun = iscsi_url->lun;
 197
 198     PDBG("Context %p created for lun 0: %s", iscsi_ctx, iscsi_uri);
 199
 200 out:
 201     if (iscsi_url) {
 202         iscsi_destroy_url(iscsi_url);
 203     }
 204     return ret;
 205
 206 fail:
 207     (void)iscsi_destroy_context(iscsi_ctx);
 208     ret = -1;
 209     goto out;
 210 }
 211
 212 static struct scsi_task *scsi_task_new(int cdb_len, uint8_t *cdb, int dir,
 213                                        int xfer_len)
 214 {
 215     struct scsi_task *task;
 216
 217     assert(cdb_len > 0);
 218     assert(cdb);
 219
 220     task = g_new0(struct scsi_task, 1);
 221     memcpy(task->cdb, cdb, cdb_len);
 222     task->cdb_size = cdb_len;
 223     task->xfer_dir = dir;
 224     task->expxferlen = xfer_len;
 225
 226     return task;
 227 }
 228
 229 static int get_cdb_len(uint8_t *cdb)
 230 {
 231     assert(cdb);
 232
 233     switch (cdb[0] >> 5) {
 234     case 0: return 6;
 235     case 1: /* fall through */
 236     case 2: return 10;
 237     case 4: return 16;
 238     case 5: return 12;
 239     }
 240     PERR("Unable to determine cdb len (0x%02hhX)", cdb[0] >> 5);
 241     return -1;
 242 }
 243
 244 static int handle_cmd_sync(struct iscsi_context *ctx,
 245                            VirtIOSCSICmdReq *req,
 246                            struct iovec *out, unsigned int out_len,
 247                            VirtIOSCSICmdResp *rsp,
 248                            struct iovec *in, unsigned int in_len)
 249 {
 250     struct scsi_task *task;
 251     uint32_t dir;
 252     uint32_t len;
 253     int cdb_len;
 254     int i;
 255
 256     assert(ctx);
 257     assert(req);
 258     assert(rsp);
 259
 260     if (!(!req->lun[1] && req->lun[2] == 0x40 && !req->lun[3])) {
 261         /* Ignore anything different than target=0, lun=0 */
 262         PDBG("Ignoring unconnected lun (0x%hhX, 0x%hhX)",
 263              req->lun[1], req->lun[3]);
 264         rsp->status = SCSI_STATUS_CHECK_CONDITION;
 265         memset(rsp->sense, 0, sizeof(rsp->sense));
 266         rsp->sense_len = 18;
 267         rsp->sense[0] = 0x70;
 268         rsp->sense[2] = SCSI_SENSE_ILLEGAL_REQUEST;
 269         rsp->sense[7] = 10;
 270         rsp->sense[12] = 0x24;
 271
 272         return 0;
 273     }
 274
 275     cdb_len = get_cdb_len(req->cdb);
 276     if (cdb_len == -1) {
 277         return -1;
 278     }
 279
 280     len = 0;
 281     if (!out_len && !in_len) {
 282         dir = SCSI_XFER_NONE;
 283     } else if (out_len) {
 284         dir = SCSI_XFER_WRITE;
 285         for (i = 0; i < out_len; i++) {
 286             len += out[i].iov_len;
 287         }
 288     } else {
 289         dir = SCSI_XFER_READ;
 290         for (i = 0; i < in_len; i++) {
 291             len += in[i].iov_len;
 292         }
 293     }
 294
 295     task = scsi_task_new(cdb_len, req->cdb, dir, len);
 296
 297     if (dir == SCSI_XFER_WRITE) {
 298         task->iovector_out.iov = (struct scsi_iovec *)out;
 299         task->iovector_out.niov = out_len;
 300     } else if (dir == SCSI_XFER_READ) {
 301         task->iovector_in.iov = (struct scsi_iovec *)in;
 302         task->iovector_in.niov = in_len;
 303     }
 304
 305     PDBG("Sending iscsi cmd (cdb_len=%d, dir=%d, task=%p)",
 306          cdb_len, dir, task);
 307     if (!iscsi_scsi_command_sync(ctx, 0, task, NULL)) {
 308         PERR("Error serving SCSI command");
 309         g_free(task);
 310         return -1;
 311     }
 312
 313     memset(rsp, 0, sizeof(*rsp));
 314
 315     rsp->status = task->status;
 316     rsp->resid  = task->residual;
 317
 318     if (task->status == SCSI_STATUS_CHECK_CONDITION) {
 319         rsp->response = VIRTIO_SCSI_S_FAILURE;
 320         rsp->sense_len = task->datain.size - 2;
 321         memcpy(rsp->sense, &task->datain.data[2], rsp->sense_len);
 322     }
 323
 324     g_free(task);
 325
 326     PDBG("Filled in rsp: status=%hhX, resid=%u, response=%hhX, sense_len=%u",
 327          rsp->status, rsp->resid, rsp->response, rsp->sense_len);
 328
 329     return 0;
 330 }
 331
 332 /** libvhost-user callbacks **/
 333
 334 static void vus_panic_cb(VuDev *vu_dev, const char *buf)
 335 {
 336     VusDev *vdev_scsi;
 337
 338     assert(vu_dev);
 339
 340     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
 341     if (buf) {
 342         PERR("vu_panic: %s", buf);
 343     }
 344
 345     g_main_loop_quit(vdev_scsi->loop);
 346 }
 347
 348 static void vus_add_watch_cb(VuDev *vu_dev, int fd, int vu_evt, vu_watch_cb cb,
 349                              void *pvt)
 350 {
 351     VusDev *vdev_scsi;
 352     guint id;
 353
 354     assert(vu_dev);
 355     assert(fd >= 0);
 356     assert(cb);
 357
 358     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
 359     id = (guint)(uintptr_t)g_tree_lookup(vdev_scsi->fdmap,
 360                                          (gpointer)(uintptr_t)fd);
 361     if (id) {
 362         GSource *vus_src = g_main_context_find_source_by_id(NULL, id);
 363         assert(vus_src);
 364         g_source_destroy(vus_src);
 365         (void)g_tree_remove(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd);
 366     }
 367
 368     vus_gsrc_new(vdev_scsi, fd, vu_evt, cb, NULL, pvt);
 369 }
 370
 371 static void vus_del_watch_cb(VuDev *vu_dev, int fd)
 372 {
 373     VusDev *vdev_scsi;
 374     guint id;
 375
 376     assert(vu_dev);
 377     assert(fd >= 0);
 378
 379     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
 380     id = (guint)(uintptr_t)g_tree_lookup(vdev_scsi->fdmap,
 381                                          (gpointer)(uintptr_t)fd);
 382     if (id) {
 383         GSource *vus_src = g_main_context_find_source_by_id(NULL, id);
 384         assert(vus_src);
 385         g_source_destroy(vus_src);
 386         (void)g_tree_remove(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd);
 387     }
 388 }
 389
 390 static void vus_proc_req(VuDev *vu_dev, int idx)
 391 {
 392     VusDev *vdev_scsi;
 393     VuVirtq *vq;
 394
 395     assert(vu_dev);
 396
 397     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
 398     if (idx < 0 || idx >= VHOST_MAX_NR_VIRTQUEUE) {
 399         PERR("VQ Index out of range: %d", idx);
 400         vus_panic_cb(vu_dev, NULL);
 401         return;
 402     }
 403
 404     vq = vu_get_queue(vu_dev, idx);
 405     if (!vq) {
 406         PERR("Error fetching VQ (dev=%p, idx=%d)", vu_dev, idx);
 407         vus_panic_cb(vu_dev, NULL);
 408         return;
 409     }
 410
 411     PDBG("Got kicked on vq[%d]@%p", idx, vq);
 412
 413     while (1) {
 414         VuVirtqElement *elem;
 415         VirtIOSCSICmdReq *req;
 416         VirtIOSCSICmdResp *rsp;
 417
 418         elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement));
 419         if (!elem) {
 420             PDBG("No more elements pending on vq[%d]@%p", idx, vq);
 421             break;
 422         }
 423         PDBG("Popped elem@%p", elem);
 424
 425         assert(!(elem->out_num > 1 && elem->in_num > 1));
 426         assert(elem->out_num > 0 && elem->in_num > 0);
 427
 428         if (elem->out_sg[0].iov_len < sizeof(VirtIOSCSICmdReq)) {
 429             PERR("Invalid virtio-scsi req header");
 430             vus_panic_cb(vu_dev, NULL);
 431             break;
 432         }
 433         req = (VirtIOSCSICmdReq *)elem->out_sg[0].iov_base;
 434
 435         if (elem->in_sg[0].iov_len < sizeof(VirtIOSCSICmdResp)) {
 436             PERR("Invalid virtio-scsi rsp header");
 437             vus_panic_cb(vu_dev, NULL);
 438             break;
 439         }
 440         rsp = (VirtIOSCSICmdResp *)elem->in_sg[0].iov_base;
 441
 442         if (handle_cmd_sync(vdev_scsi->lun.iscsi_ctx,
 443                             req, &elem->out_sg[1], elem->out_num - 1,
 444                             rsp, &elem->in_sg[1], elem->in_num - 1) != 0) {
 445             vus_panic_cb(vu_dev, NULL);
 446             break;
 447         }
 448
 449         vu_queue_push(vu_dev, vq, elem, 0);
 450         vu_queue_notify(vu_dev, vq);
 451
 452         free(elem);
 453     }
 454 }
 455
 456 static void vus_queue_set_started(VuDev *vu_dev, int idx, bool started)
 457 {
 458     VuVirtq *vq;
 459
 460     assert(vu_dev);
 461
 462     if (idx < 0 || idx >= VHOST_MAX_NR_VIRTQUEUE) {
 463         PERR("VQ Index out of range: %d", idx);
 464         vus_panic_cb(vu_dev, NULL);
 465         return;
 466     }
 467
 468     vq = vu_get_queue(vu_dev, idx);
 469
 470     if (idx == 0 || idx == 1) {
 471         PDBG("queue %d unimplemented", idx);
 472     } else {
 473         vu_set_queue_handler(vu_dev, vq, started ? vus_proc_req : NULL);
 474     }
 475 }
 476
 477 static const VuDevIface vus_iface = {
 478     .queue_set_started = vus_queue_set_started,
 479 };
 480
 481 static gboolean vus_vhost_cb(gpointer data)
 482 {
 483     VuDev *vu_dev = (VuDev *)data;
 484
 485     assert(vu_dev);
 486
 487     if (!vu_dispatch(vu_dev) != 0) {
 488         PERR("Error processing vhost message");
 489         vus_panic_cb(vu_dev, NULL);
 490         return G_SOURCE_REMOVE;
 491     }
 492
 493     return G_SOURCE_CONTINUE;
 494 }
 495
 496 /** misc helpers **/
 497
 498 static int unix_sock_new(char *unix_fn)
 499 {
 500     int sock;
 501     struct sockaddr_un un;
 502     size_t len;
 503
 504     assert(unix_fn);
 505
 506     sock = socket(AF_UNIX, SOCK_STREAM, 0);
 507     if (sock <= 0) {
 508         perror("socket");
 509         return -1;
 510     }
 511
 512     un.sun_family = AF_UNIX;
 513     (void)snprintf(un.sun_path, sizeof(un.sun_path), "%s", unix_fn);
 514     len = sizeof(un.sun_family) + strlen(un.sun_path);
 515
 516     (void)unlink(unix_fn);
 517     if (bind(sock, (struct sockaddr *)&un, len) < 0) {
 518         perror("bind");
 519         goto fail;
 520     }
 521
 522     if (listen(sock, 1) < 0) {
 523         perror("listen");
 524         goto fail;
 525     }
 526
 527     return sock;
 528
 529 fail:
 530     (void)close(sock);
 531
 532     return -1;
 533 }
 534
 535 /** vhost-user-scsi **/
 536
 537 static void vdev_scsi_free(VusDev *vdev_scsi)
 538 {
 539     if (vdev_scsi->server_sock >= 0) {
 540         close(vdev_scsi->server_sock);
 541     }
 542     g_main_loop_unref(vdev_scsi->loop);
 543     g_tree_destroy(vdev_scsi->fdmap);
 544     g_free(vdev_scsi);
 545 }
 546
 547 static VusDev *vdev_scsi_new(int server_sock)
 548 {
 549     VusDev *vdev_scsi;
 550
 551     vdev_scsi = g_new0(VusDev, 1);
 552     vdev_scsi->server_sock = server_sock;
 553     vdev_scsi->loop = g_main_loop_new(NULL, FALSE);
 554     vdev_scsi->fdmap = g_tree_new(vus_fdmap_compare);
 555
 556     return vdev_scsi;
 557 }
 558
 559 static int vdev_scsi_run(VusDev *vdev_scsi)
 560 {
 561     int cli_sock;
 562     int ret = 0;
 563
 564     assert(vdev_scsi);
 565     assert(vdev_scsi->server_sock >= 0);
 566     assert(vdev_scsi->loop);
 567
 568     cli_sock = accept(vdev_scsi->server_sock, NULL, NULL);
 569     if (cli_sock < 0) {
 570         perror("accept");
 571         return -1;
 572     }
 573
 574     vu_init(&vdev_scsi->vu_dev,
 575             cli_sock,
 576             vus_panic_cb,
 577             vus_add_watch_cb,
 578             vus_del_watch_cb,
 579             &vus_iface);
 580
 581     vus_gsrc_new(vdev_scsi, cli_sock, G_IO_IN, NULL, vus_vhost_cb,
 582                  &vdev_scsi->vu_dev);
 583
 584     g_main_loop_run(vdev_scsi->loop);
 585
 586     vu_deinit(&vdev_scsi->vu_dev);
 587
 588     return ret;
 589 }
 590
 591 int main(int argc, char **argv)
 592 {
 593     VusDev *vdev_scsi = NULL;
 594     char *unix_fn = NULL;
 595     char *iscsi_uri = NULL;
 596     int sock, opt, err = EXIT_SUCCESS;
 597
 598     while ((opt = getopt(argc, argv, "u:i:")) != -1) {
 599         switch (opt) {
 600         case 'h':
 601             goto help;
 602         case 'u':
 603             unix_fn = g_strdup(optarg);
 604             break;
 605         case 'i':
 606             iscsi_uri = g_strdup(optarg);
 607             break;
 608         default:
 609             goto help;
 610         }
 611     }
 612     if (!unix_fn || !iscsi_uri) {
 613         goto help;
 614     }
 615
 616     sock = unix_sock_new(unix_fn);
 617     if (sock < 0) {
 618         goto err;
 619     }
 620     vdev_scsi = vdev_scsi_new(sock);
 621
 622     if (vus_iscsi_add_lun(&vdev_scsi->lun, iscsi_uri) != 0) {
 623         goto err;
 624     }
 625
 626     if (vdev_scsi_run(vdev_scsi) != 0) {
 627         goto err;
 628     }
 629
 630 out:
 631     if (vdev_scsi) {
 632         vdev_scsi_free(vdev_scsi);
 633         unlink(unix_fn);
 634     }
 635     g_free(unix_fn);
 636     g_free(iscsi_uri);
 637
 638     return err;
 639
 640 err:
 641     err = EXIT_FAILURE;
 642     goto out;
 643
 644 help:
 645     fprintf(stderr, "Usage: %s [ -u unix_sock_path -i iscsi_uri ] | [ -h ]\n",
 646             argv[0]);
 647     fprintf(stderr, "          -u path to unix socket\n");
 648     fprintf(stderr, "          -i iscsi uri for lun 0\n");
 649     fprintf(stderr, "          -h print help and quit\n");
 650
 651     goto err;
 652 }