contrib/vhost-user-scsi/vhost-user-scsi.c

   1 /*
   2  * vhost-user-scsi sample application
   3  *
   4  * Copyright (c) 2016 Nutanix Inc. All rights reserved.
   5  *
   6  * Author:
   7  *  Felipe Franciosi <felipe@nutanix.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 only.
  10  * See the COPYING file in the top-level directory.
  11  */
  12
  13 #include "qemu/osdep.h"
  14 #include "contrib/libvhost-user/libvhost-user.h"
  15 #include "standard-headers/linux/virtio_scsi.h"
  16 #include "iscsi/iscsi.h"
  17 #include "iscsi/scsi-lowlevel.h"
  18
  19 #include <glib.h>
  20
  21 /* #define VUS_DEBUG 1 */
  22
  23 /** Log helpers **/
  24
  25 #define PPRE                                                          \
  26     struct timespec ts;                                               \
  27     char   timebuf[64];                                               \
  28     struct tm tm;                                                     \
  29     (void)clock_gettime(CLOCK_REALTIME, &ts);                         \
  30     (void)strftime(timebuf, 64, "%Y%m%d %T", gmtime_r(&ts.tv_sec, &tm))
  31
  32 #define PEXT(lvl, msg, ...) do {                                      \
  33     PPRE;                                                             \
  34     fprintf(stderr, "%s.%06ld " lvl ": %s:%s():%d: " msg "\n",        \
  35             timebuf, ts.tv_nsec / 1000,                               \
  36             __FILE__, __func__, __LINE__, ## __VA_ARGS__);            \
  37 } while (0)
  38
  39 #define PNOR(lvl, msg, ...) do {                                      \
  40     PPRE;                                                             \
  41     fprintf(stderr, "%s.%06ld " lvl ": " msg "\n",                    \
  42             timebuf, ts.tv_nsec / 1000, ## __VA_ARGS__);              \
  43 } while (0)
  44
  45 #ifdef VUS_DEBUG
  46 #define PDBG(msg, ...) PEXT("DBG", msg, ## __VA_ARGS__)
  47 #define PERR(msg, ...) PEXT("ERR", msg, ## __VA_ARGS__)
  48 #define PLOG(msg, ...) PEXT("LOG", msg, ## __VA_ARGS__)
  49 #else
  50 #define PDBG(msg, ...) { }
  51 #define PERR(msg, ...) PNOR("ERR", msg, ## __VA_ARGS__)
  52 #define PLOG(msg, ...) PNOR("LOG", msg, ## __VA_ARGS__)
  53 #endif
  54
  55 /** vhost-user-scsi specific definitions **/
  56
  57 #define VUS_ISCSI_INITIATOR "iqn.2016-11.com.nutanix:vhost-user-scsi"
  58
  59 typedef struct VusIscsiLun {
  60     struct iscsi_context *iscsi_ctx;
  61     int iscsi_lun;
  62 } VusIscsiLun;
  63
  64 typedef struct VusDev {
  65     VuDev vu_dev;
  66     int server_sock;
  67     GMainLoop *loop;
  68     GTree *fdmap;   /* fd -> gsource context id */
  69     VusIscsiLun lun;
  70 } VusDev;
  71
  72 /** glib event loop integration for libvhost-user and misc callbacks **/
  73
  74 QEMU_BUILD_BUG_ON((int)G_IO_IN != (int)VU_WATCH_IN);
  75 QEMU_BUILD_BUG_ON((int)G_IO_OUT != (int)VU_WATCH_OUT);
  76 QEMU_BUILD_BUG_ON((int)G_IO_PRI != (int)VU_WATCH_PRI);
  77 QEMU_BUILD_BUG_ON((int)G_IO_ERR != (int)VU_WATCH_ERR);
  78 QEMU_BUILD_BUG_ON((int)G_IO_HUP != (int)VU_WATCH_HUP);
  79
  80 typedef struct vus_gsrc {
  81     GSource parent;
  82     VusDev *vdev_scsi;
  83     GPollFD gfd;
  84     vu_watch_cb vu_cb;
  85 } vus_gsrc_t;
  86
  87 static gint vus_fdmap_compare(gconstpointer a, gconstpointer b)
  88 {
  89     return (b > a) - (b < a);
  90 }
  91
  92 static gboolean vus_gsrc_prepare(GSource *src, gint *timeout)
  93 {
  94     assert(timeout);
  95
  96     *timeout = -1;
  97     return FALSE;
  98 }
  99
 100 static gboolean vus_gsrc_check(GSource *src)
 101 {
 102     vus_gsrc_t *vus_src = (vus_gsrc_t *)src;
 103
 104     assert(vus_src);
 105
 106     return vus_src->gfd.revents & vus_src->gfd.events;
 107 }
 108
 109 static gboolean vus_gsrc_dispatch(GSource *src, GSourceFunc cb, gpointer data)
 110 {
 111     VusDev *vdev_scsi;
 112     vus_gsrc_t *vus_src = (vus_gsrc_t *)src;
 113
 114     assert(vus_src);
 115     assert(!(vus_src->vu_cb && cb));
 116
 117     vdev_scsi = vus_src->vdev_scsi;
 118
 119     assert(vdev_scsi);
 120
 121     if (cb) {
 122         return cb(data);
 123     }
 124     if (vus_src->vu_cb) {
 125         vus_src->vu_cb(&vdev_scsi->vu_dev, vus_src->gfd.revents, data);
 126     }
 127     return G_SOURCE_CONTINUE;
 128 }
 129
 130 static GSourceFuncs vus_gsrc_funcs = {
 131     vus_gsrc_prepare,
 132     vus_gsrc_check,
 133     vus_gsrc_dispatch,
 134     NULL
 135 };
 136
 137 static void vus_gsrc_new(VusDev *vdev_scsi, int fd, GIOCondition cond,
 138                          vu_watch_cb vu_cb, GSourceFunc gsrc_cb, gpointer data)
 139 {
 140     GSource *vus_gsrc;
 141     vus_gsrc_t *vus_src;
 142     guint id;
 143
 144     assert(vdev_scsi);
 145     assert(fd >= 0);
 146     assert(vu_cb || gsrc_cb);
 147     assert(!(vu_cb && gsrc_cb));
 148
 149     vus_gsrc = g_source_new(&vus_gsrc_funcs, sizeof(vus_gsrc_t));
 150     vus_src = (vus_gsrc_t *)vus_gsrc;
 151
 152     vus_src->vdev_scsi = vdev_scsi;
 153     vus_src->gfd.fd = fd;
 154     vus_src->gfd.events = cond;
 155     vus_src->vu_cb = vu_cb;
 156
 157     g_source_add_poll(vus_gsrc, &vus_src->gfd);
 158     g_source_set_callback(vus_gsrc, gsrc_cb, data, NULL);
 159     id = g_source_attach(vus_gsrc, NULL);
 160     assert(id);
 161     g_source_unref(vus_gsrc);
 162
 163     g_tree_insert(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd,
 164                                     (gpointer)(uintptr_t)id);
 165 }
 166
 167 /** libiscsi integration **/
 168
 169 typedef struct virtio_scsi_cmd_req VirtIOSCSICmdReq;
 170 typedef struct virtio_scsi_cmd_resp VirtIOSCSICmdResp;
 171
 172 static int vus_iscsi_add_lun(VusIscsiLun *lun, char *iscsi_uri)
 173 {
 174     struct iscsi_url *iscsi_url;
 175     struct iscsi_context *iscsi_ctx;
 176     int ret = 0;
 177
 178     assert(lun);
 179     assert(iscsi_uri);
 180     assert(!lun->iscsi_ctx);
 181
 182     iscsi_ctx = iscsi_create_context(VUS_ISCSI_INITIATOR);
 183     if (!iscsi_ctx) {
 184         PERR("Unable to create iSCSI context");
 185         return -1;
 186     }
 187
 188     iscsi_url = iscsi_parse_full_url(iscsi_ctx, iscsi_uri);
 189     if (!iscsi_url) {
 190         PERR("Unable to parse iSCSI URL: %s", iscsi_get_error(iscsi_ctx));
 191         goto fail;
 192     }
 193
 194     iscsi_set_session_type(iscsi_ctx, ISCSI_SESSION_NORMAL);
 195     iscsi_set_header_digest(iscsi_ctx, ISCSI_HEADER_DIGEST_NONE_CRC32C);
 196     if (iscsi_full_connect_sync(iscsi_ctx, iscsi_url->portal, iscsi_url->lun)) {
 197         PERR("Unable to login to iSCSI portal: %s", iscsi_get_error(iscsi_ctx));
 198         goto fail;
 199     }
 200
 201     lun->iscsi_ctx = iscsi_ctx;
 202     lun->iscsi_lun = iscsi_url->lun;
 203
 204     PDBG("Context %p created for lun 0: %s", iscsi_ctx, iscsi_uri);
 205
 206 out:
 207     if (iscsi_url) {
 208         iscsi_destroy_url(iscsi_url);
 209     }
 210     return ret;
 211
 212 fail:
 213     (void)iscsi_destroy_context(iscsi_ctx);
 214     ret = -1;
 215     goto out;
 216 }
 217
 218 static struct scsi_task *scsi_task_new(int cdb_len, uint8_t *cdb, int dir,
 219                                        int xfer_len)
 220 {
 221     struct scsi_task *task;
 222
 223     assert(cdb_len > 0);
 224     assert(cdb);
 225
 226     task = g_new0(struct scsi_task, 1);
 227     memcpy(task->cdb, cdb, cdb_len);
 228     task->cdb_size = cdb_len;
 229     task->xfer_dir = dir;
 230     task->expxferlen = xfer_len;
 231
 232     return task;
 233 }
 234
 235 static int get_cdb_len(uint8_t *cdb)
 236 {
 237     assert(cdb);
 238
 239     switch (cdb[0] >> 5) {
 240     case 0: return 6;
 241     case 1: /* fall through */
 242     case 2: return 10;
 243     case 4: return 16;
 244     case 5: return 12;
 245     }
 246     PERR("Unable to determine cdb len (0x%02hhX)", cdb[0] >> 5);
 247     return -1;
 248 }
 249
 250 static int handle_cmd_sync(struct iscsi_context *ctx,
 251                            VirtIOSCSICmdReq *req,
 252                            struct iovec *out, unsigned int out_len,
 253                            VirtIOSCSICmdResp *rsp,
 254                            struct iovec *in, unsigned int in_len)
 255 {
 256     struct scsi_task *task;
 257     uint32_t dir;
 258     uint32_t len;
 259     int cdb_len;
 260     int i;
 261
 262     assert(ctx);
 263     assert(req);
 264     assert(rsp);
 265
 266     if (!(!req->lun[1] && req->lun[2] == 0x40 && !req->lun[3])) {
 267         /* Ignore anything different than target=0, lun=0 */
 268         PDBG("Ignoring unconnected lun (0x%hhX, 0x%hhX)",
 269              req->lun[1], req->lun[3]);
 270         rsp->status = SCSI_STATUS_CHECK_CONDITION;
 271         memset(rsp->sense, 0, sizeof(rsp->sense));
 272         rsp->sense_len = 18;
 273         rsp->sense[0] = 0x70;
 274         rsp->sense[2] = SCSI_SENSE_ILLEGAL_REQUEST;
 275         rsp->sense[7] = 10;
 276         rsp->sense[12] = 0x24;
 277
 278         return 0;
 279     }
 280
 281     cdb_len = get_cdb_len(req->cdb);
 282     if (cdb_len == -1) {
 283         return -1;
 284     }
 285
 286     len = 0;
 287     if (!out_len && !in_len) {
 288         dir = SCSI_XFER_NONE;
 289     } else if (out_len) {
 290         dir = SCSI_XFER_WRITE;
 291         for (i = 0; i < out_len; i++) {
 292             len += out[i].iov_len;
 293         }
 294     } else {
 295         dir = SCSI_XFER_READ;
 296         for (i = 0; i < in_len; i++) {
 297             len += in[i].iov_len;
 298         }
 299     }
 300
 301     task = scsi_task_new(cdb_len, req->cdb, dir, len);
 302
 303     if (dir == SCSI_XFER_WRITE) {
 304         task->iovector_out.iov = (struct scsi_iovec *)out;
 305         task->iovector_out.niov = out_len;
 306     } else if (dir == SCSI_XFER_READ) {
 307         task->iovector_in.iov = (struct scsi_iovec *)in;
 308         task->iovector_in.niov = in_len;
 309     }
 310
 311     PDBG("Sending iscsi cmd (cdb_len=%d, dir=%d, task=%p)",
 312          cdb_len, dir, task);
 313     if (!iscsi_scsi_command_sync(ctx, 0, task, NULL)) {
 314         PERR("Error serving SCSI command");
 315         g_free(task);
 316         return -1;
 317     }
 318
 319     memset(rsp, 0, sizeof(*rsp));
 320
 321     rsp->status = task->status;
 322     rsp->resid  = task->residual;
 323
 324     if (task->status == SCSI_STATUS_CHECK_CONDITION) {
 325         rsp->response = VIRTIO_SCSI_S_FAILURE;
 326         rsp->sense_len = task->datain.size - 2;
 327         memcpy(rsp->sense, &task->datain.data[2], rsp->sense_len);
 328     }
 329
 330     g_free(task);
 331
 332     PDBG("Filled in rsp: status=%hhX, resid=%u, response=%hhX, sense_len=%u",
 333          rsp->status, rsp->resid, rsp->response, rsp->sense_len);
 334
 335     return 0;
 336 }
 337
 338 /** libvhost-user callbacks **/
 339
 340 static void vus_panic_cb(VuDev *vu_dev, const char *buf)
 341 {
 342     VusDev *vdev_scsi;
 343
 344     assert(vu_dev);
 345
 346     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
 347     if (buf) {
 348         PERR("vu_panic: %s", buf);
 349     }
 350
 351     g_main_loop_quit(vdev_scsi->loop);
 352 }
 353
 354 static void vus_add_watch_cb(VuDev *vu_dev, int fd, int vu_evt, vu_watch_cb cb,
 355                              void *pvt)
 356 {
 357     VusDev *vdev_scsi;
 358     guint id;
 359
 360     assert(vu_dev);
 361     assert(fd >= 0);
 362     assert(cb);
 363
 364     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
 365     id = (guint)(uintptr_t)g_tree_lookup(vdev_scsi->fdmap,
 366                                          (gpointer)(uintptr_t)fd);
 367     if (id) {
 368         GSource *vus_src = g_main_context_find_source_by_id(NULL, id);
 369         assert(vus_src);
 370         g_source_destroy(vus_src);
 371         (void)g_tree_remove(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd);
 372     }
 373
 374     vus_gsrc_new(vdev_scsi, fd, vu_evt, cb, NULL, pvt);
 375 }
 376
 377 static void vus_del_watch_cb(VuDev *vu_dev, int fd)
 378 {
 379     VusDev *vdev_scsi;
 380     guint id;
 381
 382     assert(vu_dev);
 383     assert(fd >= 0);
 384
 385     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
 386     id = (guint)(uintptr_t)g_tree_lookup(vdev_scsi->fdmap,
 387                                          (gpointer)(uintptr_t)fd);
 388     if (id) {
 389         GSource *vus_src = g_main_context_find_source_by_id(NULL, id);
 390         assert(vus_src);
 391         g_source_destroy(vus_src);
 392         (void)g_tree_remove(vdev_scsi->fdmap, (gpointer)(uintptr_t)fd);
 393     }
 394 }
 395
 396 static void vus_proc_req(VuDev *vu_dev, int idx)
 397 {
 398     VusDev *vdev_scsi;
 399     VuVirtq *vq;
 400
 401     assert(vu_dev);
 402
 403     vdev_scsi = container_of(vu_dev, VusDev, vu_dev);
 404     if (idx < 0 || idx >= VHOST_MAX_NR_VIRTQUEUE) {
 405         PERR("VQ Index out of range: %d", idx);
 406         vus_panic_cb(vu_dev, NULL);
 407         return;
 408     }
 409
 410     vq = vu_get_queue(vu_dev, idx);
 411     if (!vq) {
 412         PERR("Error fetching VQ (dev=%p, idx=%d)", vu_dev, idx);
 413         vus_panic_cb(vu_dev, NULL);
 414         return;
 415     }
 416
 417     PDBG("Got kicked on vq[%d]@%p", idx, vq);
 418
 419     while (1) {
 420         VuVirtqElement *elem;
 421         VirtIOSCSICmdReq *req;
 422         VirtIOSCSICmdResp *rsp;
 423
 424         elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement));
 425         if (!elem) {
 426             PDBG("No more elements pending on vq[%d]@%p", idx, vq);
 427             break;
 428         }
 429         PDBG("Popped elem@%p", elem);
 430
 431         assert(!(elem->out_num > 1 && elem->in_num > 1));
 432         assert(elem->out_num > 0 && elem->in_num > 0);
 433
 434         if (elem->out_sg[0].iov_len < sizeof(VirtIOSCSICmdReq)) {
 435             PERR("Invalid virtio-scsi req header");
 436             vus_panic_cb(vu_dev, NULL);
 437             break;
 438         }
 439         req = (VirtIOSCSICmdReq *)elem->out_sg[0].iov_base;
 440
 441         if (elem->in_sg[0].iov_len < sizeof(VirtIOSCSICmdResp)) {
 442             PERR("Invalid virtio-scsi rsp header");
 443             vus_panic_cb(vu_dev, NULL);
 444             break;
 445         }
 446         rsp = (VirtIOSCSICmdResp *)elem->in_sg[0].iov_base;
 447
 448         if (handle_cmd_sync(vdev_scsi->lun.iscsi_ctx,
 449                             req, &elem->out_sg[1], elem->out_num - 1,
 450                             rsp, &elem->in_sg[1], elem->in_num - 1) != 0) {
 451             vus_panic_cb(vu_dev, NULL);
 452             break;
 453         }
 454
 455         vu_queue_push(vu_dev, vq, elem, 0);
 456         vu_queue_notify(vu_dev, vq);
 457
 458         free(elem);
 459     }
 460 }
 461
 462 static void vus_queue_set_started(VuDev *vu_dev, int idx, bool started)
 463 {
 464     VuVirtq *vq;
 465
 466     assert(vu_dev);
 467
 468     if (idx < 0 || idx >= VHOST_MAX_NR_VIRTQUEUE) {
 469         PERR("VQ Index out of range: %d", idx);
 470         vus_panic_cb(vu_dev, NULL);
 471         return;
 472     }
 473
 474     vq = vu_get_queue(vu_dev, idx);
 475
 476     if (idx == 0 || idx == 1) {
 477         PDBG("queue %d unimplemented", idx);
 478     } else {
 479         vu_set_queue_handler(vu_dev, vq, started ? vus_proc_req : NULL);
 480     }
 481 }
 482
 483 static const VuDevIface vus_iface = {
 484     .queue_set_started = vus_queue_set_started,
 485 };
 486
 487 static gboolean vus_vhost_cb(gpointer data)
 488 {
 489     VuDev *vu_dev = (VuDev *)data;
 490
 491     assert(vu_dev);
 492
 493     if (!vu_dispatch(vu_dev) != 0) {
 494         PERR("Error processing vhost message");
 495         vus_panic_cb(vu_dev, NULL);
 496         return G_SOURCE_REMOVE;
 497     }
 498
 499     return G_SOURCE_CONTINUE;
 500 }
 501
 502 /** misc helpers **/
 503
 504 static int unix_sock_new(char *unix_fn)
 505 {
 506     int sock;
 507     struct sockaddr_un un;
 508     size_t len;
 509
 510     assert(unix_fn);
 511
 512     sock = socket(AF_UNIX, SOCK_STREAM, 0);
 513     if (sock <= 0) {
 514         perror("socket");
 515         return -1;
 516     }
 517
 518     un.sun_family = AF_UNIX;
 519     (void)snprintf(un.sun_path, sizeof(un.sun_path), "%s", unix_fn);
 520     len = sizeof(un.sun_family) + strlen(un.sun_path);
 521
 522     (void)unlink(unix_fn);
 523     if (bind(sock, (struct sockaddr *)&un, len) < 0) {
 524         perror("bind");
 525         goto fail;
 526     }
 527
 528     if (listen(sock, 1) < 0) {
 529         perror("listen");
 530         goto fail;
 531     }
 532
 533     return sock;
 534
 535 fail:
 536     (void)close(sock);
 537
 538     return -1;
 539 }
 540
 541 /** vhost-user-scsi **/
 542
 543 static void vdev_scsi_free(VusDev *vdev_scsi)
 544 {
 545     if (vdev_scsi->server_sock >= 0) {
 546         close(vdev_scsi->server_sock);
 547     }
 548     g_main_loop_unref(vdev_scsi->loop);
 549     g_tree_destroy(vdev_scsi->fdmap);
 550     g_free(vdev_scsi);
 551 }
 552
 553 static VusDev *vdev_scsi_new(int server_sock)
 554 {
 555     VusDev *vdev_scsi;
 556
 557     vdev_scsi = g_new0(VusDev, 1);
 558     vdev_scsi->server_sock = server_sock;
 559     vdev_scsi->loop = g_main_loop_new(NULL, FALSE);
 560     vdev_scsi->fdmap = g_tree_new(vus_fdmap_compare);
 561
 562     return vdev_scsi;
 563 }
 564
 565 static int vdev_scsi_run(VusDev *vdev_scsi)
 566 {
 567     int cli_sock;
 568     int ret = 0;
 569
 570     assert(vdev_scsi);
 571     assert(vdev_scsi->server_sock >= 0);
 572     assert(vdev_scsi->loop);
 573
 574     cli_sock = accept(vdev_scsi->server_sock, NULL, NULL);
 575     if (cli_sock < 0) {
 576         perror("accept");
 577         return -1;
 578     }
 579
 580     vu_init(&vdev_scsi->vu_dev,
 581             cli_sock,
 582             vus_panic_cb,
 583             vus_add_watch_cb,
 584             vus_del_watch_cb,
 585             &vus_iface);
 586
 587     vus_gsrc_new(vdev_scsi, cli_sock, G_IO_IN, NULL, vus_vhost_cb,
 588                  &vdev_scsi->vu_dev);
 589
 590     g_main_loop_run(vdev_scsi->loop);
 591
 592     vu_deinit(&vdev_scsi->vu_dev);
 593
 594     return ret;
 595 }
 596
 597 int main(int argc, char **argv)
 598 {
 599     VusDev *vdev_scsi = NULL;
 600     char *unix_fn = NULL;
 601     char *iscsi_uri = NULL;
 602     int sock, opt, err = EXIT_SUCCESS;
 603
 604     while ((opt = getopt(argc, argv, "u:i:")) != -1) {
 605         switch (opt) {
 606         case 'h':
 607             goto help;
 608         case 'u':
 609             unix_fn = g_strdup(optarg);
 610             break;
 611         case 'i':
 612             iscsi_uri = g_strdup(optarg);
 613             break;
 614         default:
 615             goto help;
 616         }
 617     }
 618     if (!unix_fn || !iscsi_uri) {
 619         goto help;
 620     }
 621
 622     sock = unix_sock_new(unix_fn);
 623     if (sock < 0) {
 624         goto err;
 625     }
 626     vdev_scsi = vdev_scsi_new(sock);
 627
 628     if (vus_iscsi_add_lun(&vdev_scsi->lun, iscsi_uri) != 0) {
 629         goto err;
 630     }
 631
 632     if (vdev_scsi_run(vdev_scsi) != 0) {
 633         goto err;
 634     }
 635
 636 out:
 637     if (vdev_scsi) {
 638         vdev_scsi_free(vdev_scsi);
 639         unlink(unix_fn);
 640     }
 641     g_free(unix_fn);
 642     g_free(iscsi_uri);
 643
 644     return err;
 645
 646 err:
 647     err = EXIT_FAILURE;
 648     goto out;
 649
 650 help:
 651     fprintf(stderr, "Usage: %s [ -u unix_sock_path -i iscsi_uri ] | [ -h ]\n",
 652             argv[0]);
 653     fprintf(stderr, "          -u path to unix socket\n");
 654     fprintf(stderr, "          -i iscsi uri for lun 0\n");
 655     fprintf(stderr, "          -h print help and quit\n");
 656
 657     goto err;
 658 }