hw/virtio/virtio-crypto.c

   1 /*
   2  * Virtio crypto Support
   3  *
   4  * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
   5  *
   6  * Authors:
   7  *    Gonglei <arei.gonglei@huawei.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * (at your option) any later version.  See the COPYING file in the
  11  * top-level directory.
  12  */
  13
  14 #include "qemu/osdep.h"
  15 #include "qemu/iov.h"
  16 #include "qemu/module.h"
  17 #include "hw/qdev.h"
  18 #include "qapi/error.h"
  19 #include "qemu/error-report.h"
  20
  21 #include "hw/virtio/virtio.h"
  22 #include "hw/virtio/virtio-crypto.h"
  23 #include "hw/virtio/virtio-access.h"
  24 #include "standard-headers/linux/virtio_ids.h"
  25 #include "sysemu/cryptodev-vhost.h"
  26
  27 #define VIRTIO_CRYPTO_VM_VERSION 1
  28
  29 /*
  30  * Transfer virtqueue index to crypto queue index.
  31  * The control virtqueue is after the data virtqueues
  32  * so the input value doesn't need to be adjusted
  33  */
  34 static inline int virtio_crypto_vq2q(int queue_index)
  35 {
  36     return queue_index;
  37 }
  38
  39 static int
  40 virtio_crypto_cipher_session_helper(VirtIODevice *vdev,
  41            CryptoDevBackendSymSessionInfo *info,
  42            struct virtio_crypto_cipher_session_para *cipher_para,
  43            struct iovec **iov, unsigned int *out_num)
  44 {
  45     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
  46     unsigned int num = *out_num;
  47
  48     info->cipher_alg = ldl_le_p(&cipher_para->algo);
  49     info->key_len = ldl_le_p(&cipher_para->keylen);
  50     info->direction = ldl_le_p(&cipher_para->op);
  51     DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n",
  52              info->cipher_alg, info->direction);
  53
  54     if (info->key_len > vcrypto->conf.max_cipher_key_len) {
  55         error_report("virtio-crypto length of cipher key is too big: %u",
  56                      info->key_len);
  57         return -VIRTIO_CRYPTO_ERR;
  58     }
  59     /* Get cipher key */
  60     if (info->key_len > 0) {
  61         size_t s;
  62         DPRINTF("keylen=%" PRIu32 "\n", info->key_len);
  63
  64         info->cipher_key = g_malloc(info->key_len);
  65         s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len);
  66         if (unlikely(s != info->key_len)) {
  67             virtio_error(vdev, "virtio-crypto cipher key incorrect");
  68             return -EFAULT;
  69         }
  70         iov_discard_front(iov, &num, info->key_len);
  71         *out_num = num;
  72     }
  73
  74     return 0;
  75 }
  76
  77 static int64_t
  78 virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto,
  79                struct virtio_crypto_sym_create_session_req *sess_req,
  80                uint32_t queue_id,
  81                uint32_t opcode,
  82                struct iovec *iov, unsigned int out_num)
  83 {
  84     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
  85     CryptoDevBackendSymSessionInfo info;
  86     int64_t session_id;
  87     int queue_index;
  88     uint32_t op_type;
  89     Error *local_err = NULL;
  90     int ret;
  91
  92     memset(&info, 0, sizeof(info));
  93     op_type = ldl_le_p(&sess_req->op_type);
  94     info.op_type = op_type;
  95     info.op_code = opcode;
  96
  97     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
  98         ret = virtio_crypto_cipher_session_helper(vdev, &info,
  99                            &sess_req->u.cipher.para,
 100                            &iov, &out_num);
 101         if (ret < 0) {
 102             goto err;
 103         }
 104     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 105         size_t s;
 106         /* cipher part */
 107         ret = virtio_crypto_cipher_session_helper(vdev, &info,
 108                            &sess_req->u.chain.para.cipher_param,
 109                            &iov, &out_num);
 110         if (ret < 0) {
 111             goto err;
 112         }
 113         /* hash part */
 114         info.alg_chain_order = ldl_le_p(
 115                                      &sess_req->u.chain.para.alg_chain_order);
 116         info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len);
 117         info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode);
 118         if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) {
 119             info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo);
 120             info.auth_key_len = ldl_le_p(
 121                              &sess_req->u.chain.para.u.mac_param.auth_key_len);
 122             info.hash_result_len = ldl_le_p(
 123                            &sess_req->u.chain.para.u.mac_param.hash_result_len);
 124             if (info.auth_key_len > vcrypto->conf.max_auth_key_len) {
 125                 error_report("virtio-crypto length of auth key is too big: %u",
 126                              info.auth_key_len);
 127                 ret = -VIRTIO_CRYPTO_ERR;
 128                 goto err;
 129             }
 130             /* get auth key */
 131             if (info.auth_key_len > 0) {
 132                 DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len);
 133                 info.auth_key = g_malloc(info.auth_key_len);
 134                 s = iov_to_buf(iov, out_num, 0, info.auth_key,
 135                                info.auth_key_len);
 136                 if (unlikely(s != info.auth_key_len)) {
 137                     virtio_error(vdev,
 138                           "virtio-crypto authenticated key incorrect");
 139                     ret = -EFAULT;
 140                     goto err;
 141                 }
 142                 iov_discard_front(&iov, &out_num, info.auth_key_len);
 143             }
 144         } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) {
 145             info.hash_alg = ldl_le_p(
 146                              &sess_req->u.chain.para.u.hash_param.algo);
 147             info.hash_result_len = ldl_le_p(
 148                         &sess_req->u.chain.para.u.hash_param.hash_result_len);
 149         } else {
 150             /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
 151             error_report("unsupported hash mode");
 152             ret = -VIRTIO_CRYPTO_NOTSUPP;
 153             goto err;
 154         }
 155     } else {
 156         /* VIRTIO_CRYPTO_SYM_OP_NONE */
 157         error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE");
 158         ret = -VIRTIO_CRYPTO_NOTSUPP;
 159         goto err;
 160     }
 161
 162     queue_index = virtio_crypto_vq2q(queue_id);
 163     session_id = cryptodev_backend_sym_create_session(
 164                                      vcrypto->cryptodev,
 165                                      &info, queue_index, &local_err);
 166     if (session_id >= 0) {
 167         DPRINTF("create session_id=%" PRIu64 " successfully\n",
 168                 session_id);
 169
 170         ret = session_id;
 171     } else {
 172         if (local_err) {
 173             error_report_err(local_err);
 174         }
 175         ret = -VIRTIO_CRYPTO_ERR;
 176     }
 177
 178 err:
 179     g_free(info.cipher_key);
 180     g_free(info.auth_key);
 181     return ret;
 182 }
 183
 184 static uint8_t
 185 virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto,
 186          struct virtio_crypto_destroy_session_req *close_sess_req,
 187          uint32_t queue_id)
 188 {
 189     int ret;
 190     uint64_t session_id;
 191     uint32_t status;
 192     Error *local_err = NULL;
 193
 194     session_id = ldq_le_p(&close_sess_req->session_id);
 195     DPRINTF("close session, id=%" PRIu64 "\n", session_id);
 196
 197     ret = cryptodev_backend_sym_close_session(
 198               vcrypto->cryptodev, session_id, queue_id, &local_err);
 199     if (ret == 0) {
 200         status = VIRTIO_CRYPTO_OK;
 201     } else {
 202         if (local_err) {
 203             error_report_err(local_err);
 204         } else {
 205             error_report("destroy session failed");
 206         }
 207         status = VIRTIO_CRYPTO_ERR;
 208     }
 209
 210     return status;
 211 }
 212
 213 static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
 214 {
 215     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 216     struct virtio_crypto_op_ctrl_req ctrl;
 217     VirtQueueElement *elem;
 218     struct iovec *in_iov;
 219     struct iovec *out_iov;
 220     unsigned in_num;
 221     unsigned out_num;
 222     uint32_t queue_id;
 223     uint32_t opcode;
 224     struct virtio_crypto_session_input input;
 225     int64_t session_id;
 226     uint8_t status;
 227     size_t s;
 228
 229     for (;;) {
 230         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
 231         if (!elem) {
 232             break;
 233         }
 234         if (elem->out_num < 1 || elem->in_num < 1) {
 235             virtio_error(vdev, "virtio-crypto ctrl missing headers");
 236             virtqueue_detach_element(vq, elem, 0);
 237             g_free(elem);
 238             break;
 239         }
 240
 241         out_num = elem->out_num;
 242         out_iov = elem->out_sg;
 243         in_num = elem->in_num;
 244         in_iov = elem->in_sg;
 245         if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl))
 246                     != sizeof(ctrl))) {
 247             virtio_error(vdev, "virtio-crypto request ctrl_hdr too short");
 248             virtqueue_detach_element(vq, elem, 0);
 249             g_free(elem);
 250             break;
 251         }
 252         iov_discard_front(&out_iov, &out_num, sizeof(ctrl));
 253
 254         opcode = ldl_le_p(&ctrl.header.opcode);
 255         queue_id = ldl_le_p(&ctrl.header.queue_id);
 256
 257         switch (opcode) {
 258         case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION:
 259             memset(&input, 0, sizeof(input));
 260             session_id = virtio_crypto_create_sym_session(vcrypto,
 261                              &ctrl.u.sym_create_session,
 262                              queue_id, opcode,
 263                              out_iov, out_num);
 264             /* Serious errors, need to reset virtio crypto device */
 265             if (session_id == -EFAULT) {
 266                 virtqueue_detach_element(vq, elem, 0);
 267                 break;
 268             } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) {
 269                 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 270             } else if (session_id == -VIRTIO_CRYPTO_ERR) {
 271                 stl_le_p(&input.status, VIRTIO_CRYPTO_ERR);
 272             } else {
 273                 /* Set the session id */
 274                 stq_le_p(&input.session_id, session_id);
 275                 stl_le_p(&input.status, VIRTIO_CRYPTO_OK);
 276             }
 277
 278             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 279             if (unlikely(s != sizeof(input))) {
 280                 virtio_error(vdev, "virtio-crypto input incorrect");
 281                 virtqueue_detach_element(vq, elem, 0);
 282                 break;
 283             }
 284             virtqueue_push(vq, elem, sizeof(input));
 285             virtio_notify(vdev, vq);
 286             break;
 287         case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION:
 288         case VIRTIO_CRYPTO_HASH_DESTROY_SESSION:
 289         case VIRTIO_CRYPTO_MAC_DESTROY_SESSION:
 290         case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION:
 291             status = virtio_crypto_handle_close_session(vcrypto,
 292                    &ctrl.u.destroy_session, queue_id);
 293             /* The status only occupy one byte, we can directly use it */
 294             s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status));
 295             if (unlikely(s != sizeof(status))) {
 296                 virtio_error(vdev, "virtio-crypto status incorrect");
 297                 virtqueue_detach_element(vq, elem, 0);
 298                 break;
 299             }
 300             virtqueue_push(vq, elem, sizeof(status));
 301             virtio_notify(vdev, vq);
 302             break;
 303         case VIRTIO_CRYPTO_HASH_CREATE_SESSION:
 304         case VIRTIO_CRYPTO_MAC_CREATE_SESSION:
 305         case VIRTIO_CRYPTO_AEAD_CREATE_SESSION:
 306         default:
 307             error_report("virtio-crypto unsupported ctrl opcode: %d", opcode);
 308             memset(&input, 0, sizeof(input));
 309             stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
 310             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
 311             if (unlikely(s != sizeof(input))) {
 312                 virtio_error(vdev, "virtio-crypto input incorrect");
 313                 virtqueue_detach_element(vq, elem, 0);
 314                 break;
 315             }
 316             virtqueue_push(vq, elem, sizeof(input));
 317             virtio_notify(vdev, vq);
 318
 319             break;
 320         } /* end switch case */
 321
 322         g_free(elem);
 323     } /* end for loop */
 324 }
 325
 326 static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq,
 327                                 VirtIOCryptoReq *req)
 328 {
 329     req->vcrypto = vcrypto;
 330     req->vq = vq;
 331     req->in = NULL;
 332     req->in_iov = NULL;
 333     req->in_num = 0;
 334     req->in_len = 0;
 335     req->flags = CRYPTODEV_BACKEND_ALG__MAX;
 336     req->u.sym_op_info = NULL;
 337 }
 338
 339 static void virtio_crypto_free_request(VirtIOCryptoReq *req)
 340 {
 341     if (req) {
 342         if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 343             size_t max_len;
 344             CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info;
 345
 346             max_len = op_info->iv_len +
 347                       op_info->aad_len +
 348                       op_info->src_len +
 349                       op_info->dst_len +
 350                       op_info->digest_result_len;
 351
 352             /* Zeroize and free request data structure */
 353             memset(op_info, 0, sizeof(*op_info) + max_len);
 354             g_free(op_info);
 355         }
 356         g_free(req);
 357     }
 358 }
 359
 360 static void
 361 virtio_crypto_sym_input_data_helper(VirtIODevice *vdev,
 362                 VirtIOCryptoReq *req,
 363                 uint32_t status,
 364                 CryptoDevBackendSymOpInfo *sym_op_info)
 365 {
 366     size_t s, len;
 367
 368     if (status != VIRTIO_CRYPTO_OK) {
 369         return;
 370     }
 371
 372     len = sym_op_info->src_len;
 373     /* Save the cipher result */
 374     s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len);
 375     if (s != len) {
 376         virtio_error(vdev, "virtio-crypto dest data incorrect");
 377         return;
 378     }
 379
 380     iov_discard_front(&req->in_iov, &req->in_num, len);
 381
 382     if (sym_op_info->op_type ==
 383                       VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 384         /* Save the digest result */
 385         s = iov_from_buf(req->in_iov, req->in_num, 0,
 386                          sym_op_info->digest_result,
 387                          sym_op_info->digest_result_len);
 388         if (s != sym_op_info->digest_result_len) {
 389             virtio_error(vdev, "virtio-crypto digest result incorrect");
 390         }
 391     }
 392 }
 393
 394 static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status)
 395 {
 396     VirtIOCrypto *vcrypto = req->vcrypto;
 397     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 398
 399     if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
 400         virtio_crypto_sym_input_data_helper(vdev, req, status,
 401                                             req->u.sym_op_info);
 402     }
 403     stb_p(&req->in->status, status);
 404     virtqueue_push(req->vq, &req->elem, req->in_len);
 405     virtio_notify(vdev, req->vq);
 406 }
 407
 408 static VirtIOCryptoReq *
 409 virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq)
 410 {
 411     VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq));
 412
 413     if (req) {
 414         virtio_crypto_init_request(s, vq, req);
 415     }
 416     return req;
 417 }
 418
 419 static CryptoDevBackendSymOpInfo *
 420 virtio_crypto_sym_op_helper(VirtIODevice *vdev,
 421            struct virtio_crypto_cipher_para *cipher_para,
 422            struct virtio_crypto_alg_chain_data_para *alg_chain_para,
 423            struct iovec *iov, unsigned int out_num)
 424 {
 425     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 426     CryptoDevBackendSymOpInfo *op_info;
 427     uint32_t src_len = 0, dst_len = 0;
 428     uint32_t iv_len = 0;
 429     uint32_t aad_len = 0, hash_result_len = 0;
 430     uint32_t hash_start_src_offset = 0, len_to_hash = 0;
 431     uint32_t cipher_start_src_offset = 0, len_to_cipher = 0;
 432
 433     uint64_t max_len, curr_size = 0;
 434     size_t s;
 435
 436     /* Plain cipher */
 437     if (cipher_para) {
 438         iv_len = ldl_le_p(&cipher_para->iv_len);
 439         src_len = ldl_le_p(&cipher_para->src_data_len);
 440         dst_len = ldl_le_p(&cipher_para->dst_data_len);
 441     } else if (alg_chain_para) { /* Algorithm chain */
 442         iv_len = ldl_le_p(&alg_chain_para->iv_len);
 443         src_len = ldl_le_p(&alg_chain_para->src_data_len);
 444         dst_len = ldl_le_p(&alg_chain_para->dst_data_len);
 445
 446         aad_len = ldl_le_p(&alg_chain_para->aad_len);
 447         hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len);
 448         hash_start_src_offset = ldl_le_p(
 449                          &alg_chain_para->hash_start_src_offset);
 450         cipher_start_src_offset = ldl_le_p(
 451                          &alg_chain_para->cipher_start_src_offset);
 452         len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher);
 453         len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash);
 454     } else {
 455         return NULL;
 456     }
 457
 458     max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len;
 459     if (unlikely(max_len > vcrypto->conf.max_size)) {
 460         virtio_error(vdev, "virtio-crypto too big length");
 461         return NULL;
 462     }
 463
 464     op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len);
 465     op_info->iv_len = iv_len;
 466     op_info->src_len = src_len;
 467     op_info->dst_len = dst_len;
 468     op_info->aad_len = aad_len;
 469     op_info->digest_result_len = hash_result_len;
 470     op_info->hash_start_src_offset = hash_start_src_offset;
 471     op_info->len_to_hash = len_to_hash;
 472     op_info->cipher_start_src_offset = cipher_start_src_offset;
 473     op_info->len_to_cipher = len_to_cipher;
 474     /* Handle the initilization vector */
 475     if (op_info->iv_len > 0) {
 476         DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len);
 477         op_info->iv = op_info->data + curr_size;
 478
 479         s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len);
 480         if (unlikely(s != op_info->iv_len)) {
 481             virtio_error(vdev, "virtio-crypto iv incorrect");
 482             goto err;
 483         }
 484         iov_discard_front(&iov, &out_num, op_info->iv_len);
 485         curr_size += op_info->iv_len;
 486     }
 487
 488     /* Handle additional authentication data if exists */
 489     if (op_info->aad_len > 0) {
 490         DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len);
 491         op_info->aad_data = op_info->data + curr_size;
 492
 493         s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len);
 494         if (unlikely(s != op_info->aad_len)) {
 495             virtio_error(vdev, "virtio-crypto additional auth data incorrect");
 496             goto err;
 497         }
 498         iov_discard_front(&iov, &out_num, op_info->aad_len);
 499
 500         curr_size += op_info->aad_len;
 501     }
 502
 503     /* Handle the source data */
 504     if (op_info->src_len > 0) {
 505         DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len);
 506         op_info->src = op_info->data + curr_size;
 507
 508         s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len);
 509         if (unlikely(s != op_info->src_len)) {
 510             virtio_error(vdev, "virtio-crypto source data incorrect");
 511             goto err;
 512         }
 513         iov_discard_front(&iov, &out_num, op_info->src_len);
 514
 515         curr_size += op_info->src_len;
 516     }
 517
 518     /* Handle the destination data */
 519     op_info->dst = op_info->data + curr_size;
 520     curr_size += op_info->dst_len;
 521
 522     DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len);
 523
 524     /* Handle the hash digest result */
 525     if (hash_result_len > 0) {
 526         DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len);
 527         op_info->digest_result = op_info->data + curr_size;
 528     }
 529
 530     return op_info;
 531
 532 err:
 533     g_free(op_info);
 534     return NULL;
 535 }
 536
 537 static int
 538 virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto,
 539                struct virtio_crypto_sym_data_req *req,
 540                CryptoDevBackendSymOpInfo **sym_op_info,
 541                struct iovec *iov, unsigned int out_num)
 542 {
 543     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 544     uint32_t op_type;
 545     CryptoDevBackendSymOpInfo *op_info;
 546
 547     op_type = ldl_le_p(&req->op_type);
 548
 549     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
 550         op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para,
 551                                               NULL, iov, out_num);
 552         if (!op_info) {
 553             return -EFAULT;
 554         }
 555         op_info->op_type = op_type;
 556     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
 557         op_info = virtio_crypto_sym_op_helper(vdev, NULL,
 558                                               &req->u.chain.para,
 559                                               iov, out_num);
 560         if (!op_info) {
 561             return -EFAULT;
 562         }
 563         op_info->op_type = op_type;
 564     } else {
 565         /* VIRTIO_CRYPTO_SYM_OP_NONE */
 566         error_report("virtio-crypto unsupported cipher type");
 567         return -VIRTIO_CRYPTO_NOTSUPP;
 568     }
 569
 570     *sym_op_info = op_info;
 571
 572     return 0;
 573 }
 574
 575 static int
 576 virtio_crypto_handle_request(VirtIOCryptoReq *request)
 577 {
 578     VirtIOCrypto *vcrypto = request->vcrypto;
 579     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 580     VirtQueueElement *elem = &request->elem;
 581     int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq));
 582     struct virtio_crypto_op_data_req req;
 583     int ret;
 584     struct iovec *in_iov;
 585     struct iovec *out_iov;
 586     unsigned in_num;
 587     unsigned out_num;
 588     uint32_t opcode;
 589     uint8_t status = VIRTIO_CRYPTO_ERR;
 590     uint64_t session_id;
 591     CryptoDevBackendSymOpInfo *sym_op_info = NULL;
 592     Error *local_err = NULL;
 593
 594     if (elem->out_num < 1 || elem->in_num < 1) {
 595         virtio_error(vdev, "virtio-crypto dataq missing headers");
 596         return -1;
 597     }
 598
 599     out_num = elem->out_num;
 600     out_iov = elem->out_sg;
 601     in_num = elem->in_num;
 602     in_iov = elem->in_sg;
 603     if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req))
 604                 != sizeof(req))) {
 605         virtio_error(vdev, "virtio-crypto request outhdr too short");
 606         return -1;
 607     }
 608     iov_discard_front(&out_iov, &out_num, sizeof(req));
 609
 610     if (in_iov[in_num - 1].iov_len <
 611             sizeof(struct virtio_crypto_inhdr)) {
 612         virtio_error(vdev, "virtio-crypto request inhdr too short");
 613         return -1;
 614     }
 615     /* We always touch the last byte, so just see how big in_iov is. */
 616     request->in_len = iov_size(in_iov, in_num);
 617     request->in = (void *)in_iov[in_num - 1].iov_base
 618               + in_iov[in_num - 1].iov_len
 619               - sizeof(struct virtio_crypto_inhdr);
 620     iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr));
 621
 622     /*
 623      * The length of operation result, including dest_data
 624      * and digest_result if exists.
 625      */
 626     request->in_num = in_num;
 627     request->in_iov = in_iov;
 628
 629     opcode = ldl_le_p(&req.header.opcode);
 630     session_id = ldq_le_p(&req.header.session_id);
 631
 632     switch (opcode) {
 633     case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
 634     case VIRTIO_CRYPTO_CIPHER_DECRYPT:
 635         ret = virtio_crypto_handle_sym_req(vcrypto,
 636                          &req.u.sym_req,
 637                          &sym_op_info,
 638                          out_iov, out_num);
 639         /* Serious errors, need to reset virtio crypto device */
 640         if (ret == -EFAULT) {
 641             return -1;
 642         } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) {
 643             virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 644             virtio_crypto_free_request(request);
 645         } else {
 646             sym_op_info->session_id = session_id;
 647
 648             /* Set request's parameter */
 649             request->flags = CRYPTODEV_BACKEND_ALG_SYM;
 650             request->u.sym_op_info = sym_op_info;
 651             ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev,
 652                                     request, queue_index, &local_err);
 653             if (ret < 0) {
 654                 status = -ret;
 655                 if (local_err) {
 656                     error_report_err(local_err);
 657                 }
 658             } else { /* ret == VIRTIO_CRYPTO_OK */
 659                 status = ret;
 660             }
 661             virtio_crypto_req_complete(request, status);
 662             virtio_crypto_free_request(request);
 663         }
 664         break;
 665     case VIRTIO_CRYPTO_HASH:
 666     case VIRTIO_CRYPTO_MAC:
 667     case VIRTIO_CRYPTO_AEAD_ENCRYPT:
 668     case VIRTIO_CRYPTO_AEAD_DECRYPT:
 669     default:
 670         error_report("virtio-crypto unsupported dataq opcode: %u",
 671                      opcode);
 672         virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
 673         virtio_crypto_free_request(request);
 674     }
 675
 676     return 0;
 677 }
 678
 679 static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq)
 680 {
 681     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 682     VirtIOCryptoReq *req;
 683
 684     while ((req = virtio_crypto_get_request(vcrypto, vq))) {
 685         if (virtio_crypto_handle_request(req) < 0) {
 686             virtqueue_detach_element(req->vq, &req->elem, 0);
 687             virtio_crypto_free_request(req);
 688             break;
 689         }
 690     }
 691 }
 692
 693 static void virtio_crypto_dataq_bh(void *opaque)
 694 {
 695     VirtIOCryptoQueue *q = opaque;
 696     VirtIOCrypto *vcrypto = q->vcrypto;
 697     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
 698
 699     /* This happens when device was stopped but BH wasn't. */
 700     if (!vdev->vm_running) {
 701         return;
 702     }
 703
 704     /* Just in case the driver is not ready on more */
 705     if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) {
 706         return;
 707     }
 708
 709     for (;;) {
 710         virtio_crypto_handle_dataq(vdev, q->dataq);
 711         virtio_queue_set_notification(q->dataq, 1);
 712
 713         /* Are we done or did the guest add more buffers? */
 714         if (virtio_queue_empty(q->dataq)) {
 715             break;
 716         }
 717
 718         virtio_queue_set_notification(q->dataq, 0);
 719     }
 720 }
 721
 722 static void
 723 virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq)
 724 {
 725     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 726     VirtIOCryptoQueue *q =
 727          &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))];
 728
 729     /* This happens when device was stopped but VCPU wasn't. */
 730     if (!vdev->vm_running) {
 731         return;
 732     }
 733     virtio_queue_set_notification(vq, 0);
 734     qemu_bh_schedule(q->dataq_bh);
 735 }
 736
 737 static uint64_t virtio_crypto_get_features(VirtIODevice *vdev,
 738                                            uint64_t features,
 739                                            Error **errp)
 740 {
 741     return features;
 742 }
 743
 744 static void virtio_crypto_reset(VirtIODevice *vdev)
 745 {
 746     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 747     /* multiqueue is disabled by default */
 748     vcrypto->curr_queues = 1;
 749     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 750         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 751     } else {
 752         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 753     }
 754 }
 755
 756 static void virtio_crypto_init_config(VirtIODevice *vdev)
 757 {
 758     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 759
 760     vcrypto->conf.crypto_services =
 761                      vcrypto->conf.cryptodev->conf.crypto_services;
 762     vcrypto->conf.cipher_algo_l =
 763                      vcrypto->conf.cryptodev->conf.cipher_algo_l;
 764     vcrypto->conf.cipher_algo_h =
 765                      vcrypto->conf.cryptodev->conf.cipher_algo_h;
 766     vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo;
 767     vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l;
 768     vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h;
 769     vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo;
 770     vcrypto->conf.max_cipher_key_len =
 771                   vcrypto->conf.cryptodev->conf.max_cipher_key_len;
 772     vcrypto->conf.max_auth_key_len =
 773                   vcrypto->conf.cryptodev->conf.max_auth_key_len;
 774     vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size;
 775 }
 776
 777 static void virtio_crypto_device_realize(DeviceState *dev, Error **errp)
 778 {
 779     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 780     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 781     int i;
 782
 783     vcrypto->cryptodev = vcrypto->conf.cryptodev;
 784     if (vcrypto->cryptodev == NULL) {
 785         error_setg(errp, "'cryptodev' parameter expects a valid object");
 786         return;
 787     } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) {
 788         char *path = object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev));
 789         error_setg(errp, "can't use already used cryptodev backend: %s", path);
 790         g_free(path);
 791         return;
 792     }
 793
 794     vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1);
 795     if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) {
 796         error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
 797                    "must be a positive integer less than %d.",
 798                    vcrypto->max_queues, VIRTIO_QUEUE_MAX);
 799         return;
 800     }
 801
 802     virtio_init(vdev, "virtio-crypto", VIRTIO_ID_CRYPTO, vcrypto->config_size);
 803     vcrypto->curr_queues = 1;
 804     vcrypto->vqs = g_malloc0(sizeof(VirtIOCryptoQueue) * vcrypto->max_queues);
 805     for (i = 0; i < vcrypto->max_queues; i++) {
 806         vcrypto->vqs[i].dataq =
 807                  virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh);
 808         vcrypto->vqs[i].dataq_bh =
 809                  qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]);
 810         vcrypto->vqs[i].vcrypto = vcrypto;
 811     }
 812
 813     vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl);
 814     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
 815         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
 816     } else {
 817         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
 818     }
 819
 820     virtio_crypto_init_config(vdev);
 821     cryptodev_backend_set_used(vcrypto->cryptodev, true);
 822 }
 823
 824 static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp)
 825 {
 826     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
 827     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
 828     VirtIOCryptoQueue *q;
 829     int i, max_queues;
 830
 831     max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1;
 832     for (i = 0; i < max_queues; i++) {
 833         virtio_del_queue(vdev, i);
 834         q = &vcrypto->vqs[i];
 835         qemu_bh_delete(q->dataq_bh);
 836     }
 837
 838     g_free(vcrypto->vqs);
 839
 840     virtio_cleanup(vdev);
 841     cryptodev_backend_set_used(vcrypto->cryptodev, false);
 842 }
 843
 844 static const VMStateDescription vmstate_virtio_crypto = {
 845     .name = "virtio-crypto",
 846     .unmigratable = 1,
 847     .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION,
 848     .version_id = VIRTIO_CRYPTO_VM_VERSION,
 849     .fields = (VMStateField[]) {
 850         VMSTATE_VIRTIO_DEVICE,
 851         VMSTATE_END_OF_LIST()
 852     },
 853 };
 854
 855 static Property virtio_crypto_properties[] = {
 856     DEFINE_PROP_LINK("cryptodev", VirtIOCrypto, conf.cryptodev,
 857                      TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *),
 858     DEFINE_PROP_END_OF_LIST(),
 859 };
 860
 861 static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config)
 862 {
 863     VirtIOCrypto *c = VIRTIO_CRYPTO(vdev);
 864     struct virtio_crypto_config crypto_cfg = {};
 865
 866     /*
 867      * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE,
 868      * so we can use LE accessors directly.
 869      */
 870     stl_le_p(&crypto_cfg.status, c->status);
 871     stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues);
 872     stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services);
 873     stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l);
 874     stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h);
 875     stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo);
 876     stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l);
 877     stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h);
 878     stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo);
 879     stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len);
 880     stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len);
 881     stq_le_p(&crypto_cfg.max_size, c->conf.max_size);
 882
 883     memcpy(config, &crypto_cfg, c->config_size);
 884 }
 885
 886 static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status)
 887 {
 888     VirtIODevice *vdev = VIRTIO_DEVICE(c);
 889     return (status & VIRTIO_CONFIG_S_DRIVER_OK) &&
 890         (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running;
 891 }
 892
 893 static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status)
 894 {
 895     VirtIODevice *vdev = VIRTIO_DEVICE(c);
 896     int queues = c->multiqueue ? c->max_queues : 1;
 897     CryptoDevBackend *b = c->cryptodev;
 898     CryptoDevBackendClient *cc = b->conf.peers.ccs[0];
 899
 900     if (!cryptodev_get_vhost(cc, b, 0)) {
 901         return;
 902     }
 903
 904     if ((virtio_crypto_started(c, status)) == !!c->vhost_started) {
 905         return;
 906     }
 907
 908     if (!c->vhost_started) {
 909         int r;
 910
 911         c->vhost_started = 1;
 912         r = cryptodev_vhost_start(vdev, queues);
 913         if (r < 0) {
 914             error_report("unable to start vhost crypto: %d: "
 915                          "falling back on userspace virtio", -r);
 916             c->vhost_started = 0;
 917         }
 918     } else {
 919         cryptodev_vhost_stop(vdev, queues);
 920         c->vhost_started = 0;
 921     }
 922 }
 923
 924 static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status)
 925 {
 926     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 927
 928     virtio_crypto_vhost_status(vcrypto, status);
 929 }
 930
 931 static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx,
 932                                            bool mask)
 933 {
 934     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 935     int queue = virtio_crypto_vq2q(idx);
 936
 937     assert(vcrypto->vhost_started);
 938
 939     cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask);
 940 }
 941
 942 static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx)
 943 {
 944     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
 945     int queue = virtio_crypto_vq2q(idx);
 946
 947     assert(vcrypto->vhost_started);
 948
 949     return cryptodev_vhost_virtqueue_pending(vdev, queue, idx);
 950 }
 951
 952 static void virtio_crypto_class_init(ObjectClass *klass, void *data)
 953 {
 954     DeviceClass *dc = DEVICE_CLASS(klass);
 955     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 956
 957     dc->props = virtio_crypto_properties;
 958     dc->vmsd = &vmstate_virtio_crypto;
 959     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 960     vdc->realize = virtio_crypto_device_realize;
 961     vdc->unrealize = virtio_crypto_device_unrealize;
 962     vdc->get_config = virtio_crypto_get_config;
 963     vdc->get_features = virtio_crypto_get_features;
 964     vdc->reset = virtio_crypto_reset;
 965     vdc->set_status = virtio_crypto_set_status;
 966     vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask;
 967     vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending;
 968 }
 969
 970 static void virtio_crypto_instance_init(Object *obj)
 971 {
 972     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj);
 973
 974     /*
 975      * The default config_size is sizeof(struct virtio_crypto_config).
 976      * Can be overriden with virtio_crypto_set_config_size.
 977      */
 978     vcrypto->config_size = sizeof(struct virtio_crypto_config);
 979 }
 980
 981 static const TypeInfo virtio_crypto_info = {
 982     .name = TYPE_VIRTIO_CRYPTO,
 983     .parent = TYPE_VIRTIO_DEVICE,
 984     .instance_size = sizeof(VirtIOCrypto),
 985     .instance_init = virtio_crypto_instance_init,
 986     .class_init = virtio_crypto_class_init,
 987 };
 988
 989 static void virtio_register_types(void)
 990 {
 991     type_register_static(&virtio_crypto_info);
 992 }
 993
 994 type_init(virtio_register_types)