drivers/crypto/cavium/nitrox/nitrox_reqmgr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/gfp.h>
   3 #include <linux/workqueue.h>
   4 #include <crypto/internal/skcipher.h>
   5
   6 #include "nitrox_dev.h"
   7 #include "nitrox_req.h"
   8 #include "nitrox_csr.h"
   9 #include "nitrox_req.h"
  10
  11 /* SLC_STORE_INFO */
  12 #define MIN_UDD_LEN 16
  13 /* PKT_IN_HDR + SLC_STORE_INFO */
  14 #define FDATA_SIZE 32
  15 /* Base destination port for the solicited requests */
  16 #define SOLICIT_BASE_DPORT 256
  17 #define PENDING_SIG     0xFFFFFFFFFFFFFFFFUL
  18
  19 #define REQ_NOT_POSTED 1
  20 #define REQ_BACKLOG    2
  21 #define REQ_POSTED     3
  22
  23 /**
  24  * Response codes from SE microcode
  25  * 0x00 - Success
  26  *   Completion with no error
  27  * 0x43 - ERR_GC_DATA_LEN_INVALID
  28  *   Invalid Data length if Encryption Data length is
  29  *   less than 16 bytes for AES-XTS and AES-CTS.
  30  * 0x45 - ERR_GC_CTX_LEN_INVALID
  31  *   Invalid context length: CTXL != 23 words.
  32  * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
  33  *   DOCSIS support is enabled with other than
  34  *   AES/DES-CBC mode encryption.
  35  * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
  36  *   Authentication offset is other than 0 with
  37  *   Encryption IV source = 0.
  38  *   Authentication offset is other than 8 (DES)/16 (AES)
  39  *   with Encryption IV source = 1
  40  * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
  41  *   CRC32 is enabled for other than DOCSIS encryption.
  42  * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
  43  *   Invalid flag options in AES-CCM IV.
  44  */
  45
  46 /**
  47  * dma_free_sglist - unmap and free the sg lists.
  48  * @ndev: N5 device
  49  * @sgtbl: SG table
  50  */
  51 static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
  52 {
  53         struct nitrox_device *ndev = sr->ndev;
  54         struct device *dev = DEV(ndev);
  55         struct nitrox_sglist *sglist;
  56
  57         /* unmap in sgbuf */
  58         sglist = sr->in.sglist;
  59         if (!sglist)
  60                 goto out_unmap;
  61
  62         /* unmap iv */
  63         dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL);
  64         /* unmpa src sglist */
  65         dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir);
  66         /* unamp gather component */
  67         dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE);
  68         kfree(sr->in.sglist);
  69         kfree(sr->in.sgcomp);
  70         sr->in.sglist = NULL;
  71         sr->in.buf = NULL;
  72         sr->in.map_bufs_cnt = 0;
  73
  74 out_unmap:
  75         /* unmap out sgbuf */
  76         sglist = sr->out.sglist;
  77         if (!sglist)
  78                 return;
  79
  80         /* unmap orh */
  81         dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
  82
  83         /* unmap dst sglist */
  84         if (!sr->inplace) {
  85                 dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3),
  86                              sr->out.dir);
  87         }
  88         /* unmap completion */
  89         dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
  90
  91         /* unmap scatter component */
  92         dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE);
  93         kfree(sr->out.sglist);
  94         kfree(sr->out.sgcomp);
  95         sr->out.sglist = NULL;
  96         sr->out.buf = NULL;
  97         sr->out.map_bufs_cnt = 0;
  98 }
  99
 100 static void softreq_destroy(struct nitrox_softreq *sr)
 101 {
 102         softreq_unmap_sgbufs(sr);
 103         kfree(sr);
 104 }
 105
 106 /**
 107  * create_sg_component - create SG componets for N5 device.
 108  * @sr: Request structure
 109  * @sgtbl: SG table
 110  * @nr_comp: total number of components required
 111  *
 112  * Component structure
 113  *
 114  *   63     48 47     32 31    16 15      0
 115  *   --------------------------------------
 116  *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
 117  *   |-------------------------------------
 118  *   |               PTR0                 |
 119  *   --------------------------------------
 120  *   |               PTR1                 |
 121  *   --------------------------------------
 122  *   |               PTR2                 |
 123  *   --------------------------------------
 124  *   |               PTR3                 |
 125  *   --------------------------------------
 126  *
 127  *   Returns 0 if success or a negative errno code on error.
 128  */
 129 static int create_sg_component(struct nitrox_softreq *sr,
 130                                struct nitrox_sgtable *sgtbl, int map_nents)
 131 {
 132         struct nitrox_device *ndev = sr->ndev;
 133         struct nitrox_sgcomp *sgcomp;
 134         struct nitrox_sglist *sglist;
 135         dma_addr_t dma;
 136         size_t sz_comp;
 137         int i, j, nr_sgcomp;
 138
 139         nr_sgcomp = roundup(map_nents, 4) / 4;
 140
 141         /* each component holds 4 dma pointers */
 142         sz_comp = nr_sgcomp * sizeof(*sgcomp);
 143         sgcomp = kzalloc(sz_comp, sr->gfp);
 144         if (!sgcomp)
 145                 return -ENOMEM;
 146
 147         sgtbl->sgcomp = sgcomp;
 148         sgtbl->nr_sgcomp = nr_sgcomp;
 149
 150         sglist = sgtbl->sglist;
 151         /* populate device sg component */
 152         for (i = 0; i < nr_sgcomp; i++) {
 153                 for (j = 0; j < 4; j++) {
 154                         sgcomp->len[j] = cpu_to_be16(sglist->len);
 155                         sgcomp->dma[j] = cpu_to_be64(sglist->dma);
 156                         sglist++;
 157                 }
 158                 sgcomp++;
 159         }
 160         /* map the device sg component */
 161         dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
 162         if (dma_mapping_error(DEV(ndev), dma)) {
 163                 kfree(sgtbl->sgcomp);
 164                 sgtbl->sgcomp = NULL;
 165                 return -ENOMEM;
 166         }
 167
 168         sgtbl->dma = dma;
 169         sgtbl->len = sz_comp;
 170
 171         return 0;
 172 }
 173
 174 /**
 175  * dma_map_inbufs - DMA map input sglist and creates sglist component
 176  *                  for N5 device.
 177  * @sr: Request structure
 178  * @req: Crypto request structre
 179  *
 180  * Returns 0 if successful or a negative errno code on error.
 181  */
 182 static int dma_map_inbufs(struct nitrox_softreq *sr,
 183                           struct se_crypto_request *req)
 184 {
 185         struct device *dev = DEV(sr->ndev);
 186         struct scatterlist *sg = req->src;
 187         struct nitrox_sglist *glist;
 188         int i, nents, ret = 0;
 189         dma_addr_t dma;
 190         size_t sz;
 191
 192         nents = sg_nents(req->src);
 193
 194         /* creater gather list IV and src entries */
 195         sz = roundup((1 + nents), 4) * sizeof(*glist);
 196         glist = kzalloc(sz, sr->gfp);
 197         if (!glist)
 198                 return -ENOMEM;
 199
 200         sr->in.sglist = glist;
 201         /* map IV */
 202         dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL);
 203         if (dma_mapping_error(dev, dma)) {
 204                 ret = -EINVAL;
 205                 goto iv_map_err;
 206         }
 207
 208         sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
 209         /* map src entries */
 210         nents = dma_map_sg(dev, req->src, nents, sr->in.dir);
 211         if (!nents) {
 212                 ret = -EINVAL;
 213                 goto src_map_err;
 214         }
 215         sr->in.buf = req->src;
 216
 217         /* store the mappings */
 218         glist->len = req->ivsize;
 219         glist->dma = dma;
 220         glist++;
 221         sr->in.total_bytes += req->ivsize;
 222
 223         for_each_sg(req->src, sg, nents, i) {
 224                 glist->len = sg_dma_len(sg);
 225                 glist->dma = sg_dma_address(sg);
 226                 sr->in.total_bytes += glist->len;
 227                 glist++;
 228         }
 229         /* roundup map count to align with entires in sg component */
 230         sr->in.map_bufs_cnt = (1 + nents);
 231
 232         /* create NITROX gather component */
 233         ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt);
 234         if (ret)
 235                 goto incomp_err;
 236
 237         return 0;
 238
 239 incomp_err:
 240         dma_unmap_sg(dev, req->src, nents, sr->in.dir);
 241         sr->in.map_bufs_cnt = 0;
 242 src_map_err:
 243         dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL);
 244 iv_map_err:
 245         kfree(sr->in.sglist);
 246         sr->in.sglist = NULL;
 247         return ret;
 248 }
 249
 250 static int dma_map_outbufs(struct nitrox_softreq *sr,
 251                            struct se_crypto_request *req)
 252 {
 253         struct device *dev = DEV(sr->ndev);
 254         struct nitrox_sglist *glist = sr->in.sglist;
 255         struct nitrox_sglist *slist;
 256         struct scatterlist *sg;
 257         int i, nents, map_bufs_cnt, ret = 0;
 258         size_t sz;
 259
 260         nents = sg_nents(req->dst);
 261
 262         /* create scatter list ORH, IV, dst entries and Completion header */
 263         sz = roundup((3 + nents), 4) * sizeof(*slist);
 264         slist = kzalloc(sz, sr->gfp);
 265         if (!slist)
 266                 return -ENOMEM;
 267
 268         sr->out.sglist = slist;
 269         sr->out.dir = DMA_BIDIRECTIONAL;
 270         /* map ORH */
 271         sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN,
 272                                           sr->out.dir);
 273         if (dma_mapping_error(dev, sr->resp.orh_dma)) {
 274                 ret = -EINVAL;
 275                 goto orh_map_err;
 276         }
 277
 278         /* map completion */
 279         sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion,
 280                                                  COMP_HLEN, sr->out.dir);
 281         if (dma_mapping_error(dev, sr->resp.completion_dma)) {
 282                 ret = -EINVAL;
 283                 goto compl_map_err;
 284         }
 285
 286         sr->inplace = (req->src == req->dst) ? true : false;
 287         /* out place */
 288         if (!sr->inplace) {
 289                 nents = dma_map_sg(dev, req->dst, nents, sr->out.dir);
 290                 if (!nents) {
 291                         ret = -EINVAL;
 292                         goto dst_map_err;
 293                 }
 294         }
 295         sr->out.buf = req->dst;
 296
 297         /* store the mappings */
 298         /* orh */
 299         slist->len = ORH_HLEN;
 300         slist->dma = sr->resp.orh_dma;
 301         slist++;
 302
 303         /* copy the glist mappings */
 304         if (sr->inplace) {
 305                 nents = sr->in.map_bufs_cnt - 1;
 306                 map_bufs_cnt = sr->in.map_bufs_cnt;
 307                 while (map_bufs_cnt--) {
 308                         slist->len = glist->len;
 309                         slist->dma = glist->dma;
 310                         slist++;
 311                         glist++;
 312                 }
 313         } else {
 314                 /* copy iv mapping */
 315                 slist->len = glist->len;
 316                 slist->dma = glist->dma;
 317                 slist++;
 318                 /* copy remaining maps */
 319                 for_each_sg(req->dst, sg, nents, i) {
 320                         slist->len = sg_dma_len(sg);
 321                         slist->dma = sg_dma_address(sg);
 322                         slist++;
 323                 }
 324         }
 325
 326         /* completion */
 327         slist->len = COMP_HLEN;
 328         slist->dma = sr->resp.completion_dma;
 329
 330         sr->out.map_bufs_cnt = (3 + nents);
 331
 332         ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt);
 333         if (ret)
 334                 goto outcomp_map_err;
 335
 336         return 0;
 337
 338 outcomp_map_err:
 339         if (!sr->inplace)
 340                 dma_unmap_sg(dev, req->dst, nents, sr->out.dir);
 341         sr->out.map_bufs_cnt = 0;
 342         sr->out.buf = NULL;
 343 dst_map_err:
 344         dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
 345         sr->resp.completion_dma = 0;
 346 compl_map_err:
 347         dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
 348         sr->resp.orh_dma = 0;
 349 orh_map_err:
 350         kfree(sr->out.sglist);
 351         sr->out.sglist = NULL;
 352         return ret;
 353 }
 354
 355 static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
 356                                     struct se_crypto_request *creq)
 357 {
 358         int ret;
 359
 360         ret = dma_map_inbufs(sr, creq);
 361         if (ret)
 362                 return ret;
 363
 364         ret = dma_map_outbufs(sr, creq);
 365         if (ret)
 366                 softreq_unmap_sgbufs(sr);
 367
 368         return ret;
 369 }
 370
 371 static inline void backlog_list_add(struct nitrox_softreq *sr,
 372                                     struct nitrox_cmdq *cmdq)
 373 {
 374         INIT_LIST_HEAD(&sr->backlog);
 375
 376         spin_lock_bh(&cmdq->backlog_lock);
 377         list_add_tail(&sr->backlog, &cmdq->backlog_head);
 378         atomic_inc(&cmdq->backlog_count);
 379         atomic_set(&sr->status, REQ_BACKLOG);
 380         spin_unlock_bh(&cmdq->backlog_lock);
 381 }
 382
 383 static inline void response_list_add(struct nitrox_softreq *sr,
 384                                      struct nitrox_cmdq *cmdq)
 385 {
 386         INIT_LIST_HEAD(&sr->response);
 387
 388         spin_lock_bh(&cmdq->response_lock);
 389         list_add_tail(&sr->response, &cmdq->response_head);
 390         spin_unlock_bh(&cmdq->response_lock);
 391 }
 392
 393 static inline void response_list_del(struct nitrox_softreq *sr,
 394                                      struct nitrox_cmdq *cmdq)
 395 {
 396         spin_lock_bh(&cmdq->response_lock);
 397         list_del(&sr->response);
 398         spin_unlock_bh(&cmdq->response_lock);
 399 }
 400
 401 static struct nitrox_softreq *
 402 get_first_response_entry(struct nitrox_cmdq *cmdq)
 403 {
 404         return list_first_entry_or_null(&cmdq->response_head,
 405                                         struct nitrox_softreq, response);
 406 }
 407
 408 static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
 409 {
 410         if (atomic_inc_return(&cmdq->pending_count) > qlen) {
 411                 atomic_dec(&cmdq->pending_count);
 412                 /* sync with other cpus */
 413                 smp_mb__after_atomic();
 414                 return true;
 415         }
 416         return false;
 417 }
 418
 419 /**
 420  * post_se_instr - Post SE instruction to Packet Input ring
 421  * @sr: Request structure
 422  *
 423  * Returns 0 if successful or a negative error code,
 424  * if no space in ring.
 425  */
 426 static void post_se_instr(struct nitrox_softreq *sr,
 427                           struct nitrox_cmdq *cmdq)
 428 {
 429         struct nitrox_device *ndev = sr->ndev;
 430         union nps_pkt_in_instr_baoff_dbell pkt_in_baoff_dbell;
 431         u64 offset;
 432         u8 *ent;
 433
 434         spin_lock_bh(&cmdq->cmdq_lock);
 435
 436         /* get the next write offset */
 437         offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(cmdq->qno);
 438         pkt_in_baoff_dbell.value = nitrox_read_csr(ndev, offset);
 439         /* copy the instruction */
 440         ent = cmdq->head + pkt_in_baoff_dbell.s.aoff;
 441         memcpy(ent, &sr->instr, cmdq->instr_size);
 442         /* flush the command queue updates */
 443         dma_wmb();
 444
 445         sr->tstamp = jiffies;
 446         atomic_set(&sr->status, REQ_POSTED);
 447         response_list_add(sr, cmdq);
 448
 449         /* Ring doorbell with count 1 */
 450         writeq(1, cmdq->dbell_csr_addr);
 451         /* orders the doorbell rings */
 452         mmiowb();
 453
 454         spin_unlock_bh(&cmdq->cmdq_lock);
 455 }
 456
 457 static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
 458 {
 459         struct nitrox_device *ndev = cmdq->ndev;
 460         struct nitrox_softreq *sr, *tmp;
 461         int ret = 0;
 462
 463         spin_lock_bh(&cmdq->backlog_lock);
 464
 465         list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
 466                 struct skcipher_request *skreq;
 467
 468                 /* submit until space available */
 469                 if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 470                         ret = -EBUSY;
 471                         break;
 472                 }
 473                 /* delete from backlog list */
 474                 list_del(&sr->backlog);
 475                 atomic_dec(&cmdq->backlog_count);
 476                 /* sync with other cpus */
 477                 smp_mb__after_atomic();
 478
 479                 skreq = sr->skreq;
 480                 /* post the command */
 481                 post_se_instr(sr, cmdq);
 482
 483                 /* backlog requests are posted, wakeup with -EINPROGRESS */
 484                 skcipher_request_complete(skreq, -EINPROGRESS);
 485         }
 486         spin_unlock_bh(&cmdq->backlog_lock);
 487
 488         return ret;
 489 }
 490
 491 static int nitrox_enqueue_request(struct nitrox_softreq *sr)
 492 {
 493         struct nitrox_cmdq *cmdq = sr->cmdq;
 494         struct nitrox_device *ndev = sr->ndev;
 495         int ret = -EBUSY;
 496
 497         if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
 498                 if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
 499                         return -EAGAIN;
 500
 501                 backlog_list_add(sr, cmdq);
 502         } else {
 503                 ret = post_backlog_cmds(cmdq);
 504                 if (ret) {
 505                         backlog_list_add(sr, cmdq);
 506                         return ret;
 507                 }
 508                 post_se_instr(sr, cmdq);
 509                 ret = -EINPROGRESS;
 510         }
 511         return ret;
 512 }
 513
 514 /**
 515  * nitrox_se_request - Send request to SE core
 516  * @ndev: NITROX device
 517  * @req: Crypto request
 518  *
 519  * Returns 0 on success, or a negative error code.
 520  */
 521 int nitrox_process_se_request(struct nitrox_device *ndev,
 522                               struct se_crypto_request *req,
 523                               completion_t callback,
 524                               struct skcipher_request *skreq)
 525 {
 526         struct nitrox_softreq *sr;
 527         dma_addr_t ctx_handle = 0;
 528         int qno, ret = 0;
 529
 530         if (!nitrox_ready(ndev))
 531                 return -ENODEV;
 532
 533         sr = kzalloc(sizeof(*sr), req->gfp);
 534         if (!sr)
 535                 return -ENOMEM;
 536
 537         sr->ndev = ndev;
 538         sr->flags = req->flags;
 539         sr->gfp = req->gfp;
 540         sr->callback = callback;
 541         sr->skreq = skreq;
 542
 543         atomic_set(&sr->status, REQ_NOT_POSTED);
 544
 545         WRITE_ONCE(sr->resp.orh, PENDING_SIG);
 546         WRITE_ONCE(sr->resp.completion, PENDING_SIG);
 547
 548         ret = softreq_map_iobuf(sr, req);
 549         if (ret) {
 550                 kfree(sr);
 551                 return ret;
 552         }
 553
 554         /* get the context handle */
 555         if (req->ctx_handle) {
 556                 struct ctx_hdr *hdr;
 557                 u8 *ctx_ptr;
 558
 559                 ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
 560                 hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
 561                 ctx_handle = hdr->ctx_dma;
 562         }
 563
 564         /* select the queue */
 565         qno = smp_processor_id() % ndev->nr_queues;
 566
 567         sr->cmdq = &ndev->pkt_cmdqs[qno];
 568
 569         /*
 570          * 64-Byte Instruction Format
 571          *
 572          *  ----------------------
 573          *  |      DPTR0         | 8 bytes
 574          *  ----------------------
 575          *  |  PKT_IN_INSTR_HDR  | 8 bytes
 576          *  ----------------------
 577          *  |    PKT_IN_HDR      | 16 bytes
 578          *  ----------------------
 579          *  |    SLC_INFO        | 16 bytes
 580          *  ----------------------
 581          *  |   Front data       | 16 bytes
 582          *  ----------------------
 583          */
 584
 585         /* fill the packet instruction */
 586         /* word 0 */
 587         sr->instr.dptr0 = cpu_to_be64(sr->in.dma);
 588
 589         /* word 1 */
 590         sr->instr.ih.value = 0;
 591         sr->instr.ih.s.g = 1;
 592         sr->instr.ih.s.gsz = sr->in.map_bufs_cnt;
 593         sr->instr.ih.s.ssz = sr->out.map_bufs_cnt;
 594         sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
 595         sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
 596         sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
 597
 598         /* word 2 */
 599         sr->instr.irh.value[0] = 0;
 600         sr->instr.irh.s.uddl = MIN_UDD_LEN;
 601         /* context length in 64-bit words */
 602         sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
 603         /* offset from solicit base port 256 */
 604         sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
 605         sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
 606         sr->instr.irh.s.arg = req->ctrl.s.arg;
 607         sr->instr.irh.s.opcode = req->opcode;
 608         sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]);
 609
 610         /* word 3 */
 611         sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
 612
 613         /* word 4 */
 614         sr->instr.slc.value[0] = 0;
 615         sr->instr.slc.s.ssz = sr->out.map_bufs_cnt;
 616         sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
 617
 618         /* word 5 */
 619         sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma);
 620
 621         /*
 622          * No conversion for front data,
 623          * It goes into payload
 624          * put GP Header in front data
 625          */
 626         sr->instr.fdata[0] = *((u64 *)&req->gph);
 627         sr->instr.fdata[1] = 0;
 628         /* flush the soft_req changes before posting the cmd */
 629         wmb();
 630
 631         ret = nitrox_enqueue_request(sr);
 632         if (ret == -EAGAIN)
 633                 goto send_fail;
 634
 635         return ret;
 636
 637 send_fail:
 638         softreq_destroy(sr);
 639         return ret;
 640 }
 641
 642 static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
 643 {
 644         return time_after_eq(jiffies, (tstamp + timeout));
 645 }
 646
 647 void backlog_qflush_work(struct work_struct *work)
 648 {
 649         struct nitrox_cmdq *cmdq;
 650
 651         cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
 652         post_backlog_cmds(cmdq);
 653 }
 654
 655 /**
 656  * process_request_list - process completed requests
 657  * @ndev: N5 device
 658  * @qno: queue to operate
 659  *
 660  * Returns the number of responses processed.
 661  */
 662 static void process_response_list(struct nitrox_cmdq *cmdq)
 663 {
 664         struct nitrox_device *ndev = cmdq->ndev;
 665         struct nitrox_softreq *sr;
 666         struct skcipher_request *skreq;
 667         completion_t callback;
 668         int req_completed = 0, err = 0, budget;
 669
 670         /* check all pending requests */
 671         budget = atomic_read(&cmdq->pending_count);
 672
 673         while (req_completed < budget) {
 674                 sr = get_first_response_entry(cmdq);
 675                 if (!sr)
 676                         break;
 677
 678                 if (atomic_read(&sr->status) != REQ_POSTED)
 679                         break;
 680
 681                 /* check orh and completion bytes updates */
 682                 if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) {
 683                         /* request not completed, check for timeout */
 684                         if (!cmd_timeout(sr->tstamp, ndev->timeout))
 685                                 break;
 686                         dev_err_ratelimited(DEV(ndev),
 687                                             "Request timeout, orh 0x%016llx\n",
 688                                             READ_ONCE(sr->resp.orh));
 689                 }
 690                 atomic_dec(&cmdq->pending_count);
 691                 /* sync with other cpus */
 692                 smp_mb__after_atomic();
 693                 /* remove from response list */
 694                 response_list_del(sr, cmdq);
 695
 696                 callback = sr->callback;
 697                 skreq = sr->skreq;
 698
 699                 /* ORH error code */
 700                 err = READ_ONCE(sr->resp.orh) & 0xff;
 701                 softreq_destroy(sr);
 702
 703                 if (callback)
 704                         callback(skreq, err);
 705
 706                 req_completed++;
 707         }
 708 }
 709
 710 /**
 711  * pkt_slc_resp_handler - post processing of SE responses
 712  */
 713 void pkt_slc_resp_handler(unsigned long data)
 714 {
 715         struct bh_data *bh = (void *)(uintptr_t)(data);
 716         struct nitrox_cmdq *cmdq = bh->cmdq;
 717         union nps_pkt_slc_cnts pkt_slc_cnts;
 718
 719         /* read completion count */
 720         pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr);
 721         /* resend the interrupt if more work to do */
 722         pkt_slc_cnts.s.resend = 1;
 723
 724         process_response_list(cmdq);
 725
 726         /*
 727          * clear the interrupt with resend bit enabled,
 728          * MSI-X interrupt generates if Completion count > Threshold
 729          */
 730         writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr);
 731         /* order the writes */
 732         mmiowb();
 733
 734         if (atomic_read(&cmdq->backlog_count))
 735                 schedule_work(&cmdq->backlog_qflush);
 736 }