]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/drivers/common/qat/qat_qp.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / spdk / dpdk / drivers / common / qat / qat_qp.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2018 Intel Corporation
3 */
4
5 #include <rte_common.h>
6 #include <rte_cycles.h>
7 #include <rte_dev.h>
8 #include <rte_malloc.h>
9 #include <rte_memzone.h>
10 #include <rte_pci.h>
11 #include <rte_bus_pci.h>
12 #include <rte_atomic.h>
13 #include <rte_prefetch.h>
14
15 #include "qat_logs.h"
16 #include "qat_device.h"
17 #include "qat_qp.h"
18 #include "qat_sym.h"
19 #include "qat_asym.h"
20 #include "qat_comp.h"
21 #include "adf_transport_access_macros.h"
22
23 #define QAT_CQ_MAX_DEQ_RETRIES 10
24
25 #define ADF_MAX_DESC 4096
26 #define ADF_MIN_DESC 128
27
28 #define ADF_ARB_REG_SLOT 0x1000
29 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C
30
31 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
32 ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
33 (ADF_ARB_REG_SLOT * index), value)
34
35 __extension__
36 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
37 [ADF_MAX_QPS_ON_ANY_SERVICE] = {
38 /* queue pairs which provide an asymmetric crypto service */
39 [QAT_SERVICE_ASYMMETRIC] = {
40 {
41 .service_type = QAT_SERVICE_ASYMMETRIC,
42 .hw_bundle_num = 0,
43 .tx_ring_num = 0,
44 .rx_ring_num = 8,
45 .tx_msg_size = 64,
46 .rx_msg_size = 32,
47
48 }, {
49 .service_type = QAT_SERVICE_ASYMMETRIC,
50 .hw_bundle_num = 0,
51 .tx_ring_num = 1,
52 .rx_ring_num = 9,
53 .tx_msg_size = 64,
54 .rx_msg_size = 32,
55 }
56 },
57 /* queue pairs which provide a symmetric crypto service */
58 [QAT_SERVICE_SYMMETRIC] = {
59 {
60 .service_type = QAT_SERVICE_SYMMETRIC,
61 .hw_bundle_num = 0,
62 .tx_ring_num = 2,
63 .rx_ring_num = 10,
64 .tx_msg_size = 128,
65 .rx_msg_size = 32,
66 },
67 {
68 .service_type = QAT_SERVICE_SYMMETRIC,
69 .hw_bundle_num = 0,
70 .tx_ring_num = 3,
71 .rx_ring_num = 11,
72 .tx_msg_size = 128,
73 .rx_msg_size = 32,
74 }
75 },
76 /* queue pairs which provide a compression service */
77 [QAT_SERVICE_COMPRESSION] = {
78 {
79 .service_type = QAT_SERVICE_COMPRESSION,
80 .hw_bundle_num = 0,
81 .tx_ring_num = 6,
82 .rx_ring_num = 14,
83 .tx_msg_size = 128,
84 .rx_msg_size = 32,
85 }, {
86 .service_type = QAT_SERVICE_COMPRESSION,
87 .hw_bundle_num = 0,
88 .tx_ring_num = 7,
89 .rx_ring_num = 15,
90 .tx_msg_size = 128,
91 .rx_msg_size = 32,
92 }
93 }
94 };
95
96 __extension__
97 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
98 [ADF_MAX_QPS_ON_ANY_SERVICE] = {
99 /* queue pairs which provide an asymmetric crypto service */
100 [QAT_SERVICE_ASYMMETRIC] = {
101 {
102 .service_type = QAT_SERVICE_ASYMMETRIC,
103 .hw_bundle_num = 0,
104 .tx_ring_num = 0,
105 .rx_ring_num = 4,
106 .tx_msg_size = 64,
107 .rx_msg_size = 32,
108 }
109 },
110 /* queue pairs which provide a symmetric crypto service */
111 [QAT_SERVICE_SYMMETRIC] = {
112 {
113 .service_type = QAT_SERVICE_SYMMETRIC,
114 .hw_bundle_num = 0,
115 .tx_ring_num = 1,
116 .rx_ring_num = 5,
117 .tx_msg_size = 128,
118 .rx_msg_size = 32,
119 }
120 },
121 /* queue pairs which provide a compression service */
122 [QAT_SERVICE_COMPRESSION] = {
123 {
124 .service_type = QAT_SERVICE_COMPRESSION,
125 .hw_bundle_num = 0,
126 .tx_ring_num = 3,
127 .rx_ring_num = 7,
128 .tx_msg_size = 128,
129 .rx_msg_size = 32,
130 }
131 }
132 };
133
134 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
135 uint32_t queue_size_bytes);
136 static void qat_queue_delete(struct qat_queue *queue);
137 static int qat_queue_create(struct qat_pci_device *qat_dev,
138 struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
139 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
140 uint32_t *queue_size_for_csr);
141 static void adf_configure_queues(struct qat_qp *queue);
142 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
143 rte_spinlock_t *lock);
144 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
145 rte_spinlock_t *lock);
146
147
148 int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
149 enum qat_service_type service)
150 {
151 int i, count;
152
153 for (i = 0, count = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++)
154 if (qp_hw_data[i].service_type == service)
155 count++;
156 return count;
157 }
158
159 static const struct rte_memzone *
160 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
161 int socket_id)
162 {
163 const struct rte_memzone *mz;
164
165 mz = rte_memzone_lookup(queue_name);
166 if (mz != 0) {
167 if (((size_t)queue_size <= mz->len) &&
168 ((socket_id == SOCKET_ID_ANY) ||
169 (socket_id == mz->socket_id))) {
170 QAT_LOG(DEBUG, "re-use memzone already "
171 "allocated for %s", queue_name);
172 return mz;
173 }
174
175 QAT_LOG(ERR, "Incompatible memzone already "
176 "allocated %s, size %u, socket %d. "
177 "Requested size %u, socket %u",
178 queue_name, (uint32_t)mz->len,
179 mz->socket_id, queue_size, socket_id);
180 return NULL;
181 }
182
183 QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
184 queue_name, queue_size, socket_id);
185 return rte_memzone_reserve_aligned(queue_name, queue_size,
186 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
187 }
188
189 int qat_qp_setup(struct qat_pci_device *qat_dev,
190 struct qat_qp **qp_addr,
191 uint16_t queue_pair_id,
192 struct qat_qp_config *qat_qp_conf)
193
194 {
195 struct qat_qp *qp;
196 struct rte_pci_device *pci_dev = qat_dev->pci_dev;
197 char op_cookie_pool_name[RTE_RING_NAMESIZE];
198 uint32_t i;
199
200 QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
201 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
202
203 if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
204 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
205 QAT_LOG(ERR, "Can't create qp for %u descriptors",
206 qat_qp_conf->nb_descriptors);
207 return -EINVAL;
208 }
209
210 if (pci_dev->mem_resource[0].addr == NULL) {
211 QAT_LOG(ERR, "Could not find VF config space "
212 "(UIO driver attached?).");
213 return -EINVAL;
214 }
215
216 /* Allocate the queue pair data structure. */
217 qp = rte_zmalloc_socket("qat PMD qp metadata",
218 sizeof(*qp), RTE_CACHE_LINE_SIZE,
219 qat_qp_conf->socket_id);
220 if (qp == NULL) {
221 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
222 return -ENOMEM;
223 }
224 qp->nb_descriptors = qat_qp_conf->nb_descriptors;
225 qp->op_cookies = rte_zmalloc_socket("qat PMD op cookie pointer",
226 qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
227 RTE_CACHE_LINE_SIZE, qat_qp_conf->socket_id);
228 if (qp->op_cookies == NULL) {
229 QAT_LOG(ERR, "Failed to alloc mem for cookie");
230 rte_free(qp);
231 return -ENOMEM;
232 }
233
234 qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
235 qp->enqueued = qp->dequeued = 0;
236
237 if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
238 ADF_RING_DIR_TX) != 0) {
239 QAT_LOG(ERR, "Tx queue create failed "
240 "queue_pair_id=%u", queue_pair_id);
241 goto create_err;
242 }
243
244 qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size,
245 ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size));
246
247 if (qp->max_inflights < 2) {
248 QAT_LOG(ERR, "Invalid num inflights");
249 qat_queue_delete(&(qp->tx_q));
250 goto create_err;
251 }
252
253 if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
254 ADF_RING_DIR_RX) != 0) {
255 QAT_LOG(ERR, "Rx queue create failed "
256 "queue_pair_id=%hu", queue_pair_id);
257 qat_queue_delete(&(qp->tx_q));
258 goto create_err;
259 }
260
261 adf_configure_queues(qp);
262 adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
263 &qat_dev->arb_csr_lock);
264
265 snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
266 "%s%d_cookies_%s_qp%hu",
267 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
268 qat_qp_conf->service_str, queue_pair_id);
269
270 QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
271 qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
272 if (qp->op_cookie_pool == NULL)
273 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
274 qp->nb_descriptors,
275 qat_qp_conf->cookie_size, 64, 0,
276 NULL, NULL, NULL, NULL,
277 qat_dev->pci_dev->device.numa_node,
278 0);
279 if (!qp->op_cookie_pool) {
280 QAT_LOG(ERR, "QAT PMD Cannot create"
281 " op mempool");
282 goto create_err;
283 }
284
285 for (i = 0; i < qp->nb_descriptors; i++) {
286 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
287 QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
288 goto create_err;
289 }
290 memset(qp->op_cookies[i], 0, qat_qp_conf->cookie_size);
291 }
292
293 qp->qat_dev_gen = qat_dev->qat_dev_gen;
294 qp->build_request = qat_qp_conf->build_request;
295 qp->service_type = qat_qp_conf->hw->service_type;
296 qp->qat_dev = qat_dev;
297
298 QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
299 queue_pair_id, op_cookie_pool_name);
300
301 *qp_addr = qp;
302 return 0;
303
304 create_err:
305 if (qp->op_cookie_pool)
306 rte_mempool_free(qp->op_cookie_pool);
307 rte_free(qp->op_cookies);
308 rte_free(qp);
309 return -EFAULT;
310 }
311
312 int qat_qp_release(struct qat_qp **qp_addr)
313 {
314 struct qat_qp *qp = *qp_addr;
315 uint32_t i;
316
317 if (qp == NULL) {
318 QAT_LOG(DEBUG, "qp already freed");
319 return 0;
320 }
321
322 QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
323 qp->qat_dev->qat_dev_id);
324
325 /* Don't free memory if there are still responses to be processed */
326 if ((qp->enqueued - qp->dequeued) == 0) {
327 qat_queue_delete(&(qp->tx_q));
328 qat_queue_delete(&(qp->rx_q));
329 } else {
330 return -EAGAIN;
331 }
332
333 adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
334 &qp->qat_dev->arb_csr_lock);
335
336 for (i = 0; i < qp->nb_descriptors; i++)
337 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
338
339 if (qp->op_cookie_pool)
340 rte_mempool_free(qp->op_cookie_pool);
341
342 rte_free(qp->op_cookies);
343 rte_free(qp);
344 *qp_addr = NULL;
345 return 0;
346 }
347
348
349 static void qat_queue_delete(struct qat_queue *queue)
350 {
351 const struct rte_memzone *mz;
352 int status = 0;
353
354 if (queue == NULL) {
355 QAT_LOG(DEBUG, "Invalid queue");
356 return;
357 }
358 QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
359 queue->hw_queue_number, queue->memz_name);
360
361 mz = rte_memzone_lookup(queue->memz_name);
362 if (mz != NULL) {
363 /* Write an unused pattern to the queue memory. */
364 memset(queue->base_addr, 0x7F, queue->queue_size);
365 status = rte_memzone_free(mz);
366 if (status != 0)
367 QAT_LOG(ERR, "Error %d on freeing queue %s",
368 status, queue->memz_name);
369 } else {
370 QAT_LOG(DEBUG, "queue %s doesn't exist",
371 queue->memz_name);
372 }
373 }
374
375 static int
376 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
377 struct qat_qp_config *qp_conf, uint8_t dir)
378 {
379 uint64_t queue_base;
380 void *io_addr;
381 const struct rte_memzone *qp_mz;
382 struct rte_pci_device *pci_dev = qat_dev->pci_dev;
383 int ret = 0;
384 uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
385 qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
386 uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
387
388 queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
389 queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
390 qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
391
392 if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
393 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
394 return -EINVAL;
395 }
396
397 /*
398 * Allocate a memzone for the queue - create a unique name.
399 */
400 snprintf(queue->memz_name, sizeof(queue->memz_name),
401 "%s_%d_%s_%s_%d_%d",
402 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
403 qp_conf->service_str, "qp_mem",
404 queue->hw_bundle_number, queue->hw_queue_number);
405 qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
406 qat_dev->pci_dev->device.numa_node);
407 if (qp_mz == NULL) {
408 QAT_LOG(ERR, "Failed to allocate ring memzone");
409 return -ENOMEM;
410 }
411
412 queue->base_addr = (char *)qp_mz->addr;
413 queue->base_phys_addr = qp_mz->iova;
414 if (qat_qp_check_queue_alignment(queue->base_phys_addr,
415 queue_size_bytes)) {
416 QAT_LOG(ERR, "Invalid alignment on queue create "
417 " 0x%"PRIx64"\n",
418 queue->base_phys_addr);
419 ret = -EFAULT;
420 goto queue_create_err;
421 }
422
423 if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
424 &(queue->queue_size)) != 0) {
425 QAT_LOG(ERR, "Invalid num inflights");
426 ret = -EINVAL;
427 goto queue_create_err;
428 }
429
430 queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
431 queue->head = 0;
432 queue->tail = 0;
433 queue->msg_size = desc_size;
434
435 /* For fast calculation of cookie index, relies on msg_size being 2^n */
436 queue->trailz = __builtin_ctz(desc_size);
437
438 /*
439 * Write an unused pattern to the queue memory.
440 */
441 memset(queue->base_addr, 0x7F, queue_size_bytes);
442
443 queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
444 queue->queue_size);
445
446 io_addr = pci_dev->mem_resource[0].addr;
447
448 WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
449 queue->hw_queue_number, queue_base);
450
451 QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
452 " nb msgs %u, msg_size %u, modulo mask %u",
453 queue->memz_name,
454 queue->queue_size, queue_size_bytes,
455 qp_conf->nb_descriptors, desc_size,
456 queue->modulo_mask);
457
458 return 0;
459
460 queue_create_err:
461 rte_memzone_free(qp_mz);
462 return ret;
463 }
464
465 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
466 uint32_t queue_size_bytes)
467 {
468 if (((queue_size_bytes - 1) & phys_addr) != 0)
469 return -EINVAL;
470 return 0;
471 }
472
473 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
474 uint32_t *p_queue_size_for_csr)
475 {
476 uint8_t i = ADF_MIN_RING_SIZE;
477
478 for (; i <= ADF_MAX_RING_SIZE; i++)
479 if ((msg_size * msg_num) ==
480 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
481 *p_queue_size_for_csr = i;
482 return 0;
483 }
484 QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
485 return -EINVAL;
486 }
487
488 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
489 rte_spinlock_t *lock)
490 {
491 uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
492 (ADF_ARB_REG_SLOT *
493 txq->hw_bundle_number);
494 uint32_t value;
495
496 rte_spinlock_lock(lock);
497 value = ADF_CSR_RD(base_addr, arb_csr_offset);
498 value |= (0x01 << txq->hw_queue_number);
499 ADF_CSR_WR(base_addr, arb_csr_offset, value);
500 rte_spinlock_unlock(lock);
501 }
502
503 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
504 rte_spinlock_t *lock)
505 {
506 uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
507 (ADF_ARB_REG_SLOT *
508 txq->hw_bundle_number);
509 uint32_t value;
510
511 rte_spinlock_lock(lock);
512 value = ADF_CSR_RD(base_addr, arb_csr_offset);
513 value &= ~(0x01 << txq->hw_queue_number);
514 ADF_CSR_WR(base_addr, arb_csr_offset, value);
515 rte_spinlock_unlock(lock);
516 }
517
518 static void adf_configure_queues(struct qat_qp *qp)
519 {
520 uint32_t queue_config;
521 struct qat_queue *queue = &qp->tx_q;
522
523 queue_config = BUILD_RING_CONFIG(queue->queue_size);
524
525 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
526 queue->hw_queue_number, queue_config);
527
528 queue = &qp->rx_q;
529 queue_config =
530 BUILD_RESP_RING_CONFIG(queue->queue_size,
531 ADF_RING_NEAR_WATERMARK_512,
532 ADF_RING_NEAR_WATERMARK_0);
533
534 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
535 queue->hw_queue_number, queue_config);
536 }
537
538 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
539 {
540 return data & modulo_mask;
541 }
542
543 static inline void
544 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
545 WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
546 q->hw_queue_number, q->tail);
547 q->csr_tail = q->tail;
548 }
549
550 static inline
551 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
552 {
553 uint32_t old_head, new_head;
554 uint32_t max_head;
555
556 old_head = q->csr_head;
557 new_head = q->head;
558 max_head = qp->nb_descriptors * q->msg_size;
559
560 /* write out free descriptors */
561 void *cur_desc = (uint8_t *)q->base_addr + old_head;
562
563 if (new_head < old_head) {
564 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
565 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
566 } else {
567 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
568 }
569 q->nb_processed_responses = 0;
570 q->csr_head = new_head;
571
572 /* write current head to CSR */
573 WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
574 q->hw_queue_number, new_head);
575 }
576
577 uint16_t
578 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
579 {
580 register struct qat_queue *queue;
581 struct qat_qp *tmp_qp = (struct qat_qp *)qp;
582 register uint32_t nb_ops_sent = 0;
583 register int ret;
584 uint16_t nb_ops_possible = nb_ops;
585 register uint8_t *base_addr;
586 register uint32_t tail;
587
588 if (unlikely(nb_ops == 0))
589 return 0;
590
591 /* read params used a lot in main loop into registers */
592 queue = &(tmp_qp->tx_q);
593 base_addr = (uint8_t *)queue->base_addr;
594 tail = queue->tail;
595
596 /* Find how many can actually fit on the ring */
597 {
598 /* dequeued can only be written by one thread, but it may not
599 * be this thread. As it's 4-byte aligned it will be read
600 * atomically here by any Intel CPU.
601 * enqueued can wrap before dequeued, but cannot
602 * lap it as var size of enq/deq (uint32_t) > var size of
603 * max_inflights (uint16_t). In reality inflights is never
604 * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
605 * On wrapping, the calculation still returns the correct
606 * positive value as all three vars are unsigned.
607 */
608 uint32_t inflights =
609 tmp_qp->enqueued - tmp_qp->dequeued;
610
611 if ((inflights + nb_ops) > tmp_qp->max_inflights) {
612 nb_ops_possible = tmp_qp->max_inflights - inflights;
613 if (nb_ops_possible == 0)
614 return 0;
615 }
616 /* QAT has plenty of work queued already, so don't waste cycles
617 * enqueueing, wait til the application has gathered a bigger
618 * burst or some completed ops have been dequeued
619 */
620 if (tmp_qp->min_enq_burst_threshold && inflights >
621 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
622 tmp_qp->min_enq_burst_threshold) {
623 tmp_qp->stats.threshold_hit_count++;
624 return 0;
625 }
626 }
627
628
629 while (nb_ops_sent != nb_ops_possible) {
630 ret = tmp_qp->build_request(*ops, base_addr + tail,
631 tmp_qp->op_cookies[tail >> queue->trailz],
632 tmp_qp->qat_dev_gen);
633 if (ret != 0) {
634 tmp_qp->stats.enqueue_err_count++;
635 /* This message cannot be enqueued */
636 if (nb_ops_sent == 0)
637 return 0;
638 goto kick_tail;
639 }
640
641 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
642 ops++;
643 nb_ops_sent++;
644 }
645 kick_tail:
646 queue->tail = tail;
647 tmp_qp->enqueued += nb_ops_sent;
648 tmp_qp->stats.enqueued_count += nb_ops_sent;
649 txq_write_tail(tmp_qp, queue);
650 return nb_ops_sent;
651 }
652
653 /* Use this for compression only - but keep consistent with above common
654 * function as much as possible.
655 */
656 uint16_t
657 qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
658 {
659 register struct qat_queue *queue;
660 struct qat_qp *tmp_qp = (struct qat_qp *)qp;
661 register uint32_t nb_ops_sent = 0;
662 register int nb_desc_to_build;
663 uint16_t nb_ops_possible = nb_ops;
664 register uint8_t *base_addr;
665 register uint32_t tail;
666
667 int descriptors_built, total_descriptors_built = 0;
668 int nb_remaining_descriptors;
669 int overflow = 0;
670
671 if (unlikely(nb_ops == 0))
672 return 0;
673
674 /* read params used a lot in main loop into registers */
675 queue = &(tmp_qp->tx_q);
676 base_addr = (uint8_t *)queue->base_addr;
677 tail = queue->tail;
678
679 /* Find how many can actually fit on the ring */
680 {
681 /* dequeued can only be written by one thread, but it may not
682 * be this thread. As it's 4-byte aligned it will be read
683 * atomically here by any Intel CPU.
684 * enqueued can wrap before dequeued, but cannot
685 * lap it as var size of enq/deq (uint32_t) > var size of
686 * max_inflights (uint16_t). In reality inflights is never
687 * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
688 * On wrapping, the calculation still returns the correct
689 * positive value as all three vars are unsigned.
690 */
691 uint32_t inflights =
692 tmp_qp->enqueued - tmp_qp->dequeued;
693
694 /* Find how many can actually fit on the ring */
695 overflow = (inflights + nb_ops) - tmp_qp->max_inflights;
696 if (overflow > 0) {
697 nb_ops_possible = nb_ops - overflow;
698 if (nb_ops_possible == 0)
699 return 0;
700 }
701
702 /* QAT has plenty of work queued already, so don't waste cycles
703 * enqueueing, wait til the application has gathered a bigger
704 * burst or some completed ops have been dequeued
705 */
706 if (tmp_qp->min_enq_burst_threshold && inflights >
707 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
708 tmp_qp->min_enq_burst_threshold) {
709 tmp_qp->stats.threshold_hit_count++;
710 return 0;
711 }
712 }
713
714 /* At this point nb_ops_possible is assuming a 1:1 mapping
715 * between ops and descriptors.
716 * Fewer may be sent if some ops have to be split.
717 * nb_ops_possible is <= burst size.
718 * Find out how many spaces are actually available on the qp in case
719 * more are needed.
720 */
721 nb_remaining_descriptors = nb_ops_possible
722 + ((overflow >= 0) ? 0 : overflow * (-1));
723 QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d",
724 nb_ops, nb_remaining_descriptors);
725
726 while (nb_ops_sent != nb_ops_possible &&
727 nb_remaining_descriptors > 0) {
728 struct qat_comp_op_cookie *cookie =
729 tmp_qp->op_cookies[tail >> queue->trailz];
730
731 descriptors_built = 0;
732
733 QAT_DP_LOG(DEBUG, "--- data length: %u",
734 ((struct rte_comp_op *)*ops)->src.length);
735
736 nb_desc_to_build = qat_comp_build_request(*ops,
737 base_addr + tail, cookie, tmp_qp->qat_dev_gen);
738 QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, "
739 "%d ops sent, %d descriptors needed",
740 total_descriptors_built, nb_remaining_descriptors,
741 nb_ops_sent, nb_desc_to_build);
742
743 if (unlikely(nb_desc_to_build < 0)) {
744 /* this message cannot be enqueued */
745 tmp_qp->stats.enqueue_err_count++;
746 if (nb_ops_sent == 0)
747 return 0;
748 goto kick_tail;
749 } else if (unlikely(nb_desc_to_build > 1)) {
750 /* this op is too big and must be split - get more
751 * descriptors and retry
752 */
753
754 QAT_DP_LOG(DEBUG, "Build %d descriptors for this op",
755 nb_desc_to_build);
756
757 nb_remaining_descriptors -= nb_desc_to_build;
758 if (nb_remaining_descriptors >= 0) {
759 /* There are enough remaining descriptors
760 * so retry
761 */
762 int ret2 = qat_comp_build_multiple_requests(
763 *ops, tmp_qp, tail,
764 nb_desc_to_build);
765
766 if (unlikely(ret2 < 1)) {
767 QAT_DP_LOG(DEBUG,
768 "Failed to build (%d) descriptors, status %d",
769 nb_desc_to_build, ret2);
770
771 qat_comp_free_split_op_memzones(cookie,
772 nb_desc_to_build - 1);
773
774 tmp_qp->stats.enqueue_err_count++;
775
776 /* This message cannot be enqueued */
777 if (nb_ops_sent == 0)
778 return 0;
779 goto kick_tail;
780 } else {
781 descriptors_built = ret2;
782 total_descriptors_built +=
783 descriptors_built;
784 nb_remaining_descriptors -=
785 descriptors_built;
786 QAT_DP_LOG(DEBUG,
787 "Multiple descriptors (%d) built ok",
788 descriptors_built);
789 }
790 } else {
791 QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) "
792 "exceeds number of available descriptors (%d)",
793 nb_desc_to_build,
794 nb_remaining_descriptors +
795 nb_desc_to_build);
796
797 qat_comp_free_split_op_memzones(cookie,
798 nb_desc_to_build - 1);
799
800 /* Not enough extra descriptors */
801 if (nb_ops_sent == 0)
802 return 0;
803 goto kick_tail;
804 }
805 } else {
806 descriptors_built = 1;
807 total_descriptors_built++;
808 nb_remaining_descriptors--;
809 QAT_DP_LOG(DEBUG, "Single descriptor built ok");
810 }
811
812 tail = adf_modulo(tail + (queue->msg_size * descriptors_built),
813 queue->modulo_mask);
814 ops++;
815 nb_ops_sent++;
816 }
817
818 kick_tail:
819 queue->tail = tail;
820 tmp_qp->enqueued += total_descriptors_built;
821 tmp_qp->stats.enqueued_count += nb_ops_sent;
822 txq_write_tail(tmp_qp, queue);
823 return nb_ops_sent;
824 }
825
826 uint16_t
827 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
828 {
829 struct qat_queue *rx_queue;
830 struct qat_qp *tmp_qp = (struct qat_qp *)qp;
831 uint32_t head;
832 uint32_t op_resp_counter = 0, fw_resp_counter = 0;
833 uint8_t *resp_msg;
834 int nb_fw_responses;
835
836 rx_queue = &(tmp_qp->rx_q);
837 head = rx_queue->head;
838 resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
839
840 while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
841 op_resp_counter != nb_ops) {
842
843 nb_fw_responses = 1;
844
845 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
846 qat_sym_process_response(ops, resp_msg);
847 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
848 nb_fw_responses = qat_comp_process_response(
849 ops, resp_msg,
850 tmp_qp->op_cookies[head >> rx_queue->trailz],
851 &tmp_qp->stats.dequeue_err_count);
852 #ifdef BUILD_QAT_ASYM
853 else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
854 qat_asym_process_response(ops, resp_msg,
855 tmp_qp->op_cookies[head >> rx_queue->trailz]);
856 #endif
857
858 head = adf_modulo(head + rx_queue->msg_size,
859 rx_queue->modulo_mask);
860
861 resp_msg = (uint8_t *)rx_queue->base_addr + head;
862
863 if (nb_fw_responses) {
864 /* only move on to next op if one was ready to return
865 * to API
866 */
867 ops++;
868 op_resp_counter++;
869 }
870
871 /* A compression op may be broken up into multiple fw requests.
872 * Only count fw responses as complete once ALL the responses
873 * associated with an op have been processed, as the cookie
874 * data from the first response must be available until
875 * finished with all firmware responses.
876 */
877 fw_resp_counter += nb_fw_responses;
878
879 rx_queue->nb_processed_responses++;
880 }
881
882 tmp_qp->dequeued += fw_resp_counter;
883 tmp_qp->stats.dequeued_count += op_resp_counter;
884
885 rx_queue->head = head;
886 if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
887 rxq_free_desc(tmp_qp, rx_queue);
888
889 QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u",
890 op_resp_counter, fw_resp_counter);
891
892 return op_resp_counter;
893 }
894
895 /* This is almost same as dequeue_op_burst, without the atomic, without stats
896 * and without the op. Dequeues one response.
897 */
898 static uint8_t
899 qat_cq_dequeue_response(struct qat_qp *qp, void *out_data)
900 {
901 uint8_t result = 0;
902 uint8_t retries = 0;
903 struct qat_queue *queue = &(qp->rx_q);
904 struct icp_qat_fw_comn_resp *resp_msg = (struct icp_qat_fw_comn_resp *)
905 ((uint8_t *)queue->base_addr + queue->head);
906
907 while (retries++ < QAT_CQ_MAX_DEQ_RETRIES &&
908 *(uint32_t *)resp_msg == ADF_RING_EMPTY_SIG) {
909 /* loop waiting for response until we reach the timeout */
910 rte_delay_ms(20);
911 }
912
913 if (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG) {
914 /* response received */
915 result = 1;
916
917 /* check status flag */
918 if (ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
919 resp_msg->comn_hdr.comn_status) ==
920 ICP_QAT_FW_COMN_STATUS_FLAG_OK) {
921 /* success */
922 memcpy(out_data, resp_msg, queue->msg_size);
923 } else {
924 memset(out_data, 0, queue->msg_size);
925 }
926
927 queue->head = adf_modulo(queue->head + queue->msg_size,
928 queue->modulo_mask);
929 rxq_free_desc(qp, queue);
930 }
931
932 return result;
933 }
934
935 /* Sends a NULL message and extracts QAT fw version from the response.
936 * Used to determine detailed capabilities based on the fw version number.
937 * This assumes that there are no inflight messages, i.e. assumes there's space
938 * on the qp, one message is sent and only one response collected.
939 * Returns fw version number or 0 for unknown version or a negative error code.
940 */
941 int
942 qat_cq_get_fw_version(struct qat_qp *qp)
943 {
944 struct qat_queue *queue = &(qp->tx_q);
945 uint8_t *base_addr = (uint8_t *)queue->base_addr;
946 struct icp_qat_fw_comn_req null_msg;
947 struct icp_qat_fw_comn_resp response;
948
949 /* prepare the NULL request */
950 memset(&null_msg, 0, sizeof(null_msg));
951 null_msg.comn_hdr.hdr_flags =
952 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
953 null_msg.comn_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
954 null_msg.comn_hdr.service_cmd_id = ICP_QAT_FW_NULL_REQ_SERV_ID;
955
956 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
957 QAT_DP_HEXDUMP_LOG(DEBUG, "NULL request", &null_msg, sizeof(null_msg));
958 #endif
959
960 /* send the NULL request */
961 memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg));
962 queue->tail = adf_modulo(queue->tail + queue->msg_size,
963 queue->modulo_mask);
964 txq_write_tail(qp, queue);
965
966 /* receive a response */
967 if (qat_cq_dequeue_response(qp, &response)) {
968
969 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
970 QAT_DP_HEXDUMP_LOG(DEBUG, "NULL response:", &response,
971 sizeof(response));
972 #endif
973 /* if LW0 bit 24 is set - then the fw version was returned */
974 if (QAT_FIELD_GET(response.comn_hdr.hdr_flags,
975 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_BITPOS,
976 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_MASK))
977 return response.resrvd[0]; /* return LW4 */
978 else
979 return 0; /* not set - we don't know fw version */
980 }
981
982 QAT_LOG(ERR, "No response received");
983 return -EINVAL;
984 }
985
986 __rte_weak int
987 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
988 void *op_cookie __rte_unused,
989 uint64_t *dequeue_err_count __rte_unused)
990 {
991 return 0;
992 }
Ceph