]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* SPDX-License-Identifier: BSD-3-Clause |
2 | * Copyright(c) 2015-2018 Intel Corporation | |
3 | */ | |
4 | ||
5 | #include <rte_common.h> | |
f67539c2 | 6 | #include <rte_cycles.h> |
11fdf7f2 TL |
7 | #include <rte_dev.h> |
8 | #include <rte_malloc.h> | |
9 | #include <rte_memzone.h> | |
10 | #include <rte_pci.h> | |
11 | #include <rte_bus_pci.h> | |
12 | #include <rte_atomic.h> | |
13 | #include <rte_prefetch.h> | |
14 | ||
15 | #include "qat_logs.h" | |
16 | #include "qat_device.h" | |
17 | #include "qat_qp.h" | |
18 | #include "qat_sym.h" | |
9f95a23c | 19 | #include "qat_asym.h" |
11fdf7f2 TL |
20 | #include "qat_comp.h" |
21 | #include "adf_transport_access_macros.h" | |
22 | ||
f67539c2 | 23 | #define QAT_CQ_MAX_DEQ_RETRIES 10 |
11fdf7f2 TL |
24 | |
25 | #define ADF_MAX_DESC 4096 | |
26 | #define ADF_MIN_DESC 128 | |
27 | ||
28 | #define ADF_ARB_REG_SLOT 0x1000 | |
29 | #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C | |
30 | ||
31 | #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ | |
32 | ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ | |
33 | (ADF_ARB_REG_SLOT * index), value) | |
34 | ||
35 | __extension__ | |
36 | const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES] | |
37 | [ADF_MAX_QPS_ON_ANY_SERVICE] = { | |
38 | /* queue pairs which provide an asymmetric crypto service */ | |
39 | [QAT_SERVICE_ASYMMETRIC] = { | |
40 | { | |
41 | .service_type = QAT_SERVICE_ASYMMETRIC, | |
42 | .hw_bundle_num = 0, | |
43 | .tx_ring_num = 0, | |
44 | .rx_ring_num = 8, | |
45 | .tx_msg_size = 64, | |
46 | .rx_msg_size = 32, | |
47 | ||
48 | }, { | |
49 | .service_type = QAT_SERVICE_ASYMMETRIC, | |
50 | .hw_bundle_num = 0, | |
51 | .tx_ring_num = 1, | |
52 | .rx_ring_num = 9, | |
53 | .tx_msg_size = 64, | |
54 | .rx_msg_size = 32, | |
55 | } | |
56 | }, | |
57 | /* queue pairs which provide a symmetric crypto service */ | |
58 | [QAT_SERVICE_SYMMETRIC] = { | |
59 | { | |
60 | .service_type = QAT_SERVICE_SYMMETRIC, | |
61 | .hw_bundle_num = 0, | |
62 | .tx_ring_num = 2, | |
63 | .rx_ring_num = 10, | |
64 | .tx_msg_size = 128, | |
65 | .rx_msg_size = 32, | |
66 | }, | |
67 | { | |
68 | .service_type = QAT_SERVICE_SYMMETRIC, | |
69 | .hw_bundle_num = 0, | |
70 | .tx_ring_num = 3, | |
71 | .rx_ring_num = 11, | |
72 | .tx_msg_size = 128, | |
73 | .rx_msg_size = 32, | |
74 | } | |
75 | }, | |
76 | /* queue pairs which provide a compression service */ | |
77 | [QAT_SERVICE_COMPRESSION] = { | |
78 | { | |
79 | .service_type = QAT_SERVICE_COMPRESSION, | |
80 | .hw_bundle_num = 0, | |
81 | .tx_ring_num = 6, | |
82 | .rx_ring_num = 14, | |
83 | .tx_msg_size = 128, | |
84 | .rx_msg_size = 32, | |
85 | }, { | |
86 | .service_type = QAT_SERVICE_COMPRESSION, | |
87 | .hw_bundle_num = 0, | |
88 | .tx_ring_num = 7, | |
89 | .rx_ring_num = 15, | |
90 | .tx_msg_size = 128, | |
91 | .rx_msg_size = 32, | |
92 | } | |
93 | } | |
94 | }; | |
95 | ||
9f95a23c TL |
96 | __extension__ |
97 | const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES] | |
98 | [ADF_MAX_QPS_ON_ANY_SERVICE] = { | |
99 | /* queue pairs which provide an asymmetric crypto service */ | |
100 | [QAT_SERVICE_ASYMMETRIC] = { | |
101 | { | |
102 | .service_type = QAT_SERVICE_ASYMMETRIC, | |
103 | .hw_bundle_num = 0, | |
104 | .tx_ring_num = 0, | |
105 | .rx_ring_num = 4, | |
106 | .tx_msg_size = 64, | |
107 | .rx_msg_size = 32, | |
108 | } | |
109 | }, | |
110 | /* queue pairs which provide a symmetric crypto service */ | |
111 | [QAT_SERVICE_SYMMETRIC] = { | |
112 | { | |
113 | .service_type = QAT_SERVICE_SYMMETRIC, | |
114 | .hw_bundle_num = 0, | |
115 | .tx_ring_num = 1, | |
116 | .rx_ring_num = 5, | |
117 | .tx_msg_size = 128, | |
118 | .rx_msg_size = 32, | |
119 | } | |
120 | }, | |
121 | /* queue pairs which provide a compression service */ | |
122 | [QAT_SERVICE_COMPRESSION] = { | |
123 | { | |
124 | .service_type = QAT_SERVICE_COMPRESSION, | |
125 | .hw_bundle_num = 0, | |
126 | .tx_ring_num = 3, | |
127 | .rx_ring_num = 7, | |
128 | .tx_msg_size = 128, | |
129 | .rx_msg_size = 32, | |
130 | } | |
131 | } | |
132 | }; | |
133 | ||
11fdf7f2 TL |
134 | static int qat_qp_check_queue_alignment(uint64_t phys_addr, |
135 | uint32_t queue_size_bytes); | |
136 | static void qat_queue_delete(struct qat_queue *queue); | |
137 | static int qat_queue_create(struct qat_pci_device *qat_dev, | |
138 | struct qat_queue *queue, struct qat_qp_config *, uint8_t dir); | |
139 | static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num, | |
140 | uint32_t *queue_size_for_csr); | |
141 | static void adf_configure_queues(struct qat_qp *queue); | |
142 | static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr, | |
143 | rte_spinlock_t *lock); | |
144 | static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr, | |
145 | rte_spinlock_t *lock); | |
146 | ||
147 | ||
148 | int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data, | |
149 | enum qat_service_type service) | |
150 | { | |
151 | int i, count; | |
152 | ||
153 | for (i = 0, count = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++) | |
154 | if (qp_hw_data[i].service_type == service) | |
155 | count++; | |
156 | return count; | |
157 | } | |
158 | ||
159 | static const struct rte_memzone * | |
160 | queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size, | |
161 | int socket_id) | |
162 | { | |
163 | const struct rte_memzone *mz; | |
164 | ||
165 | mz = rte_memzone_lookup(queue_name); | |
166 | if (mz != 0) { | |
167 | if (((size_t)queue_size <= mz->len) && | |
168 | ((socket_id == SOCKET_ID_ANY) || | |
169 | (socket_id == mz->socket_id))) { | |
170 | QAT_LOG(DEBUG, "re-use memzone already " | |
171 | "allocated for %s", queue_name); | |
172 | return mz; | |
173 | } | |
174 | ||
175 | QAT_LOG(ERR, "Incompatible memzone already " | |
176 | "allocated %s, size %u, socket %d. " | |
177 | "Requested size %u, socket %u", | |
178 | queue_name, (uint32_t)mz->len, | |
179 | mz->socket_id, queue_size, socket_id); | |
180 | return NULL; | |
181 | } | |
182 | ||
183 | QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u", | |
184 | queue_name, queue_size, socket_id); | |
185 | return rte_memzone_reserve_aligned(queue_name, queue_size, | |
186 | socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size); | |
187 | } | |
188 | ||
189 | int qat_qp_setup(struct qat_pci_device *qat_dev, | |
190 | struct qat_qp **qp_addr, | |
191 | uint16_t queue_pair_id, | |
192 | struct qat_qp_config *qat_qp_conf) | |
193 | ||
194 | { | |
195 | struct qat_qp *qp; | |
196 | struct rte_pci_device *pci_dev = qat_dev->pci_dev; | |
197 | char op_cookie_pool_name[RTE_RING_NAMESIZE]; | |
198 | uint32_t i; | |
199 | ||
200 | QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d", | |
201 | queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen); | |
202 | ||
203 | if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) || | |
204 | (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) { | |
205 | QAT_LOG(ERR, "Can't create qp for %u descriptors", | |
206 | qat_qp_conf->nb_descriptors); | |
207 | return -EINVAL; | |
208 | } | |
209 | ||
210 | if (pci_dev->mem_resource[0].addr == NULL) { | |
211 | QAT_LOG(ERR, "Could not find VF config space " | |
212 | "(UIO driver attached?)."); | |
213 | return -EINVAL; | |
214 | } | |
215 | ||
216 | /* Allocate the queue pair data structure. */ | |
9f95a23c TL |
217 | qp = rte_zmalloc_socket("qat PMD qp metadata", |
218 | sizeof(*qp), RTE_CACHE_LINE_SIZE, | |
219 | qat_qp_conf->socket_id); | |
11fdf7f2 TL |
220 | if (qp == NULL) { |
221 | QAT_LOG(ERR, "Failed to alloc mem for qp struct"); | |
222 | return -ENOMEM; | |
223 | } | |
224 | qp->nb_descriptors = qat_qp_conf->nb_descriptors; | |
9f95a23c | 225 | qp->op_cookies = rte_zmalloc_socket("qat PMD op cookie pointer", |
11fdf7f2 | 226 | qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies), |
9f95a23c | 227 | RTE_CACHE_LINE_SIZE, qat_qp_conf->socket_id); |
11fdf7f2 TL |
228 | if (qp->op_cookies == NULL) { |
229 | QAT_LOG(ERR, "Failed to alloc mem for cookie"); | |
230 | rte_free(qp); | |
231 | return -ENOMEM; | |
232 | } | |
233 | ||
234 | qp->mmap_bar_addr = pci_dev->mem_resource[0].addr; | |
f67539c2 | 235 | qp->enqueued = qp->dequeued = 0; |
11fdf7f2 TL |
236 | |
237 | if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf, | |
238 | ADF_RING_DIR_TX) != 0) { | |
239 | QAT_LOG(ERR, "Tx queue create failed " | |
240 | "queue_pair_id=%u", queue_pair_id); | |
241 | goto create_err; | |
242 | } | |
243 | ||
f67539c2 TL |
244 | qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size, |
245 | ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size)); | |
246 | ||
247 | if (qp->max_inflights < 2) { | |
248 | QAT_LOG(ERR, "Invalid num inflights"); | |
249 | qat_queue_delete(&(qp->tx_q)); | |
250 | goto create_err; | |
251 | } | |
252 | ||
11fdf7f2 TL |
253 | if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf, |
254 | ADF_RING_DIR_RX) != 0) { | |
255 | QAT_LOG(ERR, "Rx queue create failed " | |
256 | "queue_pair_id=%hu", queue_pair_id); | |
257 | qat_queue_delete(&(qp->tx_q)); | |
258 | goto create_err; | |
259 | } | |
260 | ||
261 | adf_configure_queues(qp); | |
262 | adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr, | |
263 | &qat_dev->arb_csr_lock); | |
264 | ||
265 | snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, | |
266 | "%s%d_cookies_%s_qp%hu", | |
267 | pci_dev->driver->driver.name, qat_dev->qat_dev_id, | |
268 | qat_qp_conf->service_str, queue_pair_id); | |
269 | ||
270 | QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name); | |
271 | qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name); | |
272 | if (qp->op_cookie_pool == NULL) | |
273 | qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name, | |
274 | qp->nb_descriptors, | |
275 | qat_qp_conf->cookie_size, 64, 0, | |
9f95a23c TL |
276 | NULL, NULL, NULL, NULL, |
277 | qat_dev->pci_dev->device.numa_node, | |
11fdf7f2 TL |
278 | 0); |
279 | if (!qp->op_cookie_pool) { | |
280 | QAT_LOG(ERR, "QAT PMD Cannot create" | |
281 | " op mempool"); | |
282 | goto create_err; | |
283 | } | |
284 | ||
285 | for (i = 0; i < qp->nb_descriptors; i++) { | |
286 | if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) { | |
287 | QAT_LOG(ERR, "QAT PMD Cannot get op_cookie"); | |
288 | goto create_err; | |
289 | } | |
9f95a23c | 290 | memset(qp->op_cookies[i], 0, qat_qp_conf->cookie_size); |
11fdf7f2 TL |
291 | } |
292 | ||
293 | qp->qat_dev_gen = qat_dev->qat_dev_gen; | |
294 | qp->build_request = qat_qp_conf->build_request; | |
295 | qp->service_type = qat_qp_conf->hw->service_type; | |
296 | qp->qat_dev = qat_dev; | |
297 | ||
298 | QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s", | |
299 | queue_pair_id, op_cookie_pool_name); | |
300 | ||
301 | *qp_addr = qp; | |
302 | return 0; | |
303 | ||
304 | create_err: | |
305 | if (qp->op_cookie_pool) | |
306 | rte_mempool_free(qp->op_cookie_pool); | |
307 | rte_free(qp->op_cookies); | |
308 | rte_free(qp); | |
309 | return -EFAULT; | |
310 | } | |
311 | ||
312 | int qat_qp_release(struct qat_qp **qp_addr) | |
313 | { | |
314 | struct qat_qp *qp = *qp_addr; | |
315 | uint32_t i; | |
316 | ||
317 | if (qp == NULL) { | |
318 | QAT_LOG(DEBUG, "qp already freed"); | |
319 | return 0; | |
320 | } | |
321 | ||
322 | QAT_LOG(DEBUG, "Free qp on qat_pci device %d", | |
323 | qp->qat_dev->qat_dev_id); | |
324 | ||
325 | /* Don't free memory if there are still responses to be processed */ | |
f67539c2 | 326 | if ((qp->enqueued - qp->dequeued) == 0) { |
11fdf7f2 TL |
327 | qat_queue_delete(&(qp->tx_q)); |
328 | qat_queue_delete(&(qp->rx_q)); | |
329 | } else { | |
330 | return -EAGAIN; | |
331 | } | |
332 | ||
333 | adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr, | |
334 | &qp->qat_dev->arb_csr_lock); | |
335 | ||
336 | for (i = 0; i < qp->nb_descriptors; i++) | |
337 | rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]); | |
338 | ||
339 | if (qp->op_cookie_pool) | |
340 | rte_mempool_free(qp->op_cookie_pool); | |
341 | ||
342 | rte_free(qp->op_cookies); | |
343 | rte_free(qp); | |
344 | *qp_addr = NULL; | |
345 | return 0; | |
346 | } | |
347 | ||
348 | ||
349 | static void qat_queue_delete(struct qat_queue *queue) | |
350 | { | |
351 | const struct rte_memzone *mz; | |
352 | int status = 0; | |
353 | ||
354 | if (queue == NULL) { | |
355 | QAT_LOG(DEBUG, "Invalid queue"); | |
356 | return; | |
357 | } | |
358 | QAT_LOG(DEBUG, "Free ring %d, memzone: %s", | |
359 | queue->hw_queue_number, queue->memz_name); | |
360 | ||
361 | mz = rte_memzone_lookup(queue->memz_name); | |
362 | if (mz != NULL) { | |
363 | /* Write an unused pattern to the queue memory. */ | |
364 | memset(queue->base_addr, 0x7F, queue->queue_size); | |
365 | status = rte_memzone_free(mz); | |
366 | if (status != 0) | |
367 | QAT_LOG(ERR, "Error %d on freeing queue %s", | |
368 | status, queue->memz_name); | |
369 | } else { | |
370 | QAT_LOG(DEBUG, "queue %s doesn't exist", | |
371 | queue->memz_name); | |
372 | } | |
373 | } | |
374 | ||
375 | static int | |
376 | qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, | |
377 | struct qat_qp_config *qp_conf, uint8_t dir) | |
378 | { | |
379 | uint64_t queue_base; | |
380 | void *io_addr; | |
381 | const struct rte_memzone *qp_mz; | |
382 | struct rte_pci_device *pci_dev = qat_dev->pci_dev; | |
383 | int ret = 0; | |
384 | uint16_t desc_size = (dir == ADF_RING_DIR_TX ? | |
385 | qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size); | |
386 | uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size); | |
387 | ||
388 | queue->hw_bundle_number = qp_conf->hw->hw_bundle_num; | |
389 | queue->hw_queue_number = (dir == ADF_RING_DIR_TX ? | |
390 | qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num); | |
391 | ||
392 | if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { | |
393 | QAT_LOG(ERR, "Invalid descriptor size %d", desc_size); | |
394 | return -EINVAL; | |
395 | } | |
396 | ||
397 | /* | |
398 | * Allocate a memzone for the queue - create a unique name. | |
399 | */ | |
400 | snprintf(queue->memz_name, sizeof(queue->memz_name), | |
401 | "%s_%d_%s_%s_%d_%d", | |
402 | pci_dev->driver->driver.name, qat_dev->qat_dev_id, | |
403 | qp_conf->service_str, "qp_mem", | |
404 | queue->hw_bundle_number, queue->hw_queue_number); | |
405 | qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes, | |
9f95a23c | 406 | qat_dev->pci_dev->device.numa_node); |
11fdf7f2 TL |
407 | if (qp_mz == NULL) { |
408 | QAT_LOG(ERR, "Failed to allocate ring memzone"); | |
409 | return -ENOMEM; | |
410 | } | |
411 | ||
412 | queue->base_addr = (char *)qp_mz->addr; | |
413 | queue->base_phys_addr = qp_mz->iova; | |
414 | if (qat_qp_check_queue_alignment(queue->base_phys_addr, | |
415 | queue_size_bytes)) { | |
416 | QAT_LOG(ERR, "Invalid alignment on queue create " | |
417 | " 0x%"PRIx64"\n", | |
418 | queue->base_phys_addr); | |
419 | ret = -EFAULT; | |
420 | goto queue_create_err; | |
421 | } | |
422 | ||
423 | if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors, | |
424 | &(queue->queue_size)) != 0) { | |
425 | QAT_LOG(ERR, "Invalid num inflights"); | |
426 | ret = -EINVAL; | |
427 | goto queue_create_err; | |
428 | } | |
429 | ||
11fdf7f2 | 430 | queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1; |
11fdf7f2 TL |
431 | queue->head = 0; |
432 | queue->tail = 0; | |
433 | queue->msg_size = desc_size; | |
434 | ||
f67539c2 TL |
435 | /* For fast calculation of cookie index, relies on msg_size being 2^n */ |
436 | queue->trailz = __builtin_ctz(desc_size); | |
437 | ||
11fdf7f2 TL |
438 | /* |
439 | * Write an unused pattern to the queue memory. | |
440 | */ | |
441 | memset(queue->base_addr, 0x7F, queue_size_bytes); | |
442 | ||
443 | queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr, | |
444 | queue->queue_size); | |
445 | ||
446 | io_addr = pci_dev->mem_resource[0].addr; | |
447 | ||
448 | WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number, | |
449 | queue->hw_queue_number, queue_base); | |
450 | ||
451 | QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u," | |
f67539c2 | 452 | " nb msgs %u, msg_size %u, modulo mask %u", |
11fdf7f2 TL |
453 | queue->memz_name, |
454 | queue->queue_size, queue_size_bytes, | |
455 | qp_conf->nb_descriptors, desc_size, | |
f67539c2 | 456 | queue->modulo_mask); |
11fdf7f2 TL |
457 | |
458 | return 0; | |
459 | ||
460 | queue_create_err: | |
461 | rte_memzone_free(qp_mz); | |
462 | return ret; | |
463 | } | |
464 | ||
465 | static int qat_qp_check_queue_alignment(uint64_t phys_addr, | |
466 | uint32_t queue_size_bytes) | |
467 | { | |
468 | if (((queue_size_bytes - 1) & phys_addr) != 0) | |
469 | return -EINVAL; | |
470 | return 0; | |
471 | } | |
472 | ||
473 | static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num, | |
474 | uint32_t *p_queue_size_for_csr) | |
475 | { | |
476 | uint8_t i = ADF_MIN_RING_SIZE; | |
477 | ||
478 | for (; i <= ADF_MAX_RING_SIZE; i++) | |
479 | if ((msg_size * msg_num) == | |
480 | (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) { | |
481 | *p_queue_size_for_csr = i; | |
482 | return 0; | |
483 | } | |
484 | QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num); | |
485 | return -EINVAL; | |
486 | } | |
487 | ||
488 | static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr, | |
489 | rte_spinlock_t *lock) | |
490 | { | |
491 | uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + | |
492 | (ADF_ARB_REG_SLOT * | |
493 | txq->hw_bundle_number); | |
494 | uint32_t value; | |
495 | ||
496 | rte_spinlock_lock(lock); | |
497 | value = ADF_CSR_RD(base_addr, arb_csr_offset); | |
498 | value |= (0x01 << txq->hw_queue_number); | |
499 | ADF_CSR_WR(base_addr, arb_csr_offset, value); | |
500 | rte_spinlock_unlock(lock); | |
501 | } | |
502 | ||
503 | static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr, | |
504 | rte_spinlock_t *lock) | |
505 | { | |
506 | uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + | |
507 | (ADF_ARB_REG_SLOT * | |
508 | txq->hw_bundle_number); | |
509 | uint32_t value; | |
510 | ||
511 | rte_spinlock_lock(lock); | |
512 | value = ADF_CSR_RD(base_addr, arb_csr_offset); | |
513 | value &= ~(0x01 << txq->hw_queue_number); | |
514 | ADF_CSR_WR(base_addr, arb_csr_offset, value); | |
515 | rte_spinlock_unlock(lock); | |
516 | } | |
517 | ||
518 | static void adf_configure_queues(struct qat_qp *qp) | |
519 | { | |
520 | uint32_t queue_config; | |
521 | struct qat_queue *queue = &qp->tx_q; | |
522 | ||
523 | queue_config = BUILD_RING_CONFIG(queue->queue_size); | |
524 | ||
525 | WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number, | |
526 | queue->hw_queue_number, queue_config); | |
527 | ||
528 | queue = &qp->rx_q; | |
529 | queue_config = | |
530 | BUILD_RESP_RING_CONFIG(queue->queue_size, | |
531 | ADF_RING_NEAR_WATERMARK_512, | |
532 | ADF_RING_NEAR_WATERMARK_0); | |
533 | ||
534 | WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number, | |
535 | queue->hw_queue_number, queue_config); | |
536 | } | |
537 | ||
538 | static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask) | |
539 | { | |
540 | return data & modulo_mask; | |
541 | } | |
542 | ||
543 | static inline void | |
544 | txq_write_tail(struct qat_qp *qp, struct qat_queue *q) { | |
545 | WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, | |
546 | q->hw_queue_number, q->tail); | |
11fdf7f2 TL |
547 | q->csr_tail = q->tail; |
548 | } | |
549 | ||
550 | static inline | |
551 | void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q) | |
552 | { | |
553 | uint32_t old_head, new_head; | |
554 | uint32_t max_head; | |
555 | ||
556 | old_head = q->csr_head; | |
557 | new_head = q->head; | |
558 | max_head = qp->nb_descriptors * q->msg_size; | |
559 | ||
560 | /* write out free descriptors */ | |
561 | void *cur_desc = (uint8_t *)q->base_addr + old_head; | |
562 | ||
563 | if (new_head < old_head) { | |
564 | memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head); | |
565 | memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head); | |
566 | } else { | |
567 | memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head); | |
568 | } | |
569 | q->nb_processed_responses = 0; | |
570 | q->csr_head = new_head; | |
571 | ||
572 | /* write current head to CSR */ | |
573 | WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, | |
574 | q->hw_queue_number, new_head); | |
575 | } | |
576 | ||
577 | uint16_t | |
578 | qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops) | |
579 | { | |
580 | register struct qat_queue *queue; | |
581 | struct qat_qp *tmp_qp = (struct qat_qp *)qp; | |
582 | register uint32_t nb_ops_sent = 0; | |
583 | register int ret; | |
584 | uint16_t nb_ops_possible = nb_ops; | |
585 | register uint8_t *base_addr; | |
586 | register uint32_t tail; | |
11fdf7f2 TL |
587 | |
588 | if (unlikely(nb_ops == 0)) | |
589 | return 0; | |
590 | ||
591 | /* read params used a lot in main loop into registers */ | |
592 | queue = &(tmp_qp->tx_q); | |
593 | base_addr = (uint8_t *)queue->base_addr; | |
594 | tail = queue->tail; | |
595 | ||
596 | /* Find how many can actually fit on the ring */ | |
f67539c2 TL |
597 | { |
598 | /* dequeued can only be written by one thread, but it may not | |
599 | * be this thread. As it's 4-byte aligned it will be read | |
600 | * atomically here by any Intel CPU. | |
601 | * enqueued can wrap before dequeued, but cannot | |
602 | * lap it as var size of enq/deq (uint32_t) > var size of | |
603 | * max_inflights (uint16_t). In reality inflights is never | |
604 | * even as big as max uint16_t, as it's <= ADF_MAX_DESC. | |
605 | * On wrapping, the calculation still returns the correct | |
606 | * positive value as all three vars are unsigned. | |
607 | */ | |
608 | uint32_t inflights = | |
609 | tmp_qp->enqueued - tmp_qp->dequeued; | |
610 | ||
611 | if ((inflights + nb_ops) > tmp_qp->max_inflights) { | |
612 | nb_ops_possible = tmp_qp->max_inflights - inflights; | |
613 | if (nb_ops_possible == 0) | |
614 | return 0; | |
615 | } | |
616 | /* QAT has plenty of work queued already, so don't waste cycles | |
617 | * enqueueing, wait til the application has gathered a bigger | |
618 | * burst or some completed ops have been dequeued | |
619 | */ | |
620 | if (tmp_qp->min_enq_burst_threshold && inflights > | |
621 | QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible < | |
622 | tmp_qp->min_enq_burst_threshold) { | |
623 | tmp_qp->stats.threshold_hit_count++; | |
11fdf7f2 | 624 | return 0; |
f67539c2 | 625 | } |
11fdf7f2 TL |
626 | } |
627 | ||
f67539c2 | 628 | |
11fdf7f2 TL |
629 | while (nb_ops_sent != nb_ops_possible) { |
630 | ret = tmp_qp->build_request(*ops, base_addr + tail, | |
f67539c2 | 631 | tmp_qp->op_cookies[tail >> queue->trailz], |
11fdf7f2 TL |
632 | tmp_qp->qat_dev_gen); |
633 | if (ret != 0) { | |
634 | tmp_qp->stats.enqueue_err_count++; | |
f67539c2 | 635 | /* This message cannot be enqueued */ |
11fdf7f2 TL |
636 | if (nb_ops_sent == 0) |
637 | return 0; | |
638 | goto kick_tail; | |
639 | } | |
640 | ||
641 | tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask); | |
642 | ops++; | |
643 | nb_ops_sent++; | |
644 | } | |
645 | kick_tail: | |
646 | queue->tail = tail; | |
f67539c2 | 647 | tmp_qp->enqueued += nb_ops_sent; |
11fdf7f2 | 648 | tmp_qp->stats.enqueued_count += nb_ops_sent; |
f67539c2 TL |
649 | txq_write_tail(tmp_qp, queue); |
650 | return nb_ops_sent; | |
651 | } | |
652 | ||
653 | /* Use this for compression only - but keep consistent with above common | |
654 | * function as much as possible. | |
655 | */ | |
656 | uint16_t | |
657 | qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops) | |
658 | { | |
659 | register struct qat_queue *queue; | |
660 | struct qat_qp *tmp_qp = (struct qat_qp *)qp; | |
661 | register uint32_t nb_ops_sent = 0; | |
662 | register int nb_desc_to_build; | |
663 | uint16_t nb_ops_possible = nb_ops; | |
664 | register uint8_t *base_addr; | |
665 | register uint32_t tail; | |
666 | ||
667 | int descriptors_built, total_descriptors_built = 0; | |
668 | int nb_remaining_descriptors; | |
669 | int overflow = 0; | |
670 | ||
671 | if (unlikely(nb_ops == 0)) | |
672 | return 0; | |
673 | ||
674 | /* read params used a lot in main loop into registers */ | |
675 | queue = &(tmp_qp->tx_q); | |
676 | base_addr = (uint8_t *)queue->base_addr; | |
677 | tail = queue->tail; | |
678 | ||
679 | /* Find how many can actually fit on the ring */ | |
680 | { | |
681 | /* dequeued can only be written by one thread, but it may not | |
682 | * be this thread. As it's 4-byte aligned it will be read | |
683 | * atomically here by any Intel CPU. | |
684 | * enqueued can wrap before dequeued, but cannot | |
685 | * lap it as var size of enq/deq (uint32_t) > var size of | |
686 | * max_inflights (uint16_t). In reality inflights is never | |
687 | * even as big as max uint16_t, as it's <= ADF_MAX_DESC. | |
688 | * On wrapping, the calculation still returns the correct | |
689 | * positive value as all three vars are unsigned. | |
690 | */ | |
691 | uint32_t inflights = | |
692 | tmp_qp->enqueued - tmp_qp->dequeued; | |
693 | ||
694 | /* Find how many can actually fit on the ring */ | |
695 | overflow = (inflights + nb_ops) - tmp_qp->max_inflights; | |
696 | if (overflow > 0) { | |
697 | nb_ops_possible = nb_ops - overflow; | |
698 | if (nb_ops_possible == 0) | |
699 | return 0; | |
700 | } | |
701 | ||
702 | /* QAT has plenty of work queued already, so don't waste cycles | |
703 | * enqueueing, wait til the application has gathered a bigger | |
704 | * burst or some completed ops have been dequeued | |
705 | */ | |
706 | if (tmp_qp->min_enq_burst_threshold && inflights > | |
707 | QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible < | |
708 | tmp_qp->min_enq_burst_threshold) { | |
709 | tmp_qp->stats.threshold_hit_count++; | |
710 | return 0; | |
711 | } | |
712 | } | |
713 | ||
714 | /* At this point nb_ops_possible is assuming a 1:1 mapping | |
715 | * between ops and descriptors. | |
716 | * Fewer may be sent if some ops have to be split. | |
717 | * nb_ops_possible is <= burst size. | |
718 | * Find out how many spaces are actually available on the qp in case | |
719 | * more are needed. | |
720 | */ | |
721 | nb_remaining_descriptors = nb_ops_possible | |
722 | + ((overflow >= 0) ? 0 : overflow * (-1)); | |
723 | QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d", | |
724 | nb_ops, nb_remaining_descriptors); | |
725 | ||
726 | while (nb_ops_sent != nb_ops_possible && | |
727 | nb_remaining_descriptors > 0) { | |
728 | struct qat_comp_op_cookie *cookie = | |
729 | tmp_qp->op_cookies[tail >> queue->trailz]; | |
730 | ||
731 | descriptors_built = 0; | |
732 | ||
733 | QAT_DP_LOG(DEBUG, "--- data length: %u", | |
734 | ((struct rte_comp_op *)*ops)->src.length); | |
735 | ||
736 | nb_desc_to_build = qat_comp_build_request(*ops, | |
737 | base_addr + tail, cookie, tmp_qp->qat_dev_gen); | |
738 | QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, " | |
739 | "%d ops sent, %d descriptors needed", | |
740 | total_descriptors_built, nb_remaining_descriptors, | |
741 | nb_ops_sent, nb_desc_to_build); | |
742 | ||
743 | if (unlikely(nb_desc_to_build < 0)) { | |
744 | /* this message cannot be enqueued */ | |
745 | tmp_qp->stats.enqueue_err_count++; | |
746 | if (nb_ops_sent == 0) | |
747 | return 0; | |
748 | goto kick_tail; | |
749 | } else if (unlikely(nb_desc_to_build > 1)) { | |
750 | /* this op is too big and must be split - get more | |
751 | * descriptors and retry | |
752 | */ | |
753 | ||
754 | QAT_DP_LOG(DEBUG, "Build %d descriptors for this op", | |
755 | nb_desc_to_build); | |
756 | ||
757 | nb_remaining_descriptors -= nb_desc_to_build; | |
758 | if (nb_remaining_descriptors >= 0) { | |
759 | /* There are enough remaining descriptors | |
760 | * so retry | |
761 | */ | |
762 | int ret2 = qat_comp_build_multiple_requests( | |
763 | *ops, tmp_qp, tail, | |
764 | nb_desc_to_build); | |
765 | ||
766 | if (unlikely(ret2 < 1)) { | |
767 | QAT_DP_LOG(DEBUG, | |
768 | "Failed to build (%d) descriptors, status %d", | |
769 | nb_desc_to_build, ret2); | |
770 | ||
771 | qat_comp_free_split_op_memzones(cookie, | |
772 | nb_desc_to_build - 1); | |
773 | ||
774 | tmp_qp->stats.enqueue_err_count++; | |
775 | ||
776 | /* This message cannot be enqueued */ | |
777 | if (nb_ops_sent == 0) | |
778 | return 0; | |
779 | goto kick_tail; | |
780 | } else { | |
781 | descriptors_built = ret2; | |
782 | total_descriptors_built += | |
783 | descriptors_built; | |
784 | nb_remaining_descriptors -= | |
785 | descriptors_built; | |
786 | QAT_DP_LOG(DEBUG, | |
787 | "Multiple descriptors (%d) built ok", | |
788 | descriptors_built); | |
789 | } | |
790 | } else { | |
791 | QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) " | |
792 | "exceeds number of available descriptors (%d)", | |
793 | nb_desc_to_build, | |
794 | nb_remaining_descriptors + | |
795 | nb_desc_to_build); | |
796 | ||
797 | qat_comp_free_split_op_memzones(cookie, | |
798 | nb_desc_to_build - 1); | |
799 | ||
800 | /* Not enough extra descriptors */ | |
801 | if (nb_ops_sent == 0) | |
802 | return 0; | |
803 | goto kick_tail; | |
804 | } | |
805 | } else { | |
806 | descriptors_built = 1; | |
807 | total_descriptors_built++; | |
808 | nb_remaining_descriptors--; | |
809 | QAT_DP_LOG(DEBUG, "Single descriptor built ok"); | |
810 | } | |
811 | ||
812 | tail = adf_modulo(tail + (queue->msg_size * descriptors_built), | |
813 | queue->modulo_mask); | |
814 | ops++; | |
815 | nb_ops_sent++; | |
11fdf7f2 | 816 | } |
f67539c2 TL |
817 | |
818 | kick_tail: | |
819 | queue->tail = tail; | |
820 | tmp_qp->enqueued += total_descriptors_built; | |
821 | tmp_qp->stats.enqueued_count += nb_ops_sent; | |
822 | txq_write_tail(tmp_qp, queue); | |
11fdf7f2 TL |
823 | return nb_ops_sent; |
824 | } | |
825 | ||
826 | uint16_t | |
827 | qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops) | |
828 | { | |
f67539c2 | 829 | struct qat_queue *rx_queue; |
11fdf7f2 TL |
830 | struct qat_qp *tmp_qp = (struct qat_qp *)qp; |
831 | uint32_t head; | |
f67539c2 | 832 | uint32_t op_resp_counter = 0, fw_resp_counter = 0; |
11fdf7f2 | 833 | uint8_t *resp_msg; |
f67539c2 | 834 | int nb_fw_responses; |
11fdf7f2 TL |
835 | |
836 | rx_queue = &(tmp_qp->rx_q); | |
11fdf7f2 TL |
837 | head = rx_queue->head; |
838 | resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head; | |
839 | ||
840 | while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG && | |
f67539c2 TL |
841 | op_resp_counter != nb_ops) { |
842 | ||
843 | nb_fw_responses = 1; | |
11fdf7f2 TL |
844 | |
845 | if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) | |
846 | qat_sym_process_response(ops, resp_msg); | |
847 | else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) | |
f67539c2 TL |
848 | nb_fw_responses = qat_comp_process_response( |
849 | ops, resp_msg, | |
850 | tmp_qp->op_cookies[head >> rx_queue->trailz], | |
851 | &tmp_qp->stats.dequeue_err_count); | |
9f95a23c | 852 | #ifdef BUILD_QAT_ASYM |
f67539c2 | 853 | else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) |
9f95a23c | 854 | qat_asym_process_response(ops, resp_msg, |
f67539c2 | 855 | tmp_qp->op_cookies[head >> rx_queue->trailz]); |
9f95a23c | 856 | #endif |
11fdf7f2 TL |
857 | |
858 | head = adf_modulo(head + rx_queue->msg_size, | |
859 | rx_queue->modulo_mask); | |
860 | ||
861 | resp_msg = (uint8_t *)rx_queue->base_addr + head; | |
f67539c2 TL |
862 | |
863 | if (nb_fw_responses) { | |
864 | /* only move on to next op if one was ready to return | |
865 | * to API | |
866 | */ | |
867 | ops++; | |
868 | op_resp_counter++; | |
869 | } | |
870 | ||
871 | /* A compression op may be broken up into multiple fw requests. | |
872 | * Only count fw responses as complete once ALL the responses | |
873 | * associated with an op have been processed, as the cookie | |
874 | * data from the first response must be available until | |
875 | * finished with all firmware responses. | |
876 | */ | |
877 | fw_resp_counter += nb_fw_responses; | |
878 | ||
879 | rx_queue->nb_processed_responses++; | |
11fdf7f2 | 880 | } |
f67539c2 TL |
881 | |
882 | tmp_qp->dequeued += fw_resp_counter; | |
883 | tmp_qp->stats.dequeued_count += op_resp_counter; | |
884 | ||
885 | rx_queue->head = head; | |
886 | if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) | |
887 | rxq_free_desc(tmp_qp, rx_queue); | |
888 | ||
889 | QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u", | |
890 | op_resp_counter, fw_resp_counter); | |
891 | ||
892 | return op_resp_counter; | |
893 | } | |
894 | ||
895 | /* This is almost same as dequeue_op_burst, without the atomic, without stats | |
896 | * and without the op. Dequeues one response. | |
897 | */ | |
898 | static uint8_t | |
899 | qat_cq_dequeue_response(struct qat_qp *qp, void *out_data) | |
900 | { | |
901 | uint8_t result = 0; | |
902 | uint8_t retries = 0; | |
903 | struct qat_queue *queue = &(qp->rx_q); | |
904 | struct icp_qat_fw_comn_resp *resp_msg = (struct icp_qat_fw_comn_resp *) | |
905 | ((uint8_t *)queue->base_addr + queue->head); | |
906 | ||
907 | while (retries++ < QAT_CQ_MAX_DEQ_RETRIES && | |
908 | *(uint32_t *)resp_msg == ADF_RING_EMPTY_SIG) { | |
909 | /* loop waiting for response until we reach the timeout */ | |
910 | rte_delay_ms(20); | |
11fdf7f2 | 911 | } |
f67539c2 TL |
912 | |
913 | if (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG) { | |
914 | /* response received */ | |
915 | result = 1; | |
916 | ||
917 | /* check status flag */ | |
918 | if (ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( | |
919 | resp_msg->comn_hdr.comn_status) == | |
920 | ICP_QAT_FW_COMN_STATUS_FLAG_OK) { | |
921 | /* success */ | |
922 | memcpy(out_data, resp_msg, queue->msg_size); | |
923 | } else { | |
924 | memset(out_data, 0, queue->msg_size); | |
925 | } | |
926 | ||
927 | queue->head = adf_modulo(queue->head + queue->msg_size, | |
928 | queue->modulo_mask); | |
929 | rxq_free_desc(qp, queue); | |
11fdf7f2 | 930 | } |
f67539c2 TL |
931 | |
932 | return result; | |
933 | } | |
934 | ||
935 | /* Sends a NULL message and extracts QAT fw version from the response. | |
936 | * Used to determine detailed capabilities based on the fw version number. | |
937 | * This assumes that there are no inflight messages, i.e. assumes there's space | |
938 | * on the qp, one message is sent and only one response collected. | |
939 | * Returns fw version number or 0 for unknown version or a negative error code. | |
940 | */ | |
941 | int | |
942 | qat_cq_get_fw_version(struct qat_qp *qp) | |
943 | { | |
944 | struct qat_queue *queue = &(qp->tx_q); | |
945 | uint8_t *base_addr = (uint8_t *)queue->base_addr; | |
946 | struct icp_qat_fw_comn_req null_msg; | |
947 | struct icp_qat_fw_comn_resp response; | |
948 | ||
949 | /* prepare the NULL request */ | |
950 | memset(&null_msg, 0, sizeof(null_msg)); | |
951 | null_msg.comn_hdr.hdr_flags = | |
952 | ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); | |
953 | null_msg.comn_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL; | |
954 | null_msg.comn_hdr.service_cmd_id = ICP_QAT_FW_NULL_REQ_SERV_ID; | |
955 | ||
956 | #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG | |
957 | QAT_DP_HEXDUMP_LOG(DEBUG, "NULL request", &null_msg, sizeof(null_msg)); | |
958 | #endif | |
959 | ||
960 | /* send the NULL request */ | |
961 | memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg)); | |
962 | queue->tail = adf_modulo(queue->tail + queue->msg_size, | |
963 | queue->modulo_mask); | |
964 | txq_write_tail(qp, queue); | |
965 | ||
966 | /* receive a response */ | |
967 | if (qat_cq_dequeue_response(qp, &response)) { | |
968 | ||
969 | #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG | |
970 | QAT_DP_HEXDUMP_LOG(DEBUG, "NULL response:", &response, | |
971 | sizeof(response)); | |
972 | #endif | |
973 | /* if LW0 bit 24 is set - then the fw version was returned */ | |
974 | if (QAT_FIELD_GET(response.comn_hdr.hdr_flags, | |
975 | ICP_QAT_FW_COMN_NULL_VERSION_FLAG_BITPOS, | |
976 | ICP_QAT_FW_COMN_NULL_VERSION_FLAG_MASK)) | |
977 | return response.resrvd[0]; /* return LW4 */ | |
978 | else | |
979 | return 0; /* not set - we don't know fw version */ | |
980 | } | |
981 | ||
982 | QAT_LOG(ERR, "No response received"); | |
983 | return -EINVAL; | |
11fdf7f2 TL |
984 | } |
985 | ||
9f95a23c TL |
986 | __rte_weak int |
987 | qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused, | |
f67539c2 | 988 | void *op_cookie __rte_unused, |
9f95a23c | 989 | uint64_t *dequeue_err_count __rte_unused) |
11fdf7f2 TL |
990 | { |
991 | return 0; | |
992 | } |