1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018-2019 NXP
9 #include <rte_atomic.h>
10 #include <rte_lcore.h>
11 #include <rte_rawdev.h>
12 #include <rte_rawdev_pmd.h>
13 #include <rte_malloc.h>
15 #include <rte_mempool.h>
16 #include <rte_prefetch.h>
17 #include <rte_kvargs.h>
19 #include <mc/fsl_dpdmai.h>
20 #include <portal/dpaa2_hw_pvt.h>
21 #include <portal/dpaa2_hw_dpio.h>
23 #include "rte_pmd_dpaa2_qdma.h"
24 #include "dpaa2_qdma.h"
25 #include "dpaa2_qdma_logs.h"
27 #define DPAA2_QDMA_NO_PREFETCH "no_prefetch"
29 /* Dynamic log type identifier */
30 int dpaa2_qdma_logtype
;
32 uint32_t dpaa2_coherent_no_alloc_cache
;
33 uint32_t dpaa2_coherent_alloc_cache
;
36 static struct qdma_device qdma_dev
;
38 /* QDMA H/W queues list */
39 TAILQ_HEAD(qdma_hw_queue_list
, qdma_hw_queue
);
40 static struct qdma_hw_queue_list qdma_queue_list
41 = TAILQ_HEAD_INITIALIZER(qdma_queue_list
);
43 /* QDMA Virtual Queues */
44 static struct qdma_virt_queue
*qdma_vqs
;
46 /* QDMA per core data */
47 static struct qdma_per_core_info qdma_core_info
[RTE_MAX_LCORE
];
49 typedef int (dpdmai_dev_dequeue_multijob_t
)(struct dpaa2_dpdmai_dev
*dpdmai_dev
,
52 struct rte_qdma_job
**job
,
55 dpdmai_dev_dequeue_multijob_t
*dpdmai_dev_dequeue_multijob
;
57 static struct qdma_hw_queue
*
58 alloc_hw_queue(uint32_t lcore_id
)
60 struct qdma_hw_queue
*queue
= NULL
;
62 DPAA2_QDMA_FUNC_TRACE();
64 /* Get a free queue from the list */
65 TAILQ_FOREACH(queue
, &qdma_queue_list
, next
) {
66 if (queue
->num_users
== 0) {
67 queue
->lcore_id
= lcore_id
;
77 free_hw_queue(struct qdma_hw_queue
*queue
)
79 DPAA2_QDMA_FUNC_TRACE();
85 static struct qdma_hw_queue
*
86 get_hw_queue(uint32_t lcore_id
)
88 struct qdma_per_core_info
*core_info
;
89 struct qdma_hw_queue
*queue
, *temp
;
90 uint32_t least_num_users
;
93 DPAA2_QDMA_FUNC_TRACE();
95 core_info
= &qdma_core_info
[lcore_id
];
96 num_hw_queues
= core_info
->num_hw_queues
;
99 * Allocate a HW queue if there are less queues
100 * than maximum per core queues configured
102 if (num_hw_queues
< qdma_dev
.max_hw_queues_per_core
) {
103 queue
= alloc_hw_queue(lcore_id
);
105 core_info
->hw_queues
[num_hw_queues
] = queue
;
106 core_info
->num_hw_queues
++;
111 queue
= core_info
->hw_queues
[0];
112 /* In case there is no queue associated with the core return NULL */
116 /* Fetch the least loaded H/W queue */
117 least_num_users
= core_info
->hw_queues
[0]->num_users
;
118 for (i
= 0; i
< num_hw_queues
; i
++) {
119 temp
= core_info
->hw_queues
[i
];
120 if (temp
->num_users
< least_num_users
)
131 put_hw_queue(struct qdma_hw_queue
*queue
)
133 struct qdma_per_core_info
*core_info
;
134 int lcore_id
, num_hw_queues
, i
;
136 DPAA2_QDMA_FUNC_TRACE();
139 * If this is the last user of the queue free it.
140 * Also remove it from QDMA core info.
142 if (queue
->num_users
== 1) {
143 free_hw_queue(queue
);
145 /* Remove the physical queue from core info */
146 lcore_id
= queue
->lcore_id
;
147 core_info
= &qdma_core_info
[lcore_id
];
148 num_hw_queues
= core_info
->num_hw_queues
;
149 for (i
= 0; i
< num_hw_queues
; i
++) {
150 if (queue
== core_info
->hw_queues
[i
])
153 for (; i
< num_hw_queues
- 1; i
++)
154 core_info
->hw_queues
[i
] = core_info
->hw_queues
[i
+ 1];
155 core_info
->hw_queues
[i
] = NULL
;
164 DPAA2_QDMA_FUNC_TRACE();
166 rte_spinlock_init(&qdma_dev
.lock
);
172 rte_qdma_attr_get(struct rte_qdma_attr
*qdma_attr
)
174 DPAA2_QDMA_FUNC_TRACE();
176 qdma_attr
->num_hw_queues
= qdma_dev
.num_hw_queues
;
182 struct qdma_hw_queue
*queue
;
185 DPAA2_QDMA_FUNC_TRACE();
187 /* In case QDMA device is not in stopped state, return -EBUSY */
188 if (qdma_dev
.state
== 1) {
190 "Device is in running state. Stop before reset.");
194 /* In case there are pending jobs on any VQ, return -EBUSY */
195 for (i
= 0; i
< qdma_dev
.max_vqs
; i
++) {
196 if (qdma_vqs
[i
].in_use
&& (qdma_vqs
[i
].num_enqueues
!=
197 qdma_vqs
[i
].num_dequeues
))
198 DPAA2_QDMA_ERR("Jobs are still pending on VQ: %d", i
);
202 /* Reset HW queues */
203 TAILQ_FOREACH(queue
, &qdma_queue_list
, next
)
204 queue
->num_users
= 0;
206 /* Reset and free virtual queues */
207 for (i
= 0; i
< qdma_dev
.max_vqs
; i
++) {
208 if (qdma_vqs
[i
].status_ring
)
209 rte_ring_free(qdma_vqs
[i
].status_ring
);
215 /* Reset per core info */
216 memset(&qdma_core_info
, 0,
217 sizeof(struct qdma_per_core_info
) * RTE_MAX_LCORE
);
219 /* Free the FLE pool */
220 if (qdma_dev
.fle_pool
)
221 rte_mempool_free(qdma_dev
.fle_pool
);
223 /* Reset QDMA device structure */
224 qdma_dev
.mode
= RTE_QDMA_MODE_HW
;
225 qdma_dev
.max_hw_queues_per_core
= 0;
226 qdma_dev
.fle_pool
= NULL
;
227 qdma_dev
.fle_pool_count
= 0;
228 qdma_dev
.max_vqs
= 0;
234 rte_qdma_configure(struct rte_qdma_config
*qdma_config
)
237 char fle_pool_name
[32]; /* RTE_MEMZONE_NAMESIZE = 32 */
239 DPAA2_QDMA_FUNC_TRACE();
241 /* In case QDMA device is not in stopped state, return -EBUSY */
242 if (qdma_dev
.state
== 1) {
244 "Device is in running state. Stop before config.");
248 /* Reset the QDMA device */
249 ret
= rte_qdma_reset();
251 DPAA2_QDMA_ERR("Resetting QDMA failed");
256 qdma_dev
.mode
= qdma_config
->mode
;
258 /* Set max HW queue per core */
259 if (qdma_config
->max_hw_queues_per_core
> MAX_HW_QUEUE_PER_CORE
) {
260 DPAA2_QDMA_ERR("H/W queues per core is more than: %d",
261 MAX_HW_QUEUE_PER_CORE
);
264 qdma_dev
.max_hw_queues_per_core
=
265 qdma_config
->max_hw_queues_per_core
;
267 /* Allocate Virtual Queues */
268 qdma_vqs
= rte_malloc("qdma_virtual_queues",
269 (sizeof(struct qdma_virt_queue
) * qdma_config
->max_vqs
),
270 RTE_CACHE_LINE_SIZE
);
272 DPAA2_QDMA_ERR("qdma_virtual_queues allocation failed");
275 qdma_dev
.max_vqs
= qdma_config
->max_vqs
;
277 /* Allocate FLE pool; just append PID so that in case of
278 * multiprocess, the pool's don't collide.
280 snprintf(fle_pool_name
, sizeof(fle_pool_name
), "qdma_fle_pool%u",
282 qdma_dev
.fle_pool
= rte_mempool_create(fle_pool_name
,
283 qdma_config
->fle_pool_count
, QDMA_FLE_POOL_SIZE
,
284 QDMA_FLE_CACHE_SIZE(qdma_config
->fle_pool_count
), 0,
285 NULL
, NULL
, NULL
, NULL
, SOCKET_ID_ANY
, 0);
286 if (!qdma_dev
.fle_pool
) {
287 DPAA2_QDMA_ERR("qdma_fle_pool create failed");
292 qdma_dev
.fle_pool_count
= qdma_config
->fle_pool_count
;
300 DPAA2_QDMA_FUNC_TRACE();
308 rte_qdma_vq_create(uint32_t lcore_id
, uint32_t flags
)
313 DPAA2_QDMA_FUNC_TRACE();
315 rte_spinlock_lock(&qdma_dev
.lock
);
317 /* Get a free Virtual Queue */
318 for (i
= 0; i
< qdma_dev
.max_vqs
; i
++) {
319 if (qdma_vqs
[i
].in_use
== 0)
323 /* Return in case no VQ is free */
324 if (i
== qdma_dev
.max_vqs
) {
325 rte_spinlock_unlock(&qdma_dev
.lock
);
326 DPAA2_QDMA_ERR("Unable to get lock on QDMA device");
330 if (qdma_dev
.mode
== RTE_QDMA_MODE_HW
||
331 (flags
& RTE_QDMA_VQ_EXCLUSIVE_PQ
)) {
332 /* Allocate HW queue for a VQ */
333 qdma_vqs
[i
].hw_queue
= alloc_hw_queue(lcore_id
);
334 qdma_vqs
[i
].exclusive_hw_queue
= 1;
336 /* Allocate a Ring for Virutal Queue in VQ mode */
337 snprintf(ring_name
, sizeof(ring_name
), "status ring %d", i
);
338 qdma_vqs
[i
].status_ring
= rte_ring_create(ring_name
,
339 qdma_dev
.fle_pool_count
, rte_socket_id(), 0);
340 if (!qdma_vqs
[i
].status_ring
) {
341 DPAA2_QDMA_ERR("Status ring creation failed for vq");
342 rte_spinlock_unlock(&qdma_dev
.lock
);
346 /* Get a HW queue (shared) for a VQ */
347 qdma_vqs
[i
].hw_queue
= get_hw_queue(lcore_id
);
348 qdma_vqs
[i
].exclusive_hw_queue
= 0;
351 if (qdma_vqs
[i
].hw_queue
== NULL
) {
352 DPAA2_QDMA_ERR("No H/W queue available for VQ");
353 if (qdma_vqs
[i
].status_ring
)
354 rte_ring_free(qdma_vqs
[i
].status_ring
);
355 qdma_vqs
[i
].status_ring
= NULL
;
356 rte_spinlock_unlock(&qdma_dev
.lock
);
360 qdma_vqs
[i
].in_use
= 1;
361 qdma_vqs
[i
].lcore_id
= lcore_id
;
362 memset(&qdma_vqs
[i
].rbp
, 0, sizeof(struct rte_qdma_rbp
));
363 rte_spinlock_unlock(&qdma_dev
.lock
);
368 /*create vq for route-by-port*/
370 rte_qdma_vq_create_rbp(uint32_t lcore_id
, uint32_t flags
,
371 struct rte_qdma_rbp
*rbp
)
375 i
= rte_qdma_vq_create(lcore_id
, flags
);
377 memcpy(&qdma_vqs
[i
].rbp
, rbp
, sizeof(struct rte_qdma_rbp
));
383 dpaa2_qdma_populate_fle(struct qbman_fle
*fle
,
384 struct rte_qdma_rbp
*rbp
,
385 uint64_t src
, uint64_t dest
,
386 size_t len
, uint32_t flags
)
388 struct qdma_sdd
*sdd
;
390 sdd
= (struct qdma_sdd
*)((uint8_t *)(fle
) +
391 (DPAA2_QDMA_MAX_FLE
* sizeof(struct qbman_fle
)));
393 /* first frame list to source descriptor */
394 DPAA2_SET_FLE_ADDR(fle
, DPAA2_VADDR_TO_IOVA(sdd
));
395 DPAA2_SET_FLE_LEN(fle
, (2 * (sizeof(struct qdma_sdd
))));
397 /* source and destination descriptor */
398 if (rbp
&& rbp
->enable
) {
400 sdd
->read_cmd
.portid
= rbp
->sportid
;
401 sdd
->rbpcmd_simple
.pfid
= rbp
->spfid
;
402 sdd
->rbpcmd_simple
.vfid
= rbp
->svfid
;
405 sdd
->read_cmd
.rbp
= rbp
->srbp
;
406 sdd
->read_cmd
.rdtype
= DPAA2_RBP_MEM_RW
;
408 sdd
->read_cmd
.rdtype
= dpaa2_coherent_no_alloc_cache
;
412 sdd
->write_cmd
.portid
= rbp
->dportid
;
413 sdd
->rbpcmd_simple
.pfid
= rbp
->dpfid
;
414 sdd
->rbpcmd_simple
.vfid
= rbp
->dvfid
;
417 sdd
->write_cmd
.rbp
= rbp
->drbp
;
418 sdd
->write_cmd
.wrttype
= DPAA2_RBP_MEM_RW
;
420 sdd
->write_cmd
.wrttype
= dpaa2_coherent_alloc_cache
;
424 sdd
->read_cmd
.rdtype
= dpaa2_coherent_no_alloc_cache
;
426 sdd
->write_cmd
.wrttype
= dpaa2_coherent_alloc_cache
;
429 /* source frame list to source buffer */
430 if (flags
& RTE_QDMA_JOB_SRC_PHY
) {
431 DPAA2_SET_FLE_ADDR(fle
, src
);
432 DPAA2_SET_FLE_BMT(fle
);
434 DPAA2_SET_FLE_ADDR(fle
, DPAA2_VADDR_TO_IOVA(src
));
436 DPAA2_SET_FLE_LEN(fle
, len
);
439 /* destination frame list to destination buffer */
440 if (flags
& RTE_QDMA_JOB_DEST_PHY
) {
441 DPAA2_SET_FLE_BMT(fle
);
442 DPAA2_SET_FLE_ADDR(fle
, dest
);
444 DPAA2_SET_FLE_ADDR(fle
, DPAA2_VADDR_TO_IOVA(dest
));
446 DPAA2_SET_FLE_LEN(fle
, len
);
448 /* Final bit: 1, for last frame list */
449 DPAA2_SET_FLE_FIN(fle
);
452 static inline uint16_t dpdmai_dev_set_fd(struct qbman_fd
*fd
,
453 struct rte_qdma_job
*job
,
454 struct rte_qdma_rbp
*rbp
,
457 struct qdma_io_meta
*io_meta
;
458 struct qbman_fle
*fle
;
461 * Get an FLE/SDD from FLE pool.
462 * Note: IO metadata is before the FLE and SDD memory.
464 ret
= rte_mempool_get(qdma_dev
.fle_pool
, (void **)(&io_meta
));
466 DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
470 /* Set the metadata */
471 io_meta
->cnxt
= (size_t)job
;
474 fle
= (struct qbman_fle
*)(io_meta
+ 1);
476 DPAA2_SET_FD_ADDR(fd
, DPAA2_VADDR_TO_IOVA(fle
));
477 DPAA2_SET_FD_COMPOUND_FMT(fd
);
478 DPAA2_SET_FD_FRC(fd
, QDMA_SER_CTX
);
481 memset(fle
, 0, QDMA_FLE_POOL_SIZE
);
482 dpaa2_qdma_populate_fle(fle
, rbp
, job
->src
, job
->dest
,
483 job
->len
, job
->flags
);
489 dpdmai_dev_enqueue_multi(struct dpaa2_dpdmai_dev
*dpdmai_dev
,
492 struct rte_qdma_rbp
*rbp
,
493 struct rte_qdma_job
**job
,
496 struct qbman_fd fd
[RTE_QDMA_BURST_NB_MAX
];
497 struct dpaa2_queue
*txq
;
498 struct qbman_eq_desc eqdesc
;
499 struct qbman_swp
*swp
;
501 uint32_t num_to_send
= 0;
504 if (unlikely(!DPAA2_PER_LCORE_DPIO
)) {
505 ret
= dpaa2_affine_qbman_swp();
507 DPAA2_QDMA_ERR("Failure in affining portal");
511 swp
= DPAA2_PER_LCORE_PORTAL
;
513 txq
= &(dpdmai_dev
->tx_queue
[txq_id
]);
515 /* Prepare enqueue descriptor */
516 qbman_eq_desc_clear(&eqdesc
);
517 qbman_eq_desc_set_fq(&eqdesc
, txq
->fqid
);
518 qbman_eq_desc_set_no_orp(&eqdesc
, 0);
519 qbman_eq_desc_set_response(&eqdesc
, 0, 0);
521 memset(fd
, 0, RTE_QDMA_BURST_NB_MAX
* sizeof(struct qbman_fd
));
523 while (nb_jobs
> 0) {
526 num_to_send
= (nb_jobs
> dpaa2_eqcr_size
) ?
527 dpaa2_eqcr_size
: nb_jobs
;
529 for (loop
= 0; loop
< num_to_send
; loop
++) {
530 ret
= dpdmai_dev_set_fd(&fd
[loop
],
531 job
[num_tx
], rbp
, vq_id
);
533 /* Set nb_jobs to loop, so outer while loop
543 /* Enqueue the packet to the QBMAN */
544 uint32_t enqueue_loop
= 0;
545 while (enqueue_loop
< loop
) {
546 enqueue_loop
+= qbman_swp_enqueue_multiple(swp
,
550 loop
- enqueue_loop
);
558 rte_qdma_vq_enqueue_multi(uint16_t vq_id
,
559 struct rte_qdma_job
**job
,
562 struct qdma_virt_queue
*qdma_vq
= &qdma_vqs
[vq_id
];
563 struct qdma_hw_queue
*qdma_pq
= qdma_vq
->hw_queue
;
564 struct dpaa2_dpdmai_dev
*dpdmai_dev
= qdma_pq
->dpdmai_dev
;
567 /* Return error in case of wrong lcore_id */
568 if (rte_lcore_id() != qdma_vq
->lcore_id
) {
569 DPAA2_QDMA_ERR("QDMA enqueue for vqid %d on wrong core",
574 ret
= dpdmai_dev_enqueue_multi(dpdmai_dev
,
581 DPAA2_QDMA_ERR("DPDMAI device enqueue failed: %d", ret
);
585 qdma_vq
->num_enqueues
+= ret
;
591 rte_qdma_vq_enqueue(uint16_t vq_id
,
592 struct rte_qdma_job
*job
)
594 return rte_qdma_vq_enqueue_multi(vq_id
, &job
, 1);
597 static inline uint16_t dpdmai_dev_get_job(const struct qbman_fd
*fd
,
598 struct rte_qdma_job
**job
)
600 struct qbman_fle
*fle
;
601 struct qdma_io_meta
*io_meta
;
604 * Fetch metadata from FLE. job and vq_id were set
605 * in metadata in the enqueue operation.
607 fle
= (struct qbman_fle
*)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd
));
608 io_meta
= (struct qdma_io_meta
*)(fle
) - 1;
610 *job
= (struct rte_qdma_job
*)(size_t)io_meta
->cnxt
;
611 (*job
)->status
= (DPAA2_GET_FD_ERR(fd
) << 8) |
612 (DPAA2_GET_FD_FRC(fd
) & 0xFF);
616 /* Free FLE to the pool */
617 rte_mempool_put(qdma_dev
.fle_pool
, io_meta
);
622 /* Function to receive a QDMA job for a given device and queue*/
624 dpdmai_dev_dequeue_multijob_prefetch(
625 struct dpaa2_dpdmai_dev
*dpdmai_dev
,
628 struct rte_qdma_job
**job
,
631 struct dpaa2_queue
*rxq
;
632 struct qbman_result
*dq_storage
, *dq_storage1
= NULL
;
633 struct qbman_pull_desc pulldesc
;
634 struct qbman_swp
*swp
;
635 struct queue_storage_info_t
*q_storage
;
637 uint8_t status
, pending
;
639 const struct qbman_fd
*fd
;
643 if (unlikely(!DPAA2_PER_LCORE_DPIO
)) {
644 ret
= dpaa2_affine_qbman_swp();
646 DPAA2_QDMA_ERR("Failure in affining portal");
650 swp
= DPAA2_PER_LCORE_PORTAL
;
652 pull_size
= (nb_jobs
> dpaa2_dqrr_size
) ? dpaa2_dqrr_size
: nb_jobs
;
653 rxq
= &(dpdmai_dev
->rx_queue
[rxq_id
]);
655 q_storage
= rxq
->q_storage
;
657 if (unlikely(!q_storage
->active_dqs
)) {
658 q_storage
->toggle
= 0;
659 dq_storage
= q_storage
->dq_storage
[q_storage
->toggle
];
660 q_storage
->last_num_pkts
= pull_size
;
661 qbman_pull_desc_clear(&pulldesc
);
662 qbman_pull_desc_set_numframes(&pulldesc
,
663 q_storage
->last_num_pkts
);
664 qbman_pull_desc_set_fq(&pulldesc
, fqid
);
665 qbman_pull_desc_set_storage(&pulldesc
, dq_storage
,
666 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage
)), 1);
667 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO
->index
)) {
668 while (!qbman_check_command_complete(
670 DPAA2_PER_LCORE_DPIO
->index
)))
672 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO
->index
);
675 if (qbman_swp_pull(swp
, &pulldesc
)) {
677 "VDQ command not issued.QBMAN busy\n");
678 /* Portal was busy, try again */
683 q_storage
->active_dqs
= dq_storage
;
684 q_storage
->active_dpio_id
= DPAA2_PER_LCORE_DPIO
->index
;
685 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO
->index
,
689 dq_storage
= q_storage
->active_dqs
;
690 rte_prefetch0((void *)(size_t)(dq_storage
));
691 rte_prefetch0((void *)(size_t)(dq_storage
+ 1));
693 /* Prepare next pull descriptor. This will give space for the
694 * prefething done on DQRR entries
696 q_storage
->toggle
^= 1;
697 dq_storage1
= q_storage
->dq_storage
[q_storage
->toggle
];
698 qbman_pull_desc_clear(&pulldesc
);
699 qbman_pull_desc_set_numframes(&pulldesc
, pull_size
);
700 qbman_pull_desc_set_fq(&pulldesc
, fqid
);
701 qbman_pull_desc_set_storage(&pulldesc
, dq_storage1
,
702 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1
)), 1);
704 /* Check if the previous issued command is completed.
705 * Also seems like the SWP is shared between the Ethernet Driver
706 * and the SEC driver.
708 while (!qbman_check_command_complete(dq_storage
))
710 if (dq_storage
== get_swp_active_dqs(q_storage
->active_dpio_id
))
711 clear_swp_active_dqs(q_storage
->active_dpio_id
);
716 /* Loop until the dq_storage is updated with
719 while (!qbman_check_new_result(dq_storage
))
721 rte_prefetch0((void *)((size_t)(dq_storage
+ 2)));
722 /* Check whether Last Pull command is Expired and
723 * setting Condition for Loop termination
725 if (qbman_result_DQ_is_pull_complete(dq_storage
)) {
727 /* Check for valid frame. */
728 status
= qbman_result_DQ_flags(dq_storage
);
729 if (unlikely((status
& QBMAN_DQ_STAT_VALIDFRAME
) == 0))
732 fd
= qbman_result_DQ_fd(dq_storage
);
734 vqid
= dpdmai_dev_get_job(fd
, &job
[num_rx
]);
736 vq_id
[num_rx
] = vqid
;
742 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO
->index
)) {
743 while (!qbman_check_command_complete(
744 get_swp_active_dqs(DPAA2_PER_LCORE_DPIO
->index
)))
746 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO
->index
);
748 /* issue a volatile dequeue command for next pull */
750 if (qbman_swp_pull(swp
, &pulldesc
)) {
751 DPAA2_QDMA_DP_WARN("VDQ command is not issued."
752 "QBMAN is busy (2)\n");
758 q_storage
->active_dqs
= dq_storage1
;
759 q_storage
->active_dpio_id
= DPAA2_PER_LCORE_DPIO
->index
;
760 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO
->index
, dq_storage1
);
766 dpdmai_dev_dequeue_multijob_no_prefetch(
767 struct dpaa2_dpdmai_dev
*dpdmai_dev
,
770 struct rte_qdma_job
**job
,
773 struct dpaa2_queue
*rxq
;
774 struct qbman_result
*dq_storage
;
775 struct qbman_pull_desc pulldesc
;
776 struct qbman_swp
*swp
;
778 uint8_t status
, pending
;
780 const struct qbman_fd
*fd
;
782 int ret
, next_pull
= nb_jobs
, num_pulled
= 0;
784 if (unlikely(!DPAA2_PER_LCORE_DPIO
)) {
785 ret
= dpaa2_affine_qbman_swp();
787 DPAA2_QDMA_ERR("Failure in affining portal");
791 swp
= DPAA2_PER_LCORE_PORTAL
;
793 rxq
= &(dpdmai_dev
->rx_queue
[rxq_id
]);
797 dq_storage
= rxq
->q_storage
->dq_storage
[0];
798 /* Prepare dequeue descriptor */
799 qbman_pull_desc_clear(&pulldesc
);
800 qbman_pull_desc_set_fq(&pulldesc
, fqid
);
801 qbman_pull_desc_set_storage(&pulldesc
, dq_storage
,
802 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage
)), 1);
804 if (next_pull
> dpaa2_dqrr_size
) {
805 qbman_pull_desc_set_numframes(&pulldesc
,
807 next_pull
-= dpaa2_dqrr_size
;
809 qbman_pull_desc_set_numframes(&pulldesc
, next_pull
);
814 if (qbman_swp_pull(swp
, &pulldesc
)) {
815 DPAA2_QDMA_DP_WARN("VDQ command not issued. QBMAN busy");
816 /* Portal was busy, try again */
822 rte_prefetch0((void *)((size_t)(dq_storage
+ 1)));
823 /* Check if the previous issued command is completed. */
824 while (!qbman_check_command_complete(dq_storage
))
831 /* Loop until dq_storage is updated
832 * with new token by QBMAN
834 while (!qbman_check_new_result(dq_storage
))
836 rte_prefetch0((void *)((size_t)(dq_storage
+ 2)));
838 if (qbman_result_DQ_is_pull_complete(dq_storage
)) {
840 /* Check for valid frame. */
841 status
= qbman_result_DQ_flags(dq_storage
);
842 if (unlikely((status
&
843 QBMAN_DQ_STAT_VALIDFRAME
) == 0))
846 fd
= qbman_result_DQ_fd(dq_storage
);
848 vqid
= dpdmai_dev_get_job(fd
, &job
[num_rx
]);
850 vq_id
[num_rx
] = vqid
;
857 /* Last VDQ provided all packets and more packets are requested */
858 } while (next_pull
&& num_pulled
== dpaa2_dqrr_size
);
864 rte_qdma_vq_dequeue_multi(uint16_t vq_id
,
865 struct rte_qdma_job
**job
,
868 struct qdma_virt_queue
*qdma_vq
= &qdma_vqs
[vq_id
];
869 struct qdma_hw_queue
*qdma_pq
= qdma_vq
->hw_queue
;
870 struct qdma_virt_queue
*temp_qdma_vq
;
871 struct dpaa2_dpdmai_dev
*dpdmai_dev
= qdma_pq
->dpdmai_dev
;
872 int ring_count
, ret
= 0, i
;
874 /* Return error in case of wrong lcore_id */
875 if (rte_lcore_id() != (unsigned int)(qdma_vq
->lcore_id
)) {
876 DPAA2_QDMA_WARN("QDMA dequeue for vqid %d on wrong core",
881 /* Only dequeue when there are pending jobs on VQ */
882 if (qdma_vq
->num_enqueues
== qdma_vq
->num_dequeues
)
885 if (qdma_vq
->num_enqueues
< (qdma_vq
->num_dequeues
+ nb_jobs
))
886 nb_jobs
= (qdma_vq
->num_enqueues
- qdma_vq
->num_dequeues
);
888 if (qdma_vq
->exclusive_hw_queue
) {
889 /* In case of exclusive queue directly fetch from HW queue */
890 ret
= dpdmai_dev_dequeue_multijob(dpdmai_dev
, qdma_pq
->queue_id
,
894 "Dequeue from DPDMAI device failed: %d", ret
);
897 qdma_vq
->num_dequeues
+= ret
;
899 uint16_t temp_vq_id
[RTE_QDMA_BURST_NB_MAX
];
901 * Get the QDMA completed jobs from the software ring.
902 * In case they are not available on the ring poke the HW
903 * to fetch completed jobs from corresponding HW queues
905 ring_count
= rte_ring_count(qdma_vq
->status_ring
);
906 if (ring_count
< nb_jobs
) {
907 /* TODO - How to have right budget */
908 ret
= dpdmai_dev_dequeue_multijob(dpdmai_dev
,
910 temp_vq_id
, job
, nb_jobs
);
911 for (i
= 0; i
< ret
; i
++) {
912 temp_qdma_vq
= &qdma_vqs
[temp_vq_id
[i
]];
913 rte_ring_enqueue(temp_qdma_vq
->status_ring
,
916 ring_count
= rte_ring_count(
917 qdma_vq
->status_ring
);
921 /* Dequeue job from the software ring
922 * to provide to the user
924 ret
= rte_ring_dequeue_bulk(qdma_vq
->status_ring
,
925 (void **)job
, ring_count
, NULL
);
927 qdma_vq
->num_dequeues
+= ret
;
934 struct rte_qdma_job
*
935 rte_qdma_vq_dequeue(uint16_t vq_id
)
938 struct rte_qdma_job
*job
= NULL
;
940 ret
= rte_qdma_vq_dequeue_multi(vq_id
, &job
, 1);
942 DPAA2_QDMA_DP_WARN("DPDMAI device dequeue failed: %d", ret
);
948 rte_qdma_vq_stats(uint16_t vq_id
,
949 struct rte_qdma_vq_stats
*vq_status
)
951 struct qdma_virt_queue
*qdma_vq
= &qdma_vqs
[vq_id
];
953 if (qdma_vq
->in_use
) {
954 vq_status
->exclusive_hw_queue
= qdma_vq
->exclusive_hw_queue
;
955 vq_status
->lcore_id
= qdma_vq
->lcore_id
;
956 vq_status
->num_enqueues
= qdma_vq
->num_enqueues
;
957 vq_status
->num_dequeues
= qdma_vq
->num_dequeues
;
958 vq_status
->num_pending_jobs
= vq_status
->num_enqueues
-
959 vq_status
->num_dequeues
;
964 rte_qdma_vq_destroy(uint16_t vq_id
)
966 struct qdma_virt_queue
*qdma_vq
= &qdma_vqs
[vq_id
];
968 DPAA2_QDMA_FUNC_TRACE();
970 /* In case there are pending jobs on any VQ, return -EBUSY */
971 if (qdma_vq
->num_enqueues
!= qdma_vq
->num_dequeues
)
974 rte_spinlock_lock(&qdma_dev
.lock
);
976 if (qdma_vq
->exclusive_hw_queue
)
977 free_hw_queue(qdma_vq
->hw_queue
);
979 if (qdma_vqs
->status_ring
)
980 rte_ring_free(qdma_vqs
->status_ring
);
982 put_hw_queue(qdma_vq
->hw_queue
);
985 memset(qdma_vq
, 0, sizeof(struct qdma_virt_queue
));
987 rte_spinlock_unlock(&qdma_dev
.lock
);
993 rte_qdma_vq_destroy_rbp(uint16_t vq_id
)
995 struct qdma_virt_queue
*qdma_vq
= &qdma_vqs
[vq_id
];
997 DPAA2_QDMA_FUNC_TRACE();
999 /* In case there are pending jobs on any VQ, return -EBUSY */
1000 if (qdma_vq
->num_enqueues
!= qdma_vq
->num_dequeues
)
1003 rte_spinlock_lock(&qdma_dev
.lock
);
1005 if (qdma_vq
->exclusive_hw_queue
) {
1006 free_hw_queue(qdma_vq
->hw_queue
);
1008 if (qdma_vqs
->status_ring
)
1009 rte_ring_free(qdma_vqs
->status_ring
);
1011 put_hw_queue(qdma_vq
->hw_queue
);
1014 memset(qdma_vq
, 0, sizeof(struct qdma_virt_queue
));
1016 rte_spinlock_unlock(&qdma_dev
.lock
);
1024 DPAA2_QDMA_FUNC_TRACE();
1030 rte_qdma_destroy(void)
1032 DPAA2_QDMA_FUNC_TRACE();
1037 static const struct rte_rawdev_ops dpaa2_qdma_ops
;
1040 add_hw_queues_to_list(struct dpaa2_dpdmai_dev
*dpdmai_dev
)
1042 struct qdma_hw_queue
*queue
;
1045 DPAA2_QDMA_FUNC_TRACE();
1047 for (i
= 0; i
< dpdmai_dev
->num_queues
; i
++) {
1048 queue
= rte_zmalloc(NULL
, sizeof(struct qdma_hw_queue
), 0);
1051 "Memory allocation failed for QDMA queue");
1055 queue
->dpdmai_dev
= dpdmai_dev
;
1056 queue
->queue_id
= i
;
1058 TAILQ_INSERT_TAIL(&qdma_queue_list
, queue
, next
);
1059 qdma_dev
.num_hw_queues
++;
1066 remove_hw_queues_from_list(struct dpaa2_dpdmai_dev
*dpdmai_dev
)
1068 struct qdma_hw_queue
*queue
= NULL
;
1069 struct qdma_hw_queue
*tqueue
= NULL
;
1071 DPAA2_QDMA_FUNC_TRACE();
1073 TAILQ_FOREACH_SAFE(queue
, &qdma_queue_list
, next
, tqueue
) {
1074 if (queue
->dpdmai_dev
== dpdmai_dev
) {
1075 TAILQ_REMOVE(&qdma_queue_list
, queue
, next
);
1083 dpaa2_dpdmai_dev_uninit(struct rte_rawdev
*rawdev
)
1085 struct dpaa2_dpdmai_dev
*dpdmai_dev
= rawdev
->dev_private
;
1088 DPAA2_QDMA_FUNC_TRACE();
1090 /* Remove HW queues from global list */
1091 remove_hw_queues_from_list(dpdmai_dev
);
1093 ret
= dpdmai_disable(&dpdmai_dev
->dpdmai
, CMD_PRI_LOW
,
1096 DPAA2_QDMA_ERR("dmdmai disable failed");
1098 /* Set up the DQRR storage for Rx */
1099 for (i
= 0; i
< dpdmai_dev
->num_queues
; i
++) {
1100 struct dpaa2_queue
*rxq
= &(dpdmai_dev
->rx_queue
[i
]);
1102 if (rxq
->q_storage
) {
1103 dpaa2_free_dq_storage(rxq
->q_storage
);
1104 rte_free(rxq
->q_storage
);
1108 /* Close the device at underlying layer*/
1109 ret
= dpdmai_close(&dpdmai_dev
->dpdmai
, CMD_PRI_LOW
, dpdmai_dev
->token
);
1111 DPAA2_QDMA_ERR("Failure closing dpdmai device");
1117 check_devargs_handler(__rte_unused
const char *key
, const char *value
,
1118 __rte_unused
void *opaque
)
1120 if (strcmp(value
, "1"))
1127 dpaa2_get_devargs(struct rte_devargs
*devargs
, const char *key
)
1129 struct rte_kvargs
*kvlist
;
1134 kvlist
= rte_kvargs_parse(devargs
->args
, NULL
);
1138 if (!rte_kvargs_count(kvlist
, key
)) {
1139 rte_kvargs_free(kvlist
);
1143 if (rte_kvargs_process(kvlist
, key
,
1144 check_devargs_handler
, NULL
) < 0) {
1145 rte_kvargs_free(kvlist
);
1148 rte_kvargs_free(kvlist
);
1154 dpaa2_dpdmai_dev_init(struct rte_rawdev
*rawdev
, int dpdmai_id
)
1156 struct dpaa2_dpdmai_dev
*dpdmai_dev
= rawdev
->dev_private
;
1157 struct dpdmai_rx_queue_cfg rx_queue_cfg
;
1158 struct dpdmai_attr attr
;
1159 struct dpdmai_rx_queue_attr rx_attr
;
1160 struct dpdmai_tx_queue_attr tx_attr
;
1163 DPAA2_QDMA_FUNC_TRACE();
1165 /* Open DPDMAI device */
1166 dpdmai_dev
->dpdmai_id
= dpdmai_id
;
1167 dpdmai_dev
->dpdmai
.regs
= rte_mcp_ptr_list
[MC_PORTAL_INDEX
];
1168 ret
= dpdmai_open(&dpdmai_dev
->dpdmai
, CMD_PRI_LOW
,
1169 dpdmai_dev
->dpdmai_id
, &dpdmai_dev
->token
);
1171 DPAA2_QDMA_ERR("dpdmai_open() failed with err: %d", ret
);
1175 /* Get DPDMAI attributes */
1176 ret
= dpdmai_get_attributes(&dpdmai_dev
->dpdmai
, CMD_PRI_LOW
,
1177 dpdmai_dev
->token
, &attr
);
1179 DPAA2_QDMA_ERR("dpdmai get attributes failed with err: %d",
1183 dpdmai_dev
->num_queues
= attr
.num_of_queues
;
1185 /* Set up Rx Queues */
1186 for (i
= 0; i
< dpdmai_dev
->num_queues
; i
++) {
1187 struct dpaa2_queue
*rxq
;
1189 memset(&rx_queue_cfg
, 0, sizeof(struct dpdmai_rx_queue_cfg
));
1190 ret
= dpdmai_set_rx_queue(&dpdmai_dev
->dpdmai
,
1193 i
, 0, &rx_queue_cfg
);
1195 DPAA2_QDMA_ERR("Setting Rx queue failed with err: %d",
1200 /* Allocate DQ storage for the DPDMAI Rx queues */
1201 rxq
= &(dpdmai_dev
->rx_queue
[i
]);
1202 rxq
->q_storage
= rte_malloc("dq_storage",
1203 sizeof(struct queue_storage_info_t
),
1204 RTE_CACHE_LINE_SIZE
);
1205 if (!rxq
->q_storage
) {
1206 DPAA2_QDMA_ERR("q_storage allocation failed");
1211 memset(rxq
->q_storage
, 0, sizeof(struct queue_storage_info_t
));
1212 ret
= dpaa2_alloc_dq_storage(rxq
->q_storage
);
1214 DPAA2_QDMA_ERR("dpaa2_alloc_dq_storage failed");
1219 /* Get Rx and Tx queues FQID's */
1220 for (i
= 0; i
< dpdmai_dev
->num_queues
; i
++) {
1221 ret
= dpdmai_get_rx_queue(&dpdmai_dev
->dpdmai
, CMD_PRI_LOW
,
1222 dpdmai_dev
->token
, i
, 0, &rx_attr
);
1224 DPAA2_QDMA_ERR("Reading device failed with err: %d",
1228 dpdmai_dev
->rx_queue
[i
].fqid
= rx_attr
.fqid
;
1230 ret
= dpdmai_get_tx_queue(&dpdmai_dev
->dpdmai
, CMD_PRI_LOW
,
1231 dpdmai_dev
->token
, i
, 0, &tx_attr
);
1233 DPAA2_QDMA_ERR("Reading device failed with err: %d",
1237 dpdmai_dev
->tx_queue
[i
].fqid
= tx_attr
.fqid
;
1240 /* Enable the device */
1241 ret
= dpdmai_enable(&dpdmai_dev
->dpdmai
, CMD_PRI_LOW
,
1244 DPAA2_QDMA_ERR("Enabling device failed with err: %d", ret
);
1248 /* Add the HW queue to the global list */
1249 ret
= add_hw_queues_to_list(dpdmai_dev
);
1251 DPAA2_QDMA_ERR("Adding H/W queue to list failed");
1255 if (dpaa2_get_devargs(rawdev
->device
->devargs
,
1256 DPAA2_QDMA_NO_PREFETCH
)) {
1257 /* If no prefetch is configured. */
1258 dpdmai_dev_dequeue_multijob
=
1259 dpdmai_dev_dequeue_multijob_no_prefetch
;
1260 DPAA2_QDMA_INFO("No Prefetch RX Mode enabled");
1262 dpdmai_dev_dequeue_multijob
=
1263 dpdmai_dev_dequeue_multijob_prefetch
;
1266 if (!dpaa2_coherent_no_alloc_cache
) {
1267 if (dpaa2_svr_family
== SVR_LX2160A
) {
1268 dpaa2_coherent_no_alloc_cache
=
1269 DPAA2_LX2_COHERENT_NO_ALLOCATE_CACHE
;
1270 dpaa2_coherent_alloc_cache
=
1271 DPAA2_LX2_COHERENT_ALLOCATE_CACHE
;
1273 dpaa2_coherent_no_alloc_cache
=
1274 DPAA2_COHERENT_NO_ALLOCATE_CACHE
;
1275 dpaa2_coherent_alloc_cache
=
1276 DPAA2_COHERENT_ALLOCATE_CACHE
;
1280 DPAA2_QDMA_DEBUG("Initialized dpdmai object successfully");
1284 dpaa2_dpdmai_dev_uninit(rawdev
);
1289 rte_dpaa2_qdma_probe(struct rte_dpaa2_driver
*dpaa2_drv
,
1290 struct rte_dpaa2_device
*dpaa2_dev
)
1292 struct rte_rawdev
*rawdev
;
1295 DPAA2_QDMA_FUNC_TRACE();
1297 rawdev
= rte_rawdev_pmd_allocate(dpaa2_dev
->device
.name
,
1298 sizeof(struct dpaa2_dpdmai_dev
),
1301 DPAA2_QDMA_ERR("Unable to allocate rawdevice");
1305 dpaa2_dev
->rawdev
= rawdev
;
1306 rawdev
->dev_ops
= &dpaa2_qdma_ops
;
1307 rawdev
->device
= &dpaa2_dev
->device
;
1308 rawdev
->driver_name
= dpaa2_drv
->driver
.name
;
1310 /* Invoke PMD device initialization function */
1311 ret
= dpaa2_dpdmai_dev_init(rawdev
, dpaa2_dev
->object_id
);
1313 rte_rawdev_pmd_release(rawdev
);
1321 rte_dpaa2_qdma_remove(struct rte_dpaa2_device
*dpaa2_dev
)
1323 struct rte_rawdev
*rawdev
= dpaa2_dev
->rawdev
;
1326 DPAA2_QDMA_FUNC_TRACE();
1328 dpaa2_dpdmai_dev_uninit(rawdev
);
1330 ret
= rte_rawdev_pmd_release(rawdev
);
1332 DPAA2_QDMA_ERR("Device cleanup failed");
1337 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd
= {
1338 .drv_flags
= RTE_DPAA2_DRV_IOVA_AS_VA
,
1339 .drv_type
= DPAA2_QDMA
,
1340 .probe
= rte_dpaa2_qdma_probe
,
1341 .remove
= rte_dpaa2_qdma_remove
,
1344 RTE_PMD_REGISTER_DPAA2(dpaa2_qdma
, rte_dpaa2_qdma_pmd
);
1345 RTE_PMD_REGISTER_PARAM_STRING(dpaa2_qdma
,
1346 "no_prefetch=<int> ");
1348 RTE_INIT(dpaa2_qdma_init_log
)
1350 dpaa2_qdma_logtype
= rte_log_register("pmd.raw.dpaa2.qdma");
1351 if (dpaa2_qdma_logtype
>= 0)
1352 rte_log_set_level(dpaa2_qdma_logtype
, RTE_LOG_INFO
);