1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2015-2018 Atomic Rules LLC
7 #include "ark_ethdev_rx.h"
8 #include "ark_global.h"
13 #define ARK_RX_META_SIZE 32
14 #define ARK_RX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_RX_META_SIZE)
15 #define ARK_RX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM)
17 /* Forward declarations */
21 static void dump_mbuf_data(struct rte_mbuf
*mbuf
, uint16_t lo
, uint16_t hi
);
22 static void ark_ethdev_rx_dump(const char *name
, struct ark_rx_queue
*queue
);
23 static uint32_t eth_ark_rx_jumbo(struct ark_rx_queue
*queue
,
24 struct ark_rx_meta
*meta
,
25 struct rte_mbuf
*mbuf0
,
27 static inline int eth_ark_rx_seed_mbufs(struct ark_rx_queue
*queue
);
28 static int eth_ark_rx_seed_recovery(struct ark_rx_queue
*queue
,
30 struct rte_mbuf
**mbufs
);
32 /* ************************************************************************* */
34 /* array of mbufs to populate */
35 struct rte_mbuf
**reserve_q
;
36 /* array of physical addresses of the mbuf data pointer */
37 /* This point is a virtual address */
38 rte_iova_t
*paddress_q
;
39 struct rte_mempool
*mb_pool
;
41 struct ark_udm_t
*udm
;
42 struct ark_mpu_t
*mpu
;
47 uint32_t seed_index
; /* step 1 set with empty mbuf */
48 uint32_t cons_index
; /* step 3 consumed by driver */
50 /* The queue Id is used to identify the HW Q */
53 /* The queue Index is used within the dpdk device structures */
58 /* separate cache line */
59 /* second cache line - fields only used in slow path */
60 MARKER cacheline1 __rte_cache_min_aligned
;
62 volatile uint32_t prod_index
; /* step 2 filled by FPGA */
63 } __rte_cache_aligned
;
66 /* ************************************************************************* */
68 eth_ark_rx_hw_setup(struct rte_eth_dev
*dev
,
69 struct ark_rx_queue
*queue
,
70 uint16_t rx_queue_id __rte_unused
, uint16_t rx_queue_idx
)
72 rte_iova_t queue_base
;
73 rte_iova_t phys_addr_q_base
;
74 rte_iova_t phys_addr_prod_index
;
76 queue_base
= rte_malloc_virt2iova(queue
);
77 phys_addr_prod_index
= queue_base
+
78 offsetof(struct ark_rx_queue
, prod_index
);
80 phys_addr_q_base
= rte_malloc_virt2iova(queue
->paddress_q
);
83 if (ark_mpu_verify(queue
->mpu
, sizeof(rte_iova_t
))) {
84 PMD_DRV_LOG(ERR
, "Illegal configuration rx queue\n");
88 /* Stop and Reset and configure MPU */
89 ark_mpu_configure(queue
->mpu
, phys_addr_q_base
, queue
->queue_size
, 0);
91 ark_udm_write_addr(queue
->udm
, phys_addr_prod_index
);
93 /* advance the valid pointer, but don't start until the queue starts */
94 ark_mpu_reset_stats(queue
->mpu
);
96 /* The seed is the producer index for the HW */
97 ark_mpu_set_producer(queue
->mpu
, queue
->seed_index
);
98 dev
->data
->rx_queue_state
[rx_queue_idx
] = RTE_ETH_QUEUE_STATE_STOPPED
;
104 eth_ark_rx_update_cons_index(struct ark_rx_queue
*queue
, uint32_t cons_index
)
106 queue
->cons_index
= cons_index
;
107 eth_ark_rx_seed_mbufs(queue
);
108 if (((cons_index
- queue
->last_cons
) >= 64U)) {
109 queue
->last_cons
= cons_index
;
110 ark_mpu_set_producer(queue
->mpu
, queue
->seed_index
);
114 /* ************************************************************************* */
116 eth_ark_dev_rx_queue_setup(struct rte_eth_dev
*dev
,
119 unsigned int socket_id
,
120 const struct rte_eth_rxconf
*rx_conf
,
121 struct rte_mempool
*mb_pool
)
123 static int warning1
; /* = 0 */
124 struct ark_adapter
*ark
= (struct ark_adapter
*)dev
->data
->dev_private
;
126 struct ark_rx_queue
*queue
;
130 /* Future works: divide the Q's evenly with multi-ports */
131 int port
= dev
->data
->port_id
;
132 int qidx
= port
+ queue_idx
;
134 /* We may already be setup, free memory prior to re-allocation */
135 if (dev
->data
->rx_queues
[queue_idx
] != NULL
) {
136 eth_ark_dev_rx_queue_release(dev
->data
->rx_queues
[queue_idx
]);
137 dev
->data
->rx_queues
[queue_idx
] = NULL
;
140 if (rx_conf
!= NULL
&& warning1
== 0) {
143 "Arkville ignores rte_eth_rxconf argument.\n");
146 if (RTE_PKTMBUF_HEADROOM
< ARK_RX_META_SIZE
) {
148 "Error: DPDK Arkville requires head room > %d bytes (%s)\n",
149 ARK_RX_META_SIZE
, __func__
);
150 return -1; /* ERROR CODE */
153 if (!rte_is_power_of_2(nb_desc
)) {
155 "DPDK Arkville configuration queue size must be power of two %u (%s)\n",
157 return -1; /* ERROR CODE */
160 /* Allocate queue struct */
161 queue
= rte_zmalloc_socket("Ark_rxqueue",
162 sizeof(struct ark_rx_queue
),
166 PMD_DRV_LOG(ERR
, "Failed to allocate memory in %s\n", __func__
);
170 /* NOTE zmalloc is used, no need to 0 indexes, etc. */
171 queue
->mb_pool
= mb_pool
;
172 queue
->phys_qid
= qidx
;
173 queue
->queue_index
= queue_idx
;
174 queue
->queue_size
= nb_desc
;
175 queue
->queue_mask
= nb_desc
- 1;
178 rte_zmalloc_socket("Ark_rx_queue mbuf",
179 nb_desc
* sizeof(struct rte_mbuf
*),
183 rte_zmalloc_socket("Ark_rx_queue paddr",
184 nb_desc
* sizeof(rte_iova_t
),
188 if (queue
->reserve_q
== 0 || queue
->paddress_q
== 0) {
190 "Failed to allocate queue memory in %s\n",
192 rte_free(queue
->reserve_q
);
193 rte_free(queue
->paddress_q
);
198 dev
->data
->rx_queues
[queue_idx
] = queue
;
199 queue
->udm
= RTE_PTR_ADD(ark
->udm
.v
, qidx
* ARK_UDM_QOFFSET
);
200 queue
->mpu
= RTE_PTR_ADD(ark
->mpurx
.v
, qidx
* ARK_MPU_QOFFSET
);
202 /* populate mbuf reserve */
203 status
= eth_ark_rx_seed_mbufs(queue
);
205 if (queue
->seed_index
!= nb_desc
) {
206 PMD_DRV_LOG(ERR
, "ARK: Failed to allocate %u mbufs for RX queue %d\n",
212 status
= eth_ark_rx_hw_setup(dev
, queue
, qidx
, queue_idx
);
214 if (unlikely(status
!= 0)) {
215 struct rte_mbuf
**mbuf
;
217 PMD_DRV_LOG(ERR
, "Failed to initialize RX queue %d %s\n",
220 /* Free the mbufs allocated */
221 for (i
= 0, mbuf
= queue
->reserve_q
;
222 i
< queue
->seed_index
; ++i
, mbuf
++) {
223 rte_pktmbuf_free(*mbuf
);
225 rte_free(queue
->reserve_q
);
226 rte_free(queue
->paddress_q
);
228 return -1; /* ERROR CODE */
234 /* ************************************************************************* */
236 eth_ark_recv_pkts_noop(void *rx_queue __rte_unused
,
237 struct rte_mbuf
**rx_pkts __rte_unused
,
238 uint16_t nb_pkts __rte_unused
)
243 /* ************************************************************************* */
245 eth_ark_recv_pkts(void *rx_queue
,
246 struct rte_mbuf
**rx_pkts
,
249 struct ark_rx_queue
*queue
;
250 register uint32_t cons_index
, prod_index
;
252 struct rte_mbuf
*mbuf
;
253 struct ark_rx_meta
*meta
;
255 queue
= (struct ark_rx_queue
*)rx_queue
;
256 if (unlikely(queue
== 0))
258 if (unlikely(nb_pkts
== 0))
260 prod_index
= queue
->prod_index
;
261 cons_index
= queue
->cons_index
;
264 while (prod_index
!= cons_index
) {
265 mbuf
= queue
->reserve_q
[cons_index
& queue
->queue_mask
];
267 rte_mbuf_prefetch_part1(mbuf
);
268 rte_mbuf_prefetch_part2(mbuf
);
270 /* META DATA embedded in headroom */
271 meta
= RTE_PTR_ADD(mbuf
->buf_addr
, ARK_RX_META_OFFSET
);
273 mbuf
->port
= meta
->port
;
274 mbuf
->pkt_len
= meta
->pkt_len
;
275 mbuf
->data_len
= meta
->pkt_len
;
276 mbuf
->timestamp
= meta
->timestamp
;
277 mbuf
->udata64
= meta
->user_data
;
279 if (ARK_RX_DEBUG
) { /* debug sanity checks */
280 if ((meta
->pkt_len
> (1024 * 16)) ||
281 (meta
->pkt_len
== 0)) {
282 PMD_RX_LOG(DEBUG
, "RX: Bad Meta Q: %u"
285 " seed_index %" PRIU32
293 PMD_RX_LOG(DEBUG
, " : UDM"
296 queue
->udm
->rt_cfg
.prod_idx
,
298 ark_mpu_dump(queue
->mpu
,
301 dump_mbuf_data(mbuf
, 0, 256);
302 /* its FUBAR so fix it */
306 /* seqn is only set under debug */
307 mbuf
->seqn
= cons_index
;
310 if (unlikely(meta
->pkt_len
> ARK_RX_MAX_NOCHAIN
))
311 cons_index
= eth_ark_rx_jumbo
312 (queue
, meta
, mbuf
, cons_index
+ 1);
322 if (unlikely(nb
!= 0))
323 /* report next free to FPGA */
324 eth_ark_rx_update_cons_index(queue
, cons_index
);
329 /* ************************************************************************* */
331 eth_ark_rx_jumbo(struct ark_rx_queue
*queue
,
332 struct ark_rx_meta
*meta
,
333 struct rte_mbuf
*mbuf0
,
336 struct rte_mbuf
*mbuf_prev
;
337 struct rte_mbuf
*mbuf
;
343 /* first buf populated by called */
346 data_len
= RTE_MIN(meta
->pkt_len
, RTE_MBUF_DEFAULT_DATAROOM
);
347 remaining
= meta
->pkt_len
- data_len
;
348 mbuf0
->data_len
= data_len
;
350 /* HW guarantees that the data does not exceed prod_index! */
351 while (remaining
!= 0) {
352 data_len
= RTE_MIN(remaining
,
353 RTE_MBUF_DEFAULT_DATAROOM
+
354 RTE_PKTMBUF_HEADROOM
);
356 remaining
-= data_len
;
359 mbuf
= queue
->reserve_q
[cons_index
& queue
->queue_mask
];
360 mbuf_prev
->next
= mbuf
;
362 mbuf
->data_len
= data_len
;
365 mbuf
->seqn
= cons_index
; /* for debug only */
370 mbuf0
->nb_segs
= segments
;
374 /* Drain the internal queue allowing hw to clear out. */
376 eth_ark_rx_queue_drain(struct ark_rx_queue
*queue
)
378 register uint32_t cons_index
;
379 struct rte_mbuf
*mbuf
;
381 cons_index
= queue
->cons_index
;
383 /* NOT performance optimized, since this is a one-shot call */
384 while ((cons_index
^ queue
->prod_index
) & queue
->queue_mask
) {
385 mbuf
= queue
->reserve_q
[cons_index
& queue
->queue_mask
];
386 rte_pktmbuf_free(mbuf
);
388 eth_ark_rx_update_cons_index(queue
, cons_index
);
393 eth_ark_dev_rx_queue_count(struct rte_eth_dev
*dev
, uint16_t queue_id
)
395 struct ark_rx_queue
*queue
;
397 queue
= dev
->data
->rx_queues
[queue_id
];
398 return (queue
->prod_index
- queue
->cons_index
); /* mod arith */
401 /* ************************************************************************* */
403 eth_ark_rx_start_queue(struct rte_eth_dev
*dev
, uint16_t queue_id
)
405 struct ark_rx_queue
*queue
;
407 queue
= dev
->data
->rx_queues
[queue_id
];
411 dev
->data
->rx_queue_state
[queue_id
] = RTE_ETH_QUEUE_STATE_STARTED
;
413 ark_mpu_set_producer(queue
->mpu
, queue
->seed_index
);
414 ark_mpu_start(queue
->mpu
);
416 ark_udm_queue_enable(queue
->udm
, 1);
421 /* ************************************************************************* */
423 /* Queue can be restarted. data remains
426 eth_ark_rx_stop_queue(struct rte_eth_dev
*dev
, uint16_t queue_id
)
428 struct ark_rx_queue
*queue
;
430 queue
= dev
->data
->rx_queues
[queue_id
];
434 ark_udm_queue_enable(queue
->udm
, 0);
436 dev
->data
->rx_queue_state
[queue_id
] = RTE_ETH_QUEUE_STATE_STOPPED
;
441 /* ************************************************************************* */
443 eth_ark_rx_seed_mbufs(struct ark_rx_queue
*queue
)
445 uint32_t limit
= queue
->cons_index
+ queue
->queue_size
;
446 uint32_t seed_index
= queue
->seed_index
;
449 uint32_t seed_m
= queue
->seed_index
& queue
->queue_mask
;
451 uint32_t nb
= limit
- seed_index
;
453 /* Handle wrap around -- remainder is filled on the next call */
454 if (unlikely(seed_m
+ nb
> queue
->queue_size
))
455 nb
= queue
->queue_size
- seed_m
;
457 struct rte_mbuf
**mbufs
= &queue
->reserve_q
[seed_m
];
458 int status
= rte_pktmbuf_alloc_bulk(queue
->mb_pool
, mbufs
, nb
);
460 if (unlikely(status
!= 0)) {
461 /* Try to recover from lack of mbufs in pool */
462 status
= eth_ark_rx_seed_recovery(queue
, &nb
, mbufs
);
463 if (unlikely(status
!= 0)) {
468 if (ARK_RX_DEBUG
) { /* DEBUG */
469 while (count
!= nb
) {
470 struct rte_mbuf
*mbuf_init
=
471 queue
->reserve_q
[seed_m
+ count
];
473 memset(mbuf_init
->buf_addr
, -1, 512);
474 *((uint32_t *)mbuf_init
->buf_addr
) =
476 *(uint16_t *)RTE_PTR_ADD(mbuf_init
->buf_addr
, 4) =
482 queue
->seed_index
+= nb
;
484 /* Duff's device https://en.wikipedia.org/wiki/Duff's_device */
487 while (count
!= nb
) {
488 queue
->paddress_q
[seed_m
++] =
489 (*mbufs
++)->buf_iova
;
493 queue
->paddress_q
[seed_m
++] =
494 (*mbufs
++)->buf_iova
;
498 queue
->paddress_q
[seed_m
++] =
499 (*mbufs
++)->buf_iova
;
503 queue
->paddress_q
[seed_m
++] =
504 (*mbufs
++)->buf_iova
;
508 } /* while (count != nb) */
515 eth_ark_rx_seed_recovery(struct ark_rx_queue
*queue
,
517 struct rte_mbuf
**mbufs
)
521 /* Ignore small allocation failures */
526 status
= rte_pktmbuf_alloc_bulk(queue
->mb_pool
, mbufs
, *pnb
);
529 "ARK: Could not allocate %u mbufs from pool for RX queue %u;"
530 " %u free buffers remaining in queue\n",
531 *pnb
, queue
->queue_index
,
532 queue
->seed_index
- queue
->cons_index
);
538 eth_ark_rx_dump_queue(struct rte_eth_dev
*dev
, uint16_t queue_id
,
541 struct ark_rx_queue
*queue
;
543 queue
= dev
->data
->rx_queues
[queue_id
];
545 ark_ethdev_rx_dump(msg
, queue
);
548 /* ************************************************************************* */
549 /* Call on device closed no user API, queue is stopped */
551 eth_ark_dev_rx_queue_release(void *vqueue
)
553 struct ark_rx_queue
*queue
;
556 queue
= (struct ark_rx_queue
*)vqueue
;
560 ark_udm_queue_enable(queue
->udm
, 0);
561 /* Stop the MPU since pointer are going away */
562 ark_mpu_stop(queue
->mpu
);
564 /* Need to clear out mbufs here, dropping packets along the way */
565 eth_ark_rx_queue_drain(queue
);
567 for (i
= 0; i
< queue
->queue_size
; ++i
)
568 rte_pktmbuf_free(queue
->reserve_q
[i
]);
570 rte_free(queue
->reserve_q
);
571 rte_free(queue
->paddress_q
);
576 eth_rx_queue_stats_get(void *vqueue
, struct rte_eth_stats
*stats
)
578 struct ark_rx_queue
*queue
;
579 struct ark_udm_t
*udm
;
586 uint64_t ibytes
= ark_udm_bytes(udm
);
587 uint64_t ipackets
= ark_udm_packets(udm
);
588 uint64_t idropped
= ark_udm_dropped(queue
->udm
);
590 stats
->q_ipackets
[queue
->queue_index
] = ipackets
;
591 stats
->q_ibytes
[queue
->queue_index
] = ibytes
;
592 stats
->q_errors
[queue
->queue_index
] = idropped
;
593 stats
->ipackets
+= ipackets
;
594 stats
->ibytes
+= ibytes
;
595 stats
->imissed
+= idropped
;
599 eth_rx_queue_stats_reset(void *vqueue
)
601 struct ark_rx_queue
*queue
;
607 ark_mpu_reset_stats(queue
->mpu
);
608 ark_udm_queue_stats_reset(queue
->udm
);
612 eth_ark_udm_force_close(struct rte_eth_dev
*dev
)
614 struct ark_adapter
*ark
= (struct ark_adapter
*)dev
->data
->dev_private
;
615 struct ark_rx_queue
*queue
;
619 if (!ark_udm_is_flushed(ark
->udm
.v
)) {
620 /* restart the MPUs */
621 PMD_DRV_LOG(ERR
, "ARK: %s UDM not flushed\n", __func__
);
622 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
623 queue
= (struct ark_rx_queue
*)dev
->data
->rx_queues
[i
];
627 ark_mpu_start(queue
->mpu
);
628 /* Add some buffers */
629 index
= 100000 + queue
->seed_index
;
630 ark_mpu_set_producer(queue
->mpu
, index
);
632 /* Wait to allow data to pass */
635 PMD_DEBUG_LOG(DEBUG
, "UDM forced flush attempt, stopped = %d\n",
636 ark_udm_is_flushed(ark
->udm
.v
));
638 ark_udm_reset(ark
->udm
.v
);
642 ark_ethdev_rx_dump(const char *name
, struct ark_rx_queue
*queue
)
646 PMD_DEBUG_LOG(DEBUG
, "RX QUEUE %d -- %s", queue
->phys_qid
, name
);
647 PMD_DEBUG_LOG(DEBUG
, ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32
"\n",
648 "queue_size", queue
->queue_size
,
649 "seed_index", queue
->seed_index
,
650 "prod_index", queue
->prod_index
,
651 "cons_index", queue
->cons_index
);
653 ark_mpu_dump(queue
->mpu
, name
, queue
->phys_qid
);
654 ark_mpu_dump_setup(queue
->mpu
, queue
->phys_qid
);
655 ark_udm_dump(queue
->udm
, name
);
656 ark_udm_dump_setup(queue
->udm
, queue
->phys_qid
);
659 /* Only used in debug.
660 * This function is a raw memory dump of a portion of an mbuf's memory
661 * region. The usual function, rte_pktmbuf_dump() only shows data
662 * with respect to the data_off field. This function show data
663 * anywhere in the mbuf's buffer. This is useful for examining
664 * data in the headroom or tailroom portion of an mbuf.
667 dump_mbuf_data(struct rte_mbuf
*mbuf
, uint16_t lo
, uint16_t hi
)
671 PMD_DRV_LOG(INFO
, " MBUF: %p len %d, off: %d, seq: %" PRIU32
"\n", mbuf
,
672 mbuf
->pkt_len
, mbuf
->data_off
, mbuf
->seqn
);
673 for (i
= lo
; i
< hi
; i
+= 16) {
674 uint8_t *dp
= RTE_PTR_ADD(mbuf
->buf_addr
, i
);
676 PMD_DRV_LOG(INFO
, " %6d: ", i
);
677 for (j
= 0; j
< 16; j
++)
678 PMD_DRV_LOG(INFO
, " %02x", dp
[j
]);
680 PMD_DRV_LOG(INFO
, "\n");