4 * Copyright (c) 2015-2017 Atomic Rules LLC
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of copyright holder nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include "ark_ethdev_rx.h"
37 #include "ark_global.h"
39 #include "ark_ethdev.h"
43 #define ARK_RX_META_SIZE 32
44 #define ARK_RX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_RX_META_SIZE)
45 #define ARK_RX_MAX_NOCHAIN (RTE_MBUF_DEFAULT_DATAROOM)
47 /* Forward declarations */
51 static void dump_mbuf_data(struct rte_mbuf
*mbuf
, uint16_t lo
, uint16_t hi
);
52 static void ark_ethdev_rx_dump(const char *name
, struct ark_rx_queue
*queue
);
53 static uint32_t eth_ark_rx_jumbo(struct ark_rx_queue
*queue
,
54 struct ark_rx_meta
*meta
,
55 struct rte_mbuf
*mbuf0
,
57 static inline int eth_ark_rx_seed_mbufs(struct ark_rx_queue
*queue
);
59 /* ************************************************************************* */
61 /* array of mbufs to populate */
62 struct rte_mbuf
**reserve_q
;
63 /* array of physical addresses of the mbuf data pointer */
64 /* This point is a virtual address */
65 phys_addr_t
*paddress_q
;
66 struct rte_mempool
*mb_pool
;
68 struct ark_udm_t
*udm
;
69 struct ark_mpu_t
*mpu
;
74 uint32_t seed_index
; /* step 1 set with empty mbuf */
75 uint32_t cons_index
; /* step 3 consumed by driver */
77 /* The queue Id is used to identify the HW Q */
80 /* The queue Index is used within the dpdk device structures */
85 /* separate cache line */
86 /* second cache line - fields only used in slow path */
87 MARKER cacheline1 __rte_cache_min_aligned
;
89 volatile uint32_t prod_index
; /* step 2 filled by FPGA */
90 } __rte_cache_aligned
;
93 /* ************************************************************************* */
95 eth_ark_rx_hw_setup(struct rte_eth_dev
*dev
,
96 struct ark_rx_queue
*queue
,
97 uint16_t rx_queue_id __rte_unused
, uint16_t rx_queue_idx
)
99 phys_addr_t queue_base
;
100 phys_addr_t phys_addr_q_base
;
101 phys_addr_t phys_addr_prod_index
;
103 queue_base
= rte_malloc_virt2phy(queue
);
104 phys_addr_prod_index
= queue_base
+
105 offsetof(struct ark_rx_queue
, prod_index
);
107 phys_addr_q_base
= rte_malloc_virt2phy(queue
->paddress_q
);
110 if (ark_mpu_verify(queue
->mpu
, sizeof(phys_addr_t
))) {
111 PMD_DRV_LOG(ERR
, "Illegal configuration rx queue\n");
115 /* Stop and Reset and configure MPU */
116 ark_mpu_configure(queue
->mpu
, phys_addr_q_base
, queue
->queue_size
, 0);
118 ark_udm_write_addr(queue
->udm
, phys_addr_prod_index
);
120 /* advance the valid pointer, but don't start until the queue starts */
121 ark_mpu_reset_stats(queue
->mpu
);
123 /* The seed is the producer index for the HW */
124 ark_mpu_set_producer(queue
->mpu
, queue
->seed_index
);
125 dev
->data
->rx_queue_state
[rx_queue_idx
] = RTE_ETH_QUEUE_STATE_STOPPED
;
131 eth_ark_rx_update_cons_index(struct ark_rx_queue
*queue
, uint32_t cons_index
)
133 queue
->cons_index
= cons_index
;
134 eth_ark_rx_seed_mbufs(queue
);
135 ark_mpu_set_producer(queue
->mpu
, queue
->seed_index
);
138 /* ************************************************************************* */
140 eth_ark_dev_rx_queue_setup(struct rte_eth_dev
*dev
,
143 unsigned int socket_id
,
144 const struct rte_eth_rxconf
*rx_conf
,
145 struct rte_mempool
*mb_pool
)
147 static int warning1
; /* = 0 */
148 struct ark_adapter
*ark
= (struct ark_adapter
*)dev
->data
->dev_private
;
150 struct ark_rx_queue
*queue
;
154 /* Future works: divide the Q's evenly with multi-ports */
155 int port
= dev
->data
->port_id
;
156 int qidx
= port
+ queue_idx
;
158 /* We may already be setup, free memory prior to re-allocation */
159 if (dev
->data
->rx_queues
[queue_idx
] != NULL
) {
160 eth_ark_dev_rx_queue_release(dev
->data
->rx_queues
[queue_idx
]);
161 dev
->data
->rx_queues
[queue_idx
] = NULL
;
164 if (rx_conf
!= NULL
&& warning1
== 0) {
167 "Arkville ignores rte_eth_rxconf argument.\n");
170 if (RTE_PKTMBUF_HEADROOM
< ARK_RX_META_SIZE
) {
172 "Error: DPDK Arkville requires head room > %d bytes (%s)\n",
173 ARK_RX_META_SIZE
, __func__
);
174 return -1; /* ERROR CODE */
177 if (!rte_is_power_of_2(nb_desc
)) {
179 "DPDK Arkville configuration queue size must be power of two %u (%s)\n",
181 return -1; /* ERROR CODE */
184 /* Allocate queue struct */
185 queue
= rte_zmalloc_socket("Ark_rxqueue",
186 sizeof(struct ark_rx_queue
),
190 PMD_DRV_LOG(ERR
, "Failed to allocate memory in %s\n", __func__
);
194 /* NOTE zmalloc is used, no need to 0 indexes, etc. */
195 queue
->mb_pool
= mb_pool
;
196 queue
->phys_qid
= qidx
;
197 queue
->queue_index
= queue_idx
;
198 queue
->queue_size
= nb_desc
;
199 queue
->queue_mask
= nb_desc
- 1;
202 rte_zmalloc_socket("Ark_rx_queue mbuf",
203 nb_desc
* sizeof(struct rte_mbuf
*),
207 rte_zmalloc_socket("Ark_rx_queue paddr",
208 nb_desc
* sizeof(phys_addr_t
),
212 if (queue
->reserve_q
== 0 || queue
->paddress_q
== 0) {
214 "Failed to allocate queue memory in %s\n",
216 rte_free(queue
->reserve_q
);
217 rte_free(queue
->paddress_q
);
222 dev
->data
->rx_queues
[queue_idx
] = queue
;
223 queue
->udm
= RTE_PTR_ADD(ark
->udm
.v
, qidx
* ARK_UDM_QOFFSET
);
224 queue
->mpu
= RTE_PTR_ADD(ark
->mpurx
.v
, qidx
* ARK_MPU_QOFFSET
);
226 /* populate mbuf reserve */
227 status
= eth_ark_rx_seed_mbufs(queue
);
231 status
= eth_ark_rx_hw_setup(dev
, queue
, qidx
, queue_idx
);
233 if (unlikely(status
!= 0)) {
234 struct rte_mbuf
*mbuf
;
236 PMD_DRV_LOG(ERR
, "Failed to initialize RX queue %d %s\n",
239 /* Free the mbufs allocated */
240 for (i
= 0, mbuf
= queue
->reserve_q
[0];
241 i
< nb_desc
; ++i
, mbuf
++) {
242 rte_pktmbuf_free(mbuf
);
244 rte_free(queue
->reserve_q
);
245 rte_free(queue
->paddress_q
);
247 return -1; /* ERROR CODE */
253 /* ************************************************************************* */
255 eth_ark_recv_pkts_noop(void *rx_queue __rte_unused
,
256 struct rte_mbuf
**rx_pkts __rte_unused
,
257 uint16_t nb_pkts __rte_unused
)
262 /* ************************************************************************* */
264 eth_ark_recv_pkts(void *rx_queue
,
265 struct rte_mbuf
**rx_pkts
,
268 struct ark_rx_queue
*queue
;
269 register uint32_t cons_index
, prod_index
;
271 struct rte_mbuf
*mbuf
;
272 struct ark_rx_meta
*meta
;
274 queue
= (struct ark_rx_queue
*)rx_queue
;
275 if (unlikely(queue
== 0))
277 if (unlikely(nb_pkts
== 0))
279 prod_index
= queue
->prod_index
;
280 cons_index
= queue
->cons_index
;
283 while (prod_index
!= cons_index
) {
284 mbuf
= queue
->reserve_q
[cons_index
& queue
->queue_mask
];
286 rte_mbuf_prefetch_part1(mbuf
);
287 rte_mbuf_prefetch_part2(mbuf
);
289 /* META DATA embedded in headroom */
290 meta
= RTE_PTR_ADD(mbuf
->buf_addr
, ARK_RX_META_OFFSET
);
292 mbuf
->port
= meta
->port
;
293 mbuf
->pkt_len
= meta
->pkt_len
;
294 mbuf
->data_len
= meta
->pkt_len
;
295 mbuf
->timestamp
= meta
->timestamp
;
296 mbuf
->udata64
= meta
->user_data
;
298 if (ARK_RX_DEBUG
) { /* debug sanity checks */
299 if ((meta
->pkt_len
> (1024 * 16)) ||
300 (meta
->pkt_len
== 0)) {
301 PMD_RX_LOG(DEBUG
, "RX: Bad Meta Q: %u"
304 " seed_index %" PRIU32
312 PMD_RX_LOG(DEBUG
, " : UDM"
315 queue
->udm
->rt_cfg
.prod_idx
,
317 ark_mpu_dump(queue
->mpu
,
320 dump_mbuf_data(mbuf
, 0, 256);
321 /* its FUBAR so fix it */
325 /* seqn is only set under debug */
326 mbuf
->seqn
= cons_index
;
329 if (unlikely(meta
->pkt_len
> ARK_RX_MAX_NOCHAIN
))
330 cons_index
= eth_ark_rx_jumbo
331 (queue
, meta
, mbuf
, cons_index
+ 1);
341 if (unlikely(nb
!= 0))
342 /* report next free to FPGA */
343 eth_ark_rx_update_cons_index(queue
, cons_index
);
348 /* ************************************************************************* */
350 eth_ark_rx_jumbo(struct ark_rx_queue
*queue
,
351 struct ark_rx_meta
*meta
,
352 struct rte_mbuf
*mbuf0
,
355 struct rte_mbuf
*mbuf_prev
;
356 struct rte_mbuf
*mbuf
;
362 /* first buf populated by called */
365 data_len
= RTE_MIN(meta
->pkt_len
, RTE_MBUF_DEFAULT_DATAROOM
);
366 remaining
= meta
->pkt_len
- data_len
;
367 mbuf0
->data_len
= data_len
;
369 /* HW guarantees that the data does not exceed prod_index! */
370 while (remaining
!= 0) {
371 data_len
= RTE_MIN(remaining
,
372 RTE_MBUF_DEFAULT_DATAROOM
+
373 RTE_PKTMBUF_HEADROOM
);
375 remaining
-= data_len
;
378 mbuf
= queue
->reserve_q
[cons_index
& queue
->queue_mask
];
379 mbuf_prev
->next
= mbuf
;
381 mbuf
->data_len
= data_len
;
384 mbuf
->seqn
= cons_index
; /* for debug only */
389 mbuf0
->nb_segs
= segments
;
393 /* Drain the internal queue allowing hw to clear out. */
395 eth_ark_rx_queue_drain(struct ark_rx_queue
*queue
)
397 register uint32_t cons_index
;
398 struct rte_mbuf
*mbuf
;
400 cons_index
= queue
->cons_index
;
402 /* NOT performance optimized, since this is a one-shot call */
403 while ((cons_index
^ queue
->prod_index
) & queue
->queue_mask
) {
404 mbuf
= queue
->reserve_q
[cons_index
& queue
->queue_mask
];
405 rte_pktmbuf_free(mbuf
);
407 eth_ark_rx_update_cons_index(queue
, cons_index
);
412 eth_ark_dev_rx_queue_count(struct rte_eth_dev
*dev
, uint16_t queue_id
)
414 struct ark_rx_queue
*queue
;
416 queue
= dev
->data
->rx_queues
[queue_id
];
417 return (queue
->prod_index
- queue
->cons_index
); /* mod arith */
420 /* ************************************************************************* */
422 eth_ark_rx_start_queue(struct rte_eth_dev
*dev
, uint16_t queue_id
)
424 struct ark_rx_queue
*queue
;
426 queue
= dev
->data
->rx_queues
[queue_id
];
430 dev
->data
->rx_queue_state
[queue_id
] = RTE_ETH_QUEUE_STATE_STARTED
;
432 ark_mpu_set_producer(queue
->mpu
, queue
->seed_index
);
433 ark_mpu_start(queue
->mpu
);
435 ark_udm_queue_enable(queue
->udm
, 1);
440 /* ************************************************************************* */
442 /* Queue can be restarted. data remains
445 eth_ark_rx_stop_queue(struct rte_eth_dev
*dev
, uint16_t queue_id
)
447 struct ark_rx_queue
*queue
;
449 queue
= dev
->data
->rx_queues
[queue_id
];
453 ark_udm_queue_enable(queue
->udm
, 0);
455 dev
->data
->rx_queue_state
[queue_id
] = RTE_ETH_QUEUE_STATE_STOPPED
;
460 /* ************************************************************************* */
462 eth_ark_rx_seed_mbufs(struct ark_rx_queue
*queue
)
464 uint32_t limit
= queue
->cons_index
+ queue
->queue_size
;
465 uint32_t seed_index
= queue
->seed_index
;
468 uint32_t seed_m
= queue
->seed_index
& queue
->queue_mask
;
470 uint32_t nb
= limit
- seed_index
;
472 /* Handle wrap around -- remainder is filled on the next call */
473 if (unlikely(seed_m
+ nb
> queue
->queue_size
))
474 nb
= queue
->queue_size
- seed_m
;
476 struct rte_mbuf
**mbufs
= &queue
->reserve_q
[seed_m
];
477 int status
= rte_pktmbuf_alloc_bulk(queue
->mb_pool
, mbufs
, nb
);
479 if (unlikely(status
!= 0))
482 if (ARK_RX_DEBUG
) { /* DEBUG */
483 while (count
!= nb
) {
484 struct rte_mbuf
*mbuf_init
=
485 queue
->reserve_q
[seed_m
+ count
];
487 memset(mbuf_init
->buf_addr
, -1, 512);
488 *((uint32_t *)mbuf_init
->buf_addr
) =
490 *(uint16_t *)RTE_PTR_ADD(mbuf_init
->buf_addr
, 4) =
496 queue
->seed_index
+= nb
;
498 /* Duff's device https://en.wikipedia.org/wiki/Duff's_device */
501 while (count
!= nb
) {
502 queue
->paddress_q
[seed_m
++] =
503 (*mbufs
++)->buf_physaddr
;
507 queue
->paddress_q
[seed_m
++] =
508 (*mbufs
++)->buf_physaddr
;
512 queue
->paddress_q
[seed_m
++] =
513 (*mbufs
++)->buf_physaddr
;
517 queue
->paddress_q
[seed_m
++] =
518 (*mbufs
++)->buf_physaddr
;
522 } /* while (count != nb) */
529 eth_ark_rx_dump_queue(struct rte_eth_dev
*dev
, uint16_t queue_id
,
532 struct ark_rx_queue
*queue
;
534 queue
= dev
->data
->rx_queues
[queue_id
];
536 ark_ethdev_rx_dump(msg
, queue
);
539 /* ************************************************************************* */
540 /* Call on device closed no user API, queue is stopped */
542 eth_ark_dev_rx_queue_release(void *vqueue
)
544 struct ark_rx_queue
*queue
;
547 queue
= (struct ark_rx_queue
*)vqueue
;
551 ark_udm_queue_enable(queue
->udm
, 0);
552 /* Stop the MPU since pointer are going away */
553 ark_mpu_stop(queue
->mpu
);
555 /* Need to clear out mbufs here, dropping packets along the way */
556 eth_ark_rx_queue_drain(queue
);
558 for (i
= 0; i
< queue
->queue_size
; ++i
)
559 rte_pktmbuf_free(queue
->reserve_q
[i
]);
561 rte_free(queue
->reserve_q
);
562 rte_free(queue
->paddress_q
);
567 eth_rx_queue_stats_get(void *vqueue
, struct rte_eth_stats
*stats
)
569 struct ark_rx_queue
*queue
;
570 struct ark_udm_t
*udm
;
577 uint64_t ibytes
= ark_udm_bytes(udm
);
578 uint64_t ipackets
= ark_udm_packets(udm
);
579 uint64_t idropped
= ark_udm_dropped(queue
->udm
);
581 stats
->q_ipackets
[queue
->queue_index
] = ipackets
;
582 stats
->q_ibytes
[queue
->queue_index
] = ibytes
;
583 stats
->q_errors
[queue
->queue_index
] = idropped
;
584 stats
->ipackets
+= ipackets
;
585 stats
->ibytes
+= ibytes
;
586 stats
->imissed
+= idropped
;
590 eth_rx_queue_stats_reset(void *vqueue
)
592 struct ark_rx_queue
*queue
;
598 ark_mpu_reset_stats(queue
->mpu
);
599 ark_udm_queue_stats_reset(queue
->udm
);
603 eth_ark_udm_force_close(struct rte_eth_dev
*dev
)
605 struct ark_adapter
*ark
= (struct ark_adapter
*)dev
->data
->dev_private
;
606 struct ark_rx_queue
*queue
;
610 if (!ark_udm_is_flushed(ark
->udm
.v
)) {
611 /* restart the MPUs */
612 PMD_DRV_LOG(ERR
, "ARK: %s UDM not flushed\n", __func__
);
613 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
614 queue
= (struct ark_rx_queue
*)dev
->data
->rx_queues
[i
];
618 ark_mpu_start(queue
->mpu
);
619 /* Add some buffers */
620 index
= 100000 + queue
->seed_index
;
621 ark_mpu_set_producer(queue
->mpu
, index
);
623 /* Wait to allow data to pass */
626 PMD_DEBUG_LOG(DEBUG
, "UDM forced flush attempt, stopped = %d\n",
627 ark_udm_is_flushed(ark
->udm
.v
));
629 ark_udm_reset(ark
->udm
.v
);
633 ark_ethdev_rx_dump(const char *name
, struct ark_rx_queue
*queue
)
637 PMD_DEBUG_LOG(DEBUG
, "RX QUEUE %d -- %s", queue
->phys_qid
, name
);
638 PMD_DEBUG_LOG(DEBUG
, ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32
"\n",
639 "queue_size", queue
->queue_size
,
640 "seed_index", queue
->seed_index
,
641 "prod_index", queue
->prod_index
,
642 "cons_index", queue
->cons_index
);
644 ark_mpu_dump(queue
->mpu
, name
, queue
->phys_qid
);
645 ark_mpu_dump_setup(queue
->mpu
, queue
->phys_qid
);
646 ark_udm_dump(queue
->udm
, name
);
647 ark_udm_dump_setup(queue
->udm
, queue
->phys_qid
);
650 /* Only used in debug.
651 * This function is a raw memory dump of a portion of an mbuf's memory
652 * region. The usual function, rte_pktmbuf_dump() only shows data
653 * with respect to the data_off field. This function show data
654 * anywhere in the mbuf's buffer. This is useful for examining
655 * data in the headroom or tailroom portion of an mbuf.
658 dump_mbuf_data(struct rte_mbuf
*mbuf
, uint16_t lo
, uint16_t hi
)
662 PMD_DRV_LOG(INFO
, " MBUF: %p len %d, off: %d, seq: %" PRIU32
"\n", mbuf
,
663 mbuf
->pkt_len
, mbuf
->data_off
, mbuf
->seqn
);
664 for (i
= lo
; i
< hi
; i
+= 16) {
665 uint8_t *dp
= RTE_PTR_ADD(mbuf
->buf_addr
, i
);
667 PMD_DRV_LOG(INFO
, " %6d: ", i
);
668 for (j
= 0; j
< 16; j
++)
669 PMD_DRV_LOG(INFO
, " %02x", dp
[j
]);
671 PMD_DRV_LOG(INFO
, "\n");