1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2015 Intel Corporation
5 #include "rte_eth_ring.h"
7 #include <rte_ethdev_driver.h>
8 #include <rte_malloc.h>
9 #include <rte_memcpy.h>
10 #include <rte_string_fns.h>
11 #include <rte_bus_vdev.h>
12 #include <rte_kvargs.h>
13 #include <rte_errno.h>
15 #define ETH_RING_NUMA_NODE_ACTION_ARG "nodeaction"
16 #define ETH_RING_ACTION_CREATE "CREATE"
17 #define ETH_RING_ACTION_ATTACH "ATTACH"
18 #define ETH_RING_INTERNAL_ARG "internal"
20 static const char *valid_arguments
[] = {
21 ETH_RING_NUMA_NODE_ACTION_ARG
,
22 ETH_RING_INTERNAL_ARG
,
26 struct ring_internal_args
{
27 struct rte_ring
* const *rx_queues
;
28 const unsigned int nb_rx_queues
;
29 struct rte_ring
* const *tx_queues
;
30 const unsigned int nb_tx_queues
;
31 const unsigned int numa_node
;
32 void *addr
; /* self addr for sanity check */
42 rte_atomic64_t rx_pkts
;
43 rte_atomic64_t tx_pkts
;
44 rte_atomic64_t err_pkts
;
47 struct pmd_internals
{
48 unsigned int max_rx_queues
;
49 unsigned int max_tx_queues
;
51 struct ring_queue rx_ring_queues
[RTE_PMD_RING_MAX_RX_RINGS
];
52 struct ring_queue tx_ring_queues
[RTE_PMD_RING_MAX_TX_RINGS
];
54 struct ether_addr address
;
55 enum dev_action action
;
58 static struct rte_eth_link pmd_link
= {
59 .link_speed
= ETH_SPEED_NUM_10G
,
60 .link_duplex
= ETH_LINK_FULL_DUPLEX
,
61 .link_status
= ETH_LINK_DOWN
,
62 .link_autoneg
= ETH_LINK_FIXED
,
65 static int eth_ring_logtype
;
67 #define PMD_LOG(level, fmt, args...) \
68 rte_log(RTE_LOG_ ## level, eth_ring_logtype, \
69 "%s(): " fmt "\n", __func__, ##args)
72 eth_ring_rx(void *q
, struct rte_mbuf
**bufs
, uint16_t nb_bufs
)
74 void **ptrs
= (void *)&bufs
[0];
75 struct ring_queue
*r
= q
;
76 const uint16_t nb_rx
= (uint16_t)rte_ring_dequeue_burst(r
->rng
,
78 if (r
->rng
->flags
& RING_F_SC_DEQ
)
79 r
->rx_pkts
.cnt
+= nb_rx
;
81 rte_atomic64_add(&(r
->rx_pkts
), nb_rx
);
86 eth_ring_tx(void *q
, struct rte_mbuf
**bufs
, uint16_t nb_bufs
)
88 void **ptrs
= (void *)&bufs
[0];
89 struct ring_queue
*r
= q
;
90 const uint16_t nb_tx
= (uint16_t)rte_ring_enqueue_burst(r
->rng
,
92 if (r
->rng
->flags
& RING_F_SP_ENQ
) {
93 r
->tx_pkts
.cnt
+= nb_tx
;
94 r
->err_pkts
.cnt
+= nb_bufs
- nb_tx
;
96 rte_atomic64_add(&(r
->tx_pkts
), nb_tx
);
97 rte_atomic64_add(&(r
->err_pkts
), nb_bufs
- nb_tx
);
103 eth_dev_configure(struct rte_eth_dev
*dev __rte_unused
) { return 0; }
106 eth_dev_start(struct rte_eth_dev
*dev
)
108 dev
->data
->dev_link
.link_status
= ETH_LINK_UP
;
113 eth_dev_stop(struct rte_eth_dev
*dev
)
115 dev
->data
->dev_link
.link_status
= ETH_LINK_DOWN
;
119 eth_dev_set_link_down(struct rte_eth_dev
*dev
)
121 dev
->data
->dev_link
.link_status
= ETH_LINK_DOWN
;
126 eth_dev_set_link_up(struct rte_eth_dev
*dev
)
128 dev
->data
->dev_link
.link_status
= ETH_LINK_UP
;
133 eth_rx_queue_setup(struct rte_eth_dev
*dev
, uint16_t rx_queue_id
,
134 uint16_t nb_rx_desc __rte_unused
,
135 unsigned int socket_id __rte_unused
,
136 const struct rte_eth_rxconf
*rx_conf __rte_unused
,
137 struct rte_mempool
*mb_pool __rte_unused
)
139 struct pmd_internals
*internals
= dev
->data
->dev_private
;
141 dev
->data
->rx_queues
[rx_queue_id
] = &internals
->rx_ring_queues
[rx_queue_id
];
146 eth_tx_queue_setup(struct rte_eth_dev
*dev
, uint16_t tx_queue_id
,
147 uint16_t nb_tx_desc __rte_unused
,
148 unsigned int socket_id __rte_unused
,
149 const struct rte_eth_txconf
*tx_conf __rte_unused
)
151 struct pmd_internals
*internals
= dev
->data
->dev_private
;
153 dev
->data
->tx_queues
[tx_queue_id
] = &internals
->tx_ring_queues
[tx_queue_id
];
159 eth_dev_info(struct rte_eth_dev
*dev
,
160 struct rte_eth_dev_info
*dev_info
)
162 struct pmd_internals
*internals
= dev
->data
->dev_private
;
164 dev_info
->max_mac_addrs
= 1;
165 dev_info
->max_rx_pktlen
= (uint32_t)-1;
166 dev_info
->max_rx_queues
= (uint16_t)internals
->max_rx_queues
;
167 dev_info
->max_tx_queues
= (uint16_t)internals
->max_tx_queues
;
168 dev_info
->min_rx_bufsize
= 0;
172 eth_stats_get(struct rte_eth_dev
*dev
, struct rte_eth_stats
*stats
)
175 unsigned long rx_total
= 0, tx_total
= 0, tx_err_total
= 0;
176 const struct pmd_internals
*internal
= dev
->data
->dev_private
;
178 for (i
= 0; i
< RTE_ETHDEV_QUEUE_STAT_CNTRS
&&
179 i
< dev
->data
->nb_rx_queues
; i
++) {
180 stats
->q_ipackets
[i
] = internal
->rx_ring_queues
[i
].rx_pkts
.cnt
;
181 rx_total
+= stats
->q_ipackets
[i
];
184 for (i
= 0; i
< RTE_ETHDEV_QUEUE_STAT_CNTRS
&&
185 i
< dev
->data
->nb_tx_queues
; i
++) {
186 stats
->q_opackets
[i
] = internal
->tx_ring_queues
[i
].tx_pkts
.cnt
;
187 stats
->q_errors
[i
] = internal
->tx_ring_queues
[i
].err_pkts
.cnt
;
188 tx_total
+= stats
->q_opackets
[i
];
189 tx_err_total
+= stats
->q_errors
[i
];
192 stats
->ipackets
= rx_total
;
193 stats
->opackets
= tx_total
;
194 stats
->oerrors
= tx_err_total
;
200 eth_stats_reset(struct rte_eth_dev
*dev
)
203 struct pmd_internals
*internal
= dev
->data
->dev_private
;
205 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++)
206 internal
->rx_ring_queues
[i
].rx_pkts
.cnt
= 0;
207 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
208 internal
->tx_ring_queues
[i
].tx_pkts
.cnt
= 0;
209 internal
->tx_ring_queues
[i
].err_pkts
.cnt
= 0;
214 eth_mac_addr_remove(struct rte_eth_dev
*dev __rte_unused
,
215 uint32_t index __rte_unused
)
220 eth_mac_addr_add(struct rte_eth_dev
*dev __rte_unused
,
221 struct ether_addr
*mac_addr __rte_unused
,
222 uint32_t index __rte_unused
,
223 uint32_t vmdq __rte_unused
)
229 eth_queue_release(void *q __rte_unused
) { ; }
231 eth_link_update(struct rte_eth_dev
*dev __rte_unused
,
232 int wait_to_complete __rte_unused
) { return 0; }
234 static const struct eth_dev_ops ops
= {
235 .dev_start
= eth_dev_start
,
236 .dev_stop
= eth_dev_stop
,
237 .dev_set_link_up
= eth_dev_set_link_up
,
238 .dev_set_link_down
= eth_dev_set_link_down
,
239 .dev_configure
= eth_dev_configure
,
240 .dev_infos_get
= eth_dev_info
,
241 .rx_queue_setup
= eth_rx_queue_setup
,
242 .tx_queue_setup
= eth_tx_queue_setup
,
243 .rx_queue_release
= eth_queue_release
,
244 .tx_queue_release
= eth_queue_release
,
245 .link_update
= eth_link_update
,
246 .stats_get
= eth_stats_get
,
247 .stats_reset
= eth_stats_reset
,
248 .mac_addr_remove
= eth_mac_addr_remove
,
249 .mac_addr_add
= eth_mac_addr_add
,
252 static struct rte_vdev_driver pmd_ring_drv
;
255 do_eth_dev_ring_create(const char *name
,
256 struct rte_ring
* const rx_queues
[],
257 const unsigned int nb_rx_queues
,
258 struct rte_ring
*const tx_queues
[],
259 const unsigned int nb_tx_queues
,
260 const unsigned int numa_node
, enum dev_action action
,
261 struct rte_eth_dev
**eth_dev_p
)
263 struct rte_eth_dev_data
*data
= NULL
;
264 struct pmd_internals
*internals
= NULL
;
265 struct rte_eth_dev
*eth_dev
= NULL
;
266 void **rx_queues_local
= NULL
;
267 void **tx_queues_local
= NULL
;
270 PMD_LOG(INFO
, "Creating rings-backed ethdev on numa socket %u",
273 rx_queues_local
= rte_calloc_socket(name
, nb_rx_queues
,
274 sizeof(void *), 0, numa_node
);
275 if (rx_queues_local
== NULL
) {
280 tx_queues_local
= rte_calloc_socket(name
, nb_tx_queues
,
281 sizeof(void *), 0, numa_node
);
282 if (tx_queues_local
== NULL
) {
287 internals
= rte_zmalloc_socket(name
, sizeof(*internals
), 0, numa_node
);
288 if (internals
== NULL
) {
293 /* reserve an ethdev entry */
294 eth_dev
= rte_eth_dev_allocate(name
);
295 if (eth_dev
== NULL
) {
300 /* now put it all together
301 * - store queue data in internals,
302 * - store numa_node info in eth_dev_data
303 * - point eth_dev_data to internals
304 * - and point eth_dev structure to new eth_dev_data structure
307 data
= eth_dev
->data
;
308 data
->rx_queues
= rx_queues_local
;
309 data
->tx_queues
= tx_queues_local
;
311 internals
->action
= action
;
312 internals
->max_rx_queues
= nb_rx_queues
;
313 internals
->max_tx_queues
= nb_tx_queues
;
314 for (i
= 0; i
< nb_rx_queues
; i
++) {
315 internals
->rx_ring_queues
[i
].rng
= rx_queues
[i
];
316 data
->rx_queues
[i
] = &internals
->rx_ring_queues
[i
];
318 for (i
= 0; i
< nb_tx_queues
; i
++) {
319 internals
->tx_ring_queues
[i
].rng
= tx_queues
[i
];
320 data
->tx_queues
[i
] = &internals
->tx_ring_queues
[i
];
323 data
->dev_private
= internals
;
324 data
->nb_rx_queues
= (uint16_t)nb_rx_queues
;
325 data
->nb_tx_queues
= (uint16_t)nb_tx_queues
;
326 data
->dev_link
= pmd_link
;
327 data
->mac_addrs
= &internals
->address
;
329 eth_dev
->dev_ops
= &ops
;
330 data
->kdrv
= RTE_KDRV_NONE
;
331 data
->numa_node
= numa_node
;
333 /* finally assign rx and tx ops */
334 eth_dev
->rx_pkt_burst
= eth_ring_rx
;
335 eth_dev
->tx_pkt_burst
= eth_ring_tx
;
337 rte_eth_dev_probing_finish(eth_dev
);
338 *eth_dev_p
= eth_dev
;
340 return data
->port_id
;
343 rte_free(rx_queues_local
);
344 rte_free(tx_queues_local
);
351 rte_eth_from_rings(const char *name
, struct rte_ring
*const rx_queues
[],
352 const unsigned int nb_rx_queues
,
353 struct rte_ring
*const tx_queues
[],
354 const unsigned int nb_tx_queues
,
355 const unsigned int numa_node
)
357 struct ring_internal_args args
= {
358 .rx_queues
= rx_queues
,
359 .nb_rx_queues
= nb_rx_queues
,
360 .tx_queues
= tx_queues
,
361 .nb_tx_queues
= nb_tx_queues
,
362 .numa_node
= numa_node
,
366 char ring_name
[RTE_RING_NAMESIZE
];
367 uint16_t port_id
= RTE_MAX_ETHPORTS
;
370 /* do some parameter checking */
371 if (rx_queues
== NULL
&& nb_rx_queues
> 0) {
375 if (tx_queues
== NULL
&& nb_tx_queues
> 0) {
379 if (nb_rx_queues
> RTE_PMD_RING_MAX_RX_RINGS
) {
384 snprintf(args_str
, sizeof(args_str
), "%s=%p",
385 ETH_RING_INTERNAL_ARG
, &args
);
387 ret
= snprintf(ring_name
, sizeof(ring_name
), "net_ring_%s", name
);
388 if (ret
>= (int)sizeof(ring_name
)) {
389 rte_errno
= ENAMETOOLONG
;
393 ret
= rte_vdev_init(ring_name
, args_str
);
399 ret
= rte_eth_dev_get_port_by_name(ring_name
, &port_id
);
409 rte_eth_from_ring(struct rte_ring
*r
)
411 return rte_eth_from_rings(r
->name
, &r
, 1, &r
, 1,
412 r
->memzone
? r
->memzone
->socket_id
: SOCKET_ID_ANY
);
416 eth_dev_ring_create(const char *name
, const unsigned int numa_node
,
417 enum dev_action action
, struct rte_eth_dev
**eth_dev
)
419 /* rx and tx are so-called from point of view of first port.
420 * They are inverted from the point of view of second port
422 struct rte_ring
*rxtx
[RTE_PMD_RING_MAX_RX_RINGS
];
424 char rng_name
[RTE_RING_NAMESIZE
];
425 unsigned int num_rings
= RTE_MIN(RTE_PMD_RING_MAX_RX_RINGS
,
426 RTE_PMD_RING_MAX_TX_RINGS
);
428 for (i
= 0; i
< num_rings
; i
++) {
431 cc
= snprintf(rng_name
, sizeof(rng_name
),
432 "ETH_RXTX%u_%s", i
, name
);
433 if (cc
>= (int)sizeof(rng_name
)) {
434 rte_errno
= ENAMETOOLONG
;
438 rxtx
[i
] = (action
== DEV_CREATE
) ?
439 rte_ring_create(rng_name
, 1024, numa_node
,
440 RING_F_SP_ENQ
|RING_F_SC_DEQ
) :
441 rte_ring_lookup(rng_name
);
446 if (do_eth_dev_ring_create(name
, rxtx
, num_rings
, rxtx
, num_rings
,
447 numa_node
, action
, eth_dev
) < 0)
453 struct node_action_pair
{
456 enum dev_action action
;
459 struct node_action_list
{
462 struct node_action_pair
*list
;
465 static int parse_kvlist(const char *key __rte_unused
,
466 const char *value
, void *data
)
468 struct node_action_list
*info
= data
;
475 name
= strdup(value
);
480 PMD_LOG(WARNING
, "command line parameter is empty for ring pmd!");
484 node
= strchr(name
, ':');
486 PMD_LOG(WARNING
, "could not parse node value from %s",
494 action
= strchr(node
, ':');
496 PMD_LOG(WARNING
, "could not parse action value from %s",
505 * Need to do some sanity checking here
508 if (strcmp(action
, ETH_RING_ACTION_ATTACH
) == 0)
509 info
->list
[info
->count
].action
= DEV_ATTACH
;
510 else if (strcmp(action
, ETH_RING_ACTION_CREATE
) == 0)
511 info
->list
[info
->count
].action
= DEV_CREATE
;
516 info
->list
[info
->count
].node
= strtol(node
, &end
, 10);
518 if ((errno
!= 0) || (*end
!= '\0')) {
520 "node value %s is unparseable as a number", node
);
524 strlcpy(info
->list
[info
->count
].name
, name
,
525 sizeof(info
->list
[info
->count
].name
));
536 parse_internal_args(const char *key __rte_unused
, const char *value
,
539 struct ring_internal_args
**internal_args
= data
;
542 sscanf(value
, "%p", &args
);
544 *internal_args
= args
;
546 if ((*internal_args
)->addr
!= args
)
553 rte_pmd_ring_probe(struct rte_vdev_device
*dev
)
555 const char *name
, *params
;
556 struct rte_kvargs
*kvlist
= NULL
;
558 struct node_action_list
*info
= NULL
;
559 struct rte_eth_dev
*eth_dev
= NULL
;
560 struct ring_internal_args
*internal_args
;
562 name
= rte_vdev_device_name(dev
);
563 params
= rte_vdev_device_args(dev
);
565 PMD_LOG(INFO
, "Initializing pmd_ring for %s", name
);
567 if (params
== NULL
|| params
[0] == '\0') {
568 ret
= eth_dev_ring_create(name
, rte_socket_id(), DEV_CREATE
,
572 "Attach to pmd_ring for %s", name
);
573 ret
= eth_dev_ring_create(name
, rte_socket_id(),
574 DEV_ATTACH
, ð_dev
);
577 kvlist
= rte_kvargs_parse(params
, valid_arguments
);
581 "Ignoring unsupported parameters when creatingrings-backed ethernet device");
582 ret
= eth_dev_ring_create(name
, rte_socket_id(),
583 DEV_CREATE
, ð_dev
);
586 "Attach to pmd_ring for %s",
588 ret
= eth_dev_ring_create(name
, rte_socket_id(),
589 DEV_ATTACH
, ð_dev
);
593 eth_dev
->device
= &dev
->device
;
598 if (rte_kvargs_count(kvlist
, ETH_RING_INTERNAL_ARG
) == 1) {
599 ret
= rte_kvargs_process(kvlist
, ETH_RING_INTERNAL_ARG
,
605 ret
= do_eth_dev_ring_create(name
,
606 internal_args
->rx_queues
,
607 internal_args
->nb_rx_queues
,
608 internal_args
->tx_queues
,
609 internal_args
->nb_tx_queues
,
610 internal_args
->numa_node
,
616 ret
= rte_kvargs_count(kvlist
, ETH_RING_NUMA_NODE_ACTION_ARG
);
617 info
= rte_zmalloc("struct node_action_list",
618 sizeof(struct node_action_list
) +
619 (sizeof(struct node_action_pair
) * ret
),
625 info
->list
= (struct node_action_pair
*)(info
+ 1);
627 ret
= rte_kvargs_process(kvlist
, ETH_RING_NUMA_NODE_ACTION_ARG
,
633 for (info
->count
= 0; info
->count
< info
->total
; info
->count
++) {
634 ret
= eth_dev_ring_create(info
->list
[info
->count
].name
,
635 info
->list
[info
->count
].node
,
636 info
->list
[info
->count
].action
,
639 (info
->list
[info
->count
].action
== DEV_CREATE
)) {
641 "Attach to pmd_ring for %s",
643 ret
= eth_dev_ring_create(name
,
644 info
->list
[info
->count
].node
,
653 eth_dev
->device
= &dev
->device
;
656 rte_kvargs_free(kvlist
);
662 rte_pmd_ring_remove(struct rte_vdev_device
*dev
)
664 const char *name
= rte_vdev_device_name(dev
);
665 struct rte_eth_dev
*eth_dev
= NULL
;
666 struct pmd_internals
*internals
= NULL
;
667 struct ring_queue
*r
= NULL
;
670 PMD_LOG(INFO
, "Un-Initializing pmd_ring for %s", name
);
675 /* find an ethdev entry */
676 eth_dev
= rte_eth_dev_allocated(name
);
680 eth_dev_stop(eth_dev
);
682 internals
= eth_dev
->data
->dev_private
;
683 if (internals
->action
== DEV_CREATE
) {
685 * it is only necessary to delete the rings in rx_queues because
686 * they are the same used in tx_queues
688 for (i
= 0; i
< eth_dev
->data
->nb_rx_queues
; i
++) {
689 r
= eth_dev
->data
->rx_queues
[i
];
690 rte_ring_free(r
->rng
);
694 /* mac_addrs must not be freed alone because part of dev_private */
695 eth_dev
->data
->mac_addrs
= NULL
;
696 rte_eth_dev_release_port(eth_dev
);
700 static struct rte_vdev_driver pmd_ring_drv
= {
701 .probe
= rte_pmd_ring_probe
,
702 .remove
= rte_pmd_ring_remove
,
705 RTE_PMD_REGISTER_VDEV(net_ring
, pmd_ring_drv
);
706 RTE_PMD_REGISTER_ALIAS(net_ring
, eth_ring
);
707 RTE_PMD_REGISTER_PARAM_STRING(net_ring
,
708 ETH_RING_NUMA_NODE_ACTION_ARG
"=name:node:action(ATTACH|CREATE)");
710 RTE_INIT(eth_ring_init_log
)
712 eth_ring_logtype
= rte_log_register("pmd.net.ring");
713 if (eth_ring_logtype
>= 0)
714 rte_log_set_level(eth_ring_logtype
, RTE_LOG_NOTICE
);