1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2015 Intel Corporation
7 #include <linux/if_ether.h>
8 #include <linux/if_vlan.h>
9 #include <linux/virtio_net.h>
10 #include <linux/virtio_ring.h>
13 #include <sys/eventfd.h>
14 #include <sys/param.h>
17 #include <rte_atomic.h>
18 #include <rte_cycles.h>
19 #include <rte_ethdev.h>
21 #include <rte_string_fns.h>
22 #include <rte_malloc.h>
23 #include <rte_vhost.h>
24 #include <rte_pause.h>
28 #include "vxlan_setup.h"
30 /* the maximum number of external ports supported */
31 #define MAX_SUP_PORTS 1
34 * Calculate the number of buffers needed per port
36 #define NUM_MBUFS_PER_PORT ((MAX_QUEUES * RTE_TEST_RX_DESC_DEFAULT) +\
37 (nb_switching_cores * MAX_PKT_BURST) +\
38 (nb_switching_cores * \
39 RTE_TEST_TX_DESC_DEFAULT) +\
40 (nb_switching_cores * MBUF_CACHE_SIZE))
42 #define MBUF_CACHE_SIZE 128
43 #define MBUF_DATA_SIZE RTE_MBUF_DEFAULT_BUF_SIZE
45 #define MAX_PKT_BURST 32 /* Max burst size for RX/TX */
46 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
48 /* Defines how long we wait between retries on RX */
49 #define BURST_RX_WAIT_US 15
51 #define BURST_RX_RETRIES 4 /* Number of retries on RX. */
53 #define JUMBO_FRAME_MAX_SIZE 0x2600
55 /* Config_core_flag status definitions. */
56 #define REQUEST_DEV_REMOVAL 1
57 #define ACK_DEV_REMOVAL 0
59 /* Configurable number of RX/TX ring descriptors */
60 #define RTE_TEST_RX_DESC_DEFAULT 1024
61 #define RTE_TEST_TX_DESC_DEFAULT 512
63 /* Get first 4 bytes in mbuf headroom. */
64 #define MBUF_HEADROOM_UINT32(mbuf) (*(uint32_t *)((uint8_t *)(mbuf) \
65 + sizeof(struct rte_mbuf)))
67 #define INVALID_PORT_ID 0xFFFF
69 /* Maximum character device basename size. */
70 #define MAX_BASENAME_SZ 20
72 /* Maximum long option length for option parsing. */
73 #define MAX_LONG_OPT_SZ 64
75 /* Used to compare MAC addresses. */
76 #define MAC_ADDR_CMP 0xFFFFFFFFFFFFULL
78 #define CMD_LINE_OPT_NB_DEVICES "nb-devices"
79 #define CMD_LINE_OPT_UDP_PORT "udp-port"
80 #define CMD_LINE_OPT_TX_CHECKSUM "tx-checksum"
81 #define CMD_LINE_OPT_TSO_SEGSZ "tso-segsz"
82 #define CMD_LINE_OPT_FILTER_TYPE "filter-type"
83 #define CMD_LINE_OPT_ENCAP "encap"
84 #define CMD_LINE_OPT_DECAP "decap"
85 #define CMD_LINE_OPT_RX_RETRY "rx-retry"
86 #define CMD_LINE_OPT_RX_RETRY_DELAY "rx-retry-delay"
87 #define CMD_LINE_OPT_RX_RETRY_NUM "rx-retry-num"
88 #define CMD_LINE_OPT_STATS "stats"
89 #define CMD_LINE_OPT_DEV_BASENAME "dev-basename"
91 /* mask of enabled ports */
92 static uint32_t enabled_port_mask
;
94 /*Number of switching cores enabled*/
95 static uint32_t nb_switching_cores
;
97 /* number of devices/queues to support*/
98 uint16_t nb_devices
= 2;
100 /* max ring descriptor, ixgbe, i40e, e1000 all are 4096. */
101 #define MAX_RING_DESC 4096
104 struct rte_mempool
*pool
;
105 struct rte_ring
*ring
;
107 } vpool_array
[MAX_QUEUES
+MAX_QUEUES
];
109 /* UDP tunneling port */
110 uint16_t udp_port
= 4789;
112 /* enable/disable inner TX checksum */
113 uint8_t tx_checksum
= 0;
115 /* TCP segment size */
116 uint16_t tso_segsz
= 0;
118 /* enable/disable decapsulation */
119 uint8_t rx_decap
= 1;
121 /* enable/disable encapsulation */
122 uint8_t tx_encap
= 1;
124 /* RX filter type for tunneling packet */
125 uint8_t filter_idx
= 1;
127 /* overlay packet operation */
128 struct ol_switch_ops overlay_options
= {
129 .port_configure
= vxlan_port_init
,
130 .tunnel_setup
= vxlan_link
,
131 .tunnel_destroy
= vxlan_unlink
,
132 .tx_handle
= vxlan_tx_pkts
,
133 .rx_handle
= vxlan_rx_pkts
,
134 .param_handle
= NULL
,
138 uint32_t enable_stats
= 0;
139 /* Enable retries on RX. */
140 static uint32_t enable_retry
= 1;
141 /* Specify timeout (in useconds) between retries on RX. */
142 static uint32_t burst_rx_delay_time
= BURST_RX_WAIT_US
;
143 /* Specify the number of retries on RX. */
144 static uint32_t burst_rx_retry_num
= BURST_RX_RETRIES
;
146 /* Character device basename. Can be set by user. */
147 static char dev_basename
[MAX_BASENAME_SZ
] = "vhost-net";
149 static unsigned lcore_ids
[RTE_MAX_LCORE
];
150 uint16_t ports
[RTE_MAX_ETHPORTS
];
152 static unsigned nb_ports
; /**< The number of ports specified in command line */
154 /* ethernet addresses of ports */
155 struct rte_ether_addr ports_eth_addr
[RTE_MAX_ETHPORTS
];
157 /* heads for the main used and free linked lists for the data path. */
158 static struct virtio_net_data_ll
*ll_root_used
;
159 static struct virtio_net_data_ll
*ll_root_free
;
162 * Array of data core structures containing information on
163 * individual core linked lists.
165 static struct lcore_info lcore_info
[RTE_MAX_LCORE
];
167 /* Used for queueing bursts of TX packets. */
171 struct rte_mbuf
*m_table
[MAX_PKT_BURST
];
174 /* TX queue for each data core. */
175 struct mbuf_table lcore_tx_queue
[RTE_MAX_LCORE
];
177 struct device_statistics dev_statistics
[MAX_DEVICES
];
180 * Set character device basename.
183 us_vhost_parse_basename(const char *q_arg
)
185 /* parse number string */
186 if (strlen(q_arg
) >= MAX_BASENAME_SZ
)
189 strlcpy((char *)&dev_basename
, q_arg
, MAX_BASENAME_SZ
);
195 * Parse the portmask provided at run time.
198 parse_portmask(const char *portmask
)
203 /* parse hexadecimal string */
204 pm
= strtoul(portmask
, &end
, 16);
205 if ((portmask
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
215 * Parse num options at run time.
218 parse_num_opt(const char *q_arg
, uint32_t max_valid_value
)
223 /* parse unsigned int string */
224 num
= strtoul(q_arg
, &end
, 10);
225 if ((q_arg
[0] == '\0') || (end
== NULL
) || (*end
!= '\0'))
228 if (num
> max_valid_value
)
238 tep_termination_usage(const char *prgname
)
240 RTE_LOG(INFO
, VHOST_CONFIG
, "%s [EAL options] -- -p PORTMASK\n"
241 " --udp-port: UDP destination port for VXLAN packet\n"
242 " --nb-devices[1-64]: The number of virtIO device\n"
243 " --tx-checksum [0|1]: inner Tx checksum offload\n"
244 " --tso-segsz [0-N]: TCP segment size\n"
245 " --decap [0|1]: tunneling packet decapsulation\n"
246 " --encap [0|1]: tunneling packet encapsulation\n"
247 " --filter-type[1-3]: filter type for tunneling packet\n"
248 " 1: Inner MAC and tenent ID\n"
249 " 2: Inner MAC and VLAN, and tenent ID\n"
250 " 3: Outer MAC, Inner MAC and tenent ID\n"
251 " -p PORTMASK: Set mask for ports to be used by application\n"
252 " --rx-retry [0|1]: disable/enable(default) retries on rx."
253 " Enable retry if destintation queue is full\n"
254 " --rx-retry-delay [0-N]: timeout(in usecond) between retries on RX."
255 " This makes effect only if retries on rx enabled\n"
256 " --rx-retry-num [0-N]: the number of retries on rx."
257 " This makes effect only if retries on rx enabled\n"
258 " --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n"
259 " --dev-basename: The basename to be used for the character device.\n",
264 * Parse the arguments given in the command line of the application.
267 tep_termination_parse_args(int argc
, char **argv
)
272 const char *prgname
= argv
[0];
273 static struct option long_option
[] = {
274 {CMD_LINE_OPT_NB_DEVICES
, required_argument
, NULL
, 0},
275 {CMD_LINE_OPT_UDP_PORT
, required_argument
, NULL
, 0},
276 {CMD_LINE_OPT_TX_CHECKSUM
, required_argument
, NULL
, 0},
277 {CMD_LINE_OPT_TSO_SEGSZ
, required_argument
, NULL
, 0},
278 {CMD_LINE_OPT_DECAP
, required_argument
, NULL
, 0},
279 {CMD_LINE_OPT_ENCAP
, required_argument
, NULL
, 0},
280 {CMD_LINE_OPT_FILTER_TYPE
, required_argument
, NULL
, 0},
281 {CMD_LINE_OPT_RX_RETRY
, required_argument
, NULL
, 0},
282 {CMD_LINE_OPT_RX_RETRY_DELAY
, required_argument
, NULL
, 0},
283 {CMD_LINE_OPT_RX_RETRY_NUM
, required_argument
, NULL
, 0},
284 {CMD_LINE_OPT_STATS
, required_argument
, NULL
, 0},
285 {CMD_LINE_OPT_DEV_BASENAME
, required_argument
, NULL
, 0},
289 /* Parse command line */
290 while ((opt
= getopt_long(argc
, argv
, "p:",
291 long_option
, &option_index
)) != EOF
) {
295 enabled_port_mask
= parse_portmask(optarg
);
296 if (enabled_port_mask
== 0) {
297 RTE_LOG(INFO
, VHOST_CONFIG
,
298 "Invalid portmask\n");
299 tep_termination_usage(prgname
);
304 if (!strncmp(long_option
[option_index
].name
,
305 CMD_LINE_OPT_NB_DEVICES
,
306 sizeof(CMD_LINE_OPT_NB_DEVICES
))) {
307 ret
= parse_num_opt(optarg
, MAX_DEVICES
);
309 RTE_LOG(INFO
, VHOST_CONFIG
,
310 "Invalid argument for nb-devices [0-%d]\n",
312 tep_termination_usage(prgname
);
318 /* Enable/disable retries on RX. */
319 if (!strncmp(long_option
[option_index
].name
,
320 CMD_LINE_OPT_RX_RETRY
,
321 sizeof(CMD_LINE_OPT_RX_RETRY
))) {
322 ret
= parse_num_opt(optarg
, 1);
324 RTE_LOG(INFO
, VHOST_CONFIG
,
325 "Invalid argument for rx-retry [0|1]\n");
326 tep_termination_usage(prgname
);
332 if (!strncmp(long_option
[option_index
].name
,
333 CMD_LINE_OPT_TSO_SEGSZ
,
334 sizeof(CMD_LINE_OPT_TSO_SEGSZ
))) {
335 ret
= parse_num_opt(optarg
, INT16_MAX
);
337 RTE_LOG(INFO
, VHOST_CONFIG
,
338 "Invalid argument for TCP segment size [0-N]\n");
339 tep_termination_usage(prgname
);
345 if (!strncmp(long_option
[option_index
].name
,
346 CMD_LINE_OPT_UDP_PORT
,
347 sizeof(CMD_LINE_OPT_UDP_PORT
))) {
348 ret
= parse_num_opt(optarg
, INT16_MAX
);
350 RTE_LOG(INFO
, VHOST_CONFIG
,
351 "Invalid argument for UDP port [0-N]\n");
352 tep_termination_usage(prgname
);
358 /* Specify the retries delay time (in useconds) on RX.*/
359 if (!strncmp(long_option
[option_index
].name
,
360 CMD_LINE_OPT_RX_RETRY_DELAY
,
361 sizeof(CMD_LINE_OPT_RX_RETRY_DELAY
))) {
362 ret
= parse_num_opt(optarg
, INT32_MAX
);
364 RTE_LOG(INFO
, VHOST_CONFIG
,
365 "Invalid argument for rx-retry-delay [0-N]\n");
366 tep_termination_usage(prgname
);
369 burst_rx_delay_time
= ret
;
372 /* Specify the retries number on RX. */
373 if (!strncmp(long_option
[option_index
].name
,
374 CMD_LINE_OPT_RX_RETRY_NUM
,
375 sizeof(CMD_LINE_OPT_RX_RETRY_NUM
))) {
376 ret
= parse_num_opt(optarg
, INT32_MAX
);
378 RTE_LOG(INFO
, VHOST_CONFIG
,
379 "Invalid argument for rx-retry-num [0-N]\n");
380 tep_termination_usage(prgname
);
383 burst_rx_retry_num
= ret
;
386 if (!strncmp(long_option
[option_index
].name
,
387 CMD_LINE_OPT_TX_CHECKSUM
,
388 sizeof(CMD_LINE_OPT_TX_CHECKSUM
))) {
389 ret
= parse_num_opt(optarg
, 1);
391 RTE_LOG(INFO
, VHOST_CONFIG
,
392 "Invalid argument for tx-checksum [0|1]\n");
393 tep_termination_usage(prgname
);
399 if (!strncmp(long_option
[option_index
].name
,
400 CMD_LINE_OPT_FILTER_TYPE
,
401 sizeof(CMD_LINE_OPT_FILTER_TYPE
))) {
402 ret
= parse_num_opt(optarg
, 3);
403 if ((ret
== -1) || (ret
== 0)) {
404 RTE_LOG(INFO
, VHOST_CONFIG
,
405 "Invalid argument for filter type [1-3]\n");
406 tep_termination_usage(prgname
);
409 filter_idx
= ret
- 1;
412 /* Enable/disable encapsulation on RX. */
413 if (!strncmp(long_option
[option_index
].name
,
415 sizeof(CMD_LINE_OPT_DECAP
))) {
416 ret
= parse_num_opt(optarg
, 1);
418 RTE_LOG(INFO
, VHOST_CONFIG
,
419 "Invalid argument for decap [0|1]\n");
420 tep_termination_usage(prgname
);
426 /* Enable/disable encapsulation on TX. */
427 if (!strncmp(long_option
[option_index
].name
,
429 sizeof(CMD_LINE_OPT_ENCAP
))) {
430 ret
= parse_num_opt(optarg
, 1);
432 RTE_LOG(INFO
, VHOST_CONFIG
,
433 "Invalid argument for encap [0|1]\n");
434 tep_termination_usage(prgname
);
440 /* Enable/disable stats. */
441 if (!strncmp(long_option
[option_index
].name
,
443 sizeof(CMD_LINE_OPT_STATS
))) {
444 ret
= parse_num_opt(optarg
, INT32_MAX
);
446 RTE_LOG(INFO
, VHOST_CONFIG
,
447 "Invalid argument for stats [0..N]\n");
448 tep_termination_usage(prgname
);
454 /* Set character device basename. */
455 if (!strncmp(long_option
[option_index
].name
,
456 CMD_LINE_OPT_DEV_BASENAME
,
457 sizeof(CMD_LINE_OPT_DEV_BASENAME
))) {
458 if (us_vhost_parse_basename(optarg
) == -1) {
459 RTE_LOG(INFO
, VHOST_CONFIG
,
460 "Invalid argument for character "
461 "device basename (Max %d characters)\n",
463 tep_termination_usage(prgname
);
470 /* Invalid option - print options. */
472 tep_termination_usage(prgname
);
477 for (i
= 0; i
< RTE_MAX_ETHPORTS
; i
++) {
478 if (enabled_port_mask
& (1 << i
))
479 ports
[nb_ports
++] = (uint8_t)i
;
482 if ((nb_ports
== 0) || (nb_ports
> MAX_SUP_PORTS
)) {
483 RTE_LOG(INFO
, VHOST_PORT
, "Current enabled port number is %u,"
484 "but only %u port can be enabled\n", nb_ports
,
493 * Update the global var NB_PORTS and array PORTS
494 * according to system ports number and return valid ports number
497 check_ports_num(unsigned max_nb_ports
)
499 unsigned valid_nb_ports
= nb_ports
;
502 if (nb_ports
> max_nb_ports
) {
503 RTE_LOG(INFO
, VHOST_PORT
, "\nSpecified port number(%u) "
504 " exceeds total system port number(%u)\n",
505 nb_ports
, max_nb_ports
);
506 nb_ports
= max_nb_ports
;
509 for (portid
= 0; portid
< nb_ports
; portid
++) {
510 if (!rte_eth_dev_is_valid_port(ports
[portid
])) {
511 RTE_LOG(INFO
, VHOST_PORT
,
512 "\nSpecified port ID(%u) is not valid\n",
514 ports
[portid
] = INVALID_PORT_ID
;
518 return valid_nb_ports
;
522 * This function routes the TX packet to the correct interface. This may be a local device
523 * or the physical port.
525 static __rte_always_inline
void
526 virtio_tx_route(struct vhost_dev
*vdev
, struct rte_mbuf
*m
)
528 struct mbuf_table
*tx_q
;
529 struct rte_mbuf
**m_table
;
530 unsigned len
, ret
= 0;
531 const uint16_t lcore_id
= rte_lcore_id();
533 RTE_LOG_DP(DEBUG
, VHOST_DATA
, "(%d) TX: MAC address is external\n",
536 /* Add packet to the port tx queue */
537 tx_q
= &lcore_tx_queue
[lcore_id
];
540 tx_q
->m_table
[len
] = m
;
543 dev_statistics
[vdev
->vid
].tx_total
++;
544 dev_statistics
[vdev
->vid
].tx
++;
547 if (unlikely(len
== MAX_PKT_BURST
)) {
548 m_table
= (struct rte_mbuf
**)tx_q
->m_table
;
549 ret
= overlay_options
.tx_handle(ports
[0],
550 (uint16_t)tx_q
->txq_id
, m_table
,
551 (uint16_t)tx_q
->len
);
553 /* Free any buffers not handled by TX and update
556 if (unlikely(ret
< len
)) {
558 rte_pktmbuf_free(m_table
[ret
]);
559 } while (++ret
< len
);
570 * This function is called by each data core. It handles all
571 * RX/TX registered with the core. For TX the specific lcore
572 * linked list is used. For RX, MAC addresses are compared
573 * with all devices in the main linked list.
576 switch_worker(__rte_unused
void *arg
)
578 struct rte_mempool
*mbuf_pool
= arg
;
579 struct vhost_dev
*vdev
= NULL
;
580 struct rte_mbuf
*pkts_burst
[MAX_PKT_BURST
];
581 struct virtio_net_data_ll
*dev_ll
;
582 struct mbuf_table
*tx_q
;
583 volatile struct lcore_ll_info
*lcore_ll
;
584 const uint64_t drain_tsc
= (rte_get_tsc_hz() + US_PER_S
- 1)
585 / US_PER_S
* BURST_TX_DRAIN_US
;
586 uint64_t prev_tsc
, diff_tsc
, cur_tsc
, ret_count
= 0;
588 const uint16_t lcore_id
= rte_lcore_id();
589 const uint16_t num_cores
= (uint16_t)rte_lcore_count();
590 uint16_t rx_count
= 0;
594 RTE_LOG(INFO
, VHOST_DATA
, "Procesing on Core %u started\n", lcore_id
);
595 lcore_ll
= lcore_info
[lcore_id
].lcore_ll
;
598 tx_q
= &lcore_tx_queue
[lcore_id
];
599 for (i
= 0; i
< num_cores
; i
++) {
600 if (lcore_ids
[i
] == lcore_id
) {
607 cur_tsc
= rte_rdtsc();
609 * TX burst queue drain
611 diff_tsc
= cur_tsc
- prev_tsc
;
612 if (unlikely(diff_tsc
> drain_tsc
)) {
615 RTE_LOG_DP(DEBUG
, VHOST_DATA
, "TX queue drained after "
616 "timeout with burst size %u\n",
618 ret
= overlay_options
.tx_handle(ports
[0],
619 (uint16_t)tx_q
->txq_id
,
620 (struct rte_mbuf
**)tx_q
->m_table
,
621 (uint16_t)tx_q
->len
);
622 if (unlikely(ret
< tx_q
->len
)) {
624 rte_pktmbuf_free(tx_q
->m_table
[ret
]);
625 } while (++ret
< tx_q
->len
);
635 rte_prefetch0(lcore_ll
->ll_root_used
);
638 * Inform the configuration core that we have exited
639 * the linked list and that no devices are
640 * in use if requested.
642 if (lcore_ll
->dev_removal_flag
== REQUEST_DEV_REMOVAL
)
643 lcore_ll
->dev_removal_flag
= ACK_DEV_REMOVAL
;
648 dev_ll
= lcore_ll
->ll_root_used
;
650 while (dev_ll
!= NULL
) {
653 if (unlikely(vdev
->remove
)) {
654 dev_ll
= dev_ll
->next
;
655 overlay_options
.tunnel_destroy(vdev
);
656 vdev
->ready
= DEVICE_SAFE_REMOVE
;
659 if (likely(vdev
->ready
== DEVICE_RX
)) {
660 /* Handle guest RX */
661 rx_count
= rte_eth_rx_burst(ports
[0],
662 vdev
->rx_q
, pkts_burst
, MAX_PKT_BURST
);
666 * Retry is enabled and the queue is
667 * full then we wait and retry to
668 * avoid packet loss. Here MAX_PKT_BURST
669 * must be less than virtio queue size
671 if (enable_retry
&& unlikely(rx_count
>
672 rte_vhost_avail_entries(vdev
->vid
, VIRTIO_RXQ
))) {
673 for (retry
= 0; retry
< burst_rx_retry_num
;
675 rte_delay_us(burst_rx_delay_time
);
676 if (rx_count
<= rte_vhost_avail_entries(vdev
->vid
, VIRTIO_RXQ
))
681 ret_count
= overlay_options
.rx_handle(vdev
->vid
, pkts_burst
, rx_count
);
684 &dev_statistics
[vdev
->vid
].rx_total_atomic
,
687 &dev_statistics
[vdev
->vid
].rx_atomic
, ret_count
);
689 while (likely(rx_count
)) {
691 rte_pktmbuf_free(pkts_burst
[rx_count
]);
697 if (likely(!vdev
->remove
)) {
699 tx_count
= rte_vhost_dequeue_burst(vdev
->vid
,
700 VIRTIO_TXQ
, mbuf_pool
,
701 pkts_burst
, MAX_PKT_BURST
);
702 /* If this is the first received packet we need to learn the MAC */
703 if (unlikely(vdev
->ready
== DEVICE_MAC_LEARNING
) && tx_count
) {
705 (overlay_options
.tunnel_setup(vdev
, pkts_burst
[0]) == -1)) {
707 rte_pktmbuf_free(pkts_burst
[--tx_count
]);
711 virtio_tx_route(vdev
, pkts_burst
[--tx_count
]);
714 /* move to the next device in the list */
715 dev_ll
= dev_ll
->next
;
723 * Add an entry to a used linked list. A free entry must first be found
724 * in the free linked list using get_data_ll_free_entry();
727 add_data_ll_entry(struct virtio_net_data_ll
**ll_root_addr
,
728 struct virtio_net_data_ll
*ll_dev
)
730 struct virtio_net_data_ll
*ll
= *ll_root_addr
;
732 /* Set next as NULL and use a compiler barrier to avoid reordering. */
734 rte_compiler_barrier();
736 /* If ll == NULL then this is the first device. */
738 /* Increment to the tail of the linked list. */
739 while (ll
->next
!= NULL
)
744 *ll_root_addr
= ll_dev
;
749 * Remove an entry from a used linked list. The entry must then be added to
750 * the free linked list using put_data_ll_free_entry().
753 rm_data_ll_entry(struct virtio_net_data_ll
**ll_root_addr
,
754 struct virtio_net_data_ll
*ll_dev
,
755 struct virtio_net_data_ll
*ll_dev_last
)
757 struct virtio_net_data_ll
*ll
= *ll_root_addr
;
759 if (unlikely((ll
== NULL
) || (ll_dev
== NULL
)))
763 *ll_root_addr
= ll_dev
->next
;
765 if (likely(ll_dev_last
!= NULL
))
766 ll_dev_last
->next
= ll_dev
->next
;
768 RTE_LOG(ERR
, VHOST_CONFIG
,
769 "Remove entry form ll failed.\n");
773 * Find and return an entry from the free linked list.
775 static struct virtio_net_data_ll
*
776 get_data_ll_free_entry(struct virtio_net_data_ll
**ll_root_addr
)
778 struct virtio_net_data_ll
*ll_free
= *ll_root_addr
;
779 struct virtio_net_data_ll
*ll_dev
;
785 *ll_root_addr
= ll_free
->next
;
791 * Place an entry back on to the free linked list.
794 put_data_ll_free_entry(struct virtio_net_data_ll
**ll_root_addr
,
795 struct virtio_net_data_ll
*ll_dev
)
797 struct virtio_net_data_ll
*ll_free
= *ll_root_addr
;
802 ll_dev
->next
= ll_free
;
803 *ll_root_addr
= ll_dev
;
807 * Creates a linked list of a given size.
809 static struct virtio_net_data_ll
*
810 alloc_data_ll(uint32_t size
)
812 struct virtio_net_data_ll
*ll_new
;
815 /* Malloc and then chain the linked list. */
816 ll_new
= malloc(size
* sizeof(struct virtio_net_data_ll
));
817 if (ll_new
== NULL
) {
818 RTE_LOG(ERR
, VHOST_CONFIG
,
819 "Failed to allocate memory for ll_new.\n");
823 for (i
= 0; i
< size
- 1; i
++) {
824 ll_new
[i
].vdev
= NULL
;
825 ll_new
[i
].next
= &ll_new
[i
+1];
827 ll_new
[i
].next
= NULL
;
833 * Create the main linked list along with each individual cores
834 * linked list. A used and a free list are created to manage entries.
841 RTE_LCORE_FOREACH_SLAVE(lcore
) {
842 lcore_info
[lcore
].lcore_ll
=
843 malloc(sizeof(struct lcore_ll_info
));
844 if (lcore_info
[lcore
].lcore_ll
== NULL
) {
845 RTE_LOG(ERR
, VHOST_CONFIG
,
846 "Failed to allocate memory for lcore_ll.\n");
850 lcore_info
[lcore
].lcore_ll
->device_num
= 0;
851 lcore_info
[lcore
].lcore_ll
->dev_removal_flag
= ACK_DEV_REMOVAL
;
852 lcore_info
[lcore
].lcore_ll
->ll_root_used
= NULL
;
853 if (nb_devices
% nb_switching_cores
)
854 lcore_info
[lcore
].lcore_ll
->ll_root_free
=
855 alloc_data_ll((nb_devices
/ nb_switching_cores
)
858 lcore_info
[lcore
].lcore_ll
->ll_root_free
=
859 alloc_data_ll(nb_devices
/ nb_switching_cores
);
862 /* Allocate devices up to a maximum of MAX_DEVICES. */
863 ll_root_free
= alloc_data_ll(MIN((nb_devices
), MAX_DEVICES
));
869 * Remove a device from the specific data core linked list and
870 * from the main linked list. Synchonization occurs through the use
871 * of the lcore dev_removal_flag.
874 destroy_device(int vid
)
876 struct virtio_net_data_ll
*ll_lcore_dev_cur
;
877 struct virtio_net_data_ll
*ll_main_dev_cur
;
878 struct virtio_net_data_ll
*ll_lcore_dev_last
= NULL
;
879 struct virtio_net_data_ll
*ll_main_dev_last
= NULL
;
880 struct vhost_dev
*vdev
= NULL
;
883 ll_main_dev_cur
= ll_root_used
;
884 while (ll_main_dev_cur
!= NULL
) {
885 if (ll_main_dev_cur
->vdev
->vid
== vid
) {
886 vdev
= ll_main_dev_cur
->vdev
;
893 /* set the remove flag. */
895 while (vdev
->ready
!= DEVICE_SAFE_REMOVE
)
898 /* Search for entry to be removed from lcore ll */
899 ll_lcore_dev_cur
= lcore_info
[vdev
->coreid
].lcore_ll
->ll_root_used
;
900 while (ll_lcore_dev_cur
!= NULL
) {
901 if (ll_lcore_dev_cur
->vdev
== vdev
) {
904 ll_lcore_dev_last
= ll_lcore_dev_cur
;
905 ll_lcore_dev_cur
= ll_lcore_dev_cur
->next
;
909 if (ll_lcore_dev_cur
== NULL
) {
910 RTE_LOG(ERR
, VHOST_CONFIG
,
911 "(%d) Failed to find the dev to be destroy.\n", vid
);
915 /* Search for entry to be removed from main ll */
916 ll_main_dev_cur
= ll_root_used
;
917 ll_main_dev_last
= NULL
;
918 while (ll_main_dev_cur
!= NULL
) {
919 if (ll_main_dev_cur
->vdev
== vdev
) {
922 ll_main_dev_last
= ll_main_dev_cur
;
923 ll_main_dev_cur
= ll_main_dev_cur
->next
;
927 /* Remove entries from the lcore and main ll. */
928 rm_data_ll_entry(&lcore_info
[vdev
->coreid
].lcore_ll
->ll_root_used
,
929 ll_lcore_dev_cur
, ll_lcore_dev_last
);
930 rm_data_ll_entry(&ll_root_used
, ll_main_dev_cur
, ll_main_dev_last
);
932 /* Set the dev_removal_flag on each lcore. */
933 RTE_LCORE_FOREACH_SLAVE(lcore
) {
934 lcore_info
[lcore
].lcore_ll
->dev_removal_flag
=
939 * Once each core has set the dev_removal_flag to
940 * ACK_DEV_REMOVAL we can be sure that they can no longer access
941 * the device removed from the linked lists and that the devices
942 * are no longer in use.
944 RTE_LCORE_FOREACH_SLAVE(lcore
) {
945 while (lcore_info
[lcore
].lcore_ll
->dev_removal_flag
950 /* Add the entries back to the lcore and main free ll.*/
951 put_data_ll_free_entry(&lcore_info
[vdev
->coreid
].lcore_ll
->ll_root_free
,
953 put_data_ll_free_entry(&ll_root_free
, ll_main_dev_cur
);
955 /* Decrement number of device on the lcore. */
956 lcore_info
[vdev
->coreid
].lcore_ll
->device_num
--;
958 RTE_LOG(INFO
, VHOST_DATA
, "(%d) Device has been removed "
959 "from data core\n", vid
);
966 * A new device is added to a data core. First the device is added
967 * to the main linked list and the allocated to a specific data core.
972 struct virtio_net_data_ll
*ll_dev
;
973 int lcore
, core_add
= 0;
974 uint32_t device_num_min
= nb_devices
;
975 struct vhost_dev
*vdev
;
977 vdev
= rte_zmalloc("vhost device", sizeof(*vdev
), RTE_CACHE_LINE_SIZE
);
979 RTE_LOG(INFO
, VHOST_DATA
,
980 "(%d) Couldn't allocate memory for vhost dev\n", vid
);
984 /* Add device to main ll */
985 ll_dev
= get_data_ll_free_entry(&ll_root_free
);
986 if (ll_dev
== NULL
) {
987 RTE_LOG(INFO
, VHOST_DATA
, "(%d) No free entry found in"
988 " linked list Device limit of %d devices per core"
989 " has been reached\n", vid
, nb_devices
);
990 if (vdev
->regions_hpa
)
991 rte_free(vdev
->regions_hpa
);
996 add_data_ll_entry(&ll_root_used
, ll_dev
);
999 /* reset ready flag */
1000 vdev
->ready
= DEVICE_MAC_LEARNING
;
1003 /* Find a suitable lcore to add the device. */
1004 RTE_LCORE_FOREACH_SLAVE(lcore
) {
1005 if (lcore_info
[lcore
].lcore_ll
->device_num
< device_num_min
) {
1006 device_num_min
= lcore_info
[lcore
].lcore_ll
->device_num
;
1010 /* Add device to lcore ll */
1011 ll_dev
= get_data_ll_free_entry(&lcore_info
[core_add
].lcore_ll
->ll_root_free
);
1012 if (ll_dev
== NULL
) {
1013 RTE_LOG(INFO
, VHOST_DATA
,
1014 "(%d) Failed to add device to data core\n",
1016 vdev
->ready
= DEVICE_SAFE_REMOVE
;
1017 destroy_device(vid
);
1018 rte_free(vdev
->regions_hpa
);
1022 ll_dev
->vdev
= vdev
;
1023 vdev
->coreid
= core_add
;
1025 add_data_ll_entry(&lcore_info
[vdev
->coreid
].lcore_ll
->ll_root_used
,
1028 /* Initialize device stats */
1029 memset(&dev_statistics
[vid
], 0,
1030 sizeof(struct device_statistics
));
1032 /* Disable notifications. */
1033 rte_vhost_enable_guest_notification(vid
, VIRTIO_RXQ
, 0);
1034 rte_vhost_enable_guest_notification(vid
, VIRTIO_TXQ
, 0);
1035 lcore_info
[vdev
->coreid
].lcore_ll
->device_num
++;
1037 RTE_LOG(INFO
, VHOST_DATA
, "(%d) Device has been added to data core %d\n",
1044 * These callback allow devices to be added to the data core when configuration
1045 * has been fully complete.
1047 static const struct vhost_device_ops virtio_net_device_ops
= {
1048 .new_device
= new_device
,
1049 .destroy_device
= destroy_device
,
1053 * This is a thread will wake up after a period to print stats if the user has
1057 print_stats(__rte_unused
void *arg
)
1059 struct virtio_net_data_ll
*dev_ll
;
1060 uint64_t tx_dropped
, rx_dropped
;
1061 uint64_t tx
, tx_total
, rx
, rx_total
, rx_ip_csum
, rx_l4_csum
;
1063 const char clr
[] = { 27, '[', '2', 'J', '\0' };
1064 const char top_left
[] = { 27, '[', '1', ';', '1', 'H', '\0' };
1067 sleep(enable_stats
);
1069 /* Clear screen and move to top left */
1070 printf("%s%s", clr
, top_left
);
1072 printf("\nDevice statistics ================================");
1074 dev_ll
= ll_root_used
;
1075 while (dev_ll
!= NULL
) {
1076 vid
= dev_ll
->vdev
->vid
;
1077 tx_total
= dev_statistics
[vid
].tx_total
;
1078 tx
= dev_statistics
[vid
].tx
;
1079 tx_dropped
= tx_total
- tx
;
1081 rx_total
= rte_atomic64_read(
1082 &dev_statistics
[vid
].rx_total_atomic
);
1083 rx
= rte_atomic64_read(
1084 &dev_statistics
[vid
].rx_atomic
);
1085 rx_dropped
= rx_total
- rx
;
1086 rx_ip_csum
= rte_atomic64_read(
1087 &dev_statistics
[vid
].rx_bad_ip_csum
);
1088 rx_l4_csum
= rte_atomic64_read(
1089 &dev_statistics
[vid
].rx_bad_l4_csum
);
1091 printf("\nStatistics for device %d ----------"
1092 "\nTX total: %"PRIu64
""
1093 "\nTX dropped: %"PRIu64
""
1094 "\nTX successful: %"PRIu64
""
1095 "\nRX total: %"PRIu64
""
1096 "\nRX bad IP csum: %"PRIu64
""
1097 "\nRX bad L4 csum: %"PRIu64
""
1098 "\nRX dropped: %"PRIu64
""
1099 "\nRX successful: %"PRIu64
"",
1110 dev_ll
= dev_ll
->next
;
1112 printf("\n================================================\n");
1119 * Main function, does initialisation and calls the per-lcore functions.
1122 main(int argc
, char *argv
[])
1124 struct rte_mempool
*mbuf_pool
= NULL
;
1125 unsigned lcore_id
, core_id
= 0;
1126 unsigned nb_ports
, valid_nb_ports
;
1130 static pthread_t tid
;
1133 ret
= rte_eal_init(argc
, argv
);
1135 rte_exit(EXIT_FAILURE
, "Error with EAL initialization\n");
1139 /* parse app arguments */
1140 ret
= tep_termination_parse_args(argc
, argv
);
1142 rte_exit(EXIT_FAILURE
, "Invalid argument\n");
1144 for (lcore_id
= 0; lcore_id
< RTE_MAX_LCORE
; lcore_id
++)
1145 if (rte_lcore_is_enabled(lcore_id
))
1146 lcore_ids
[core_id
++] = lcore_id
;
1148 /* set the number of swithcing cores available */
1149 nb_switching_cores
= rte_lcore_count()-1;
1151 /* Get the number of physical ports. */
1152 nb_ports
= rte_eth_dev_count_avail();
1155 * Update the global var NB_PORTS and global array PORTS
1156 * and get value of var VALID_NB_PORTS according to system ports number
1158 valid_nb_ports
= check_ports_num(nb_ports
);
1160 if ((valid_nb_ports
== 0) || (valid_nb_ports
> MAX_SUP_PORTS
)) {
1161 rte_exit(EXIT_FAILURE
, "Current enabled port number is %u,"
1162 "but only %u port can be enabled\n", nb_ports
,
1165 /* Create the mbuf pool. */
1166 mbuf_pool
= rte_pktmbuf_pool_create(
1168 NUM_MBUFS_PER_PORT
* valid_nb_ports
,
1173 if (mbuf_pool
== NULL
)
1174 rte_exit(EXIT_FAILURE
, "Cannot create mbuf pool\n");
1176 for (queue_id
= 0; queue_id
< MAX_QUEUES
+ 1; queue_id
++)
1177 vpool_array
[queue_id
].pool
= mbuf_pool
;
1179 /* initialize all ports */
1180 RTE_ETH_FOREACH_DEV(portid
) {
1181 /* skip ports that are not enabled */
1182 if ((enabled_port_mask
& (1 << portid
)) == 0) {
1183 RTE_LOG(INFO
, VHOST_PORT
,
1184 "Skipping disabled port %d\n", portid
);
1187 if (overlay_options
.port_configure(portid
, mbuf_pool
) != 0)
1188 rte_exit(EXIT_FAILURE
,
1189 "Cannot initialize network ports\n");
1192 /* Initialise all linked lists. */
1193 if (init_data_ll() == -1)
1194 rte_exit(EXIT_FAILURE
, "Failed to initialize linked list\n");
1196 /* Initialize device stats */
1197 memset(&dev_statistics
, 0, sizeof(dev_statistics
));
1199 /* Enable stats if the user option is set. */
1201 ret
= rte_ctrl_thread_create(&tid
, "print-stats", NULL
,
1204 rte_exit(EXIT_FAILURE
, "Cannot create print-stats thread\n");
1207 /* Launch all data cores. */
1208 RTE_LCORE_FOREACH_SLAVE(lcore_id
) {
1209 rte_eal_remote_launch(switch_worker
,
1210 mbuf_pool
, lcore_id
);
1213 ret
= rte_vhost_driver_register((char *)&dev_basename
, 0);
1215 rte_exit(EXIT_FAILURE
, "failed to register vhost driver.\n");
1217 rte_vhost_driver_disable_features(dev_basename
,
1218 1ULL << VIRTIO_NET_F_MRG_RXBUF
);
1220 ret
= rte_vhost_driver_callback_register(dev_basename
,
1221 &virtio_net_device_ops
);
1223 rte_exit(EXIT_FAILURE
,
1224 "failed to register vhost driver callbacks.\n");
1227 if (rte_vhost_driver_start(dev_basename
) < 0) {
1228 rte_exit(EXIT_FAILURE
,
1229 "failed to start vhost driver.\n");
1232 RTE_LCORE_FOREACH_SLAVE(lcore_id
)
1233 rte_eal_wait_lcore(lcore_id
);