1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
11 #include <rte_ethdev_driver.h>
12 #include <rte_ethdev_pci.h>
13 #include <rte_memcpy.h>
14 #include <rte_string_fns.h>
15 #include <rte_memzone.h>
16 #include <rte_malloc.h>
17 #include <rte_branch_prediction.h>
19 #include <rte_bus_pci.h>
20 #include <rte_ether.h>
23 #include <rte_common.h>
24 #include <rte_errno.h>
25 #include <rte_cpuflags.h>
27 #include <rte_memory.h>
30 #include <rte_cycles.h>
31 #include <rte_kvargs.h>
33 #include "virtio_ethdev.h"
34 #include "virtio_pci.h"
35 #include "virtio_logs.h"
36 #include "virtqueue.h"
37 #include "virtio_rxtx.h"
39 static int eth_virtio_dev_uninit(struct rte_eth_dev
*eth_dev
);
40 static int virtio_dev_configure(struct rte_eth_dev
*dev
);
41 static int virtio_dev_start(struct rte_eth_dev
*dev
);
42 static void virtio_dev_stop(struct rte_eth_dev
*dev
);
43 static void virtio_dev_promiscuous_enable(struct rte_eth_dev
*dev
);
44 static void virtio_dev_promiscuous_disable(struct rte_eth_dev
*dev
);
45 static void virtio_dev_allmulticast_enable(struct rte_eth_dev
*dev
);
46 static void virtio_dev_allmulticast_disable(struct rte_eth_dev
*dev
);
47 static void virtio_dev_info_get(struct rte_eth_dev
*dev
,
48 struct rte_eth_dev_info
*dev_info
);
49 static int virtio_dev_link_update(struct rte_eth_dev
*dev
,
50 int wait_to_complete
);
51 static int virtio_dev_vlan_offload_set(struct rte_eth_dev
*dev
, int mask
);
53 static void virtio_set_hwaddr(struct virtio_hw
*hw
);
54 static void virtio_get_hwaddr(struct virtio_hw
*hw
);
56 static int virtio_dev_stats_get(struct rte_eth_dev
*dev
,
57 struct rte_eth_stats
*stats
);
58 static int virtio_dev_xstats_get(struct rte_eth_dev
*dev
,
59 struct rte_eth_xstat
*xstats
, unsigned n
);
60 static int virtio_dev_xstats_get_names(struct rte_eth_dev
*dev
,
61 struct rte_eth_xstat_name
*xstats_names
,
63 static void virtio_dev_stats_reset(struct rte_eth_dev
*dev
);
64 static void virtio_dev_free_mbufs(struct rte_eth_dev
*dev
);
65 static int virtio_vlan_filter_set(struct rte_eth_dev
*dev
,
66 uint16_t vlan_id
, int on
);
67 static int virtio_mac_addr_add(struct rte_eth_dev
*dev
,
68 struct ether_addr
*mac_addr
,
69 uint32_t index
, uint32_t vmdq
);
70 static void virtio_mac_addr_remove(struct rte_eth_dev
*dev
, uint32_t index
);
71 static int virtio_mac_addr_set(struct rte_eth_dev
*dev
,
72 struct ether_addr
*mac_addr
);
74 static int virtio_intr_disable(struct rte_eth_dev
*dev
);
76 static int virtio_dev_queue_stats_mapping_set(
77 struct rte_eth_dev
*eth_dev
,
82 int virtio_logtype_init
;
83 int virtio_logtype_driver
;
85 static void virtio_notify_peers(struct rte_eth_dev
*dev
);
86 static void virtio_ack_link_announce(struct rte_eth_dev
*dev
);
89 * The set of PCI devices this driver supports
91 static const struct rte_pci_id pci_id_virtio_map
[] = {
92 { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID
, VIRTIO_PCI_LEGACY_DEVICEID_NET
) },
93 { RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID
, VIRTIO_PCI_MODERN_DEVICEID_NET
) },
94 { .vendor_id
= 0, /* sentinel */ },
97 struct rte_virtio_xstats_name_off
{
98 char name
[RTE_ETH_XSTATS_NAME_SIZE
];
102 /* [rt]x_qX_ is prepended to the name string here */
103 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings
[] = {
104 {"good_packets", offsetof(struct virtnet_rx
, stats
.packets
)},
105 {"good_bytes", offsetof(struct virtnet_rx
, stats
.bytes
)},
106 {"errors", offsetof(struct virtnet_rx
, stats
.errors
)},
107 {"multicast_packets", offsetof(struct virtnet_rx
, stats
.multicast
)},
108 {"broadcast_packets", offsetof(struct virtnet_rx
, stats
.broadcast
)},
109 {"undersize_packets", offsetof(struct virtnet_rx
, stats
.size_bins
[0])},
110 {"size_64_packets", offsetof(struct virtnet_rx
, stats
.size_bins
[1])},
111 {"size_65_127_packets", offsetof(struct virtnet_rx
, stats
.size_bins
[2])},
112 {"size_128_255_packets", offsetof(struct virtnet_rx
, stats
.size_bins
[3])},
113 {"size_256_511_packets", offsetof(struct virtnet_rx
, stats
.size_bins
[4])},
114 {"size_512_1023_packets", offsetof(struct virtnet_rx
, stats
.size_bins
[5])},
115 {"size_1024_1518_packets", offsetof(struct virtnet_rx
, stats
.size_bins
[6])},
116 {"size_1519_max_packets", offsetof(struct virtnet_rx
, stats
.size_bins
[7])},
119 /* [rt]x_qX_ is prepended to the name string here */
120 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings
[] = {
121 {"good_packets", offsetof(struct virtnet_tx
, stats
.packets
)},
122 {"good_bytes", offsetof(struct virtnet_tx
, stats
.bytes
)},
123 {"errors", offsetof(struct virtnet_tx
, stats
.errors
)},
124 {"multicast_packets", offsetof(struct virtnet_tx
, stats
.multicast
)},
125 {"broadcast_packets", offsetof(struct virtnet_tx
, stats
.broadcast
)},
126 {"undersize_packets", offsetof(struct virtnet_tx
, stats
.size_bins
[0])},
127 {"size_64_packets", offsetof(struct virtnet_tx
, stats
.size_bins
[1])},
128 {"size_65_127_packets", offsetof(struct virtnet_tx
, stats
.size_bins
[2])},
129 {"size_128_255_packets", offsetof(struct virtnet_tx
, stats
.size_bins
[3])},
130 {"size_256_511_packets", offsetof(struct virtnet_tx
, stats
.size_bins
[4])},
131 {"size_512_1023_packets", offsetof(struct virtnet_tx
, stats
.size_bins
[5])},
132 {"size_1024_1518_packets", offsetof(struct virtnet_tx
, stats
.size_bins
[6])},
133 {"size_1519_max_packets", offsetof(struct virtnet_tx
, stats
.size_bins
[7])},
136 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
137 sizeof(rte_virtio_rxq_stat_strings[0]))
138 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
139 sizeof(rte_virtio_txq_stat_strings[0]))
141 struct virtio_hw_internal virtio_hw_internal
[RTE_MAX_ETHPORTS
];
143 static struct virtio_pmd_ctrl
*
144 virtio_send_command_packed(struct virtnet_ctl
*cvq
,
145 struct virtio_pmd_ctrl
*ctrl
,
146 int *dlen
, int pkt_num
)
148 struct virtqueue
*vq
= cvq
->vq
;
150 struct vring_packed_desc
*desc
= vq
->vq_packed
.ring
.desc
;
151 struct virtio_pmd_ctrl
*result
;
158 * Format is enforced in qemu code:
159 * One TX packet for header;
160 * At least one TX packet per argument;
161 * One RX packet for ACK.
163 head
= vq
->vq_avail_idx
;
164 flags
= vq
->vq_packed
.cached_flags
;
165 desc
[head
].addr
= cvq
->virtio_net_hdr_mem
;
166 desc
[head
].len
= sizeof(struct virtio_net_ctrl_hdr
);
169 if (++vq
->vq_avail_idx
>= vq
->vq_nentries
) {
170 vq
->vq_avail_idx
-= vq
->vq_nentries
;
171 vq
->vq_packed
.cached_flags
^= VRING_PACKED_DESC_F_AVAIL_USED
;
174 for (k
= 0; k
< pkt_num
; k
++) {
175 desc
[vq
->vq_avail_idx
].addr
= cvq
->virtio_net_hdr_mem
176 + sizeof(struct virtio_net_ctrl_hdr
)
177 + sizeof(ctrl
->status
) + sizeof(uint8_t) * sum
;
178 desc
[vq
->vq_avail_idx
].len
= dlen
[k
];
179 desc
[vq
->vq_avail_idx
].flags
= VRING_DESC_F_NEXT
|
180 vq
->vq_packed
.cached_flags
;
184 if (++vq
->vq_avail_idx
>= vq
->vq_nentries
) {
185 vq
->vq_avail_idx
-= vq
->vq_nentries
;
186 vq
->vq_packed
.cached_flags
^=
187 VRING_PACKED_DESC_F_AVAIL_USED
;
191 desc
[vq
->vq_avail_idx
].addr
= cvq
->virtio_net_hdr_mem
192 + sizeof(struct virtio_net_ctrl_hdr
);
193 desc
[vq
->vq_avail_idx
].len
= sizeof(ctrl
->status
);
194 desc
[vq
->vq_avail_idx
].flags
= VRING_DESC_F_WRITE
|
195 vq
->vq_packed
.cached_flags
;
198 if (++vq
->vq_avail_idx
>= vq
->vq_nentries
) {
199 vq
->vq_avail_idx
-= vq
->vq_nentries
;
200 vq
->vq_packed
.cached_flags
^= VRING_PACKED_DESC_F_AVAIL_USED
;
203 virtio_wmb(vq
->hw
->weak_barriers
);
204 desc
[head
].flags
= VRING_DESC_F_NEXT
| flags
;
206 virtio_wmb(vq
->hw
->weak_barriers
);
207 virtqueue_notify(vq
);
209 /* wait for used descriptors in virtqueue */
210 while (!desc_is_used(&desc
[head
], vq
))
213 virtio_rmb(vq
->hw
->weak_barriers
);
215 /* now get used descriptors */
216 vq
->vq_free_cnt
+= nb_descs
;
217 vq
->vq_used_cons_idx
+= nb_descs
;
218 if (vq
->vq_used_cons_idx
>= vq
->vq_nentries
) {
219 vq
->vq_used_cons_idx
-= vq
->vq_nentries
;
220 vq
->vq_packed
.used_wrap_counter
^= 1;
223 PMD_INIT_LOG(DEBUG
, "vq->vq_free_cnt=%d\n"
224 "vq->vq_avail_idx=%d\n"
225 "vq->vq_used_cons_idx=%d\n"
226 "vq->vq_packed.cached_flags=0x%x\n"
227 "vq->vq_packed.used_wrap_counter=%d\n",
230 vq
->vq_used_cons_idx
,
231 vq
->vq_packed
.cached_flags
,
232 vq
->vq_packed
.used_wrap_counter
);
234 result
= cvq
->virtio_net_hdr_mz
->addr
;
238 static struct virtio_pmd_ctrl
*
239 virtio_send_command_split(struct virtnet_ctl
*cvq
,
240 struct virtio_pmd_ctrl
*ctrl
,
241 int *dlen
, int pkt_num
)
243 struct virtio_pmd_ctrl
*result
;
244 struct virtqueue
*vq
= cvq
->vq
;
248 head
= vq
->vq_desc_head_idx
;
251 * Format is enforced in qemu code:
252 * One TX packet for header;
253 * At least one TX packet per argument;
254 * One RX packet for ACK.
256 vq
->vq_split
.ring
.desc
[head
].flags
= VRING_DESC_F_NEXT
;
257 vq
->vq_split
.ring
.desc
[head
].addr
= cvq
->virtio_net_hdr_mem
;
258 vq
->vq_split
.ring
.desc
[head
].len
= sizeof(struct virtio_net_ctrl_hdr
);
260 i
= vq
->vq_split
.ring
.desc
[head
].next
;
262 for (k
= 0; k
< pkt_num
; k
++) {
263 vq
->vq_split
.ring
.desc
[i
].flags
= VRING_DESC_F_NEXT
;
264 vq
->vq_split
.ring
.desc
[i
].addr
= cvq
->virtio_net_hdr_mem
265 + sizeof(struct virtio_net_ctrl_hdr
)
266 + sizeof(ctrl
->status
) + sizeof(uint8_t)*sum
;
267 vq
->vq_split
.ring
.desc
[i
].len
= dlen
[k
];
270 i
= vq
->vq_split
.ring
.desc
[i
].next
;
273 vq
->vq_split
.ring
.desc
[i
].flags
= VRING_DESC_F_WRITE
;
274 vq
->vq_split
.ring
.desc
[i
].addr
= cvq
->virtio_net_hdr_mem
275 + sizeof(struct virtio_net_ctrl_hdr
);
276 vq
->vq_split
.ring
.desc
[i
].len
= sizeof(ctrl
->status
);
279 vq
->vq_desc_head_idx
= vq
->vq_split
.ring
.desc
[i
].next
;
281 vq_update_avail_ring(vq
, head
);
282 vq_update_avail_idx(vq
);
284 PMD_INIT_LOG(DEBUG
, "vq->vq_queue_index = %d", vq
->vq_queue_index
);
286 virtqueue_notify(vq
);
289 while (VIRTQUEUE_NUSED(vq
) == 0) {
294 while (VIRTQUEUE_NUSED(vq
)) {
295 uint32_t idx
, desc_idx
, used_idx
;
296 struct vring_used_elem
*uep
;
298 used_idx
= (uint32_t)(vq
->vq_used_cons_idx
299 & (vq
->vq_nentries
- 1));
300 uep
= &vq
->vq_split
.ring
.used
->ring
[used_idx
];
301 idx
= (uint32_t) uep
->id
;
304 while (vq
->vq_split
.ring
.desc
[desc_idx
].flags
&
306 desc_idx
= vq
->vq_split
.ring
.desc
[desc_idx
].next
;
310 vq
->vq_split
.ring
.desc
[desc_idx
].next
= vq
->vq_desc_head_idx
;
311 vq
->vq_desc_head_idx
= idx
;
313 vq
->vq_used_cons_idx
++;
317 PMD_INIT_LOG(DEBUG
, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
318 vq
->vq_free_cnt
, vq
->vq_desc_head_idx
);
320 result
= cvq
->virtio_net_hdr_mz
->addr
;
325 virtio_send_command(struct virtnet_ctl
*cvq
, struct virtio_pmd_ctrl
*ctrl
,
326 int *dlen
, int pkt_num
)
328 virtio_net_ctrl_ack status
= ~0;
329 struct virtio_pmd_ctrl
*result
;
330 struct virtqueue
*vq
;
332 ctrl
->status
= status
;
334 if (!cvq
|| !cvq
->vq
) {
335 PMD_INIT_LOG(ERR
, "Control queue is not supported.");
339 rte_spinlock_lock(&cvq
->lock
);
342 PMD_INIT_LOG(DEBUG
, "vq->vq_desc_head_idx = %d, status = %d, "
343 "vq->hw->cvq = %p vq = %p",
344 vq
->vq_desc_head_idx
, status
, vq
->hw
->cvq
, vq
);
346 if (vq
->vq_free_cnt
< pkt_num
+ 2 || pkt_num
< 1) {
347 rte_spinlock_unlock(&cvq
->lock
);
351 memcpy(cvq
->virtio_net_hdr_mz
->addr
, ctrl
,
352 sizeof(struct virtio_pmd_ctrl
));
354 if (vtpci_packed_queue(vq
->hw
))
355 result
= virtio_send_command_packed(cvq
, ctrl
, dlen
, pkt_num
);
357 result
= virtio_send_command_split(cvq
, ctrl
, dlen
, pkt_num
);
359 rte_spinlock_unlock(&cvq
->lock
);
360 return result
->status
;
364 virtio_set_multiple_queues(struct rte_eth_dev
*dev
, uint16_t nb_queues
)
366 struct virtio_hw
*hw
= dev
->data
->dev_private
;
367 struct virtio_pmd_ctrl ctrl
;
371 ctrl
.hdr
.class = VIRTIO_NET_CTRL_MQ
;
372 ctrl
.hdr
.cmd
= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
;
373 memcpy(ctrl
.data
, &nb_queues
, sizeof(uint16_t));
375 dlen
[0] = sizeof(uint16_t);
377 ret
= virtio_send_command(hw
->cvq
, &ctrl
, dlen
, 1);
379 PMD_INIT_LOG(ERR
, "Multiqueue configured but send command "
380 "failed, this is too late now...");
388 virtio_dev_queue_release(void *queue __rte_unused
)
394 virtio_get_nr_vq(struct virtio_hw
*hw
)
396 uint16_t nr_vq
= hw
->max_queue_pairs
* 2;
398 if (vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_VQ
))
405 virtio_init_vring(struct virtqueue
*vq
)
407 int size
= vq
->vq_nentries
;
408 uint8_t *ring_mem
= vq
->vq_ring_virt_mem
;
410 PMD_INIT_FUNC_TRACE();
412 memset(ring_mem
, 0, vq
->vq_ring_size
);
414 vq
->vq_used_cons_idx
= 0;
415 vq
->vq_desc_head_idx
= 0;
416 vq
->vq_avail_idx
= 0;
417 vq
->vq_desc_tail_idx
= (uint16_t)(vq
->vq_nentries
- 1);
418 vq
->vq_free_cnt
= vq
->vq_nentries
;
419 memset(vq
->vq_descx
, 0, sizeof(struct vq_desc_extra
) * vq
->vq_nentries
);
420 if (vtpci_packed_queue(vq
->hw
)) {
421 vring_init_packed(&vq
->vq_packed
.ring
, ring_mem
,
422 VIRTIO_PCI_VRING_ALIGN
, size
);
423 vring_desc_init_packed(vq
, size
);
425 struct vring
*vr
= &vq
->vq_split
.ring
;
427 vring_init_split(vr
, ring_mem
, VIRTIO_PCI_VRING_ALIGN
, size
);
428 vring_desc_init_split(vr
->desc
, size
);
431 * Disable device(host) interrupting guest
433 virtqueue_disable_intr(vq
);
437 virtio_init_queue(struct rte_eth_dev
*dev
, uint16_t vtpci_queue_idx
)
439 char vq_name
[VIRTQUEUE_MAX_NAME_SZ
];
440 char vq_hdr_name
[VIRTQUEUE_MAX_NAME_SZ
];
441 const struct rte_memzone
*mz
= NULL
, *hdr_mz
= NULL
;
442 unsigned int vq_size
, size
;
443 struct virtio_hw
*hw
= dev
->data
->dev_private
;
444 struct virtnet_rx
*rxvq
= NULL
;
445 struct virtnet_tx
*txvq
= NULL
;
446 struct virtnet_ctl
*cvq
= NULL
;
447 struct virtqueue
*vq
;
448 size_t sz_hdr_mz
= 0;
449 void *sw_ring
= NULL
;
450 int queue_type
= virtio_get_queue_type(hw
, vtpci_queue_idx
);
452 int numa_node
= dev
->device
->numa_node
;
454 PMD_INIT_LOG(INFO
, "setting up queue: %u on NUMA node %d",
455 vtpci_queue_idx
, numa_node
);
458 * Read the virtqueue size from the Queue Size field
459 * Always power of 2 and if 0 virtqueue does not exist
461 vq_size
= VTPCI_OPS(hw
)->get_queue_num(hw
, vtpci_queue_idx
);
462 PMD_INIT_LOG(DEBUG
, "vq_size: %u", vq_size
);
464 PMD_INIT_LOG(ERR
, "virtqueue does not exist");
468 if (!rte_is_power_of_2(vq_size
)) {
469 PMD_INIT_LOG(ERR
, "virtqueue size is not powerof 2");
473 snprintf(vq_name
, sizeof(vq_name
), "port%d_vq%d",
474 dev
->data
->port_id
, vtpci_queue_idx
);
476 size
= RTE_ALIGN_CEIL(sizeof(*vq
) +
477 vq_size
* sizeof(struct vq_desc_extra
),
478 RTE_CACHE_LINE_SIZE
);
479 if (queue_type
== VTNET_TQ
) {
481 * For each xmit packet, allocate a virtio_net_hdr
482 * and indirect ring elements
484 sz_hdr_mz
= vq_size
* sizeof(struct virtio_tx_region
);
485 } else if (queue_type
== VTNET_CQ
) {
486 /* Allocate a page for control vq command, data and status */
487 sz_hdr_mz
= PAGE_SIZE
;
490 vq
= rte_zmalloc_socket(vq_name
, size
, RTE_CACHE_LINE_SIZE
,
493 PMD_INIT_LOG(ERR
, "can not allocate vq");
496 hw
->vqs
[vtpci_queue_idx
] = vq
;
499 vq
->vq_queue_index
= vtpci_queue_idx
;
500 vq
->vq_nentries
= vq_size
;
501 if (vtpci_packed_queue(hw
)) {
502 vq
->vq_packed
.used_wrap_counter
= 1;
503 vq
->vq_packed
.cached_flags
= VRING_PACKED_DESC_F_AVAIL
;
504 vq
->vq_packed
.event_flags_shadow
= 0;
505 if (queue_type
== VTNET_RQ
)
506 vq
->vq_packed
.cached_flags
|= VRING_DESC_F_WRITE
;
510 * Reserve a memzone for vring elements
512 size
= vring_size(hw
, vq_size
, VIRTIO_PCI_VRING_ALIGN
);
513 vq
->vq_ring_size
= RTE_ALIGN_CEIL(size
, VIRTIO_PCI_VRING_ALIGN
);
514 PMD_INIT_LOG(DEBUG
, "vring_size: %d, rounded_vring_size: %d",
515 size
, vq
->vq_ring_size
);
517 mz
= rte_memzone_reserve_aligned(vq_name
, vq
->vq_ring_size
,
518 numa_node
, RTE_MEMZONE_IOVA_CONTIG
,
519 VIRTIO_PCI_VRING_ALIGN
);
521 if (rte_errno
== EEXIST
)
522 mz
= rte_memzone_lookup(vq_name
);
529 memset(mz
->addr
, 0, mz
->len
);
531 vq
->vq_ring_mem
= mz
->iova
;
532 vq
->vq_ring_virt_mem
= mz
->addr
;
533 PMD_INIT_LOG(DEBUG
, "vq->vq_ring_mem: 0x%" PRIx64
,
535 PMD_INIT_LOG(DEBUG
, "vq->vq_ring_virt_mem: 0x%" PRIx64
,
536 (uint64_t)(uintptr_t)mz
->addr
);
538 virtio_init_vring(vq
);
541 snprintf(vq_hdr_name
, sizeof(vq_hdr_name
), "port%d_vq%d_hdr",
542 dev
->data
->port_id
, vtpci_queue_idx
);
543 hdr_mz
= rte_memzone_reserve_aligned(vq_hdr_name
, sz_hdr_mz
,
544 numa_node
, RTE_MEMZONE_IOVA_CONTIG
,
545 RTE_CACHE_LINE_SIZE
);
546 if (hdr_mz
== NULL
) {
547 if (rte_errno
== EEXIST
)
548 hdr_mz
= rte_memzone_lookup(vq_hdr_name
);
549 if (hdr_mz
== NULL
) {
556 if (queue_type
== VTNET_RQ
) {
557 size_t sz_sw
= (RTE_PMD_VIRTIO_RX_MAX_BURST
+ vq_size
) *
558 sizeof(vq
->sw_ring
[0]);
560 sw_ring
= rte_zmalloc_socket("sw_ring", sz_sw
,
561 RTE_CACHE_LINE_SIZE
, numa_node
);
563 PMD_INIT_LOG(ERR
, "can not allocate RX soft ring");
568 vq
->sw_ring
= sw_ring
;
571 rxvq
->port_id
= dev
->data
->port_id
;
573 } else if (queue_type
== VTNET_TQ
) {
576 txvq
->port_id
= dev
->data
->port_id
;
578 txvq
->virtio_net_hdr_mz
= hdr_mz
;
579 txvq
->virtio_net_hdr_mem
= hdr_mz
->iova
;
580 } else if (queue_type
== VTNET_CQ
) {
584 cvq
->virtio_net_hdr_mz
= hdr_mz
;
585 cvq
->virtio_net_hdr_mem
= hdr_mz
->iova
;
586 memset(cvq
->virtio_net_hdr_mz
->addr
, 0, PAGE_SIZE
);
591 /* For virtio_user case (that is when hw->dev is NULL), we use
592 * virtual address. And we need properly set _offset_, please see
593 * VIRTIO_MBUF_DATA_DMA_ADDR in virtqueue.h for more information.
595 if (!hw
->virtio_user_dev
)
596 vq
->offset
= offsetof(struct rte_mbuf
, buf_iova
);
598 vq
->vq_ring_mem
= (uintptr_t)mz
->addr
;
599 vq
->offset
= offsetof(struct rte_mbuf
, buf_addr
);
600 if (queue_type
== VTNET_TQ
)
601 txvq
->virtio_net_hdr_mem
= (uintptr_t)hdr_mz
->addr
;
602 else if (queue_type
== VTNET_CQ
)
603 cvq
->virtio_net_hdr_mem
= (uintptr_t)hdr_mz
->addr
;
606 if (queue_type
== VTNET_TQ
) {
607 struct virtio_tx_region
*txr
;
611 memset(txr
, 0, vq_size
* sizeof(*txr
));
612 for (i
= 0; i
< vq_size
; i
++) {
613 struct vring_desc
*start_dp
= txr
[i
].tx_indir
;
615 /* first indirect descriptor is always the tx header */
616 if (!vtpci_packed_queue(hw
)) {
617 vring_desc_init_split(start_dp
,
618 RTE_DIM(txr
[i
].tx_indir
));
619 start_dp
->addr
= txvq
->virtio_net_hdr_mem
621 + offsetof(struct virtio_tx_region
,
623 start_dp
->len
= hw
->vtnet_hdr_size
;
624 start_dp
->flags
= VRING_DESC_F_NEXT
;
629 if (VTPCI_OPS(hw
)->setup_queue(hw
, vq
) < 0) {
630 PMD_INIT_LOG(ERR
, "setup_queue failed");
638 rte_memzone_free(hdr_mz
);
639 rte_memzone_free(mz
);
646 virtio_free_queues(struct virtio_hw
*hw
)
648 uint16_t nr_vq
= virtio_get_nr_vq(hw
);
649 struct virtqueue
*vq
;
656 for (i
= 0; i
< nr_vq
; i
++) {
661 queue_type
= virtio_get_queue_type(hw
, i
);
662 if (queue_type
== VTNET_RQ
) {
663 rte_free(vq
->sw_ring
);
664 rte_memzone_free(vq
->rxq
.mz
);
665 } else if (queue_type
== VTNET_TQ
) {
666 rte_memzone_free(vq
->txq
.mz
);
667 rte_memzone_free(vq
->txq
.virtio_net_hdr_mz
);
669 rte_memzone_free(vq
->cq
.mz
);
670 rte_memzone_free(vq
->cq
.virtio_net_hdr_mz
);
682 virtio_alloc_queues(struct rte_eth_dev
*dev
)
684 struct virtio_hw
*hw
= dev
->data
->dev_private
;
685 uint16_t nr_vq
= virtio_get_nr_vq(hw
);
689 hw
->vqs
= rte_zmalloc(NULL
, sizeof(struct virtqueue
*) * nr_vq
, 0);
691 PMD_INIT_LOG(ERR
, "failed to allocate vqs");
695 for (i
= 0; i
< nr_vq
; i
++) {
696 ret
= virtio_init_queue(dev
, i
);
698 virtio_free_queues(hw
);
706 static void virtio_queues_unbind_intr(struct rte_eth_dev
*dev
);
709 virtio_dev_close(struct rte_eth_dev
*dev
)
711 struct virtio_hw
*hw
= dev
->data
->dev_private
;
712 struct rte_intr_conf
*intr_conf
= &dev
->data
->dev_conf
.intr_conf
;
714 PMD_INIT_LOG(DEBUG
, "virtio_dev_close");
721 if (dev
->data
->dev_flags
& RTE_ETH_DEV_INTR_LSC
)
722 VTPCI_OPS(hw
)->set_config_irq(hw
, VIRTIO_MSI_NO_VECTOR
);
724 virtio_queues_unbind_intr(dev
);
726 if (intr_conf
->lsc
|| intr_conf
->rxq
) {
727 virtio_intr_disable(dev
);
728 rte_intr_efd_disable(dev
->intr_handle
);
729 rte_free(dev
->intr_handle
->intr_vec
);
730 dev
->intr_handle
->intr_vec
= NULL
;
734 virtio_dev_free_mbufs(dev
);
735 virtio_free_queues(hw
);
739 virtio_dev_promiscuous_enable(struct rte_eth_dev
*dev
)
741 struct virtio_hw
*hw
= dev
->data
->dev_private
;
742 struct virtio_pmd_ctrl ctrl
;
746 if (!vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_RX
)) {
747 PMD_INIT_LOG(INFO
, "host does not support rx control");
751 ctrl
.hdr
.class = VIRTIO_NET_CTRL_RX
;
752 ctrl
.hdr
.cmd
= VIRTIO_NET_CTRL_RX_PROMISC
;
756 ret
= virtio_send_command(hw
->cvq
, &ctrl
, dlen
, 1);
758 PMD_INIT_LOG(ERR
, "Failed to enable promisc");
762 virtio_dev_promiscuous_disable(struct rte_eth_dev
*dev
)
764 struct virtio_hw
*hw
= dev
->data
->dev_private
;
765 struct virtio_pmd_ctrl ctrl
;
769 if (!vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_RX
)) {
770 PMD_INIT_LOG(INFO
, "host does not support rx control");
774 ctrl
.hdr
.class = VIRTIO_NET_CTRL_RX
;
775 ctrl
.hdr
.cmd
= VIRTIO_NET_CTRL_RX_PROMISC
;
779 ret
= virtio_send_command(hw
->cvq
, &ctrl
, dlen
, 1);
781 PMD_INIT_LOG(ERR
, "Failed to disable promisc");
785 virtio_dev_allmulticast_enable(struct rte_eth_dev
*dev
)
787 struct virtio_hw
*hw
= dev
->data
->dev_private
;
788 struct virtio_pmd_ctrl ctrl
;
792 if (!vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_RX
)) {
793 PMD_INIT_LOG(INFO
, "host does not support rx control");
797 ctrl
.hdr
.class = VIRTIO_NET_CTRL_RX
;
798 ctrl
.hdr
.cmd
= VIRTIO_NET_CTRL_RX_ALLMULTI
;
802 ret
= virtio_send_command(hw
->cvq
, &ctrl
, dlen
, 1);
804 PMD_INIT_LOG(ERR
, "Failed to enable allmulticast");
808 virtio_dev_allmulticast_disable(struct rte_eth_dev
*dev
)
810 struct virtio_hw
*hw
= dev
->data
->dev_private
;
811 struct virtio_pmd_ctrl ctrl
;
815 if (!vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_RX
)) {
816 PMD_INIT_LOG(INFO
, "host does not support rx control");
820 ctrl
.hdr
.class = VIRTIO_NET_CTRL_RX
;
821 ctrl
.hdr
.cmd
= VIRTIO_NET_CTRL_RX_ALLMULTI
;
825 ret
= virtio_send_command(hw
->cvq
, &ctrl
, dlen
, 1);
827 PMD_INIT_LOG(ERR
, "Failed to disable allmulticast");
830 #define VLAN_TAG_LEN 4 /* 802.3ac tag (not DMA'd) */
832 virtio_mtu_set(struct rte_eth_dev
*dev
, uint16_t mtu
)
834 struct virtio_hw
*hw
= dev
->data
->dev_private
;
835 uint32_t ether_hdr_len
= ETHER_HDR_LEN
+ VLAN_TAG_LEN
+
837 uint32_t frame_size
= mtu
+ ether_hdr_len
;
838 uint32_t max_frame_size
= hw
->max_mtu
+ ether_hdr_len
;
840 max_frame_size
= RTE_MIN(max_frame_size
, VIRTIO_MAX_RX_PKTLEN
);
842 if (mtu
< ETHER_MIN_MTU
|| frame_size
> max_frame_size
) {
843 PMD_INIT_LOG(ERR
, "MTU should be between %d and %d",
844 ETHER_MIN_MTU
, max_frame_size
- ether_hdr_len
);
851 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev
*dev
, uint16_t queue_id
)
853 struct virtio_hw
*hw
= dev
->data
->dev_private
;
854 struct virtnet_rx
*rxvq
= dev
->data
->rx_queues
[queue_id
];
855 struct virtqueue
*vq
= rxvq
->vq
;
857 virtqueue_enable_intr(vq
);
858 virtio_mb(hw
->weak_barriers
);
863 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev
*dev
, uint16_t queue_id
)
865 struct virtnet_rx
*rxvq
= dev
->data
->rx_queues
[queue_id
];
866 struct virtqueue
*vq
= rxvq
->vq
;
868 virtqueue_disable_intr(vq
);
873 * dev_ops for virtio, bare necessities for basic operation
875 static const struct eth_dev_ops virtio_eth_dev_ops
= {
876 .dev_configure
= virtio_dev_configure
,
877 .dev_start
= virtio_dev_start
,
878 .dev_stop
= virtio_dev_stop
,
879 .dev_close
= virtio_dev_close
,
880 .promiscuous_enable
= virtio_dev_promiscuous_enable
,
881 .promiscuous_disable
= virtio_dev_promiscuous_disable
,
882 .allmulticast_enable
= virtio_dev_allmulticast_enable
,
883 .allmulticast_disable
= virtio_dev_allmulticast_disable
,
884 .mtu_set
= virtio_mtu_set
,
885 .dev_infos_get
= virtio_dev_info_get
,
886 .stats_get
= virtio_dev_stats_get
,
887 .xstats_get
= virtio_dev_xstats_get
,
888 .xstats_get_names
= virtio_dev_xstats_get_names
,
889 .stats_reset
= virtio_dev_stats_reset
,
890 .xstats_reset
= virtio_dev_stats_reset
,
891 .link_update
= virtio_dev_link_update
,
892 .vlan_offload_set
= virtio_dev_vlan_offload_set
,
893 .rx_queue_setup
= virtio_dev_rx_queue_setup
,
894 .rx_queue_intr_enable
= virtio_dev_rx_queue_intr_enable
,
895 .rx_queue_intr_disable
= virtio_dev_rx_queue_intr_disable
,
896 .rx_queue_release
= virtio_dev_queue_release
,
897 .rx_descriptor_done
= virtio_dev_rx_queue_done
,
898 .tx_queue_setup
= virtio_dev_tx_queue_setup
,
899 .tx_queue_release
= virtio_dev_queue_release
,
900 /* collect stats per queue */
901 .queue_stats_mapping_set
= virtio_dev_queue_stats_mapping_set
,
902 .vlan_filter_set
= virtio_vlan_filter_set
,
903 .mac_addr_add
= virtio_mac_addr_add
,
904 .mac_addr_remove
= virtio_mac_addr_remove
,
905 .mac_addr_set
= virtio_mac_addr_set
,
909 * dev_ops for virtio-user in secondary processes, as we just have
910 * some limited supports currently.
912 const struct eth_dev_ops virtio_user_secondary_eth_dev_ops
= {
913 .dev_infos_get
= virtio_dev_info_get
,
914 .stats_get
= virtio_dev_stats_get
,
915 .xstats_get
= virtio_dev_xstats_get
,
916 .xstats_get_names
= virtio_dev_xstats_get_names
,
917 .stats_reset
= virtio_dev_stats_reset
,
918 .xstats_reset
= virtio_dev_stats_reset
,
919 /* collect stats per queue */
920 .queue_stats_mapping_set
= virtio_dev_queue_stats_mapping_set
,
924 virtio_update_stats(struct rte_eth_dev
*dev
, struct rte_eth_stats
*stats
)
928 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
929 const struct virtnet_tx
*txvq
= dev
->data
->tx_queues
[i
];
933 stats
->opackets
+= txvq
->stats
.packets
;
934 stats
->obytes
+= txvq
->stats
.bytes
;
935 stats
->oerrors
+= txvq
->stats
.errors
;
937 if (i
< RTE_ETHDEV_QUEUE_STAT_CNTRS
) {
938 stats
->q_opackets
[i
] = txvq
->stats
.packets
;
939 stats
->q_obytes
[i
] = txvq
->stats
.bytes
;
943 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
944 const struct virtnet_rx
*rxvq
= dev
->data
->rx_queues
[i
];
948 stats
->ipackets
+= rxvq
->stats
.packets
;
949 stats
->ibytes
+= rxvq
->stats
.bytes
;
950 stats
->ierrors
+= rxvq
->stats
.errors
;
952 if (i
< RTE_ETHDEV_QUEUE_STAT_CNTRS
) {
953 stats
->q_ipackets
[i
] = rxvq
->stats
.packets
;
954 stats
->q_ibytes
[i
] = rxvq
->stats
.bytes
;
958 stats
->rx_nombuf
= dev
->data
->rx_mbuf_alloc_failed
;
961 static int virtio_dev_xstats_get_names(struct rte_eth_dev
*dev
,
962 struct rte_eth_xstat_name
*xstats_names
,
963 __rte_unused
unsigned limit
)
969 unsigned nstats
= dev
->data
->nb_tx_queues
* VIRTIO_NB_TXQ_XSTATS
+
970 dev
->data
->nb_rx_queues
* VIRTIO_NB_RXQ_XSTATS
;
972 if (xstats_names
!= NULL
) {
973 /* Note: limit checked in rte_eth_xstats_names() */
975 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
976 struct virtnet_rx
*rxvq
= dev
->data
->rx_queues
[i
];
979 for (t
= 0; t
< VIRTIO_NB_RXQ_XSTATS
; t
++) {
980 snprintf(xstats_names
[count
].name
,
981 sizeof(xstats_names
[count
].name
),
983 rte_virtio_rxq_stat_strings
[t
].name
);
988 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
989 struct virtnet_tx
*txvq
= dev
->data
->tx_queues
[i
];
992 for (t
= 0; t
< VIRTIO_NB_TXQ_XSTATS
; t
++) {
993 snprintf(xstats_names
[count
].name
,
994 sizeof(xstats_names
[count
].name
),
996 rte_virtio_txq_stat_strings
[t
].name
);
1006 virtio_dev_xstats_get(struct rte_eth_dev
*dev
, struct rte_eth_xstat
*xstats
,
1012 unsigned nstats
= dev
->data
->nb_tx_queues
* VIRTIO_NB_TXQ_XSTATS
+
1013 dev
->data
->nb_rx_queues
* VIRTIO_NB_RXQ_XSTATS
;
1018 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
1019 struct virtnet_rx
*rxvq
= dev
->data
->rx_queues
[i
];
1026 for (t
= 0; t
< VIRTIO_NB_RXQ_XSTATS
; t
++) {
1027 xstats
[count
].value
= *(uint64_t *)(((char *)rxvq
) +
1028 rte_virtio_rxq_stat_strings
[t
].offset
);
1029 xstats
[count
].id
= count
;
1034 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
1035 struct virtnet_tx
*txvq
= dev
->data
->tx_queues
[i
];
1042 for (t
= 0; t
< VIRTIO_NB_TXQ_XSTATS
; t
++) {
1043 xstats
[count
].value
= *(uint64_t *)(((char *)txvq
) +
1044 rte_virtio_txq_stat_strings
[t
].offset
);
1045 xstats
[count
].id
= count
;
1054 virtio_dev_stats_get(struct rte_eth_dev
*dev
, struct rte_eth_stats
*stats
)
1056 virtio_update_stats(dev
, stats
);
1062 virtio_dev_stats_reset(struct rte_eth_dev
*dev
)
1066 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
1067 struct virtnet_tx
*txvq
= dev
->data
->tx_queues
[i
];
1071 txvq
->stats
.packets
= 0;
1072 txvq
->stats
.bytes
= 0;
1073 txvq
->stats
.errors
= 0;
1074 txvq
->stats
.multicast
= 0;
1075 txvq
->stats
.broadcast
= 0;
1076 memset(txvq
->stats
.size_bins
, 0,
1077 sizeof(txvq
->stats
.size_bins
[0]) * 8);
1080 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
1081 struct virtnet_rx
*rxvq
= dev
->data
->rx_queues
[i
];
1085 rxvq
->stats
.packets
= 0;
1086 rxvq
->stats
.bytes
= 0;
1087 rxvq
->stats
.errors
= 0;
1088 rxvq
->stats
.multicast
= 0;
1089 rxvq
->stats
.broadcast
= 0;
1090 memset(rxvq
->stats
.size_bins
, 0,
1091 sizeof(rxvq
->stats
.size_bins
[0]) * 8);
1096 virtio_set_hwaddr(struct virtio_hw
*hw
)
1098 vtpci_write_dev_config(hw
,
1099 offsetof(struct virtio_net_config
, mac
),
1100 &hw
->mac_addr
, ETHER_ADDR_LEN
);
1104 virtio_get_hwaddr(struct virtio_hw
*hw
)
1106 if (vtpci_with_feature(hw
, VIRTIO_NET_F_MAC
)) {
1107 vtpci_read_dev_config(hw
,
1108 offsetof(struct virtio_net_config
, mac
),
1109 &hw
->mac_addr
, ETHER_ADDR_LEN
);
1111 eth_random_addr(&hw
->mac_addr
[0]);
1112 virtio_set_hwaddr(hw
);
1117 virtio_mac_table_set(struct virtio_hw
*hw
,
1118 const struct virtio_net_ctrl_mac
*uc
,
1119 const struct virtio_net_ctrl_mac
*mc
)
1121 struct virtio_pmd_ctrl ctrl
;
1124 if (!vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_MAC_ADDR
)) {
1125 PMD_DRV_LOG(INFO
, "host does not support mac table");
1129 ctrl
.hdr
.class = VIRTIO_NET_CTRL_MAC
;
1130 ctrl
.hdr
.cmd
= VIRTIO_NET_CTRL_MAC_TABLE_SET
;
1132 len
[0] = uc
->entries
* ETHER_ADDR_LEN
+ sizeof(uc
->entries
);
1133 memcpy(ctrl
.data
, uc
, len
[0]);
1135 len
[1] = mc
->entries
* ETHER_ADDR_LEN
+ sizeof(mc
->entries
);
1136 memcpy(ctrl
.data
+ len
[0], mc
, len
[1]);
1138 err
= virtio_send_command(hw
->cvq
, &ctrl
, len
, 2);
1140 PMD_DRV_LOG(NOTICE
, "mac table set failed: %d", err
);
1145 virtio_mac_addr_add(struct rte_eth_dev
*dev
, struct ether_addr
*mac_addr
,
1146 uint32_t index
, uint32_t vmdq __rte_unused
)
1148 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1149 const struct ether_addr
*addrs
= dev
->data
->mac_addrs
;
1151 struct virtio_net_ctrl_mac
*uc
, *mc
;
1153 if (index
>= VIRTIO_MAX_MAC_ADDRS
) {
1154 PMD_DRV_LOG(ERR
, "mac address index %u out of range", index
);
1158 uc
= alloca(VIRTIO_MAX_MAC_ADDRS
* ETHER_ADDR_LEN
+ sizeof(uc
->entries
));
1160 mc
= alloca(VIRTIO_MAX_MAC_ADDRS
* ETHER_ADDR_LEN
+ sizeof(mc
->entries
));
1163 for (i
= 0; i
< VIRTIO_MAX_MAC_ADDRS
; i
++) {
1164 const struct ether_addr
*addr
1165 = (i
== index
) ? mac_addr
: addrs
+ i
;
1166 struct virtio_net_ctrl_mac
*tbl
1167 = is_multicast_ether_addr(addr
) ? mc
: uc
;
1169 memcpy(&tbl
->macs
[tbl
->entries
++], addr
, ETHER_ADDR_LEN
);
1172 return virtio_mac_table_set(hw
, uc
, mc
);
1176 virtio_mac_addr_remove(struct rte_eth_dev
*dev
, uint32_t index
)
1178 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1179 struct ether_addr
*addrs
= dev
->data
->mac_addrs
;
1180 struct virtio_net_ctrl_mac
*uc
, *mc
;
1183 if (index
>= VIRTIO_MAX_MAC_ADDRS
) {
1184 PMD_DRV_LOG(ERR
, "mac address index %u out of range", index
);
1188 uc
= alloca(VIRTIO_MAX_MAC_ADDRS
* ETHER_ADDR_LEN
+ sizeof(uc
->entries
));
1190 mc
= alloca(VIRTIO_MAX_MAC_ADDRS
* ETHER_ADDR_LEN
+ sizeof(mc
->entries
));
1193 for (i
= 0; i
< VIRTIO_MAX_MAC_ADDRS
; i
++) {
1194 struct virtio_net_ctrl_mac
*tbl
;
1196 if (i
== index
|| is_zero_ether_addr(addrs
+ i
))
1199 tbl
= is_multicast_ether_addr(addrs
+ i
) ? mc
: uc
;
1200 memcpy(&tbl
->macs
[tbl
->entries
++], addrs
+ i
, ETHER_ADDR_LEN
);
1203 virtio_mac_table_set(hw
, uc
, mc
);
1207 virtio_mac_addr_set(struct rte_eth_dev
*dev
, struct ether_addr
*mac_addr
)
1209 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1211 memcpy(hw
->mac_addr
, mac_addr
, ETHER_ADDR_LEN
);
1213 /* Use atomic update if available */
1214 if (vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_MAC_ADDR
)) {
1215 struct virtio_pmd_ctrl ctrl
;
1216 int len
= ETHER_ADDR_LEN
;
1218 ctrl
.hdr
.class = VIRTIO_NET_CTRL_MAC
;
1219 ctrl
.hdr
.cmd
= VIRTIO_NET_CTRL_MAC_ADDR_SET
;
1221 memcpy(ctrl
.data
, mac_addr
, ETHER_ADDR_LEN
);
1222 return virtio_send_command(hw
->cvq
, &ctrl
, &len
, 1);
1225 if (!vtpci_with_feature(hw
, VIRTIO_NET_F_MAC
))
1228 virtio_set_hwaddr(hw
);
1233 virtio_vlan_filter_set(struct rte_eth_dev
*dev
, uint16_t vlan_id
, int on
)
1235 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1236 struct virtio_pmd_ctrl ctrl
;
1239 if (!vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_VLAN
))
1242 ctrl
.hdr
.class = VIRTIO_NET_CTRL_VLAN
;
1243 ctrl
.hdr
.cmd
= on
? VIRTIO_NET_CTRL_VLAN_ADD
: VIRTIO_NET_CTRL_VLAN_DEL
;
1244 memcpy(ctrl
.data
, &vlan_id
, sizeof(vlan_id
));
1245 len
= sizeof(vlan_id
);
1247 return virtio_send_command(hw
->cvq
, &ctrl
, &len
, 1);
1251 virtio_intr_enable(struct rte_eth_dev
*dev
)
1253 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1255 if (rte_intr_enable(dev
->intr_handle
) < 0)
1258 if (!hw
->virtio_user_dev
)
1259 hw
->use_msix
= vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev
));
1265 virtio_intr_disable(struct rte_eth_dev
*dev
)
1267 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1269 if (rte_intr_disable(dev
->intr_handle
) < 0)
1272 if (!hw
->virtio_user_dev
)
1273 hw
->use_msix
= vtpci_msix_detect(RTE_ETH_DEV_TO_PCI(dev
));
1279 virtio_negotiate_features(struct virtio_hw
*hw
, uint64_t req_features
)
1281 uint64_t host_features
;
1283 /* Prepare guest_features: feature that driver wants to support */
1284 PMD_INIT_LOG(DEBUG
, "guest_features before negotiate = %" PRIx64
,
1287 /* Read device(host) feature bits */
1288 host_features
= VTPCI_OPS(hw
)->get_features(hw
);
1289 PMD_INIT_LOG(DEBUG
, "host_features before negotiate = %" PRIx64
,
1292 /* If supported, ensure MTU value is valid before acknowledging it. */
1293 if (host_features
& req_features
& (1ULL << VIRTIO_NET_F_MTU
)) {
1294 struct virtio_net_config config
;
1296 vtpci_read_dev_config(hw
,
1297 offsetof(struct virtio_net_config
, mtu
),
1298 &config
.mtu
, sizeof(config
.mtu
));
1300 if (config
.mtu
< ETHER_MIN_MTU
)
1301 req_features
&= ~(1ULL << VIRTIO_NET_F_MTU
);
1305 * Negotiate features: Subset of device feature bits are written back
1306 * guest feature bits.
1308 hw
->guest_features
= req_features
;
1309 hw
->guest_features
= vtpci_negotiate_features(hw
, host_features
);
1310 PMD_INIT_LOG(DEBUG
, "features after negotiate = %" PRIx64
,
1311 hw
->guest_features
);
1314 if (!vtpci_with_feature(hw
, VIRTIO_F_VERSION_1
)) {
1316 "VIRTIO_F_VERSION_1 features is not enabled.");
1319 vtpci_set_status(hw
, VIRTIO_CONFIG_STATUS_FEATURES_OK
);
1320 if (!(vtpci_get_status(hw
) & VIRTIO_CONFIG_STATUS_FEATURES_OK
)) {
1322 "failed to set FEATURES_OK status!");
1327 hw
->req_guest_features
= req_features
;
1333 virtio_dev_pause(struct rte_eth_dev
*dev
)
1335 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1337 rte_spinlock_lock(&hw
->state_lock
);
1339 if (hw
->started
== 0) {
1340 /* Device is just stopped. */
1341 rte_spinlock_unlock(&hw
->state_lock
);
1346 * Prevent the worker threads from touching queues to avoid contention,
1347 * 1 ms should be enough for the ongoing Tx function to finish.
1354 * Recover hw state to let the worker threads continue.
1357 virtio_dev_resume(struct rte_eth_dev
*dev
)
1359 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1362 rte_spinlock_unlock(&hw
->state_lock
);
1366 * Should be called only after device is paused.
1369 virtio_inject_pkts(struct rte_eth_dev
*dev
, struct rte_mbuf
**tx_pkts
,
1372 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1373 struct virtnet_tx
*txvq
= dev
->data
->tx_queues
[0];
1376 hw
->inject_pkts
= tx_pkts
;
1377 ret
= dev
->tx_pkt_burst(txvq
, tx_pkts
, nb_pkts
);
1378 hw
->inject_pkts
= NULL
;
1384 virtio_notify_peers(struct rte_eth_dev
*dev
)
1386 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1387 struct virtnet_rx
*rxvq
;
1388 struct rte_mbuf
*rarp_mbuf
;
1390 if (!dev
->data
->rx_queues
)
1393 rxvq
= dev
->data
->rx_queues
[0];
1397 rarp_mbuf
= rte_net_make_rarp_packet(rxvq
->mpool
,
1398 (struct ether_addr
*)hw
->mac_addr
);
1399 if (rarp_mbuf
== NULL
) {
1400 PMD_DRV_LOG(ERR
, "failed to make RARP packet.");
1404 /* If virtio port just stopped, no need to send RARP */
1405 if (virtio_dev_pause(dev
) < 0) {
1406 rte_pktmbuf_free(rarp_mbuf
);
1410 virtio_inject_pkts(dev
, &rarp_mbuf
, 1);
1411 virtio_dev_resume(dev
);
1415 virtio_ack_link_announce(struct rte_eth_dev
*dev
)
1417 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1418 struct virtio_pmd_ctrl ctrl
;
1420 ctrl
.hdr
.class = VIRTIO_NET_CTRL_ANNOUNCE
;
1421 ctrl
.hdr
.cmd
= VIRTIO_NET_CTRL_ANNOUNCE_ACK
;
1423 virtio_send_command(hw
->cvq
, &ctrl
, NULL
, 0);
1427 * Process virtio config changed interrupt. Call the callback
1428 * if link state changed, generate gratuitous RARP packet if
1429 * the status indicates an ANNOUNCE.
1432 virtio_interrupt_handler(void *param
)
1434 struct rte_eth_dev
*dev
= param
;
1435 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1439 /* Read interrupt status which clears interrupt */
1440 isr
= vtpci_isr(hw
);
1441 PMD_DRV_LOG(INFO
, "interrupt status = %#x", isr
);
1443 if (virtio_intr_enable(dev
) < 0)
1444 PMD_DRV_LOG(ERR
, "interrupt enable failed");
1446 if (isr
& VIRTIO_PCI_ISR_CONFIG
) {
1447 if (virtio_dev_link_update(dev
, 0) == 0)
1448 _rte_eth_dev_callback_process(dev
,
1449 RTE_ETH_EVENT_INTR_LSC
,
1452 if (vtpci_with_feature(hw
, VIRTIO_NET_F_STATUS
)) {
1453 vtpci_read_dev_config(hw
,
1454 offsetof(struct virtio_net_config
, status
),
1455 &status
, sizeof(status
));
1456 if (status
& VIRTIO_NET_S_ANNOUNCE
) {
1457 virtio_notify_peers(dev
);
1459 virtio_ack_link_announce(dev
);
1465 /* set rx and tx handlers according to what is supported */
1467 set_rxtx_funcs(struct rte_eth_dev
*eth_dev
)
1469 struct virtio_hw
*hw
= eth_dev
->data
->dev_private
;
1471 if (vtpci_packed_queue(hw
)) {
1473 "virtio: using packed ring %s Tx path on port %u",
1474 hw
->use_inorder_tx
? "inorder" : "standard",
1475 eth_dev
->data
->port_id
);
1476 eth_dev
->tx_pkt_burst
= virtio_xmit_pkts_packed
;
1478 if (hw
->use_inorder_tx
) {
1479 PMD_INIT_LOG(INFO
, "virtio: using inorder Tx path on port %u",
1480 eth_dev
->data
->port_id
);
1481 eth_dev
->tx_pkt_burst
= virtio_xmit_pkts_inorder
;
1483 PMD_INIT_LOG(INFO
, "virtio: using standard Tx path on port %u",
1484 eth_dev
->data
->port_id
);
1485 eth_dev
->tx_pkt_burst
= virtio_xmit_pkts
;
1489 if (vtpci_packed_queue(hw
)) {
1490 if (vtpci_with_feature(hw
, VIRTIO_NET_F_MRG_RXBUF
)) {
1492 "virtio: using packed ring mergeable buffer Rx path on port %u",
1493 eth_dev
->data
->port_id
);
1494 eth_dev
->rx_pkt_burst
=
1495 &virtio_recv_mergeable_pkts_packed
;
1498 "virtio: using packed ring standard Rx path on port %u",
1499 eth_dev
->data
->port_id
);
1500 eth_dev
->rx_pkt_burst
= &virtio_recv_pkts_packed
;
1503 if (hw
->use_simple_rx
) {
1504 PMD_INIT_LOG(INFO
, "virtio: using simple Rx path on port %u",
1505 eth_dev
->data
->port_id
);
1506 eth_dev
->rx_pkt_burst
= virtio_recv_pkts_vec
;
1507 } else if (hw
->use_inorder_rx
) {
1509 "virtio: using inorder Rx path on port %u",
1510 eth_dev
->data
->port_id
);
1511 eth_dev
->rx_pkt_burst
= &virtio_recv_pkts_inorder
;
1512 } else if (vtpci_with_feature(hw
, VIRTIO_NET_F_MRG_RXBUF
)) {
1514 "virtio: using mergeable buffer Rx path on port %u",
1515 eth_dev
->data
->port_id
);
1516 eth_dev
->rx_pkt_burst
= &virtio_recv_mergeable_pkts
;
1518 PMD_INIT_LOG(INFO
, "virtio: using standard Rx path on port %u",
1519 eth_dev
->data
->port_id
);
1520 eth_dev
->rx_pkt_burst
= &virtio_recv_pkts
;
1526 /* Only support 1:1 queue/interrupt mapping so far.
1527 * TODO: support n:1 queue/interrupt mapping when there are limited number of
1528 * interrupt vectors (<N+1).
1531 virtio_queues_bind_intr(struct rte_eth_dev
*dev
)
1534 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1536 PMD_INIT_LOG(INFO
, "queue/interrupt binding");
1537 for (i
= 0; i
< dev
->data
->nb_rx_queues
; ++i
) {
1538 dev
->intr_handle
->intr_vec
[i
] = i
+ 1;
1539 if (VTPCI_OPS(hw
)->set_queue_irq(hw
, hw
->vqs
[i
* 2], i
+ 1) ==
1540 VIRTIO_MSI_NO_VECTOR
) {
1541 PMD_DRV_LOG(ERR
, "failed to set queue vector");
1550 virtio_queues_unbind_intr(struct rte_eth_dev
*dev
)
1553 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1555 PMD_INIT_LOG(INFO
, "queue/interrupt unbinding");
1556 for (i
= 0; i
< dev
->data
->nb_rx_queues
; ++i
)
1557 VTPCI_OPS(hw
)->set_queue_irq(hw
,
1558 hw
->vqs
[i
* VTNET_CQ
],
1559 VIRTIO_MSI_NO_VECTOR
);
1563 virtio_configure_intr(struct rte_eth_dev
*dev
)
1565 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1567 if (!rte_intr_cap_multiple(dev
->intr_handle
)) {
1568 PMD_INIT_LOG(ERR
, "Multiple intr vector not supported");
1572 if (rte_intr_efd_enable(dev
->intr_handle
, dev
->data
->nb_rx_queues
)) {
1573 PMD_INIT_LOG(ERR
, "Fail to create eventfd");
1577 if (!dev
->intr_handle
->intr_vec
) {
1578 dev
->intr_handle
->intr_vec
=
1579 rte_zmalloc("intr_vec",
1580 hw
->max_queue_pairs
* sizeof(int), 0);
1581 if (!dev
->intr_handle
->intr_vec
) {
1582 PMD_INIT_LOG(ERR
, "Failed to allocate %u rxq vectors",
1583 hw
->max_queue_pairs
);
1588 /* Re-register callback to update max_intr */
1589 rte_intr_callback_unregister(dev
->intr_handle
,
1590 virtio_interrupt_handler
,
1592 rte_intr_callback_register(dev
->intr_handle
,
1593 virtio_interrupt_handler
,
1596 /* DO NOT try to remove this! This function will enable msix, or QEMU
1597 * will encounter SIGSEGV when DRIVER_OK is sent.
1598 * And for legacy devices, this should be done before queue/vec binding
1599 * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR
1600 * (22) will be ignored.
1602 if (virtio_intr_enable(dev
) < 0) {
1603 PMD_DRV_LOG(ERR
, "interrupt enable failed");
1607 if (virtio_queues_bind_intr(dev
) < 0) {
1608 PMD_INIT_LOG(ERR
, "Failed to bind queue/interrupt");
1615 /* reset device and renegotiate features if needed */
1617 virtio_init_device(struct rte_eth_dev
*eth_dev
, uint64_t req_features
)
1619 struct virtio_hw
*hw
= eth_dev
->data
->dev_private
;
1620 struct virtio_net_config
*config
;
1621 struct virtio_net_config local_config
;
1622 struct rte_pci_device
*pci_dev
= NULL
;
1625 /* Reset the device although not necessary at startup */
1629 virtio_dev_free_mbufs(eth_dev
);
1630 virtio_free_queues(hw
);
1633 /* Tell the host we've noticed this device. */
1634 vtpci_set_status(hw
, VIRTIO_CONFIG_STATUS_ACK
);
1636 /* Tell the host we've known how to drive the device. */
1637 vtpci_set_status(hw
, VIRTIO_CONFIG_STATUS_DRIVER
);
1638 if (virtio_negotiate_features(hw
, req_features
) < 0)
1641 hw
->weak_barriers
= !vtpci_with_feature(hw
, VIRTIO_F_ORDER_PLATFORM
);
1643 if (!hw
->virtio_user_dev
) {
1644 pci_dev
= RTE_ETH_DEV_TO_PCI(eth_dev
);
1645 rte_eth_copy_pci_info(eth_dev
, pci_dev
);
1648 /* If host does not support both status and MSI-X then disable LSC */
1649 if (vtpci_with_feature(hw
, VIRTIO_NET_F_STATUS
) &&
1650 hw
->use_msix
!= VIRTIO_MSIX_NONE
)
1651 eth_dev
->data
->dev_flags
|= RTE_ETH_DEV_INTR_LSC
;
1653 eth_dev
->data
->dev_flags
&= ~RTE_ETH_DEV_INTR_LSC
;
1655 /* Setting up rx_header size for the device */
1656 if (vtpci_with_feature(hw
, VIRTIO_NET_F_MRG_RXBUF
) ||
1657 vtpci_with_feature(hw
, VIRTIO_F_VERSION_1
) ||
1658 vtpci_with_feature(hw
, VIRTIO_F_RING_PACKED
))
1659 hw
->vtnet_hdr_size
= sizeof(struct virtio_net_hdr_mrg_rxbuf
);
1661 hw
->vtnet_hdr_size
= sizeof(struct virtio_net_hdr
);
1663 /* Copy the permanent MAC address to: virtio_hw */
1664 virtio_get_hwaddr(hw
);
1665 ether_addr_copy((struct ether_addr
*) hw
->mac_addr
,
1666 ð_dev
->data
->mac_addrs
[0]);
1668 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1669 hw
->mac_addr
[0], hw
->mac_addr
[1], hw
->mac_addr
[2],
1670 hw
->mac_addr
[3], hw
->mac_addr
[4], hw
->mac_addr
[5]);
1672 if (vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_VQ
)) {
1673 config
= &local_config
;
1675 vtpci_read_dev_config(hw
,
1676 offsetof(struct virtio_net_config
, mac
),
1677 &config
->mac
, sizeof(config
->mac
));
1679 if (vtpci_with_feature(hw
, VIRTIO_NET_F_STATUS
)) {
1680 vtpci_read_dev_config(hw
,
1681 offsetof(struct virtio_net_config
, status
),
1682 &config
->status
, sizeof(config
->status
));
1685 "VIRTIO_NET_F_STATUS is not supported");
1689 if (vtpci_with_feature(hw
, VIRTIO_NET_F_MQ
)) {
1690 vtpci_read_dev_config(hw
,
1691 offsetof(struct virtio_net_config
, max_virtqueue_pairs
),
1692 &config
->max_virtqueue_pairs
,
1693 sizeof(config
->max_virtqueue_pairs
));
1696 "VIRTIO_NET_F_MQ is not supported");
1697 config
->max_virtqueue_pairs
= 1;
1700 hw
->max_queue_pairs
= config
->max_virtqueue_pairs
;
1702 if (vtpci_with_feature(hw
, VIRTIO_NET_F_MTU
)) {
1703 vtpci_read_dev_config(hw
,
1704 offsetof(struct virtio_net_config
, mtu
),
1706 sizeof(config
->mtu
));
1709 * MTU value has already been checked at negotiation
1710 * time, but check again in case it has changed since
1711 * then, which should not happen.
1713 if (config
->mtu
< ETHER_MIN_MTU
) {
1714 PMD_INIT_LOG(ERR
, "invalid max MTU value (%u)",
1719 hw
->max_mtu
= config
->mtu
;
1720 /* Set initial MTU to maximum one supported by vhost */
1721 eth_dev
->data
->mtu
= config
->mtu
;
1724 hw
->max_mtu
= VIRTIO_MAX_RX_PKTLEN
- ETHER_HDR_LEN
-
1725 VLAN_TAG_LEN
- hw
->vtnet_hdr_size
;
1728 PMD_INIT_LOG(DEBUG
, "config->max_virtqueue_pairs=%d",
1729 config
->max_virtqueue_pairs
);
1730 PMD_INIT_LOG(DEBUG
, "config->status=%d", config
->status
);
1732 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1733 config
->mac
[0], config
->mac
[1],
1734 config
->mac
[2], config
->mac
[3],
1735 config
->mac
[4], config
->mac
[5]);
1737 PMD_INIT_LOG(DEBUG
, "config->max_virtqueue_pairs=1");
1738 hw
->max_queue_pairs
= 1;
1739 hw
->max_mtu
= VIRTIO_MAX_RX_PKTLEN
- ETHER_HDR_LEN
-
1740 VLAN_TAG_LEN
- hw
->vtnet_hdr_size
;
1743 ret
= virtio_alloc_queues(eth_dev
);
1747 if (eth_dev
->data
->dev_conf
.intr_conf
.rxq
) {
1748 if (virtio_configure_intr(eth_dev
) < 0) {
1749 PMD_INIT_LOG(ERR
, "failed to configure interrupt");
1754 vtpci_reinit_complete(hw
);
1757 PMD_INIT_LOG(DEBUG
, "port %d vendorID=0x%x deviceID=0x%x",
1758 eth_dev
->data
->port_id
, pci_dev
->id
.vendor_id
,
1759 pci_dev
->id
.device_id
);
1765 * Remap the PCI device again (IO port map for legacy device and
1766 * memory map for modern device), so that the secondary process
1767 * could have the PCI initiated correctly.
1770 virtio_remap_pci(struct rte_pci_device
*pci_dev
, struct virtio_hw
*hw
)
1774 * We don't have to re-parse the PCI config space, since
1775 * rte_pci_map_device() makes sure the mapped address
1776 * in secondary process would equal to the one mapped in
1777 * the primary process: error will be returned if that
1778 * requirement is not met.
1780 * That said, we could simply reuse all cap pointers
1781 * (such as dev_cfg, common_cfg, etc.) parsed from the
1782 * primary process, which is stored in shared memory.
1784 if (rte_pci_map_device(pci_dev
)) {
1785 PMD_INIT_LOG(DEBUG
, "failed to map pci device!");
1789 if (rte_pci_ioport_map(pci_dev
, 0, VTPCI_IO(hw
)) < 0)
1797 virtio_set_vtpci_ops(struct virtio_hw
*hw
)
1799 #ifdef RTE_VIRTIO_USER
1800 if (hw
->virtio_user_dev
)
1801 VTPCI_OPS(hw
) = &virtio_user_ops
;
1805 VTPCI_OPS(hw
) = &modern_ops
;
1807 VTPCI_OPS(hw
) = &legacy_ops
;
1811 * This function is based on probe() function in virtio_pci.c
1812 * It returns 0 on success.
1815 eth_virtio_dev_init(struct rte_eth_dev
*eth_dev
)
1817 struct virtio_hw
*hw
= eth_dev
->data
->dev_private
;
1820 RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM
< sizeof(struct virtio_net_hdr_mrg_rxbuf
));
1822 eth_dev
->dev_ops
= &virtio_eth_dev_ops
;
1824 if (rte_eal_process_type() == RTE_PROC_SECONDARY
) {
1825 if (!hw
->virtio_user_dev
) {
1826 ret
= virtio_remap_pci(RTE_ETH_DEV_TO_PCI(eth_dev
), hw
);
1831 virtio_set_vtpci_ops(hw
);
1832 set_rxtx_funcs(eth_dev
);
1837 /* Allocate memory for storing MAC addresses */
1838 eth_dev
->data
->mac_addrs
= rte_zmalloc("virtio", VIRTIO_MAX_MAC_ADDRS
* ETHER_ADDR_LEN
, 0);
1839 if (eth_dev
->data
->mac_addrs
== NULL
) {
1841 "Failed to allocate %d bytes needed to store MAC addresses",
1842 VIRTIO_MAX_MAC_ADDRS
* ETHER_ADDR_LEN
);
1846 hw
->port_id
= eth_dev
->data
->port_id
;
1847 /* For virtio_user case the hw->virtio_user_dev is populated by
1848 * virtio_user_eth_dev_alloc() before eth_virtio_dev_init() is called.
1850 if (!hw
->virtio_user_dev
) {
1851 ret
= vtpci_init(RTE_ETH_DEV_TO_PCI(eth_dev
), hw
);
1856 /* reset device and negotiate default features */
1857 ret
= virtio_init_device(eth_dev
, VIRTIO_PMD_DEFAULT_GUEST_FEATURES
);
1864 rte_free(eth_dev
->data
->mac_addrs
);
1865 eth_dev
->data
->mac_addrs
= NULL
;
1870 eth_virtio_dev_uninit(struct rte_eth_dev
*eth_dev
)
1872 PMD_INIT_FUNC_TRACE();
1874 if (rte_eal_process_type() == RTE_PROC_SECONDARY
)
1877 virtio_dev_stop(eth_dev
);
1878 virtio_dev_close(eth_dev
);
1880 eth_dev
->dev_ops
= NULL
;
1881 eth_dev
->tx_pkt_burst
= NULL
;
1882 eth_dev
->rx_pkt_burst
= NULL
;
1884 if (eth_dev
->device
)
1885 rte_pci_unmap_device(RTE_ETH_DEV_TO_PCI(eth_dev
));
1887 PMD_INIT_LOG(DEBUG
, "dev_uninit completed");
1892 static int vdpa_check_handler(__rte_unused
const char *key
,
1893 const char *value
, __rte_unused
void *opaque
)
1895 if (strcmp(value
, "1"))
1902 vdpa_mode_selected(struct rte_devargs
*devargs
)
1904 struct rte_kvargs
*kvlist
;
1905 const char *key
= "vdpa";
1908 if (devargs
== NULL
)
1911 kvlist
= rte_kvargs_parse(devargs
->args
, NULL
);
1915 if (!rte_kvargs_count(kvlist
, key
))
1918 /* vdpa mode selected when there's a key-value pair: vdpa=1 */
1919 if (rte_kvargs_process(kvlist
, key
,
1920 vdpa_check_handler
, NULL
) < 0) {
1926 rte_kvargs_free(kvlist
);
1930 static int eth_virtio_pci_probe(struct rte_pci_driver
*pci_drv __rte_unused
,
1931 struct rte_pci_device
*pci_dev
)
1933 if (rte_eal_iopl_init() != 0) {
1934 PMD_INIT_LOG(ERR
, "IOPL call failed - cannot use virtio PMD");
1938 /* virtio pmd skips probe if device needs to work in vdpa mode */
1939 if (vdpa_mode_selected(pci_dev
->device
.devargs
))
1942 return rte_eth_dev_pci_generic_probe(pci_dev
, sizeof(struct virtio_hw
),
1943 eth_virtio_dev_init
);
1946 static int eth_virtio_pci_remove(struct rte_pci_device
*pci_dev
)
1948 return rte_eth_dev_pci_generic_remove(pci_dev
, eth_virtio_dev_uninit
);
1951 static struct rte_pci_driver rte_virtio_pmd
= {
1953 .name
= "net_virtio",
1955 .id_table
= pci_id_virtio_map
,
1957 .probe
= eth_virtio_pci_probe
,
1958 .remove
= eth_virtio_pci_remove
,
1961 RTE_INIT(rte_virtio_pmd_init
)
1963 rte_eal_iopl_init();
1964 rte_pci_register(&rte_virtio_pmd
);
1968 rx_offload_enabled(struct virtio_hw
*hw
)
1970 return vtpci_with_feature(hw
, VIRTIO_NET_F_GUEST_CSUM
) ||
1971 vtpci_with_feature(hw
, VIRTIO_NET_F_GUEST_TSO4
) ||
1972 vtpci_with_feature(hw
, VIRTIO_NET_F_GUEST_TSO6
);
1976 tx_offload_enabled(struct virtio_hw
*hw
)
1978 return vtpci_with_feature(hw
, VIRTIO_NET_F_CSUM
) ||
1979 vtpci_with_feature(hw
, VIRTIO_NET_F_HOST_TSO4
) ||
1980 vtpci_with_feature(hw
, VIRTIO_NET_F_HOST_TSO6
);
1984 * Configure virtio device
1985 * It returns 0 on success.
1988 virtio_dev_configure(struct rte_eth_dev
*dev
)
1990 const struct rte_eth_rxmode
*rxmode
= &dev
->data
->dev_conf
.rxmode
;
1991 const struct rte_eth_txmode
*txmode
= &dev
->data
->dev_conf
.txmode
;
1992 struct virtio_hw
*hw
= dev
->data
->dev_private
;
1993 uint32_t ether_hdr_len
= ETHER_HDR_LEN
+ VLAN_TAG_LEN
+
1995 uint64_t rx_offloads
= rxmode
->offloads
;
1996 uint64_t tx_offloads
= txmode
->offloads
;
1997 uint64_t req_features
;
2000 PMD_INIT_LOG(DEBUG
, "configure");
2001 req_features
= VIRTIO_PMD_DEFAULT_GUEST_FEATURES
;
2003 if (dev
->data
->dev_conf
.intr_conf
.rxq
) {
2004 ret
= virtio_init_device(dev
, hw
->req_guest_features
);
2009 if (rxmode
->max_rx_pkt_len
> hw
->max_mtu
+ ether_hdr_len
)
2010 req_features
&= ~(1ULL << VIRTIO_NET_F_MTU
);
2012 if (rx_offloads
& (DEV_RX_OFFLOAD_UDP_CKSUM
|
2013 DEV_RX_OFFLOAD_TCP_CKSUM
))
2014 req_features
|= (1ULL << VIRTIO_NET_F_GUEST_CSUM
);
2016 if (rx_offloads
& DEV_RX_OFFLOAD_TCP_LRO
)
2018 (1ULL << VIRTIO_NET_F_GUEST_TSO4
) |
2019 (1ULL << VIRTIO_NET_F_GUEST_TSO6
);
2021 if (tx_offloads
& (DEV_TX_OFFLOAD_UDP_CKSUM
|
2022 DEV_TX_OFFLOAD_TCP_CKSUM
))
2023 req_features
|= (1ULL << VIRTIO_NET_F_CSUM
);
2025 if (tx_offloads
& DEV_TX_OFFLOAD_TCP_TSO
)
2027 (1ULL << VIRTIO_NET_F_HOST_TSO4
) |
2028 (1ULL << VIRTIO_NET_F_HOST_TSO6
);
2030 /* if request features changed, reinit the device */
2031 if (req_features
!= hw
->req_guest_features
) {
2032 ret
= virtio_init_device(dev
, req_features
);
2037 if ((rx_offloads
& (DEV_RX_OFFLOAD_UDP_CKSUM
|
2038 DEV_RX_OFFLOAD_TCP_CKSUM
)) &&
2039 !vtpci_with_feature(hw
, VIRTIO_NET_F_GUEST_CSUM
)) {
2041 "rx checksum not available on this host");
2045 if ((rx_offloads
& DEV_RX_OFFLOAD_TCP_LRO
) &&
2046 (!vtpci_with_feature(hw
, VIRTIO_NET_F_GUEST_TSO4
) ||
2047 !vtpci_with_feature(hw
, VIRTIO_NET_F_GUEST_TSO6
))) {
2049 "Large Receive Offload not available on this host");
2053 /* start control queue */
2054 if (vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_VQ
))
2055 virtio_dev_cq_start(dev
);
2057 if (rx_offloads
& DEV_RX_OFFLOAD_VLAN_STRIP
)
2060 if ((rx_offloads
& DEV_RX_OFFLOAD_VLAN_FILTER
)
2061 && !vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_VLAN
)) {
2063 "vlan filtering not available on this host");
2067 hw
->has_tx_offload
= tx_offload_enabled(hw
);
2068 hw
->has_rx_offload
= rx_offload_enabled(hw
);
2070 if (dev
->data
->dev_flags
& RTE_ETH_DEV_INTR_LSC
)
2071 /* Enable vector (0) for Link State Intrerrupt */
2072 if (VTPCI_OPS(hw
)->set_config_irq(hw
, 0) ==
2073 VIRTIO_MSI_NO_VECTOR
) {
2074 PMD_DRV_LOG(ERR
, "failed to set config vector");
2078 rte_spinlock_init(&hw
->state_lock
);
2080 hw
->use_simple_rx
= 1;
2082 if (vtpci_with_feature(hw
, VIRTIO_F_IN_ORDER
)) {
2083 hw
->use_inorder_tx
= 1;
2084 hw
->use_inorder_rx
= 1;
2085 hw
->use_simple_rx
= 0;
2088 if (vtpci_packed_queue(hw
)) {
2089 hw
->use_simple_rx
= 0;
2090 hw
->use_inorder_rx
= 0;
2093 #if defined RTE_ARCH_ARM64 || defined RTE_ARCH_ARM
2094 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON
)) {
2095 hw
->use_simple_rx
= 0;
2098 if (vtpci_with_feature(hw
, VIRTIO_NET_F_MRG_RXBUF
)) {
2099 hw
->use_simple_rx
= 0;
2102 if (rx_offloads
& (DEV_RX_OFFLOAD_UDP_CKSUM
|
2103 DEV_RX_OFFLOAD_TCP_CKSUM
|
2104 DEV_RX_OFFLOAD_TCP_LRO
|
2105 DEV_RX_OFFLOAD_VLAN_STRIP
))
2106 hw
->use_simple_rx
= 0;
2115 virtio_dev_start(struct rte_eth_dev
*dev
)
2117 uint16_t nb_queues
, i
;
2118 struct virtnet_rx
*rxvq
;
2119 struct virtnet_tx
*txvq __rte_unused
;
2120 struct virtio_hw
*hw
= dev
->data
->dev_private
;
2123 /* Finish the initialization of the queues */
2124 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
2125 ret
= virtio_dev_rx_queue_setup_finish(dev
, i
);
2129 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
2130 ret
= virtio_dev_tx_queue_setup_finish(dev
, i
);
2135 /* check if lsc interrupt feature is enabled */
2136 if (dev
->data
->dev_conf
.intr_conf
.lsc
) {
2137 if (!(dev
->data
->dev_flags
& RTE_ETH_DEV_INTR_LSC
)) {
2138 PMD_DRV_LOG(ERR
, "link status not supported by host");
2143 /* Enable uio/vfio intr/eventfd mapping: althrough we already did that
2144 * in device configure, but it could be unmapped when device is
2147 if (dev
->data
->dev_conf
.intr_conf
.lsc
||
2148 dev
->data
->dev_conf
.intr_conf
.rxq
) {
2149 virtio_intr_disable(dev
);
2151 /* Setup interrupt callback */
2152 if (dev
->data
->dev_flags
& RTE_ETH_DEV_INTR_LSC
)
2153 rte_intr_callback_register(dev
->intr_handle
,
2154 virtio_interrupt_handler
,
2157 if (virtio_intr_enable(dev
) < 0) {
2158 PMD_DRV_LOG(ERR
, "interrupt enable failed");
2163 /*Notify the backend
2164 *Otherwise the tap backend might already stop its queue due to fullness.
2165 *vhost backend will have no chance to be waked up
2167 nb_queues
= RTE_MAX(dev
->data
->nb_rx_queues
, dev
->data
->nb_tx_queues
);
2168 if (hw
->max_queue_pairs
> 1) {
2169 if (virtio_set_multiple_queues(dev
, nb_queues
) != 0)
2173 PMD_INIT_LOG(DEBUG
, "nb_queues=%d", nb_queues
);
2175 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
2176 rxvq
= dev
->data
->rx_queues
[i
];
2177 /* Flush the old packets */
2178 virtqueue_rxvq_flush(rxvq
->vq
);
2179 virtqueue_notify(rxvq
->vq
);
2182 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
2183 txvq
= dev
->data
->tx_queues
[i
];
2184 virtqueue_notify(txvq
->vq
);
2187 PMD_INIT_LOG(DEBUG
, "Notified backend at initialization");
2189 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
2190 rxvq
= dev
->data
->rx_queues
[i
];
2191 VIRTQUEUE_DUMP(rxvq
->vq
);
2194 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
2195 txvq
= dev
->data
->tx_queues
[i
];
2196 VIRTQUEUE_DUMP(txvq
->vq
);
2199 set_rxtx_funcs(dev
);
2202 /* Initialize Link state */
2203 virtio_dev_link_update(dev
, 0);
2208 static void virtio_dev_free_mbufs(struct rte_eth_dev
*dev
)
2210 struct virtio_hw
*hw
= dev
->data
->dev_private
;
2211 uint16_t nr_vq
= virtio_get_nr_vq(hw
);
2212 const char *type __rte_unused
;
2213 unsigned int i
, mbuf_num
= 0;
2214 struct virtqueue
*vq
;
2215 struct rte_mbuf
*buf
;
2218 if (hw
->vqs
== NULL
)
2221 for (i
= 0; i
< nr_vq
; i
++) {
2226 queue_type
= virtio_get_queue_type(hw
, i
);
2227 if (queue_type
== VTNET_RQ
)
2229 else if (queue_type
== VTNET_TQ
)
2235 "Before freeing %s[%d] used and unused buf",
2239 while ((buf
= virtqueue_detach_unused(vq
)) != NULL
) {
2240 rte_pktmbuf_free(buf
);
2245 "After freeing %s[%d] used and unused buf",
2250 PMD_INIT_LOG(DEBUG
, "%d mbufs freed", mbuf_num
);
2254 * Stop device: disable interrupt and mark link down
2257 virtio_dev_stop(struct rte_eth_dev
*dev
)
2259 struct virtio_hw
*hw
= dev
->data
->dev_private
;
2260 struct rte_eth_link link
;
2261 struct rte_intr_conf
*intr_conf
= &dev
->data
->dev_conf
.intr_conf
;
2263 PMD_INIT_LOG(DEBUG
, "stop");
2265 rte_spinlock_lock(&hw
->state_lock
);
2268 hw
->started
= false;
2270 if (intr_conf
->lsc
|| intr_conf
->rxq
) {
2271 virtio_intr_disable(dev
);
2273 /* Reset interrupt callback */
2274 if (dev
->data
->dev_flags
& RTE_ETH_DEV_INTR_LSC
) {
2275 rte_intr_callback_unregister(dev
->intr_handle
,
2276 virtio_interrupt_handler
,
2281 memset(&link
, 0, sizeof(link
));
2282 rte_eth_linkstatus_set(dev
, &link
);
2284 rte_spinlock_unlock(&hw
->state_lock
);
2288 virtio_dev_link_update(struct rte_eth_dev
*dev
, __rte_unused
int wait_to_complete
)
2290 struct rte_eth_link link
;
2292 struct virtio_hw
*hw
= dev
->data
->dev_private
;
2294 memset(&link
, 0, sizeof(link
));
2295 link
.link_duplex
= ETH_LINK_FULL_DUPLEX
;
2296 link
.link_speed
= ETH_SPEED_NUM_10G
;
2297 link
.link_autoneg
= ETH_LINK_FIXED
;
2300 link
.link_status
= ETH_LINK_DOWN
;
2301 } else if (vtpci_with_feature(hw
, VIRTIO_NET_F_STATUS
)) {
2302 PMD_INIT_LOG(DEBUG
, "Get link status from hw");
2303 vtpci_read_dev_config(hw
,
2304 offsetof(struct virtio_net_config
, status
),
2305 &status
, sizeof(status
));
2306 if ((status
& VIRTIO_NET_S_LINK_UP
) == 0) {
2307 link
.link_status
= ETH_LINK_DOWN
;
2308 PMD_INIT_LOG(DEBUG
, "Port %d is down",
2309 dev
->data
->port_id
);
2311 link
.link_status
= ETH_LINK_UP
;
2312 PMD_INIT_LOG(DEBUG
, "Port %d is up",
2313 dev
->data
->port_id
);
2316 link
.link_status
= ETH_LINK_UP
;
2319 return rte_eth_linkstatus_set(dev
, &link
);
2323 virtio_dev_vlan_offload_set(struct rte_eth_dev
*dev
, int mask
)
2325 const struct rte_eth_rxmode
*rxmode
= &dev
->data
->dev_conf
.rxmode
;
2326 struct virtio_hw
*hw
= dev
->data
->dev_private
;
2327 uint64_t offloads
= rxmode
->offloads
;
2329 if (mask
& ETH_VLAN_FILTER_MASK
) {
2330 if ((offloads
& DEV_RX_OFFLOAD_VLAN_FILTER
) &&
2331 !vtpci_with_feature(hw
, VIRTIO_NET_F_CTRL_VLAN
)) {
2334 "vlan filtering not available on this host");
2340 if (mask
& ETH_VLAN_STRIP_MASK
)
2341 hw
->vlan_strip
= !!(offloads
& DEV_RX_OFFLOAD_VLAN_STRIP
);
2347 virtio_dev_info_get(struct rte_eth_dev
*dev
, struct rte_eth_dev_info
*dev_info
)
2349 uint64_t tso_mask
, host_features
;
2350 struct virtio_hw
*hw
= dev
->data
->dev_private
;
2352 dev_info
->speed_capa
= ETH_LINK_SPEED_10G
; /* fake value */
2354 dev_info
->max_rx_queues
=
2355 RTE_MIN(hw
->max_queue_pairs
, VIRTIO_MAX_RX_QUEUES
);
2356 dev_info
->max_tx_queues
=
2357 RTE_MIN(hw
->max_queue_pairs
, VIRTIO_MAX_TX_QUEUES
);
2358 dev_info
->min_rx_bufsize
= VIRTIO_MIN_RX_BUFSIZE
;
2359 dev_info
->max_rx_pktlen
= VIRTIO_MAX_RX_PKTLEN
;
2360 dev_info
->max_mac_addrs
= VIRTIO_MAX_MAC_ADDRS
;
2362 host_features
= VTPCI_OPS(hw
)->get_features(hw
);
2363 dev_info
->rx_offload_capa
= DEV_RX_OFFLOAD_VLAN_STRIP
;
2364 dev_info
->rx_offload_capa
|= DEV_RX_OFFLOAD_JUMBO_FRAME
;
2365 if (host_features
& (1ULL << VIRTIO_NET_F_GUEST_CSUM
)) {
2366 dev_info
->rx_offload_capa
|=
2367 DEV_RX_OFFLOAD_TCP_CKSUM
|
2368 DEV_RX_OFFLOAD_UDP_CKSUM
;
2370 if (host_features
& (1ULL << VIRTIO_NET_F_CTRL_VLAN
))
2371 dev_info
->rx_offload_capa
|= DEV_RX_OFFLOAD_VLAN_FILTER
;
2372 tso_mask
= (1ULL << VIRTIO_NET_F_GUEST_TSO4
) |
2373 (1ULL << VIRTIO_NET_F_GUEST_TSO6
);
2374 if ((host_features
& tso_mask
) == tso_mask
)
2375 dev_info
->rx_offload_capa
|= DEV_RX_OFFLOAD_TCP_LRO
;
2377 dev_info
->tx_offload_capa
= DEV_TX_OFFLOAD_MULTI_SEGS
|
2378 DEV_TX_OFFLOAD_VLAN_INSERT
;
2379 if (host_features
& (1ULL << VIRTIO_NET_F_CSUM
)) {
2380 dev_info
->tx_offload_capa
|=
2381 DEV_TX_OFFLOAD_UDP_CKSUM
|
2382 DEV_TX_OFFLOAD_TCP_CKSUM
;
2384 tso_mask
= (1ULL << VIRTIO_NET_F_HOST_TSO4
) |
2385 (1ULL << VIRTIO_NET_F_HOST_TSO6
);
2386 if ((host_features
& tso_mask
) == tso_mask
)
2387 dev_info
->tx_offload_capa
|= DEV_TX_OFFLOAD_TCP_TSO
;
2391 * It enables testpmd to collect per queue stats.
2394 virtio_dev_queue_stats_mapping_set(__rte_unused
struct rte_eth_dev
*eth_dev
,
2395 __rte_unused
uint16_t queue_id
, __rte_unused
uint8_t stat_idx
,
2396 __rte_unused
uint8_t is_rx
)
2401 RTE_PMD_EXPORT_NAME(net_virtio
, __COUNTER__
);
2402 RTE_PMD_REGISTER_PCI_TABLE(net_virtio
, pci_id_virtio_map
);
2403 RTE_PMD_REGISTER_KMOD_DEP(net_virtio
, "* igb_uio | uio_pci_generic | vfio-pci");
2405 RTE_INIT(virtio_init_log
)
2407 virtio_logtype_init
= rte_log_register("pmd.net.virtio.init");
2408 if (virtio_logtype_init
>= 0)
2409 rte_log_set_level(virtio_logtype_init
, RTE_LOG_NOTICE
);
2410 virtio_logtype_driver
= rte_log_register("pmd.net.virtio.driver");
2411 if (virtio_logtype_driver
>= 0)
2412 rte_log_set_level(virtio_logtype_driver
, RTE_LOG_NOTICE
);