1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* A network driver using virtio.
4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/ethtool.h>
10 #include <linux/module.h>
11 #include <linux/virtio.h>
12 #include <linux/virtio_net.h>
13 #include <linux/bpf.h>
14 #include <linux/bpf_trace.h>
15 #include <linux/scatterlist.h>
16 #include <linux/if_vlan.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/average.h>
20 #include <linux/filter.h>
21 #include <linux/kernel.h>
22 #include <net/route.h>
24 #include <net/net_failover.h>
26 static int napi_weight
= NAPI_POLL_WEIGHT
;
27 module_param(napi_weight
, int, 0444);
29 static bool csum
= true, gso
= true, napi_tx
= true;
30 module_param(csum
, bool, 0444);
31 module_param(gso
, bool, 0444);
32 module_param(napi_tx
, bool, 0644);
34 /* FIXME: MTU in config. */
35 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
36 #define GOOD_COPY_LEN 128
38 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
40 /* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
41 #define VIRTIO_XDP_HEADROOM 256
43 /* Separating two types of XDP xmit */
44 #define VIRTIO_XDP_TX BIT(0)
45 #define VIRTIO_XDP_REDIR BIT(1)
47 #define VIRTIO_XDP_FLAG BIT(0)
49 /* RX packet size EWMA. The average packet size is used to determine the packet
50 * buffer size when refilling RX rings. As the entire RX ring may be refilled
51 * at once, the weight is chosen so that the EWMA will be insensitive to short-
52 * term, transient changes in packet size.
54 DECLARE_EWMA(pkt_len
, 0, 64)
56 #define VIRTNET_DRIVER_VERSION "1.0.0"
58 static const unsigned long guest_offloads
[] = {
59 VIRTIO_NET_F_GUEST_TSO4
,
60 VIRTIO_NET_F_GUEST_TSO6
,
61 VIRTIO_NET_F_GUEST_ECN
,
62 VIRTIO_NET_F_GUEST_UFO
,
63 VIRTIO_NET_F_GUEST_CSUM
66 #define GUEST_OFFLOAD_LRO_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
67 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
68 (1ULL << VIRTIO_NET_F_GUEST_ECN) | \
69 (1ULL << VIRTIO_NET_F_GUEST_UFO))
71 struct virtnet_stat_desc
{
72 char desc
[ETH_GSTRING_LEN
];
76 struct virtnet_sq_stats
{
77 struct u64_stats_sync syncp
;
85 struct virtnet_rq_stats
{
86 struct u64_stats_sync syncp
;
97 #define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m)
98 #define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m)
100 static const struct virtnet_stat_desc virtnet_sq_stats_desc
[] = {
101 { "packets", VIRTNET_SQ_STAT(packets
) },
102 { "bytes", VIRTNET_SQ_STAT(bytes
) },
103 { "xdp_tx", VIRTNET_SQ_STAT(xdp_tx
) },
104 { "xdp_tx_drops", VIRTNET_SQ_STAT(xdp_tx_drops
) },
105 { "kicks", VIRTNET_SQ_STAT(kicks
) },
108 static const struct virtnet_stat_desc virtnet_rq_stats_desc
[] = {
109 { "packets", VIRTNET_RQ_STAT(packets
) },
110 { "bytes", VIRTNET_RQ_STAT(bytes
) },
111 { "drops", VIRTNET_RQ_STAT(drops
) },
112 { "xdp_packets", VIRTNET_RQ_STAT(xdp_packets
) },
113 { "xdp_tx", VIRTNET_RQ_STAT(xdp_tx
) },
114 { "xdp_redirects", VIRTNET_RQ_STAT(xdp_redirects
) },
115 { "xdp_drops", VIRTNET_RQ_STAT(xdp_drops
) },
116 { "kicks", VIRTNET_RQ_STAT(kicks
) },
119 #define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc)
120 #define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc)
122 /* Internal representation of a send virtqueue */
124 /* Virtqueue associated with this send _queue */
125 struct virtqueue
*vq
;
127 /* TX: fragments + linear part + virtio header */
128 struct scatterlist sg
[MAX_SKB_FRAGS
+ 2];
130 /* Name of the send queue: output.$index */
133 struct virtnet_sq_stats stats
;
135 struct napi_struct napi
;
138 /* Internal representation of a receive virtqueue */
139 struct receive_queue
{
140 /* Virtqueue associated with this receive_queue */
141 struct virtqueue
*vq
;
143 struct napi_struct napi
;
145 struct bpf_prog __rcu
*xdp_prog
;
147 struct virtnet_rq_stats stats
;
149 /* Chain pages by the private ptr. */
152 /* Average packet length for mergeable receive buffers. */
153 struct ewma_pkt_len mrg_avg_pkt_len
;
155 /* Page frag for packet buffer allocation. */
156 struct page_frag alloc_frag
;
158 /* RX: fragments + linear part + virtio header */
159 struct scatterlist sg
[MAX_SKB_FRAGS
+ 2];
161 /* Min single buffer size for mergeable buffers case. */
162 unsigned int min_buf_len
;
164 /* Name of this receive queue: input.$index */
167 struct xdp_rxq_info xdp_rxq
;
170 /* Control VQ buffers: protected by the rtnl lock */
172 struct virtio_net_ctrl_hdr hdr
;
173 virtio_net_ctrl_ack status
;
174 struct virtio_net_ctrl_mq mq
;
181 struct virtnet_info
{
182 struct virtio_device
*vdev
;
183 struct virtqueue
*cvq
;
184 struct net_device
*dev
;
185 struct send_queue
*sq
;
186 struct receive_queue
*rq
;
189 /* Max # of queue pairs supported by the device */
192 /* # of queue pairs currently used by the driver */
193 u16 curr_queue_pairs
;
195 /* # of XDP queue pairs currently used by the driver */
198 /* I like... big packets and I cannot lie! */
201 /* Host will merge rx buffers for big packets (shake it! shake it!) */
202 bool mergeable_rx_bufs
;
204 /* Has control virtqueue */
207 /* Host can handle any s/g split between our header and packet data */
210 /* Packet virtio header size */
213 /* Work struct for refilling if we run low on memory. */
214 struct delayed_work refill
;
216 /* Work struct for config space updates */
217 struct work_struct config_work
;
219 /* Does the affinity hint is set for virtqueues? */
220 bool affinity_hint_set
;
222 /* CPU hotplug instances for online & dead */
223 struct hlist_node node
;
224 struct hlist_node node_dead
;
226 struct control_buf
*ctrl
;
228 /* Ethtool settings */
232 unsigned long guest_offloads
;
233 unsigned long guest_offloads_capable
;
235 /* failover when STANDBY feature enabled */
236 struct failover
*failover
;
239 struct padded_vnet_hdr
{
240 struct virtio_net_hdr_mrg_rxbuf hdr
;
242 * hdr is in a separate sg buffer, and data sg buffer shares same page
243 * with this header sg. This padding makes next sg 16 byte aligned
249 static bool is_xdp_frame(void *ptr
)
251 return (unsigned long)ptr
& VIRTIO_XDP_FLAG
;
254 static void *xdp_to_ptr(struct xdp_frame
*ptr
)
256 return (void *)((unsigned long)ptr
| VIRTIO_XDP_FLAG
);
259 static struct xdp_frame
*ptr_to_xdp(void *ptr
)
261 return (struct xdp_frame
*)((unsigned long)ptr
& ~VIRTIO_XDP_FLAG
);
264 /* Converting between virtqueue no. and kernel tx/rx queue no.
265 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
267 static int vq2txq(struct virtqueue
*vq
)
269 return (vq
->index
- 1) / 2;
272 static int txq2vq(int txq
)
277 static int vq2rxq(struct virtqueue
*vq
)
279 return vq
->index
/ 2;
282 static int rxq2vq(int rxq
)
287 static inline struct virtio_net_hdr_mrg_rxbuf
*skb_vnet_hdr(struct sk_buff
*skb
)
289 return (struct virtio_net_hdr_mrg_rxbuf
*)skb
->cb
;
293 * private is used to chain pages for big packets, put the whole
294 * most recent used list in the beginning for reuse
296 static void give_pages(struct receive_queue
*rq
, struct page
*page
)
300 /* Find end of list, sew whole thing into vi->rq.pages. */
301 for (end
= page
; end
->private; end
= (struct page
*)end
->private);
302 end
->private = (unsigned long)rq
->pages
;
306 static struct page
*get_a_page(struct receive_queue
*rq
, gfp_t gfp_mask
)
308 struct page
*p
= rq
->pages
;
311 rq
->pages
= (struct page
*)p
->private;
312 /* clear private here, it is used to chain pages */
315 p
= alloc_page(gfp_mask
);
319 static void virtqueue_napi_schedule(struct napi_struct
*napi
,
320 struct virtqueue
*vq
)
322 if (napi_schedule_prep(napi
)) {
323 virtqueue_disable_cb(vq
);
324 __napi_schedule(napi
);
328 static void virtqueue_napi_complete(struct napi_struct
*napi
,
329 struct virtqueue
*vq
, int processed
)
333 opaque
= virtqueue_enable_cb_prepare(vq
);
334 if (napi_complete_done(napi
, processed
)) {
335 if (unlikely(virtqueue_poll(vq
, opaque
)))
336 virtqueue_napi_schedule(napi
, vq
);
338 virtqueue_disable_cb(vq
);
342 static void skb_xmit_done(struct virtqueue
*vq
)
344 struct virtnet_info
*vi
= vq
->vdev
->priv
;
345 struct napi_struct
*napi
= &vi
->sq
[vq2txq(vq
)].napi
;
347 /* Suppress further interrupts. */
348 virtqueue_disable_cb(vq
);
351 virtqueue_napi_schedule(napi
, vq
);
353 /* We were probably waiting for more output buffers. */
354 netif_wake_subqueue(vi
->dev
, vq2txq(vq
));
357 #define MRG_CTX_HEADER_SHIFT 22
358 static void *mergeable_len_to_ctx(unsigned int truesize
,
359 unsigned int headroom
)
361 return (void *)(unsigned long)((headroom
<< MRG_CTX_HEADER_SHIFT
) | truesize
);
364 static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx
)
366 return (unsigned long)mrg_ctx
>> MRG_CTX_HEADER_SHIFT
;
369 static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx
)
371 return (unsigned long)mrg_ctx
& ((1 << MRG_CTX_HEADER_SHIFT
) - 1);
374 /* Called from bottom half context */
375 static struct sk_buff
*page_to_skb(struct virtnet_info
*vi
,
376 struct receive_queue
*rq
,
377 struct page
*page
, unsigned int offset
,
378 unsigned int len
, unsigned int truesize
,
379 bool hdr_valid
, unsigned int metasize
)
382 struct virtio_net_hdr_mrg_rxbuf
*hdr
;
383 unsigned int copy
, hdr_len
, hdr_padded_len
;
386 p
= page_address(page
) + offset
;
388 /* copy small packet so we can reuse these pages for small data */
389 skb
= napi_alloc_skb(&rq
->napi
, GOOD_COPY_LEN
);
393 hdr
= skb_vnet_hdr(skb
);
395 hdr_len
= vi
->hdr_len
;
396 if (vi
->mergeable_rx_bufs
)
397 hdr_padded_len
= sizeof(*hdr
);
399 hdr_padded_len
= sizeof(struct padded_vnet_hdr
);
401 /* hdr_valid means no XDP, so we can copy the vnet header */
403 memcpy(hdr
, p
, hdr_len
);
406 offset
+= hdr_padded_len
;
410 if (copy
> skb_tailroom(skb
))
411 copy
= skb_tailroom(skb
);
412 skb_put_data(skb
, p
, copy
);
415 __skb_pull(skb
, metasize
);
416 skb_metadata_set(skb
, metasize
);
422 if (vi
->mergeable_rx_bufs
) {
424 skb_add_rx_frag(skb
, 0, page
, offset
, len
, truesize
);
431 * Verify that we can indeed put this data into a skb.
432 * This is here to handle cases when the device erroneously
433 * tries to receive more than is possible. This is usually
434 * the case of a broken device.
436 if (unlikely(len
> MAX_SKB_FRAGS
* PAGE_SIZE
)) {
437 net_dbg_ratelimited("%s: too much data\n", skb
->dev
->name
);
441 BUG_ON(offset
>= PAGE_SIZE
);
443 unsigned int frag_size
= min((unsigned)PAGE_SIZE
- offset
, len
);
444 skb_add_rx_frag(skb
, skb_shinfo(skb
)->nr_frags
, page
, offset
,
445 frag_size
, truesize
);
447 page
= (struct page
*)page
->private;
452 give_pages(rq
, page
);
457 static int __virtnet_xdp_xmit_one(struct virtnet_info
*vi
,
458 struct send_queue
*sq
,
459 struct xdp_frame
*xdpf
)
461 struct virtio_net_hdr_mrg_rxbuf
*hdr
;
464 if (unlikely(xdpf
->headroom
< vi
->hdr_len
))
467 /* Make room for virtqueue hdr (also change xdpf->headroom?) */
468 xdpf
->data
-= vi
->hdr_len
;
469 /* Zero header and leave csum up to XDP layers */
471 memset(hdr
, 0, vi
->hdr_len
);
472 xdpf
->len
+= vi
->hdr_len
;
474 sg_init_one(sq
->sg
, xdpf
->data
, xdpf
->len
);
476 err
= virtqueue_add_outbuf(sq
->vq
, sq
->sg
, 1, xdp_to_ptr(xdpf
),
479 return -ENOSPC
; /* Caller handle free/refcnt */
484 static struct send_queue
*virtnet_xdp_sq(struct virtnet_info
*vi
)
488 qp
= vi
->curr_queue_pairs
- vi
->xdp_queue_pairs
+ smp_processor_id();
492 static int virtnet_xdp_xmit(struct net_device
*dev
,
493 int n
, struct xdp_frame
**frames
, u32 flags
)
495 struct virtnet_info
*vi
= netdev_priv(dev
);
496 struct receive_queue
*rq
= vi
->rq
;
497 struct bpf_prog
*xdp_prog
;
498 struct send_queue
*sq
;
508 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
509 * indicate XDP resources have been successfully allocated.
511 xdp_prog
= rcu_access_pointer(rq
->xdp_prog
);
515 sq
= virtnet_xdp_sq(vi
);
517 if (unlikely(flags
& ~XDP_XMIT_FLAGS_MASK
)) {
523 /* Free up any pending old buffers before queueing new ones. */
524 while ((ptr
= virtqueue_get_buf(sq
->vq
, &len
)) != NULL
) {
525 if (likely(is_xdp_frame(ptr
))) {
526 struct xdp_frame
*frame
= ptr_to_xdp(ptr
);
529 xdp_return_frame(frame
);
531 struct sk_buff
*skb
= ptr
;
534 napi_consume_skb(skb
, false);
539 for (i
= 0; i
< n
; i
++) {
540 struct xdp_frame
*xdpf
= frames
[i
];
542 err
= __virtnet_xdp_xmit_one(vi
, sq
, xdpf
);
544 xdp_return_frame_rx_napi(xdpf
);
550 if (flags
& XDP_XMIT_FLUSH
) {
551 if (virtqueue_kick_prepare(sq
->vq
) && virtqueue_notify(sq
->vq
))
555 u64_stats_update_begin(&sq
->stats
.syncp
);
556 sq
->stats
.bytes
+= bytes
;
557 sq
->stats
.packets
+= packets
;
558 sq
->stats
.xdp_tx
+= n
;
559 sq
->stats
.xdp_tx_drops
+= drops
;
560 sq
->stats
.kicks
+= kicks
;
561 u64_stats_update_end(&sq
->stats
.syncp
);
566 static unsigned int virtnet_get_headroom(struct virtnet_info
*vi
)
568 return vi
->xdp_queue_pairs
? VIRTIO_XDP_HEADROOM
: 0;
571 /* We copy the packet for XDP in the following cases:
573 * 1) Packet is scattered across multiple rx buffers.
574 * 2) Headroom space is insufficient.
576 * This is inefficient but it's a temporary condition that
577 * we hit right after XDP is enabled and until queue is refilled
578 * with large buffers with sufficient headroom - so it should affect
579 * at most queue size packets.
580 * Afterwards, the conditions to enable
581 * XDP should preclude the underlying device from sending packets
582 * across multiple buffers (num_buf > 1), and we make sure buffers
583 * have enough headroom.
585 static struct page
*xdp_linearize_page(struct receive_queue
*rq
,
592 struct page
*page
= alloc_page(GFP_ATOMIC
);
597 memcpy(page_address(page
) + page_off
, page_address(p
) + offset
, *len
);
601 int tailroom
= SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
606 buf
= virtqueue_get_buf(rq
->vq
, &buflen
);
610 p
= virt_to_head_page(buf
);
611 off
= buf
- page_address(p
);
613 /* guard against a misconfigured or uncooperative backend that
614 * is sending packet larger than the MTU.
616 if ((page_off
+ buflen
+ tailroom
) > PAGE_SIZE
) {
621 memcpy(page_address(page
) + page_off
,
622 page_address(p
) + off
, buflen
);
627 /* Headroom does not contribute to packet length */
628 *len
= page_off
- VIRTIO_XDP_HEADROOM
;
631 __free_pages(page
, 0);
635 static struct sk_buff
*receive_small(struct net_device
*dev
,
636 struct virtnet_info
*vi
,
637 struct receive_queue
*rq
,
638 void *buf
, void *ctx
,
640 unsigned int *xdp_xmit
,
641 struct virtnet_rq_stats
*stats
)
644 struct bpf_prog
*xdp_prog
;
645 unsigned int xdp_headroom
= (unsigned long)ctx
;
646 unsigned int header_offset
= VIRTNET_RX_PAD
+ xdp_headroom
;
647 unsigned int headroom
= vi
->hdr_len
+ header_offset
;
648 unsigned int buflen
= SKB_DATA_ALIGN(GOOD_PACKET_LEN
+ headroom
) +
649 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
650 struct page
*page
= virt_to_head_page(buf
);
651 unsigned int delta
= 0;
652 struct page
*xdp_page
;
654 unsigned int metasize
= 0;
660 xdp_prog
= rcu_dereference(rq
->xdp_prog
);
662 struct virtio_net_hdr_mrg_rxbuf
*hdr
= buf
+ header_offset
;
663 struct xdp_frame
*xdpf
;
668 if (unlikely(hdr
->hdr
.gso_type
))
671 if (unlikely(xdp_headroom
< virtnet_get_headroom(vi
))) {
672 int offset
= buf
- page_address(page
) + header_offset
;
673 unsigned int tlen
= len
+ vi
->hdr_len
;
676 xdp_headroom
= virtnet_get_headroom(vi
);
677 header_offset
= VIRTNET_RX_PAD
+ xdp_headroom
;
678 headroom
= vi
->hdr_len
+ header_offset
;
679 buflen
= SKB_DATA_ALIGN(GOOD_PACKET_LEN
+ headroom
) +
680 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
681 xdp_page
= xdp_linearize_page(rq
, &num_buf
, page
,
682 offset
, header_offset
,
687 buf
= page_address(xdp_page
);
692 xdp
.data_hard_start
= buf
+ VIRTNET_RX_PAD
+ vi
->hdr_len
;
693 xdp
.data
= xdp
.data_hard_start
+ xdp_headroom
;
694 xdp
.data_end
= xdp
.data
+ len
;
695 xdp
.data_meta
= xdp
.data
;
696 xdp
.rxq
= &rq
->xdp_rxq
;
697 xdp
.frame_sz
= buflen
;
698 orig_data
= xdp
.data
;
699 act
= bpf_prog_run_xdp(xdp_prog
, &xdp
);
700 stats
->xdp_packets
++;
704 /* Recalculate length in case bpf program changed it */
705 delta
= orig_data
- xdp
.data
;
706 len
= xdp
.data_end
- xdp
.data
;
707 metasize
= xdp
.data
- xdp
.data_meta
;
711 xdpf
= xdp_convert_buff_to_frame(&xdp
);
714 err
= virtnet_xdp_xmit(dev
, 1, &xdpf
, 0);
715 if (unlikely(err
< 0)) {
716 trace_xdp_exception(vi
->dev
, xdp_prog
, act
);
719 *xdp_xmit
|= VIRTIO_XDP_TX
;
723 stats
->xdp_redirects
++;
724 err
= xdp_do_redirect(dev
, &xdp
, xdp_prog
);
727 *xdp_xmit
|= VIRTIO_XDP_REDIR
;
731 bpf_warn_invalid_xdp_action(act
);
734 trace_xdp_exception(vi
->dev
, xdp_prog
, act
);
741 skb
= build_skb(buf
, buflen
);
746 skb_reserve(skb
, headroom
- delta
);
749 buf
+= header_offset
;
750 memcpy(skb_vnet_hdr(skb
), buf
, vi
->hdr_len
);
751 } /* keep zeroed vnet hdr since XDP is loaded */
754 skb_metadata_set(skb
, metasize
);
768 static struct sk_buff
*receive_big(struct net_device
*dev
,
769 struct virtnet_info
*vi
,
770 struct receive_queue
*rq
,
773 struct virtnet_rq_stats
*stats
)
775 struct page
*page
= buf
;
776 struct sk_buff
*skb
=
777 page_to_skb(vi
, rq
, page
, 0, len
, PAGE_SIZE
, true, 0);
779 stats
->bytes
+= len
- vi
->hdr_len
;
787 give_pages(rq
, page
);
791 static struct sk_buff
*receive_mergeable(struct net_device
*dev
,
792 struct virtnet_info
*vi
,
793 struct receive_queue
*rq
,
797 unsigned int *xdp_xmit
,
798 struct virtnet_rq_stats
*stats
)
800 struct virtio_net_hdr_mrg_rxbuf
*hdr
= buf
;
801 u16 num_buf
= virtio16_to_cpu(vi
->vdev
, hdr
->num_buffers
);
802 struct page
*page
= virt_to_head_page(buf
);
803 int offset
= buf
- page_address(page
);
804 struct sk_buff
*head_skb
, *curr_skb
;
805 struct bpf_prog
*xdp_prog
;
806 unsigned int truesize
= mergeable_ctx_to_truesize(ctx
);
807 unsigned int headroom
= mergeable_ctx_to_headroom(ctx
);
808 unsigned int metasize
= 0;
809 unsigned int frame_sz
;
813 stats
->bytes
+= len
- vi
->hdr_len
;
816 xdp_prog
= rcu_dereference(rq
->xdp_prog
);
818 struct xdp_frame
*xdpf
;
819 struct page
*xdp_page
;
824 /* Transient failure which in theory could occur if
825 * in-flight packets from before XDP was enabled reach
826 * the receive path after XDP is loaded.
828 if (unlikely(hdr
->hdr
.gso_type
))
831 /* Buffers with headroom use PAGE_SIZE as alloc size,
832 * see add_recvbuf_mergeable() + get_mergeable_buf_len()
834 frame_sz
= headroom
? PAGE_SIZE
: truesize
;
836 /* This happens when rx buffer size is underestimated
837 * or headroom is not enough because of the buffer
838 * was refilled before XDP is set. This should only
839 * happen for the first several packets, so we don't
840 * care much about its performance.
842 if (unlikely(num_buf
> 1 ||
843 headroom
< virtnet_get_headroom(vi
))) {
844 /* linearize data for XDP */
845 xdp_page
= xdp_linearize_page(rq
, &num_buf
,
849 frame_sz
= PAGE_SIZE
;
853 offset
= VIRTIO_XDP_HEADROOM
;
858 /* Allow consuming headroom but reserve enough space to push
859 * the descriptor on if we get an XDP_TX return code.
861 data
= page_address(xdp_page
) + offset
;
862 xdp
.data_hard_start
= data
- VIRTIO_XDP_HEADROOM
+ vi
->hdr_len
;
863 xdp
.data
= data
+ vi
->hdr_len
;
864 xdp
.data_end
= xdp
.data
+ (len
- vi
->hdr_len
);
865 xdp
.data_meta
= xdp
.data
;
866 xdp
.rxq
= &rq
->xdp_rxq
;
867 xdp
.frame_sz
= frame_sz
- vi
->hdr_len
;
869 act
= bpf_prog_run_xdp(xdp_prog
, &xdp
);
870 stats
->xdp_packets
++;
874 metasize
= xdp
.data
- xdp
.data_meta
;
876 /* recalculate offset to account for any header
877 * adjustments and minus the metasize to copy the
878 * metadata in page_to_skb(). Note other cases do not
879 * build an skb and avoid using offset
881 offset
= xdp
.data
- page_address(xdp_page
) -
882 vi
->hdr_len
- metasize
;
884 /* recalculate len if xdp.data, xdp.data_end or
885 * xdp.data_meta were adjusted
887 len
= xdp
.data_end
- xdp
.data
+ vi
->hdr_len
+ metasize
;
888 /* We can only create skb based on xdp_page. */
889 if (unlikely(xdp_page
!= page
)) {
892 head_skb
= page_to_skb(vi
, rq
, xdp_page
, offset
,
893 len
, PAGE_SIZE
, false,
900 xdpf
= xdp_convert_buff_to_frame(&xdp
);
903 err
= virtnet_xdp_xmit(dev
, 1, &xdpf
, 0);
904 if (unlikely(err
< 0)) {
905 trace_xdp_exception(vi
->dev
, xdp_prog
, act
);
906 if (unlikely(xdp_page
!= page
))
910 *xdp_xmit
|= VIRTIO_XDP_TX
;
911 if (unlikely(xdp_page
!= page
))
916 stats
->xdp_redirects
++;
917 err
= xdp_do_redirect(dev
, &xdp
, xdp_prog
);
919 if (unlikely(xdp_page
!= page
))
923 *xdp_xmit
|= VIRTIO_XDP_REDIR
;
924 if (unlikely(xdp_page
!= page
))
929 bpf_warn_invalid_xdp_action(act
);
932 trace_xdp_exception(vi
->dev
, xdp_prog
, act
);
935 if (unlikely(xdp_page
!= page
))
936 __free_pages(xdp_page
, 0);
942 if (unlikely(len
> truesize
)) {
943 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
944 dev
->name
, len
, (unsigned long)ctx
);
945 dev
->stats
.rx_length_errors
++;
949 head_skb
= page_to_skb(vi
, rq
, page
, offset
, len
, truesize
, !xdp_prog
,
953 if (unlikely(!curr_skb
))
958 buf
= virtqueue_get_buf_ctx(rq
->vq
, &len
, &ctx
);
959 if (unlikely(!buf
)) {
960 pr_debug("%s: rx error: %d buffers out of %d missing\n",
962 virtio16_to_cpu(vi
->vdev
,
964 dev
->stats
.rx_length_errors
++;
969 page
= virt_to_head_page(buf
);
971 truesize
= mergeable_ctx_to_truesize(ctx
);
972 if (unlikely(len
> truesize
)) {
973 pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
974 dev
->name
, len
, (unsigned long)ctx
);
975 dev
->stats
.rx_length_errors
++;
979 num_skb_frags
= skb_shinfo(curr_skb
)->nr_frags
;
980 if (unlikely(num_skb_frags
== MAX_SKB_FRAGS
)) {
981 struct sk_buff
*nskb
= alloc_skb(0, GFP_ATOMIC
);
985 if (curr_skb
== head_skb
)
986 skb_shinfo(curr_skb
)->frag_list
= nskb
;
988 curr_skb
->next
= nskb
;
990 head_skb
->truesize
+= nskb
->truesize
;
993 if (curr_skb
!= head_skb
) {
994 head_skb
->data_len
+= len
;
995 head_skb
->len
+= len
;
996 head_skb
->truesize
+= truesize
;
998 offset
= buf
- page_address(page
);
999 if (skb_can_coalesce(curr_skb
, num_skb_frags
, page
, offset
)) {
1001 skb_coalesce_rx_frag(curr_skb
, num_skb_frags
- 1,
1004 skb_add_rx_frag(curr_skb
, num_skb_frags
, page
,
1005 offset
, len
, truesize
);
1009 ewma_pkt_len_add(&rq
->mrg_avg_pkt_len
, head_skb
->len
);
1017 while (num_buf
-- > 1) {
1018 buf
= virtqueue_get_buf(rq
->vq
, &len
);
1019 if (unlikely(!buf
)) {
1020 pr_debug("%s: rx error: %d buffers missing\n",
1021 dev
->name
, num_buf
);
1022 dev
->stats
.rx_length_errors
++;
1025 stats
->bytes
+= len
;
1026 page
= virt_to_head_page(buf
);
1031 dev_kfree_skb(head_skb
);
1036 static void receive_buf(struct virtnet_info
*vi
, struct receive_queue
*rq
,
1037 void *buf
, unsigned int len
, void **ctx
,
1038 unsigned int *xdp_xmit
,
1039 struct virtnet_rq_stats
*stats
)
1041 struct net_device
*dev
= vi
->dev
;
1042 struct sk_buff
*skb
;
1043 struct virtio_net_hdr_mrg_rxbuf
*hdr
;
1045 if (unlikely(len
< vi
->hdr_len
+ ETH_HLEN
)) {
1046 pr_debug("%s: short packet %i\n", dev
->name
, len
);
1047 dev
->stats
.rx_length_errors
++;
1048 if (vi
->mergeable_rx_bufs
) {
1049 put_page(virt_to_head_page(buf
));
1050 } else if (vi
->big_packets
) {
1051 give_pages(rq
, buf
);
1053 put_page(virt_to_head_page(buf
));
1058 if (vi
->mergeable_rx_bufs
)
1059 skb
= receive_mergeable(dev
, vi
, rq
, buf
, ctx
, len
, xdp_xmit
,
1061 else if (vi
->big_packets
)
1062 skb
= receive_big(dev
, vi
, rq
, buf
, len
, stats
);
1064 skb
= receive_small(dev
, vi
, rq
, buf
, ctx
, len
, xdp_xmit
, stats
);
1069 hdr
= skb_vnet_hdr(skb
);
1071 if (hdr
->hdr
.flags
& VIRTIO_NET_HDR_F_DATA_VALID
)
1072 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1074 if (virtio_net_hdr_to_skb(skb
, &hdr
->hdr
,
1075 virtio_is_little_endian(vi
->vdev
))) {
1076 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n",
1077 dev
->name
, hdr
->hdr
.gso_type
,
1082 skb_record_rx_queue(skb
, vq2rxq(rq
->vq
));
1083 skb
->protocol
= eth_type_trans(skb
, dev
);
1084 pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
1085 ntohs(skb
->protocol
), skb
->len
, skb
->pkt_type
);
1087 napi_gro_receive(&rq
->napi
, skb
);
1091 dev
->stats
.rx_frame_errors
++;
1095 /* Unlike mergeable buffers, all buffers are allocated to the
1096 * same size, except for the headroom. For this reason we do
1097 * not need to use mergeable_len_to_ctx here - it is enough
1098 * to store the headroom as the context ignoring the truesize.
1100 static int add_recvbuf_small(struct virtnet_info
*vi
, struct receive_queue
*rq
,
1103 struct page_frag
*alloc_frag
= &rq
->alloc_frag
;
1105 unsigned int xdp_headroom
= virtnet_get_headroom(vi
);
1106 void *ctx
= (void *)(unsigned long)xdp_headroom
;
1107 int len
= vi
->hdr_len
+ VIRTNET_RX_PAD
+ GOOD_PACKET_LEN
+ xdp_headroom
;
1110 len
= SKB_DATA_ALIGN(len
) +
1111 SKB_DATA_ALIGN(sizeof(struct skb_shared_info
));
1112 if (unlikely(!skb_page_frag_refill(len
, alloc_frag
, gfp
)))
1115 buf
= (char *)page_address(alloc_frag
->page
) + alloc_frag
->offset
;
1116 get_page(alloc_frag
->page
);
1117 alloc_frag
->offset
+= len
;
1118 sg_init_one(rq
->sg
, buf
+ VIRTNET_RX_PAD
+ xdp_headroom
,
1119 vi
->hdr_len
+ GOOD_PACKET_LEN
);
1120 err
= virtqueue_add_inbuf_ctx(rq
->vq
, rq
->sg
, 1, buf
, ctx
, gfp
);
1122 put_page(virt_to_head_page(buf
));
1126 static int add_recvbuf_big(struct virtnet_info
*vi
, struct receive_queue
*rq
,
1129 struct page
*first
, *list
= NULL
;
1133 sg_init_table(rq
->sg
, MAX_SKB_FRAGS
+ 2);
1135 /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */
1136 for (i
= MAX_SKB_FRAGS
+ 1; i
> 1; --i
) {
1137 first
= get_a_page(rq
, gfp
);
1140 give_pages(rq
, list
);
1143 sg_set_buf(&rq
->sg
[i
], page_address(first
), PAGE_SIZE
);
1145 /* chain new page in list head to match sg */
1146 first
->private = (unsigned long)list
;
1150 first
= get_a_page(rq
, gfp
);
1152 give_pages(rq
, list
);
1155 p
= page_address(first
);
1157 /* rq->sg[0], rq->sg[1] share the same page */
1158 /* a separated rq->sg[0] for header - required in case !any_header_sg */
1159 sg_set_buf(&rq
->sg
[0], p
, vi
->hdr_len
);
1161 /* rq->sg[1] for data packet, from offset */
1162 offset
= sizeof(struct padded_vnet_hdr
);
1163 sg_set_buf(&rq
->sg
[1], p
+ offset
, PAGE_SIZE
- offset
);
1165 /* chain first in list head */
1166 first
->private = (unsigned long)list
;
1167 err
= virtqueue_add_inbuf(rq
->vq
, rq
->sg
, MAX_SKB_FRAGS
+ 2,
1170 give_pages(rq
, first
);
1175 static unsigned int get_mergeable_buf_len(struct receive_queue
*rq
,
1176 struct ewma_pkt_len
*avg_pkt_len
,
1179 const size_t hdr_len
= sizeof(struct virtio_net_hdr_mrg_rxbuf
);
1183 return PAGE_SIZE
- room
;
1185 len
= hdr_len
+ clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len
),
1186 rq
->min_buf_len
, PAGE_SIZE
- hdr_len
);
1188 return ALIGN(len
, L1_CACHE_BYTES
);
1191 static int add_recvbuf_mergeable(struct virtnet_info
*vi
,
1192 struct receive_queue
*rq
, gfp_t gfp
)
1194 struct page_frag
*alloc_frag
= &rq
->alloc_frag
;
1195 unsigned int headroom
= virtnet_get_headroom(vi
);
1196 unsigned int tailroom
= headroom
? sizeof(struct skb_shared_info
) : 0;
1197 unsigned int room
= SKB_DATA_ALIGN(headroom
+ tailroom
);
1201 unsigned int len
, hole
;
1203 /* Extra tailroom is needed to satisfy XDP's assumption. This
1204 * means rx frags coalescing won't work, but consider we've
1205 * disabled GSO for XDP, it won't be a big issue.
1207 len
= get_mergeable_buf_len(rq
, &rq
->mrg_avg_pkt_len
, room
);
1208 if (unlikely(!skb_page_frag_refill(len
+ room
, alloc_frag
, gfp
)))
1211 buf
= (char *)page_address(alloc_frag
->page
) + alloc_frag
->offset
;
1212 buf
+= headroom
; /* advance address leaving hole at front of pkt */
1213 get_page(alloc_frag
->page
);
1214 alloc_frag
->offset
+= len
+ room
;
1215 hole
= alloc_frag
->size
- alloc_frag
->offset
;
1216 if (hole
< len
+ room
) {
1217 /* To avoid internal fragmentation, if there is very likely not
1218 * enough space for another buffer, add the remaining space to
1219 * the current buffer.
1222 alloc_frag
->offset
+= hole
;
1225 sg_init_one(rq
->sg
, buf
, len
);
1226 ctx
= mergeable_len_to_ctx(len
, headroom
);
1227 err
= virtqueue_add_inbuf_ctx(rq
->vq
, rq
->sg
, 1, buf
, ctx
, gfp
);
1229 put_page(virt_to_head_page(buf
));
1235 * Returns false if we couldn't fill entirely (OOM).
1237 * Normally run in the receive path, but can also be run from ndo_open
1238 * before we're receiving packets, or from refill_work which is
1239 * careful to disable receiving (using napi_disable).
1241 static bool try_fill_recv(struct virtnet_info
*vi
, struct receive_queue
*rq
,
1248 if (vi
->mergeable_rx_bufs
)
1249 err
= add_recvbuf_mergeable(vi
, rq
, gfp
);
1250 else if (vi
->big_packets
)
1251 err
= add_recvbuf_big(vi
, rq
, gfp
);
1253 err
= add_recvbuf_small(vi
, rq
, gfp
);
1255 oom
= err
== -ENOMEM
;
1258 } while (rq
->vq
->num_free
);
1259 if (virtqueue_kick_prepare(rq
->vq
) && virtqueue_notify(rq
->vq
)) {
1260 unsigned long flags
;
1262 flags
= u64_stats_update_begin_irqsave(&rq
->stats
.syncp
);
1264 u64_stats_update_end_irqrestore(&rq
->stats
.syncp
, flags
);
1270 static void skb_recv_done(struct virtqueue
*rvq
)
1272 struct virtnet_info
*vi
= rvq
->vdev
->priv
;
1273 struct receive_queue
*rq
= &vi
->rq
[vq2rxq(rvq
)];
1275 virtqueue_napi_schedule(&rq
->napi
, rvq
);
1278 static void virtnet_napi_enable(struct virtqueue
*vq
, struct napi_struct
*napi
)
1282 /* If all buffers were filled by other side before we napi_enabled, we
1283 * won't get another interrupt, so process any outstanding packets now.
1284 * Call local_bh_enable after to trigger softIRQ processing.
1287 virtqueue_napi_schedule(napi
, vq
);
1291 static void virtnet_napi_tx_enable(struct virtnet_info
*vi
,
1292 struct virtqueue
*vq
,
1293 struct napi_struct
*napi
)
1298 /* Tx napi touches cachelines on the cpu handling tx interrupts. Only
1299 * enable the feature if this is likely affine with the transmit path.
1301 if (!vi
->affinity_hint_set
) {
1306 return virtnet_napi_enable(vq
, napi
);
1309 static void virtnet_napi_tx_disable(struct napi_struct
*napi
)
1315 static void refill_work(struct work_struct
*work
)
1317 struct virtnet_info
*vi
=
1318 container_of(work
, struct virtnet_info
, refill
.work
);
1322 for (i
= 0; i
< vi
->curr_queue_pairs
; i
++) {
1323 struct receive_queue
*rq
= &vi
->rq
[i
];
1325 napi_disable(&rq
->napi
);
1326 still_empty
= !try_fill_recv(vi
, rq
, GFP_KERNEL
);
1327 virtnet_napi_enable(rq
->vq
, &rq
->napi
);
1329 /* In theory, this can happen: if we don't get any buffers in
1330 * we will *never* try to fill again.
1333 schedule_delayed_work(&vi
->refill
, HZ
/2);
1337 static int virtnet_receive(struct receive_queue
*rq
, int budget
,
1338 unsigned int *xdp_xmit
)
1340 struct virtnet_info
*vi
= rq
->vq
->vdev
->priv
;
1341 struct virtnet_rq_stats stats
= {};
1346 if (!vi
->big_packets
|| vi
->mergeable_rx_bufs
) {
1349 while (stats
.packets
< budget
&&
1350 (buf
= virtqueue_get_buf_ctx(rq
->vq
, &len
, &ctx
))) {
1351 receive_buf(vi
, rq
, buf
, len
, ctx
, xdp_xmit
, &stats
);
1355 while (stats
.packets
< budget
&&
1356 (buf
= virtqueue_get_buf(rq
->vq
, &len
)) != NULL
) {
1357 receive_buf(vi
, rq
, buf
, len
, NULL
, xdp_xmit
, &stats
);
1362 if (rq
->vq
->num_free
> min((unsigned int)budget
, virtqueue_get_vring_size(rq
->vq
)) / 2) {
1363 if (!try_fill_recv(vi
, rq
, GFP_ATOMIC
))
1364 schedule_delayed_work(&vi
->refill
, 0);
1367 u64_stats_update_begin(&rq
->stats
.syncp
);
1368 for (i
= 0; i
< VIRTNET_RQ_STATS_LEN
; i
++) {
1369 size_t offset
= virtnet_rq_stats_desc
[i
].offset
;
1372 item
= (u64
*)((u8
*)&rq
->stats
+ offset
);
1373 *item
+= *(u64
*)((u8
*)&stats
+ offset
);
1375 u64_stats_update_end(&rq
->stats
.syncp
);
1377 return stats
.packets
;
1380 static void free_old_xmit_skbs(struct send_queue
*sq
, bool in_napi
)
1383 unsigned int packets
= 0;
1384 unsigned int bytes
= 0;
1387 while ((ptr
= virtqueue_get_buf(sq
->vq
, &len
)) != NULL
) {
1388 if (likely(!is_xdp_frame(ptr
))) {
1389 struct sk_buff
*skb
= ptr
;
1391 pr_debug("Sent skb %p\n", skb
);
1394 napi_consume_skb(skb
, in_napi
);
1396 struct xdp_frame
*frame
= ptr_to_xdp(ptr
);
1398 bytes
+= frame
->len
;
1399 xdp_return_frame(frame
);
1404 /* Avoid overhead when no packets have been processed
1405 * happens when called speculatively from start_xmit.
1410 u64_stats_update_begin(&sq
->stats
.syncp
);
1411 sq
->stats
.bytes
+= bytes
;
1412 sq
->stats
.packets
+= packets
;
1413 u64_stats_update_end(&sq
->stats
.syncp
);
1416 static bool is_xdp_raw_buffer_queue(struct virtnet_info
*vi
, int q
)
1418 if (q
< (vi
->curr_queue_pairs
- vi
->xdp_queue_pairs
))
1420 else if (q
< vi
->curr_queue_pairs
)
1426 static void virtnet_poll_cleantx(struct receive_queue
*rq
)
1428 struct virtnet_info
*vi
= rq
->vq
->vdev
->priv
;
1429 unsigned int index
= vq2rxq(rq
->vq
);
1430 struct send_queue
*sq
= &vi
->sq
[index
];
1431 struct netdev_queue
*txq
= netdev_get_tx_queue(vi
->dev
, index
);
1433 if (!sq
->napi
.weight
|| is_xdp_raw_buffer_queue(vi
, index
))
1436 if (__netif_tx_trylock(txq
)) {
1437 free_old_xmit_skbs(sq
, true);
1438 __netif_tx_unlock(txq
);
1441 if (sq
->vq
->num_free
>= 2 + MAX_SKB_FRAGS
)
1442 netif_tx_wake_queue(txq
);
1445 static int virtnet_poll(struct napi_struct
*napi
, int budget
)
1447 struct receive_queue
*rq
=
1448 container_of(napi
, struct receive_queue
, napi
);
1449 struct virtnet_info
*vi
= rq
->vq
->vdev
->priv
;
1450 struct send_queue
*sq
;
1451 unsigned int received
;
1452 unsigned int xdp_xmit
= 0;
1454 virtnet_poll_cleantx(rq
);
1456 received
= virtnet_receive(rq
, budget
, &xdp_xmit
);
1458 /* Out of packets? */
1459 if (received
< budget
)
1460 virtqueue_napi_complete(napi
, rq
->vq
, received
);
1462 if (xdp_xmit
& VIRTIO_XDP_REDIR
)
1465 if (xdp_xmit
& VIRTIO_XDP_TX
) {
1466 sq
= virtnet_xdp_sq(vi
);
1467 if (virtqueue_kick_prepare(sq
->vq
) && virtqueue_notify(sq
->vq
)) {
1468 u64_stats_update_begin(&sq
->stats
.syncp
);
1470 u64_stats_update_end(&sq
->stats
.syncp
);
1477 static int virtnet_open(struct net_device
*dev
)
1479 struct virtnet_info
*vi
= netdev_priv(dev
);
1482 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
1483 if (i
< vi
->curr_queue_pairs
)
1484 /* Make sure we have some buffers: if oom use wq. */
1485 if (!try_fill_recv(vi
, &vi
->rq
[i
], GFP_KERNEL
))
1486 schedule_delayed_work(&vi
->refill
, 0);
1488 err
= xdp_rxq_info_reg(&vi
->rq
[i
].xdp_rxq
, dev
, i
, vi
->rq
[i
].napi
.napi_id
);
1492 err
= xdp_rxq_info_reg_mem_model(&vi
->rq
[i
].xdp_rxq
,
1493 MEM_TYPE_PAGE_SHARED
, NULL
);
1495 xdp_rxq_info_unreg(&vi
->rq
[i
].xdp_rxq
);
1499 virtnet_napi_enable(vi
->rq
[i
].vq
, &vi
->rq
[i
].napi
);
1500 virtnet_napi_tx_enable(vi
, vi
->sq
[i
].vq
, &vi
->sq
[i
].napi
);
1506 static int virtnet_poll_tx(struct napi_struct
*napi
, int budget
)
1508 struct send_queue
*sq
= container_of(napi
, struct send_queue
, napi
);
1509 struct virtnet_info
*vi
= sq
->vq
->vdev
->priv
;
1510 unsigned int index
= vq2txq(sq
->vq
);
1511 struct netdev_queue
*txq
;
1513 if (unlikely(is_xdp_raw_buffer_queue(vi
, index
))) {
1514 /* We don't need to enable cb for XDP */
1515 napi_complete_done(napi
, 0);
1519 txq
= netdev_get_tx_queue(vi
->dev
, index
);
1520 __netif_tx_lock(txq
, raw_smp_processor_id());
1521 free_old_xmit_skbs(sq
, true);
1522 __netif_tx_unlock(txq
);
1524 virtqueue_napi_complete(napi
, sq
->vq
, 0);
1526 if (sq
->vq
->num_free
>= 2 + MAX_SKB_FRAGS
)
1527 netif_tx_wake_queue(txq
);
1532 static int xmit_skb(struct send_queue
*sq
, struct sk_buff
*skb
)
1534 struct virtio_net_hdr_mrg_rxbuf
*hdr
;
1535 const unsigned char *dest
= ((struct ethhdr
*)skb
->data
)->h_dest
;
1536 struct virtnet_info
*vi
= sq
->vq
->vdev
->priv
;
1538 unsigned hdr_len
= vi
->hdr_len
;
1541 pr_debug("%s: xmit %p %pM\n", vi
->dev
->name
, skb
, dest
);
1543 can_push
= vi
->any_header_sg
&&
1544 !((unsigned long)skb
->data
& (__alignof__(*hdr
) - 1)) &&
1545 !skb_header_cloned(skb
) && skb_headroom(skb
) >= hdr_len
;
1546 /* Even if we can, don't push here yet as this would skew
1547 * csum_start offset below. */
1549 hdr
= (struct virtio_net_hdr_mrg_rxbuf
*)(skb
->data
- hdr_len
);
1551 hdr
= skb_vnet_hdr(skb
);
1553 if (virtio_net_hdr_from_skb(skb
, &hdr
->hdr
,
1554 virtio_is_little_endian(vi
->vdev
), false,
1558 if (vi
->mergeable_rx_bufs
)
1559 hdr
->num_buffers
= 0;
1561 sg_init_table(sq
->sg
, skb_shinfo(skb
)->nr_frags
+ (can_push
? 1 : 2));
1563 __skb_push(skb
, hdr_len
);
1564 num_sg
= skb_to_sgvec(skb
, sq
->sg
, 0, skb
->len
);
1565 if (unlikely(num_sg
< 0))
1567 /* Pull header back to avoid skew in tx bytes calculations. */
1568 __skb_pull(skb
, hdr_len
);
1570 sg_set_buf(sq
->sg
, hdr
, hdr_len
);
1571 num_sg
= skb_to_sgvec(skb
, sq
->sg
+ 1, 0, skb
->len
);
1572 if (unlikely(num_sg
< 0))
1576 return virtqueue_add_outbuf(sq
->vq
, sq
->sg
, num_sg
, skb
, GFP_ATOMIC
);
1579 static netdev_tx_t
start_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
1581 struct virtnet_info
*vi
= netdev_priv(dev
);
1582 int qnum
= skb_get_queue_mapping(skb
);
1583 struct send_queue
*sq
= &vi
->sq
[qnum
];
1585 struct netdev_queue
*txq
= netdev_get_tx_queue(dev
, qnum
);
1586 bool kick
= !netdev_xmit_more();
1587 bool use_napi
= sq
->napi
.weight
;
1589 /* Free up any pending old buffers before queueing new ones. */
1590 free_old_xmit_skbs(sq
, false);
1592 if (use_napi
&& kick
)
1593 virtqueue_enable_cb_delayed(sq
->vq
);
1595 /* timestamp packet in software */
1596 skb_tx_timestamp(skb
);
1598 /* Try to transmit */
1599 err
= xmit_skb(sq
, skb
);
1601 /* This should not happen! */
1602 if (unlikely(err
)) {
1603 dev
->stats
.tx_fifo_errors
++;
1604 if (net_ratelimit())
1606 "Unexpected TXQ (%d) queue failure: %d\n",
1608 dev
->stats
.tx_dropped
++;
1609 dev_kfree_skb_any(skb
);
1610 return NETDEV_TX_OK
;
1613 /* Don't wait up for transmitted skbs to be freed. */
1619 /* If running out of space, stop queue to avoid getting packets that we
1620 * are then unable to transmit.
1621 * An alternative would be to force queuing layer to requeue the skb by
1622 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be
1623 * returned in a normal path of operation: it means that driver is not
1624 * maintaining the TX queue stop/start state properly, and causes
1625 * the stack to do a non-trivial amount of useless work.
1626 * Since most packets only take 1 or 2 ring slots, stopping the queue
1627 * early means 16 slots are typically wasted.
1629 if (sq
->vq
->num_free
< 2+MAX_SKB_FRAGS
) {
1630 netif_stop_subqueue(dev
, qnum
);
1632 unlikely(!virtqueue_enable_cb_delayed(sq
->vq
))) {
1633 /* More just got used, free them then recheck. */
1634 free_old_xmit_skbs(sq
, false);
1635 if (sq
->vq
->num_free
>= 2+MAX_SKB_FRAGS
) {
1636 netif_start_subqueue(dev
, qnum
);
1637 virtqueue_disable_cb(sq
->vq
);
1642 if (kick
|| netif_xmit_stopped(txq
)) {
1643 if (virtqueue_kick_prepare(sq
->vq
) && virtqueue_notify(sq
->vq
)) {
1644 u64_stats_update_begin(&sq
->stats
.syncp
);
1646 u64_stats_update_end(&sq
->stats
.syncp
);
1650 return NETDEV_TX_OK
;
1654 * Send command via the control virtqueue and check status. Commands
1655 * supported by the hypervisor, as indicated by feature bits, should
1656 * never fail unless improperly formatted.
1658 static bool virtnet_send_command(struct virtnet_info
*vi
, u8
class, u8 cmd
,
1659 struct scatterlist
*out
)
1661 struct scatterlist
*sgs
[4], hdr
, stat
;
1662 unsigned out_num
= 0, tmp
;
1664 /* Caller should know better */
1665 BUG_ON(!virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_CTRL_VQ
));
1667 vi
->ctrl
->status
= ~0;
1668 vi
->ctrl
->hdr
.class = class;
1669 vi
->ctrl
->hdr
.cmd
= cmd
;
1671 sg_init_one(&hdr
, &vi
->ctrl
->hdr
, sizeof(vi
->ctrl
->hdr
));
1672 sgs
[out_num
++] = &hdr
;
1675 sgs
[out_num
++] = out
;
1677 /* Add return status. */
1678 sg_init_one(&stat
, &vi
->ctrl
->status
, sizeof(vi
->ctrl
->status
));
1679 sgs
[out_num
] = &stat
;
1681 BUG_ON(out_num
+ 1 > ARRAY_SIZE(sgs
));
1682 virtqueue_add_sgs(vi
->cvq
, sgs
, out_num
, 1, vi
, GFP_ATOMIC
);
1684 if (unlikely(!virtqueue_kick(vi
->cvq
)))
1685 return vi
->ctrl
->status
== VIRTIO_NET_OK
;
1687 /* Spin for a response, the kick causes an ioport write, trapping
1688 * into the hypervisor, so the request should be handled immediately.
1690 while (!virtqueue_get_buf(vi
->cvq
, &tmp
) &&
1691 !virtqueue_is_broken(vi
->cvq
))
1694 return vi
->ctrl
->status
== VIRTIO_NET_OK
;
1697 static int virtnet_set_mac_address(struct net_device
*dev
, void *p
)
1699 struct virtnet_info
*vi
= netdev_priv(dev
);
1700 struct virtio_device
*vdev
= vi
->vdev
;
1702 struct sockaddr
*addr
;
1703 struct scatterlist sg
;
1705 if (virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_STANDBY
))
1708 addr
= kmemdup(p
, sizeof(*addr
), GFP_KERNEL
);
1712 ret
= eth_prepare_mac_addr_change(dev
, addr
);
1716 if (virtio_has_feature(vdev
, VIRTIO_NET_F_CTRL_MAC_ADDR
)) {
1717 sg_init_one(&sg
, addr
->sa_data
, dev
->addr_len
);
1718 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_MAC
,
1719 VIRTIO_NET_CTRL_MAC_ADDR_SET
, &sg
)) {
1720 dev_warn(&vdev
->dev
,
1721 "Failed to set mac address by vq command.\n");
1725 } else if (virtio_has_feature(vdev
, VIRTIO_NET_F_MAC
) &&
1726 !virtio_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1729 /* Naturally, this has an atomicity problem. */
1730 for (i
= 0; i
< dev
->addr_len
; i
++)
1731 virtio_cwrite8(vdev
,
1732 offsetof(struct virtio_net_config
, mac
) +
1733 i
, addr
->sa_data
[i
]);
1736 eth_commit_mac_addr_change(dev
, p
);
1744 static void virtnet_stats(struct net_device
*dev
,
1745 struct rtnl_link_stats64
*tot
)
1747 struct virtnet_info
*vi
= netdev_priv(dev
);
1751 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
1752 u64 tpackets
, tbytes
, rpackets
, rbytes
, rdrops
;
1753 struct receive_queue
*rq
= &vi
->rq
[i
];
1754 struct send_queue
*sq
= &vi
->sq
[i
];
1757 start
= u64_stats_fetch_begin_irq(&sq
->stats
.syncp
);
1758 tpackets
= sq
->stats
.packets
;
1759 tbytes
= sq
->stats
.bytes
;
1760 } while (u64_stats_fetch_retry_irq(&sq
->stats
.syncp
, start
));
1763 start
= u64_stats_fetch_begin_irq(&rq
->stats
.syncp
);
1764 rpackets
= rq
->stats
.packets
;
1765 rbytes
= rq
->stats
.bytes
;
1766 rdrops
= rq
->stats
.drops
;
1767 } while (u64_stats_fetch_retry_irq(&rq
->stats
.syncp
, start
));
1769 tot
->rx_packets
+= rpackets
;
1770 tot
->tx_packets
+= tpackets
;
1771 tot
->rx_bytes
+= rbytes
;
1772 tot
->tx_bytes
+= tbytes
;
1773 tot
->rx_dropped
+= rdrops
;
1776 tot
->tx_dropped
= dev
->stats
.tx_dropped
;
1777 tot
->tx_fifo_errors
= dev
->stats
.tx_fifo_errors
;
1778 tot
->rx_length_errors
= dev
->stats
.rx_length_errors
;
1779 tot
->rx_frame_errors
= dev
->stats
.rx_frame_errors
;
1782 static void virtnet_ack_link_announce(struct virtnet_info
*vi
)
1785 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_ANNOUNCE
,
1786 VIRTIO_NET_CTRL_ANNOUNCE_ACK
, NULL
))
1787 dev_warn(&vi
->dev
->dev
, "Failed to ack link announce.\n");
1791 static int _virtnet_set_queues(struct virtnet_info
*vi
, u16 queue_pairs
)
1793 struct scatterlist sg
;
1794 struct net_device
*dev
= vi
->dev
;
1796 if (!vi
->has_cvq
|| !virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_MQ
))
1799 vi
->ctrl
->mq
.virtqueue_pairs
= cpu_to_virtio16(vi
->vdev
, queue_pairs
);
1800 sg_init_one(&sg
, &vi
->ctrl
->mq
, sizeof(vi
->ctrl
->mq
));
1802 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_MQ
,
1803 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
, &sg
)) {
1804 dev_warn(&dev
->dev
, "Fail to set num of queue pairs to %d\n",
1808 vi
->curr_queue_pairs
= queue_pairs
;
1809 /* virtnet_open() will refill when device is going to up. */
1810 if (dev
->flags
& IFF_UP
)
1811 schedule_delayed_work(&vi
->refill
, 0);
1817 static int virtnet_set_queues(struct virtnet_info
*vi
, u16 queue_pairs
)
1822 err
= _virtnet_set_queues(vi
, queue_pairs
);
1827 static int virtnet_close(struct net_device
*dev
)
1829 struct virtnet_info
*vi
= netdev_priv(dev
);
1832 /* Make sure refill_work doesn't re-enable napi! */
1833 cancel_delayed_work_sync(&vi
->refill
);
1835 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
1836 xdp_rxq_info_unreg(&vi
->rq
[i
].xdp_rxq
);
1837 napi_disable(&vi
->rq
[i
].napi
);
1838 virtnet_napi_tx_disable(&vi
->sq
[i
].napi
);
1844 static void virtnet_set_rx_mode(struct net_device
*dev
)
1846 struct virtnet_info
*vi
= netdev_priv(dev
);
1847 struct scatterlist sg
[2];
1848 struct virtio_net_ctrl_mac
*mac_data
;
1849 struct netdev_hw_addr
*ha
;
1855 /* We can't dynamically set ndo_set_rx_mode, so return gracefully */
1856 if (!virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_CTRL_RX
))
1859 vi
->ctrl
->promisc
= ((dev
->flags
& IFF_PROMISC
) != 0);
1860 vi
->ctrl
->allmulti
= ((dev
->flags
& IFF_ALLMULTI
) != 0);
1862 sg_init_one(sg
, &vi
->ctrl
->promisc
, sizeof(vi
->ctrl
->promisc
));
1864 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_RX
,
1865 VIRTIO_NET_CTRL_RX_PROMISC
, sg
))
1866 dev_warn(&dev
->dev
, "Failed to %sable promisc mode.\n",
1867 vi
->ctrl
->promisc
? "en" : "dis");
1869 sg_init_one(sg
, &vi
->ctrl
->allmulti
, sizeof(vi
->ctrl
->allmulti
));
1871 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_RX
,
1872 VIRTIO_NET_CTRL_RX_ALLMULTI
, sg
))
1873 dev_warn(&dev
->dev
, "Failed to %sable allmulti mode.\n",
1874 vi
->ctrl
->allmulti
? "en" : "dis");
1876 uc_count
= netdev_uc_count(dev
);
1877 mc_count
= netdev_mc_count(dev
);
1878 /* MAC filter - use one buffer for both lists */
1879 buf
= kzalloc(((uc_count
+ mc_count
) * ETH_ALEN
) +
1880 (2 * sizeof(mac_data
->entries
)), GFP_ATOMIC
);
1885 sg_init_table(sg
, 2);
1887 /* Store the unicast list and count in the front of the buffer */
1888 mac_data
->entries
= cpu_to_virtio32(vi
->vdev
, uc_count
);
1890 netdev_for_each_uc_addr(ha
, dev
)
1891 memcpy(&mac_data
->macs
[i
++][0], ha
->addr
, ETH_ALEN
);
1893 sg_set_buf(&sg
[0], mac_data
,
1894 sizeof(mac_data
->entries
) + (uc_count
* ETH_ALEN
));
1896 /* multicast list and count fill the end */
1897 mac_data
= (void *)&mac_data
->macs
[uc_count
][0];
1899 mac_data
->entries
= cpu_to_virtio32(vi
->vdev
, mc_count
);
1901 netdev_for_each_mc_addr(ha
, dev
)
1902 memcpy(&mac_data
->macs
[i
++][0], ha
->addr
, ETH_ALEN
);
1904 sg_set_buf(&sg
[1], mac_data
,
1905 sizeof(mac_data
->entries
) + (mc_count
* ETH_ALEN
));
1907 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_MAC
,
1908 VIRTIO_NET_CTRL_MAC_TABLE_SET
, sg
))
1909 dev_warn(&dev
->dev
, "Failed to set MAC filter table.\n");
1914 static int virtnet_vlan_rx_add_vid(struct net_device
*dev
,
1915 __be16 proto
, u16 vid
)
1917 struct virtnet_info
*vi
= netdev_priv(dev
);
1918 struct scatterlist sg
;
1920 vi
->ctrl
->vid
= cpu_to_virtio16(vi
->vdev
, vid
);
1921 sg_init_one(&sg
, &vi
->ctrl
->vid
, sizeof(vi
->ctrl
->vid
));
1923 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_VLAN
,
1924 VIRTIO_NET_CTRL_VLAN_ADD
, &sg
))
1925 dev_warn(&dev
->dev
, "Failed to add VLAN ID %d.\n", vid
);
1929 static int virtnet_vlan_rx_kill_vid(struct net_device
*dev
,
1930 __be16 proto
, u16 vid
)
1932 struct virtnet_info
*vi
= netdev_priv(dev
);
1933 struct scatterlist sg
;
1935 vi
->ctrl
->vid
= cpu_to_virtio16(vi
->vdev
, vid
);
1936 sg_init_one(&sg
, &vi
->ctrl
->vid
, sizeof(vi
->ctrl
->vid
));
1938 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_VLAN
,
1939 VIRTIO_NET_CTRL_VLAN_DEL
, &sg
))
1940 dev_warn(&dev
->dev
, "Failed to kill VLAN ID %d.\n", vid
);
1944 static void virtnet_clean_affinity(struct virtnet_info
*vi
)
1948 if (vi
->affinity_hint_set
) {
1949 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
1950 virtqueue_set_affinity(vi
->rq
[i
].vq
, NULL
);
1951 virtqueue_set_affinity(vi
->sq
[i
].vq
, NULL
);
1954 vi
->affinity_hint_set
= false;
1958 static void virtnet_set_affinity(struct virtnet_info
*vi
)
1967 if (!zalloc_cpumask_var(&mask
, GFP_KERNEL
)) {
1968 virtnet_clean_affinity(vi
);
1972 num_cpu
= num_online_cpus();
1973 stride
= max_t(int, num_cpu
/ vi
->curr_queue_pairs
, 1);
1974 stragglers
= num_cpu
>= vi
->curr_queue_pairs
?
1975 num_cpu
% vi
->curr_queue_pairs
:
1977 cpu
= cpumask_next(-1, cpu_online_mask
);
1979 for (i
= 0; i
< vi
->curr_queue_pairs
; i
++) {
1980 group_size
= stride
+ (i
< stragglers
? 1 : 0);
1982 for (j
= 0; j
< group_size
; j
++) {
1983 cpumask_set_cpu(cpu
, mask
);
1984 cpu
= cpumask_next_wrap(cpu
, cpu_online_mask
,
1987 virtqueue_set_affinity(vi
->rq
[i
].vq
, mask
);
1988 virtqueue_set_affinity(vi
->sq
[i
].vq
, mask
);
1989 __netif_set_xps_queue(vi
->dev
, cpumask_bits(mask
), i
, false);
1990 cpumask_clear(mask
);
1993 vi
->affinity_hint_set
= true;
1994 free_cpumask_var(mask
);
1997 static int virtnet_cpu_online(unsigned int cpu
, struct hlist_node
*node
)
1999 struct virtnet_info
*vi
= hlist_entry_safe(node
, struct virtnet_info
,
2001 virtnet_set_affinity(vi
);
2005 static int virtnet_cpu_dead(unsigned int cpu
, struct hlist_node
*node
)
2007 struct virtnet_info
*vi
= hlist_entry_safe(node
, struct virtnet_info
,
2009 virtnet_set_affinity(vi
);
2013 static int virtnet_cpu_down_prep(unsigned int cpu
, struct hlist_node
*node
)
2015 struct virtnet_info
*vi
= hlist_entry_safe(node
, struct virtnet_info
,
2018 virtnet_clean_affinity(vi
);
2022 static enum cpuhp_state virtionet_online
;
2024 static int virtnet_cpu_notif_add(struct virtnet_info
*vi
)
2028 ret
= cpuhp_state_add_instance_nocalls(virtionet_online
, &vi
->node
);
2031 ret
= cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD
,
2035 cpuhp_state_remove_instance_nocalls(virtionet_online
, &vi
->node
);
2039 static void virtnet_cpu_notif_remove(struct virtnet_info
*vi
)
2041 cpuhp_state_remove_instance_nocalls(virtionet_online
, &vi
->node
);
2042 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD
,
2046 static void virtnet_get_ringparam(struct net_device
*dev
,
2047 struct ethtool_ringparam
*ring
)
2049 struct virtnet_info
*vi
= netdev_priv(dev
);
2051 ring
->rx_max_pending
= virtqueue_get_vring_size(vi
->rq
[0].vq
);
2052 ring
->tx_max_pending
= virtqueue_get_vring_size(vi
->sq
[0].vq
);
2053 ring
->rx_pending
= ring
->rx_max_pending
;
2054 ring
->tx_pending
= ring
->tx_max_pending
;
2058 static void virtnet_get_drvinfo(struct net_device
*dev
,
2059 struct ethtool_drvinfo
*info
)
2061 struct virtnet_info
*vi
= netdev_priv(dev
);
2062 struct virtio_device
*vdev
= vi
->vdev
;
2064 strlcpy(info
->driver
, KBUILD_MODNAME
, sizeof(info
->driver
));
2065 strlcpy(info
->version
, VIRTNET_DRIVER_VERSION
, sizeof(info
->version
));
2066 strlcpy(info
->bus_info
, virtio_bus_name(vdev
), sizeof(info
->bus_info
));
2070 /* TODO: Eliminate OOO packets during switching */
2071 static int virtnet_set_channels(struct net_device
*dev
,
2072 struct ethtool_channels
*channels
)
2074 struct virtnet_info
*vi
= netdev_priv(dev
);
2075 u16 queue_pairs
= channels
->combined_count
;
2078 /* We don't support separate rx/tx channels.
2079 * We don't allow setting 'other' channels.
2081 if (channels
->rx_count
|| channels
->tx_count
|| channels
->other_count
)
2084 if (queue_pairs
> vi
->max_queue_pairs
|| queue_pairs
== 0)
2087 /* For now we don't support modifying channels while XDP is loaded
2088 * also when XDP is loaded all RX queues have XDP programs so we only
2089 * need to check a single RX queue.
2091 if (vi
->rq
[0].xdp_prog
)
2095 err
= _virtnet_set_queues(vi
, queue_pairs
);
2100 virtnet_set_affinity(vi
);
2103 netif_set_real_num_tx_queues(dev
, queue_pairs
);
2104 netif_set_real_num_rx_queues(dev
, queue_pairs
);
2109 static void virtnet_get_strings(struct net_device
*dev
, u32 stringset
, u8
*data
)
2111 struct virtnet_info
*vi
= netdev_priv(dev
);
2112 char *p
= (char *)data
;
2115 switch (stringset
) {
2117 for (i
= 0; i
< vi
->curr_queue_pairs
; i
++) {
2118 for (j
= 0; j
< VIRTNET_RQ_STATS_LEN
; j
++) {
2119 snprintf(p
, ETH_GSTRING_LEN
, "rx_queue_%u_%s",
2120 i
, virtnet_rq_stats_desc
[j
].desc
);
2121 p
+= ETH_GSTRING_LEN
;
2125 for (i
= 0; i
< vi
->curr_queue_pairs
; i
++) {
2126 for (j
= 0; j
< VIRTNET_SQ_STATS_LEN
; j
++) {
2127 snprintf(p
, ETH_GSTRING_LEN
, "tx_queue_%u_%s",
2128 i
, virtnet_sq_stats_desc
[j
].desc
);
2129 p
+= ETH_GSTRING_LEN
;
2136 static int virtnet_get_sset_count(struct net_device
*dev
, int sset
)
2138 struct virtnet_info
*vi
= netdev_priv(dev
);
2142 return vi
->curr_queue_pairs
* (VIRTNET_RQ_STATS_LEN
+
2143 VIRTNET_SQ_STATS_LEN
);
2149 static void virtnet_get_ethtool_stats(struct net_device
*dev
,
2150 struct ethtool_stats
*stats
, u64
*data
)
2152 struct virtnet_info
*vi
= netdev_priv(dev
);
2153 unsigned int idx
= 0, start
, i
, j
;
2154 const u8
*stats_base
;
2157 for (i
= 0; i
< vi
->curr_queue_pairs
; i
++) {
2158 struct receive_queue
*rq
= &vi
->rq
[i
];
2160 stats_base
= (u8
*)&rq
->stats
;
2162 start
= u64_stats_fetch_begin_irq(&rq
->stats
.syncp
);
2163 for (j
= 0; j
< VIRTNET_RQ_STATS_LEN
; j
++) {
2164 offset
= virtnet_rq_stats_desc
[j
].offset
;
2165 data
[idx
+ j
] = *(u64
*)(stats_base
+ offset
);
2167 } while (u64_stats_fetch_retry_irq(&rq
->stats
.syncp
, start
));
2168 idx
+= VIRTNET_RQ_STATS_LEN
;
2171 for (i
= 0; i
< vi
->curr_queue_pairs
; i
++) {
2172 struct send_queue
*sq
= &vi
->sq
[i
];
2174 stats_base
= (u8
*)&sq
->stats
;
2176 start
= u64_stats_fetch_begin_irq(&sq
->stats
.syncp
);
2177 for (j
= 0; j
< VIRTNET_SQ_STATS_LEN
; j
++) {
2178 offset
= virtnet_sq_stats_desc
[j
].offset
;
2179 data
[idx
+ j
] = *(u64
*)(stats_base
+ offset
);
2181 } while (u64_stats_fetch_retry_irq(&sq
->stats
.syncp
, start
));
2182 idx
+= VIRTNET_SQ_STATS_LEN
;
2186 static void virtnet_get_channels(struct net_device
*dev
,
2187 struct ethtool_channels
*channels
)
2189 struct virtnet_info
*vi
= netdev_priv(dev
);
2191 channels
->combined_count
= vi
->curr_queue_pairs
;
2192 channels
->max_combined
= vi
->max_queue_pairs
;
2193 channels
->max_other
= 0;
2194 channels
->rx_count
= 0;
2195 channels
->tx_count
= 0;
2196 channels
->other_count
= 0;
2199 static int virtnet_set_link_ksettings(struct net_device
*dev
,
2200 const struct ethtool_link_ksettings
*cmd
)
2202 struct virtnet_info
*vi
= netdev_priv(dev
);
2204 return ethtool_virtdev_set_link_ksettings(dev
, cmd
,
2205 &vi
->speed
, &vi
->duplex
);
2208 static int virtnet_get_link_ksettings(struct net_device
*dev
,
2209 struct ethtool_link_ksettings
*cmd
)
2211 struct virtnet_info
*vi
= netdev_priv(dev
);
2213 cmd
->base
.speed
= vi
->speed
;
2214 cmd
->base
.duplex
= vi
->duplex
;
2215 cmd
->base
.port
= PORT_OTHER
;
2220 static int virtnet_set_coalesce(struct net_device
*dev
,
2221 struct ethtool_coalesce
*ec
)
2223 struct virtnet_info
*vi
= netdev_priv(dev
);
2226 if (ec
->tx_max_coalesced_frames
> 1 ||
2227 ec
->rx_max_coalesced_frames
!= 1)
2230 napi_weight
= ec
->tx_max_coalesced_frames
? NAPI_POLL_WEIGHT
: 0;
2231 if (napi_weight
^ vi
->sq
[0].napi
.weight
) {
2232 if (dev
->flags
& IFF_UP
)
2234 for (i
= 0; i
< vi
->max_queue_pairs
; i
++)
2235 vi
->sq
[i
].napi
.weight
= napi_weight
;
2241 static int virtnet_get_coalesce(struct net_device
*dev
,
2242 struct ethtool_coalesce
*ec
)
2244 struct ethtool_coalesce ec_default
= {
2245 .cmd
= ETHTOOL_GCOALESCE
,
2246 .rx_max_coalesced_frames
= 1,
2248 struct virtnet_info
*vi
= netdev_priv(dev
);
2250 memcpy(ec
, &ec_default
, sizeof(ec_default
));
2252 if (vi
->sq
[0].napi
.weight
)
2253 ec
->tx_max_coalesced_frames
= 1;
2258 static void virtnet_init_settings(struct net_device
*dev
)
2260 struct virtnet_info
*vi
= netdev_priv(dev
);
2262 vi
->speed
= SPEED_UNKNOWN
;
2263 vi
->duplex
= DUPLEX_UNKNOWN
;
2266 static void virtnet_update_settings(struct virtnet_info
*vi
)
2271 if (!virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_SPEED_DUPLEX
))
2274 virtio_cread_le(vi
->vdev
, struct virtio_net_config
, speed
, &speed
);
2276 if (ethtool_validate_speed(speed
))
2279 virtio_cread_le(vi
->vdev
, struct virtio_net_config
, duplex
, &duplex
);
2281 if (ethtool_validate_duplex(duplex
))
2282 vi
->duplex
= duplex
;
2285 static const struct ethtool_ops virtnet_ethtool_ops
= {
2286 .supported_coalesce_params
= ETHTOOL_COALESCE_MAX_FRAMES
,
2287 .get_drvinfo
= virtnet_get_drvinfo
,
2288 .get_link
= ethtool_op_get_link
,
2289 .get_ringparam
= virtnet_get_ringparam
,
2290 .get_strings
= virtnet_get_strings
,
2291 .get_sset_count
= virtnet_get_sset_count
,
2292 .get_ethtool_stats
= virtnet_get_ethtool_stats
,
2293 .set_channels
= virtnet_set_channels
,
2294 .get_channels
= virtnet_get_channels
,
2295 .get_ts_info
= ethtool_op_get_ts_info
,
2296 .get_link_ksettings
= virtnet_get_link_ksettings
,
2297 .set_link_ksettings
= virtnet_set_link_ksettings
,
2298 .set_coalesce
= virtnet_set_coalesce
,
2299 .get_coalesce
= virtnet_get_coalesce
,
2302 static void virtnet_freeze_down(struct virtio_device
*vdev
)
2304 struct virtnet_info
*vi
= vdev
->priv
;
2307 /* Make sure no work handler is accessing the device */
2308 flush_work(&vi
->config_work
);
2310 netif_tx_lock_bh(vi
->dev
);
2311 netif_device_detach(vi
->dev
);
2312 netif_tx_unlock_bh(vi
->dev
);
2313 cancel_delayed_work_sync(&vi
->refill
);
2315 if (netif_running(vi
->dev
)) {
2316 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2317 napi_disable(&vi
->rq
[i
].napi
);
2318 virtnet_napi_tx_disable(&vi
->sq
[i
].napi
);
2323 static int init_vqs(struct virtnet_info
*vi
);
2325 static int virtnet_restore_up(struct virtio_device
*vdev
)
2327 struct virtnet_info
*vi
= vdev
->priv
;
2334 virtio_device_ready(vdev
);
2336 if (netif_running(vi
->dev
)) {
2337 for (i
= 0; i
< vi
->curr_queue_pairs
; i
++)
2338 if (!try_fill_recv(vi
, &vi
->rq
[i
], GFP_KERNEL
))
2339 schedule_delayed_work(&vi
->refill
, 0);
2341 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2342 virtnet_napi_enable(vi
->rq
[i
].vq
, &vi
->rq
[i
].napi
);
2343 virtnet_napi_tx_enable(vi
, vi
->sq
[i
].vq
,
2348 netif_tx_lock_bh(vi
->dev
);
2349 netif_device_attach(vi
->dev
);
2350 netif_tx_unlock_bh(vi
->dev
);
2354 static int virtnet_set_guest_offloads(struct virtnet_info
*vi
, u64 offloads
)
2356 struct scatterlist sg
;
2357 vi
->ctrl
->offloads
= cpu_to_virtio64(vi
->vdev
, offloads
);
2359 sg_init_one(&sg
, &vi
->ctrl
->offloads
, sizeof(vi
->ctrl
->offloads
));
2361 if (!virtnet_send_command(vi
, VIRTIO_NET_CTRL_GUEST_OFFLOADS
,
2362 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
, &sg
)) {
2363 dev_warn(&vi
->dev
->dev
, "Fail to set guest offload.\n");
2370 static int virtnet_clear_guest_offloads(struct virtnet_info
*vi
)
2374 if (!vi
->guest_offloads
)
2377 return virtnet_set_guest_offloads(vi
, offloads
);
2380 static int virtnet_restore_guest_offloads(struct virtnet_info
*vi
)
2382 u64 offloads
= vi
->guest_offloads
;
2384 if (!vi
->guest_offloads
)
2387 return virtnet_set_guest_offloads(vi
, offloads
);
2390 static int virtnet_xdp_set(struct net_device
*dev
, struct bpf_prog
*prog
,
2391 struct netlink_ext_ack
*extack
)
2393 unsigned long int max_sz
= PAGE_SIZE
- sizeof(struct padded_vnet_hdr
);
2394 struct virtnet_info
*vi
= netdev_priv(dev
);
2395 struct bpf_prog
*old_prog
;
2396 u16 xdp_qp
= 0, curr_qp
;
2399 if (!virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
)
2400 && (virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_GUEST_TSO4
) ||
2401 virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_GUEST_TSO6
) ||
2402 virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_GUEST_ECN
) ||
2403 virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_GUEST_UFO
) ||
2404 virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_GUEST_CSUM
))) {
2405 NL_SET_ERR_MSG_MOD(extack
, "Can't set XDP while host is implementing LRO/CSUM, disable LRO/CSUM first");
2409 if (vi
->mergeable_rx_bufs
&& !vi
->any_header_sg
) {
2410 NL_SET_ERR_MSG_MOD(extack
, "XDP expects header/data in single page, any_header_sg required");
2414 if (dev
->mtu
> max_sz
) {
2415 NL_SET_ERR_MSG_MOD(extack
, "MTU too large to enable XDP");
2416 netdev_warn(dev
, "XDP requires MTU less than %lu\n", max_sz
);
2420 curr_qp
= vi
->curr_queue_pairs
- vi
->xdp_queue_pairs
;
2422 xdp_qp
= nr_cpu_ids
;
2424 /* XDP requires extra queues for XDP_TX */
2425 if (curr_qp
+ xdp_qp
> vi
->max_queue_pairs
) {
2426 NL_SET_ERR_MSG_MOD(extack
, "Too few free TX rings available");
2427 netdev_warn(dev
, "request %i queues but max is %i\n",
2428 curr_qp
+ xdp_qp
, vi
->max_queue_pairs
);
2432 old_prog
= rtnl_dereference(vi
->rq
[0].xdp_prog
);
2433 if (!prog
&& !old_prog
)
2437 bpf_prog_add(prog
, vi
->max_queue_pairs
- 1);
2439 /* Make sure NAPI is not using any XDP TX queues for RX. */
2440 if (netif_running(dev
)) {
2441 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2442 napi_disable(&vi
->rq
[i
].napi
);
2443 virtnet_napi_tx_disable(&vi
->sq
[i
].napi
);
2448 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2449 rcu_assign_pointer(vi
->rq
[i
].xdp_prog
, prog
);
2451 virtnet_restore_guest_offloads(vi
);
2456 err
= _virtnet_set_queues(vi
, curr_qp
+ xdp_qp
);
2459 netif_set_real_num_rx_queues(dev
, curr_qp
+ xdp_qp
);
2460 vi
->xdp_queue_pairs
= xdp_qp
;
2463 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2464 rcu_assign_pointer(vi
->rq
[i
].xdp_prog
, prog
);
2465 if (i
== 0 && !old_prog
)
2466 virtnet_clear_guest_offloads(vi
);
2470 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2472 bpf_prog_put(old_prog
);
2473 if (netif_running(dev
)) {
2474 virtnet_napi_enable(vi
->rq
[i
].vq
, &vi
->rq
[i
].napi
);
2475 virtnet_napi_tx_enable(vi
, vi
->sq
[i
].vq
,
2484 virtnet_clear_guest_offloads(vi
);
2485 for (i
= 0; i
< vi
->max_queue_pairs
; i
++)
2486 rcu_assign_pointer(vi
->rq
[i
].xdp_prog
, old_prog
);
2489 if (netif_running(dev
)) {
2490 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2491 virtnet_napi_enable(vi
->rq
[i
].vq
, &vi
->rq
[i
].napi
);
2492 virtnet_napi_tx_enable(vi
, vi
->sq
[i
].vq
,
2497 bpf_prog_sub(prog
, vi
->max_queue_pairs
- 1);
2501 static int virtnet_xdp(struct net_device
*dev
, struct netdev_bpf
*xdp
)
2503 switch (xdp
->command
) {
2504 case XDP_SETUP_PROG
:
2505 return virtnet_xdp_set(dev
, xdp
->prog
, xdp
->extack
);
2511 static int virtnet_get_phys_port_name(struct net_device
*dev
, char *buf
,
2514 struct virtnet_info
*vi
= netdev_priv(dev
);
2517 if (!virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_STANDBY
))
2520 ret
= snprintf(buf
, len
, "sby");
2527 static int virtnet_set_features(struct net_device
*dev
,
2528 netdev_features_t features
)
2530 struct virtnet_info
*vi
= netdev_priv(dev
);
2534 if ((dev
->features
^ features
) & NETIF_F_LRO
) {
2535 if (vi
->xdp_queue_pairs
)
2538 if (features
& NETIF_F_LRO
)
2539 offloads
= vi
->guest_offloads_capable
;
2541 offloads
= vi
->guest_offloads_capable
&
2542 ~GUEST_OFFLOAD_LRO_MASK
;
2544 err
= virtnet_set_guest_offloads(vi
, offloads
);
2547 vi
->guest_offloads
= offloads
;
2553 static const struct net_device_ops virtnet_netdev
= {
2554 .ndo_open
= virtnet_open
,
2555 .ndo_stop
= virtnet_close
,
2556 .ndo_start_xmit
= start_xmit
,
2557 .ndo_validate_addr
= eth_validate_addr
,
2558 .ndo_set_mac_address
= virtnet_set_mac_address
,
2559 .ndo_set_rx_mode
= virtnet_set_rx_mode
,
2560 .ndo_get_stats64
= virtnet_stats
,
2561 .ndo_vlan_rx_add_vid
= virtnet_vlan_rx_add_vid
,
2562 .ndo_vlan_rx_kill_vid
= virtnet_vlan_rx_kill_vid
,
2563 .ndo_bpf
= virtnet_xdp
,
2564 .ndo_xdp_xmit
= virtnet_xdp_xmit
,
2565 .ndo_features_check
= passthru_features_check
,
2566 .ndo_get_phys_port_name
= virtnet_get_phys_port_name
,
2567 .ndo_set_features
= virtnet_set_features
,
2570 static void virtnet_config_changed_work(struct work_struct
*work
)
2572 struct virtnet_info
*vi
=
2573 container_of(work
, struct virtnet_info
, config_work
);
2576 if (virtio_cread_feature(vi
->vdev
, VIRTIO_NET_F_STATUS
,
2577 struct virtio_net_config
, status
, &v
) < 0)
2580 if (v
& VIRTIO_NET_S_ANNOUNCE
) {
2581 netdev_notify_peers(vi
->dev
);
2582 virtnet_ack_link_announce(vi
);
2585 /* Ignore unknown (future) status bits */
2586 v
&= VIRTIO_NET_S_LINK_UP
;
2588 if (vi
->status
== v
)
2593 if (vi
->status
& VIRTIO_NET_S_LINK_UP
) {
2594 virtnet_update_settings(vi
);
2595 netif_carrier_on(vi
->dev
);
2596 netif_tx_wake_all_queues(vi
->dev
);
2598 netif_carrier_off(vi
->dev
);
2599 netif_tx_stop_all_queues(vi
->dev
);
2603 static void virtnet_config_changed(struct virtio_device
*vdev
)
2605 struct virtnet_info
*vi
= vdev
->priv
;
2607 schedule_work(&vi
->config_work
);
2610 static void virtnet_free_queues(struct virtnet_info
*vi
)
2614 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2615 __netif_napi_del(&vi
->rq
[i
].napi
);
2616 __netif_napi_del(&vi
->sq
[i
].napi
);
2619 /* We called __netif_napi_del(),
2620 * we need to respect an RCU grace period before freeing vi->rq
2629 static void _free_receive_bufs(struct virtnet_info
*vi
)
2631 struct bpf_prog
*old_prog
;
2634 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2635 while (vi
->rq
[i
].pages
)
2636 __free_pages(get_a_page(&vi
->rq
[i
], GFP_KERNEL
), 0);
2638 old_prog
= rtnl_dereference(vi
->rq
[i
].xdp_prog
);
2639 RCU_INIT_POINTER(vi
->rq
[i
].xdp_prog
, NULL
);
2641 bpf_prog_put(old_prog
);
2645 static void free_receive_bufs(struct virtnet_info
*vi
)
2648 _free_receive_bufs(vi
);
2652 static void free_receive_page_frags(struct virtnet_info
*vi
)
2655 for (i
= 0; i
< vi
->max_queue_pairs
; i
++)
2656 if (vi
->rq
[i
].alloc_frag
.page
)
2657 put_page(vi
->rq
[i
].alloc_frag
.page
);
2660 static void free_unused_bufs(struct virtnet_info
*vi
)
2665 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2666 struct virtqueue
*vq
= vi
->sq
[i
].vq
;
2667 while ((buf
= virtqueue_detach_unused_buf(vq
)) != NULL
) {
2668 if (!is_xdp_frame(buf
))
2671 xdp_return_frame(ptr_to_xdp(buf
));
2675 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2676 struct virtqueue
*vq
= vi
->rq
[i
].vq
;
2678 while ((buf
= virtqueue_detach_unused_buf(vq
)) != NULL
) {
2679 if (vi
->mergeable_rx_bufs
) {
2680 put_page(virt_to_head_page(buf
));
2681 } else if (vi
->big_packets
) {
2682 give_pages(&vi
->rq
[i
], buf
);
2684 put_page(virt_to_head_page(buf
));
2690 static void virtnet_del_vqs(struct virtnet_info
*vi
)
2692 struct virtio_device
*vdev
= vi
->vdev
;
2694 virtnet_clean_affinity(vi
);
2696 vdev
->config
->del_vqs(vdev
);
2698 virtnet_free_queues(vi
);
2701 /* How large should a single buffer be so a queue full of these can fit at
2702 * least one full packet?
2703 * Logic below assumes the mergeable buffer header is used.
2705 static unsigned int mergeable_min_buf_len(struct virtnet_info
*vi
, struct virtqueue
*vq
)
2707 const unsigned int hdr_len
= sizeof(struct virtio_net_hdr_mrg_rxbuf
);
2708 unsigned int rq_size
= virtqueue_get_vring_size(vq
);
2709 unsigned int packet_len
= vi
->big_packets
? IP_MAX_MTU
: vi
->dev
->max_mtu
;
2710 unsigned int buf_len
= hdr_len
+ ETH_HLEN
+ VLAN_HLEN
+ packet_len
;
2711 unsigned int min_buf_len
= DIV_ROUND_UP(buf_len
, rq_size
);
2713 return max(max(min_buf_len
, hdr_len
) - hdr_len
,
2714 (unsigned int)GOOD_PACKET_LEN
);
2717 static int virtnet_find_vqs(struct virtnet_info
*vi
)
2719 vq_callback_t
**callbacks
;
2720 struct virtqueue
**vqs
;
2726 /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by
2727 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by
2728 * possible control vq.
2730 total_vqs
= vi
->max_queue_pairs
* 2 +
2731 virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_CTRL_VQ
);
2733 /* Allocate space for find_vqs parameters */
2734 vqs
= kcalloc(total_vqs
, sizeof(*vqs
), GFP_KERNEL
);
2737 callbacks
= kmalloc_array(total_vqs
, sizeof(*callbacks
), GFP_KERNEL
);
2740 names
= kmalloc_array(total_vqs
, sizeof(*names
), GFP_KERNEL
);
2743 if (!vi
->big_packets
|| vi
->mergeable_rx_bufs
) {
2744 ctx
= kcalloc(total_vqs
, sizeof(*ctx
), GFP_KERNEL
);
2751 /* Parameters for control virtqueue, if any */
2753 callbacks
[total_vqs
- 1] = NULL
;
2754 names
[total_vqs
- 1] = "control";
2757 /* Allocate/initialize parameters for send/receive virtqueues */
2758 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2759 callbacks
[rxq2vq(i
)] = skb_recv_done
;
2760 callbacks
[txq2vq(i
)] = skb_xmit_done
;
2761 sprintf(vi
->rq
[i
].name
, "input.%d", i
);
2762 sprintf(vi
->sq
[i
].name
, "output.%d", i
);
2763 names
[rxq2vq(i
)] = vi
->rq
[i
].name
;
2764 names
[txq2vq(i
)] = vi
->sq
[i
].name
;
2766 ctx
[rxq2vq(i
)] = true;
2769 ret
= vi
->vdev
->config
->find_vqs(vi
->vdev
, total_vqs
, vqs
, callbacks
,
2775 vi
->cvq
= vqs
[total_vqs
- 1];
2776 if (virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_CTRL_VLAN
))
2777 vi
->dev
->features
|= NETIF_F_HW_VLAN_CTAG_FILTER
;
2780 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2781 vi
->rq
[i
].vq
= vqs
[rxq2vq(i
)];
2782 vi
->rq
[i
].min_buf_len
= mergeable_min_buf_len(vi
, vi
->rq
[i
].vq
);
2783 vi
->sq
[i
].vq
= vqs
[txq2vq(i
)];
2786 /* run here: ret == 0. */
2801 static int virtnet_alloc_queues(struct virtnet_info
*vi
)
2805 vi
->ctrl
= kzalloc(sizeof(*vi
->ctrl
), GFP_KERNEL
);
2808 vi
->sq
= kcalloc(vi
->max_queue_pairs
, sizeof(*vi
->sq
), GFP_KERNEL
);
2811 vi
->rq
= kcalloc(vi
->max_queue_pairs
, sizeof(*vi
->rq
), GFP_KERNEL
);
2815 INIT_DELAYED_WORK(&vi
->refill
, refill_work
);
2816 for (i
= 0; i
< vi
->max_queue_pairs
; i
++) {
2817 vi
->rq
[i
].pages
= NULL
;
2818 netif_napi_add(vi
->dev
, &vi
->rq
[i
].napi
, virtnet_poll
,
2820 netif_tx_napi_add(vi
->dev
, &vi
->sq
[i
].napi
, virtnet_poll_tx
,
2821 napi_tx
? napi_weight
: 0);
2823 sg_init_table(vi
->rq
[i
].sg
, ARRAY_SIZE(vi
->rq
[i
].sg
));
2824 ewma_pkt_len_init(&vi
->rq
[i
].mrg_avg_pkt_len
);
2825 sg_init_table(vi
->sq
[i
].sg
, ARRAY_SIZE(vi
->sq
[i
].sg
));
2827 u64_stats_init(&vi
->rq
[i
].stats
.syncp
);
2828 u64_stats_init(&vi
->sq
[i
].stats
.syncp
);
2841 static int init_vqs(struct virtnet_info
*vi
)
2845 /* Allocate send & receive queues */
2846 ret
= virtnet_alloc_queues(vi
);
2850 ret
= virtnet_find_vqs(vi
);
2855 virtnet_set_affinity(vi
);
2861 virtnet_free_queues(vi
);
2867 static ssize_t
mergeable_rx_buffer_size_show(struct netdev_rx_queue
*queue
,
2870 struct virtnet_info
*vi
= netdev_priv(queue
->dev
);
2871 unsigned int queue_index
= get_netdev_rx_queue_index(queue
);
2872 unsigned int headroom
= virtnet_get_headroom(vi
);
2873 unsigned int tailroom
= headroom
? sizeof(struct skb_shared_info
) : 0;
2874 struct ewma_pkt_len
*avg
;
2876 BUG_ON(queue_index
>= vi
->max_queue_pairs
);
2877 avg
= &vi
->rq
[queue_index
].mrg_avg_pkt_len
;
2878 return sprintf(buf
, "%u\n",
2879 get_mergeable_buf_len(&vi
->rq
[queue_index
], avg
,
2880 SKB_DATA_ALIGN(headroom
+ tailroom
)));
2883 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute
=
2884 __ATTR_RO(mergeable_rx_buffer_size
);
2886 static struct attribute
*virtio_net_mrg_rx_attrs
[] = {
2887 &mergeable_rx_buffer_size_attribute
.attr
,
2891 static const struct attribute_group virtio_net_mrg_rx_group
= {
2892 .name
= "virtio_net",
2893 .attrs
= virtio_net_mrg_rx_attrs
2897 static bool virtnet_fail_on_feature(struct virtio_device
*vdev
,
2899 const char *fname
, const char *dname
)
2901 if (!virtio_has_feature(vdev
, fbit
))
2904 dev_err(&vdev
->dev
, "device advertises feature %s but not %s",
2910 #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \
2911 virtnet_fail_on_feature(vdev, fbit, #fbit, dbit)
2913 static bool virtnet_validate_features(struct virtio_device
*vdev
)
2915 if (!virtio_has_feature(vdev
, VIRTIO_NET_F_CTRL_VQ
) &&
2916 (VIRTNET_FAIL_ON(vdev
, VIRTIO_NET_F_CTRL_RX
,
2917 "VIRTIO_NET_F_CTRL_VQ") ||
2918 VIRTNET_FAIL_ON(vdev
, VIRTIO_NET_F_CTRL_VLAN
,
2919 "VIRTIO_NET_F_CTRL_VQ") ||
2920 VIRTNET_FAIL_ON(vdev
, VIRTIO_NET_F_GUEST_ANNOUNCE
,
2921 "VIRTIO_NET_F_CTRL_VQ") ||
2922 VIRTNET_FAIL_ON(vdev
, VIRTIO_NET_F_MQ
, "VIRTIO_NET_F_CTRL_VQ") ||
2923 VIRTNET_FAIL_ON(vdev
, VIRTIO_NET_F_CTRL_MAC_ADDR
,
2924 "VIRTIO_NET_F_CTRL_VQ"))) {
2931 #define MIN_MTU ETH_MIN_MTU
2932 #define MAX_MTU ETH_MAX_MTU
2934 static int virtnet_validate(struct virtio_device
*vdev
)
2936 if (!vdev
->config
->get
) {
2937 dev_err(&vdev
->dev
, "%s failure: config access disabled\n",
2942 if (!virtnet_validate_features(vdev
))
2945 if (virtio_has_feature(vdev
, VIRTIO_NET_F_MTU
)) {
2946 int mtu
= virtio_cread16(vdev
,
2947 offsetof(struct virtio_net_config
,
2950 __virtio_clear_bit(vdev
, VIRTIO_NET_F_MTU
);
2956 static int virtnet_probe(struct virtio_device
*vdev
)
2958 int i
, err
= -ENOMEM
;
2959 struct net_device
*dev
;
2960 struct virtnet_info
*vi
;
2961 u16 max_queue_pairs
;
2964 /* Find if host supports multiqueue virtio_net device */
2965 err
= virtio_cread_feature(vdev
, VIRTIO_NET_F_MQ
,
2966 struct virtio_net_config
,
2967 max_virtqueue_pairs
, &max_queue_pairs
);
2969 /* We need at least 2 queue's */
2970 if (err
|| max_queue_pairs
< VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN
||
2971 max_queue_pairs
> VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX
||
2972 !virtio_has_feature(vdev
, VIRTIO_NET_F_CTRL_VQ
))
2973 max_queue_pairs
= 1;
2975 /* Allocate ourselves a network device with room for our info */
2976 dev
= alloc_etherdev_mq(sizeof(struct virtnet_info
), max_queue_pairs
);
2980 /* Set up network device as normal. */
2981 dev
->priv_flags
|= IFF_UNICAST_FLT
| IFF_LIVE_ADDR_CHANGE
;
2982 dev
->netdev_ops
= &virtnet_netdev
;
2983 dev
->features
= NETIF_F_HIGHDMA
;
2985 dev
->ethtool_ops
= &virtnet_ethtool_ops
;
2986 SET_NETDEV_DEV(dev
, &vdev
->dev
);
2988 /* Do we support "hardware" checksums? */
2989 if (virtio_has_feature(vdev
, VIRTIO_NET_F_CSUM
)) {
2990 /* This opens up the world of extra features. */
2991 dev
->hw_features
|= NETIF_F_HW_CSUM
| NETIF_F_SG
;
2993 dev
->features
|= NETIF_F_HW_CSUM
| NETIF_F_SG
;
2995 if (virtio_has_feature(vdev
, VIRTIO_NET_F_GSO
)) {
2996 dev
->hw_features
|= NETIF_F_TSO
2997 | NETIF_F_TSO_ECN
| NETIF_F_TSO6
;
2999 /* Individual feature bits: what can host handle? */
3000 if (virtio_has_feature(vdev
, VIRTIO_NET_F_HOST_TSO4
))
3001 dev
->hw_features
|= NETIF_F_TSO
;
3002 if (virtio_has_feature(vdev
, VIRTIO_NET_F_HOST_TSO6
))
3003 dev
->hw_features
|= NETIF_F_TSO6
;
3004 if (virtio_has_feature(vdev
, VIRTIO_NET_F_HOST_ECN
))
3005 dev
->hw_features
|= NETIF_F_TSO_ECN
;
3007 dev
->features
|= NETIF_F_GSO_ROBUST
;
3010 dev
->features
|= dev
->hw_features
& NETIF_F_ALL_TSO
;
3011 /* (!csum && gso) case will be fixed by register_netdev() */
3013 if (virtio_has_feature(vdev
, VIRTIO_NET_F_GUEST_CSUM
))
3014 dev
->features
|= NETIF_F_RXCSUM
;
3015 if (virtio_has_feature(vdev
, VIRTIO_NET_F_GUEST_TSO4
) ||
3016 virtio_has_feature(vdev
, VIRTIO_NET_F_GUEST_TSO6
))
3017 dev
->features
|= NETIF_F_LRO
;
3018 if (virtio_has_feature(vdev
, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
))
3019 dev
->hw_features
|= NETIF_F_LRO
;
3021 dev
->vlan_features
= dev
->features
;
3023 /* MTU range: 68 - 65535 */
3024 dev
->min_mtu
= MIN_MTU
;
3025 dev
->max_mtu
= MAX_MTU
;
3027 /* Configuration may specify what MAC to use. Otherwise random. */
3028 if (virtio_has_feature(vdev
, VIRTIO_NET_F_MAC
))
3029 virtio_cread_bytes(vdev
,
3030 offsetof(struct virtio_net_config
, mac
),
3031 dev
->dev_addr
, dev
->addr_len
);
3033 eth_hw_addr_random(dev
);
3035 /* Set up our device-specific information */
3036 vi
= netdev_priv(dev
);
3041 INIT_WORK(&vi
->config_work
, virtnet_config_changed_work
);
3043 /* If we can receive ANY GSO packets, we must allocate large ones. */
3044 if (virtio_has_feature(vdev
, VIRTIO_NET_F_GUEST_TSO4
) ||
3045 virtio_has_feature(vdev
, VIRTIO_NET_F_GUEST_TSO6
) ||
3046 virtio_has_feature(vdev
, VIRTIO_NET_F_GUEST_ECN
) ||
3047 virtio_has_feature(vdev
, VIRTIO_NET_F_GUEST_UFO
))
3048 vi
->big_packets
= true;
3050 if (virtio_has_feature(vdev
, VIRTIO_NET_F_MRG_RXBUF
))
3051 vi
->mergeable_rx_bufs
= true;
3053 if (virtio_has_feature(vdev
, VIRTIO_NET_F_MRG_RXBUF
) ||
3054 virtio_has_feature(vdev
, VIRTIO_F_VERSION_1
))
3055 vi
->hdr_len
= sizeof(struct virtio_net_hdr_mrg_rxbuf
);
3057 vi
->hdr_len
= sizeof(struct virtio_net_hdr
);
3059 if (virtio_has_feature(vdev
, VIRTIO_F_ANY_LAYOUT
) ||
3060 virtio_has_feature(vdev
, VIRTIO_F_VERSION_1
))
3061 vi
->any_header_sg
= true;
3063 if (virtio_has_feature(vdev
, VIRTIO_NET_F_CTRL_VQ
))
3066 if (virtio_has_feature(vdev
, VIRTIO_NET_F_MTU
)) {
3067 mtu
= virtio_cread16(vdev
,
3068 offsetof(struct virtio_net_config
,
3070 if (mtu
< dev
->min_mtu
) {
3071 /* Should never trigger: MTU was previously validated
3072 * in virtnet_validate.
3075 "device MTU appears to have changed it is now %d < %d",
3084 /* TODO: size buffers correctly in this case. */
3085 if (dev
->mtu
> ETH_DATA_LEN
)
3086 vi
->big_packets
= true;
3089 if (vi
->any_header_sg
)
3090 dev
->needed_headroom
= vi
->hdr_len
;
3092 /* Enable multiqueue by default */
3093 if (num_online_cpus() >= max_queue_pairs
)
3094 vi
->curr_queue_pairs
= max_queue_pairs
;
3096 vi
->curr_queue_pairs
= num_online_cpus();
3097 vi
->max_queue_pairs
= max_queue_pairs
;
3099 /* Allocate/initialize the rx/tx queues, and invoke find_vqs */
3105 if (vi
->mergeable_rx_bufs
)
3106 dev
->sysfs_rx_queue_group
= &virtio_net_mrg_rx_group
;
3108 netif_set_real_num_tx_queues(dev
, vi
->curr_queue_pairs
);
3109 netif_set_real_num_rx_queues(dev
, vi
->curr_queue_pairs
);
3111 virtnet_init_settings(dev
);
3113 if (virtio_has_feature(vdev
, VIRTIO_NET_F_STANDBY
)) {
3114 vi
->failover
= net_failover_create(vi
->dev
);
3115 if (IS_ERR(vi
->failover
)) {
3116 err
= PTR_ERR(vi
->failover
);
3121 err
= register_netdev(dev
);
3123 pr_debug("virtio_net: registering device failed\n");
3127 virtio_device_ready(vdev
);
3129 err
= virtnet_cpu_notif_add(vi
);
3131 pr_debug("virtio_net: registering cpu notifier failed\n");
3132 goto free_unregister_netdev
;
3135 virtnet_set_queues(vi
, vi
->curr_queue_pairs
);
3137 /* Assume link up if device can't report link status,
3138 otherwise get link status from config. */
3139 netif_carrier_off(dev
);
3140 if (virtio_has_feature(vi
->vdev
, VIRTIO_NET_F_STATUS
)) {
3141 schedule_work(&vi
->config_work
);
3143 vi
->status
= VIRTIO_NET_S_LINK_UP
;
3144 virtnet_update_settings(vi
);
3145 netif_carrier_on(dev
);
3148 for (i
= 0; i
< ARRAY_SIZE(guest_offloads
); i
++)
3149 if (virtio_has_feature(vi
->vdev
, guest_offloads
[i
]))
3150 set_bit(guest_offloads
[i
], &vi
->guest_offloads
);
3151 vi
->guest_offloads_capable
= vi
->guest_offloads
;
3153 pr_debug("virtnet: registered device %s with %d RX and TX vq's\n",
3154 dev
->name
, max_queue_pairs
);
3158 free_unregister_netdev
:
3159 vi
->vdev
->config
->reset(vdev
);
3161 unregister_netdev(dev
);
3163 net_failover_destroy(vi
->failover
);
3165 cancel_delayed_work_sync(&vi
->refill
);
3166 free_receive_page_frags(vi
);
3167 virtnet_del_vqs(vi
);
3173 static void remove_vq_common(struct virtnet_info
*vi
)
3175 vi
->vdev
->config
->reset(vi
->vdev
);
3177 /* Free unused buffers in both send and recv, if any. */
3178 free_unused_bufs(vi
);
3180 free_receive_bufs(vi
);
3182 free_receive_page_frags(vi
);
3184 virtnet_del_vqs(vi
);
3187 static void virtnet_remove(struct virtio_device
*vdev
)
3189 struct virtnet_info
*vi
= vdev
->priv
;
3191 virtnet_cpu_notif_remove(vi
);
3193 /* Make sure no work handler is accessing the device. */
3194 flush_work(&vi
->config_work
);
3196 unregister_netdev(vi
->dev
);
3198 net_failover_destroy(vi
->failover
);
3200 remove_vq_common(vi
);
3202 free_netdev(vi
->dev
);
3205 static __maybe_unused
int virtnet_freeze(struct virtio_device
*vdev
)
3207 struct virtnet_info
*vi
= vdev
->priv
;
3209 virtnet_cpu_notif_remove(vi
);
3210 virtnet_freeze_down(vdev
);
3211 remove_vq_common(vi
);
3216 static __maybe_unused
int virtnet_restore(struct virtio_device
*vdev
)
3218 struct virtnet_info
*vi
= vdev
->priv
;
3221 err
= virtnet_restore_up(vdev
);
3224 virtnet_set_queues(vi
, vi
->curr_queue_pairs
);
3226 err
= virtnet_cpu_notif_add(vi
);
3233 static struct virtio_device_id id_table
[] = {
3234 { VIRTIO_ID_NET
, VIRTIO_DEV_ANY_ID
},
3238 #define VIRTNET_FEATURES \
3239 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \
3241 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \
3242 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \
3243 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \
3244 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \
3245 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
3246 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
3247 VIRTIO_NET_F_CTRL_MAC_ADDR, \
3248 VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3249 VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY
3251 static unsigned int features
[] = {
3255 static unsigned int features_legacy
[] = {
3258 VIRTIO_F_ANY_LAYOUT
,
3261 static struct virtio_driver virtio_net_driver
= {
3262 .feature_table
= features
,
3263 .feature_table_size
= ARRAY_SIZE(features
),
3264 .feature_table_legacy
= features_legacy
,
3265 .feature_table_size_legacy
= ARRAY_SIZE(features_legacy
),
3266 .driver
.name
= KBUILD_MODNAME
,
3267 .driver
.owner
= THIS_MODULE
,
3268 .id_table
= id_table
,
3269 .validate
= virtnet_validate
,
3270 .probe
= virtnet_probe
,
3271 .remove
= virtnet_remove
,
3272 .config_changed
= virtnet_config_changed
,
3273 #ifdef CONFIG_PM_SLEEP
3274 .freeze
= virtnet_freeze
,
3275 .restore
= virtnet_restore
,
3279 static __init
int virtio_net_driver_init(void)
3283 ret
= cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN
, "virtio/net:online",
3285 virtnet_cpu_down_prep
);
3288 virtionet_online
= ret
;
3289 ret
= cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD
, "virtio/net:dead",
3290 NULL
, virtnet_cpu_dead
);
3294 ret
= register_virtio_driver(&virtio_net_driver
);
3299 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD
);
3301 cpuhp_remove_multi_state(virtionet_online
);
3305 module_init(virtio_net_driver_init
);
3307 static __exit
void virtio_net_driver_exit(void)
3309 unregister_virtio_driver(&virtio_net_driver
);
3310 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD
);
3311 cpuhp_remove_multi_state(virtionet_online
);
3313 module_exit(virtio_net_driver_exit
);
3315 MODULE_DEVICE_TABLE(virtio
, id_table
);
3316 MODULE_DESCRIPTION("Virtio network driver");
3317 MODULE_LICENSE("GPL");