1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2009, Microsoft Corporation.
6 * Haiyang Zhang <haiyangz@microsoft.com>
7 * Hank Janssen <hjanssen@microsoft.com>
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 #include <linux/init.h>
12 #include <linux/atomic.h>
13 #include <linux/module.h>
14 #include <linux/highmem.h>
15 #include <linux/device.h>
17 #include <linux/delay.h>
18 #include <linux/netdevice.h>
19 #include <linux/inetdevice.h>
20 #include <linux/etherdevice.h>
21 #include <linux/pci.h>
22 #include <linux/skbuff.h>
23 #include <linux/if_vlan.h>
25 #include <linux/slab.h>
26 #include <linux/rtnetlink.h>
27 #include <linux/netpoll.h>
30 #include <net/route.h>
32 #include <net/pkt_sched.h>
33 #include <net/checksum.h>
34 #include <net/ip6_checksum.h>
36 #include "hyperv_net.h"
38 #define RING_SIZE_MIN 64
39 #define RETRY_US_LO 5000
40 #define RETRY_US_HI 10000
41 #define RETRY_MAX 2000 /* >10 sec */
43 #define LINKCHANGE_INT (2 * HZ)
44 #define VF_TAKEOVER_INT (HZ / 10)
46 static unsigned int ring_size __ro_after_init
= 128;
47 module_param(ring_size
, uint
, 0444);
48 MODULE_PARM_DESC(ring_size
, "Ring buffer size (# of pages)");
49 unsigned int netvsc_ring_bytes __ro_after_init
;
51 static const u32 default_msg
= NETIF_MSG_DRV
| NETIF_MSG_PROBE
|
52 NETIF_MSG_LINK
| NETIF_MSG_IFUP
|
53 NETIF_MSG_IFDOWN
| NETIF_MSG_RX_ERR
|
56 static int debug
= -1;
57 module_param(debug
, int, 0444);
58 MODULE_PARM_DESC(debug
, "Debug level (0=none,...,16=all)");
60 static LIST_HEAD(netvsc_dev_list
);
62 static void netvsc_change_rx_flags(struct net_device
*net
, int change
)
64 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
65 struct net_device
*vf_netdev
= rtnl_dereference(ndev_ctx
->vf_netdev
);
71 if (change
& IFF_PROMISC
) {
72 inc
= (net
->flags
& IFF_PROMISC
) ? 1 : -1;
73 dev_set_promiscuity(vf_netdev
, inc
);
76 if (change
& IFF_ALLMULTI
) {
77 inc
= (net
->flags
& IFF_ALLMULTI
) ? 1 : -1;
78 dev_set_allmulti(vf_netdev
, inc
);
82 static void netvsc_set_rx_mode(struct net_device
*net
)
84 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
85 struct net_device
*vf_netdev
;
86 struct netvsc_device
*nvdev
;
89 vf_netdev
= rcu_dereference(ndev_ctx
->vf_netdev
);
91 dev_uc_sync(vf_netdev
, net
);
92 dev_mc_sync(vf_netdev
, net
);
95 nvdev
= rcu_dereference(ndev_ctx
->nvdev
);
97 rndis_filter_update(nvdev
);
101 static void netvsc_tx_enable(struct netvsc_device
*nvscdev
,
102 struct net_device
*ndev
)
104 nvscdev
->tx_disable
= false;
105 virt_wmb(); /* ensure queue wake up mechanism is on */
107 netif_tx_wake_all_queues(ndev
);
110 static int netvsc_open(struct net_device
*net
)
112 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
113 struct net_device
*vf_netdev
= rtnl_dereference(ndev_ctx
->vf_netdev
);
114 struct netvsc_device
*nvdev
= rtnl_dereference(ndev_ctx
->nvdev
);
115 struct rndis_device
*rdev
;
118 netif_carrier_off(net
);
120 /* Open up the device */
121 ret
= rndis_filter_open(nvdev
);
123 netdev_err(net
, "unable to open device (ret %d).\n", ret
);
127 rdev
= nvdev
->extension
;
128 if (!rdev
->link_state
) {
129 netif_carrier_on(net
);
130 netvsc_tx_enable(nvdev
, net
);
134 /* Setting synthetic device up transparently sets
135 * slave as up. If open fails, then slave will be
136 * still be offline (and not used).
138 ret
= dev_open(vf_netdev
, NULL
);
141 "unable to open slave: %s: %d\n",
142 vf_netdev
->name
, ret
);
147 static int netvsc_wait_until_empty(struct netvsc_device
*nvdev
)
149 unsigned int retry
= 0;
152 /* Ensure pending bytes in ring are read */
156 for (i
= 0; i
< nvdev
->num_chn
; i
++) {
157 struct vmbus_channel
*chn
158 = nvdev
->chan_table
[i
].channel
;
163 /* make sure receive not running now */
164 napi_synchronize(&nvdev
->chan_table
[i
].napi
);
166 aread
= hv_get_bytes_to_read(&chn
->inbound
);
170 aread
= hv_get_bytes_to_read(&chn
->outbound
);
178 if (++retry
> RETRY_MAX
)
181 usleep_range(RETRY_US_LO
, RETRY_US_HI
);
185 static void netvsc_tx_disable(struct netvsc_device
*nvscdev
,
186 struct net_device
*ndev
)
189 nvscdev
->tx_disable
= true;
190 virt_wmb(); /* ensure txq will not wake up after stop */
193 netif_tx_disable(ndev
);
196 static int netvsc_close(struct net_device
*net
)
198 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
199 struct net_device
*vf_netdev
200 = rtnl_dereference(net_device_ctx
->vf_netdev
);
201 struct netvsc_device
*nvdev
= rtnl_dereference(net_device_ctx
->nvdev
);
204 netvsc_tx_disable(nvdev
, net
);
206 /* No need to close rndis filter if it is removed already */
210 ret
= rndis_filter_close(nvdev
);
212 netdev_err(net
, "unable to close device (ret %d).\n", ret
);
216 ret
= netvsc_wait_until_empty(nvdev
);
218 netdev_err(net
, "Ring buffer not empty after closing rndis\n");
221 dev_close(vf_netdev
);
226 static inline void *init_ppi_data(struct rndis_message
*msg
,
227 u32 ppi_size
, u32 pkt_type
)
229 struct rndis_packet
*rndis_pkt
= &msg
->msg
.pkt
;
230 struct rndis_per_packet_info
*ppi
;
232 rndis_pkt
->data_offset
+= ppi_size
;
233 ppi
= (void *)rndis_pkt
+ rndis_pkt
->per_pkt_info_offset
234 + rndis_pkt
->per_pkt_info_len
;
236 ppi
->size
= ppi_size
;
237 ppi
->type
= pkt_type
;
239 ppi
->ppi_offset
= sizeof(struct rndis_per_packet_info
);
241 rndis_pkt
->per_pkt_info_len
+= ppi_size
;
246 /* Azure hosts don't support non-TCP port numbers in hashing for fragmented
247 * packets. We can use ethtool to change UDP hash level when necessary.
249 static inline u32
netvsc_get_hash(
251 const struct net_device_context
*ndc
)
253 struct flow_keys flow
;
254 u32 hash
, pkt_proto
= 0;
255 static u32 hashrnd __read_mostly
;
257 net_get_random_once(&hashrnd
, sizeof(hashrnd
));
259 if (!skb_flow_dissect_flow_keys(skb
, &flow
, 0))
262 switch (flow
.basic
.ip_proto
) {
264 if (flow
.basic
.n_proto
== htons(ETH_P_IP
))
265 pkt_proto
= HV_TCP4_L4HASH
;
266 else if (flow
.basic
.n_proto
== htons(ETH_P_IPV6
))
267 pkt_proto
= HV_TCP6_L4HASH
;
272 if (flow
.basic
.n_proto
== htons(ETH_P_IP
))
273 pkt_proto
= HV_UDP4_L4HASH
;
274 else if (flow
.basic
.n_proto
== htons(ETH_P_IPV6
))
275 pkt_proto
= HV_UDP6_L4HASH
;
280 if (pkt_proto
& ndc
->l4_hash
) {
281 return skb_get_hash(skb
);
283 if (flow
.basic
.n_proto
== htons(ETH_P_IP
))
284 hash
= jhash2((u32
*)&flow
.addrs
.v4addrs
, 2, hashrnd
);
285 else if (flow
.basic
.n_proto
== htons(ETH_P_IPV6
))
286 hash
= jhash2((u32
*)&flow
.addrs
.v6addrs
, 8, hashrnd
);
290 __skb_set_sw_hash(skb
, hash
, false);
296 static inline int netvsc_get_tx_queue(struct net_device
*ndev
,
297 struct sk_buff
*skb
, int old_idx
)
299 const struct net_device_context
*ndc
= netdev_priv(ndev
);
300 struct sock
*sk
= skb
->sk
;
303 q_idx
= ndc
->tx_table
[netvsc_get_hash(skb
, ndc
) &
304 (VRSS_SEND_TAB_SIZE
- 1)];
306 /* If queue index changed record the new value */
307 if (q_idx
!= old_idx
&&
308 sk
&& sk_fullsock(sk
) && rcu_access_pointer(sk
->sk_dst_cache
))
309 sk_tx_queue_set(sk
, q_idx
);
315 * Select queue for transmit.
317 * If a valid queue has already been assigned, then use that.
318 * Otherwise compute tx queue based on hash and the send table.
320 * This is basically similar to default (netdev_pick_tx) with the added step
321 * of using the host send_table when no other queue has been assigned.
323 * TODO support XPS - but get_xps_queue not exported
325 static u16
netvsc_pick_tx(struct net_device
*ndev
, struct sk_buff
*skb
)
327 int q_idx
= sk_tx_queue_get(skb
->sk
);
329 if (q_idx
< 0 || skb
->ooo_okay
|| q_idx
>= ndev
->real_num_tx_queues
) {
330 /* If forwarding a packet, we use the recorded queue when
331 * available for better cache locality.
333 if (skb_rx_queue_recorded(skb
))
334 q_idx
= skb_get_rx_queue(skb
);
336 q_idx
= netvsc_get_tx_queue(ndev
, skb
, q_idx
);
342 static u16
netvsc_select_queue(struct net_device
*ndev
, struct sk_buff
*skb
,
343 struct net_device
*sb_dev
)
345 struct net_device_context
*ndc
= netdev_priv(ndev
);
346 struct net_device
*vf_netdev
;
350 vf_netdev
= rcu_dereference(ndc
->vf_netdev
);
352 const struct net_device_ops
*vf_ops
= vf_netdev
->netdev_ops
;
354 if (vf_ops
->ndo_select_queue
)
355 txq
= vf_ops
->ndo_select_queue(vf_netdev
, skb
, sb_dev
);
357 txq
= netdev_pick_tx(vf_netdev
, skb
, NULL
);
359 /* Record the queue selected by VF so that it can be
360 * used for common case where VF has more queues than
361 * the synthetic device.
363 qdisc_skb_cb(skb
)->slave_dev_queue_mapping
= txq
;
365 txq
= netvsc_pick_tx(ndev
, skb
);
369 while (unlikely(txq
>= ndev
->real_num_tx_queues
))
370 txq
-= ndev
->real_num_tx_queues
;
375 static u32
fill_pg_buf(struct page
*page
, u32 offset
, u32 len
,
376 struct hv_page_buffer
*pb
)
380 /* Deal with compound pages by ignoring unused part
383 page
+= (offset
>> PAGE_SHIFT
);
384 offset
&= ~PAGE_MASK
;
389 bytes
= PAGE_SIZE
- offset
;
392 pb
[j
].pfn
= page_to_pfn(page
);
393 pb
[j
].offset
= offset
;
399 if (offset
== PAGE_SIZE
&& len
) {
409 static u32
init_page_array(void *hdr
, u32 len
, struct sk_buff
*skb
,
410 struct hv_netvsc_packet
*packet
,
411 struct hv_page_buffer
*pb
)
414 char *data
= skb
->data
;
415 int frags
= skb_shinfo(skb
)->nr_frags
;
418 /* The packet is laid out thus:
419 * 1. hdr: RNDIS header and PPI
421 * 3. skb fragment data
423 slots_used
+= fill_pg_buf(virt_to_page(hdr
),
425 len
, &pb
[slots_used
]);
427 packet
->rmsg_size
= len
;
428 packet
->rmsg_pgcnt
= slots_used
;
430 slots_used
+= fill_pg_buf(virt_to_page(data
),
431 offset_in_page(data
),
432 skb_headlen(skb
), &pb
[slots_used
]);
434 for (i
= 0; i
< frags
; i
++) {
435 skb_frag_t
*frag
= skb_shinfo(skb
)->frags
+ i
;
437 slots_used
+= fill_pg_buf(skb_frag_page(frag
),
439 skb_frag_size(frag
), &pb
[slots_used
]);
444 static int count_skb_frag_slots(struct sk_buff
*skb
)
446 int i
, frags
= skb_shinfo(skb
)->nr_frags
;
449 for (i
= 0; i
< frags
; i
++) {
450 skb_frag_t
*frag
= skb_shinfo(skb
)->frags
+ i
;
451 unsigned long size
= skb_frag_size(frag
);
452 unsigned long offset
= skb_frag_off(frag
);
454 /* Skip unused frames from start of page */
455 offset
&= ~PAGE_MASK
;
456 pages
+= PFN_UP(offset
+ size
);
461 static int netvsc_get_slots(struct sk_buff
*skb
)
463 char *data
= skb
->data
;
464 unsigned int offset
= offset_in_page(data
);
465 unsigned int len
= skb_headlen(skb
);
469 slots
= DIV_ROUND_UP(offset
+ len
, PAGE_SIZE
);
470 frag_slots
= count_skb_frag_slots(skb
);
471 return slots
+ frag_slots
;
474 static u32
net_checksum_info(struct sk_buff
*skb
)
476 if (skb
->protocol
== htons(ETH_P_IP
)) {
477 struct iphdr
*ip
= ip_hdr(skb
);
479 if (ip
->protocol
== IPPROTO_TCP
)
480 return TRANSPORT_INFO_IPV4_TCP
;
481 else if (ip
->protocol
== IPPROTO_UDP
)
482 return TRANSPORT_INFO_IPV4_UDP
;
484 struct ipv6hdr
*ip6
= ipv6_hdr(skb
);
486 if (ip6
->nexthdr
== IPPROTO_TCP
)
487 return TRANSPORT_INFO_IPV6_TCP
;
488 else if (ip6
->nexthdr
== IPPROTO_UDP
)
489 return TRANSPORT_INFO_IPV6_UDP
;
492 return TRANSPORT_INFO_NOT_IP
;
495 /* Send skb on the slave VF device. */
496 static int netvsc_vf_xmit(struct net_device
*net
, struct net_device
*vf_netdev
,
499 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
500 unsigned int len
= skb
->len
;
503 skb
->dev
= vf_netdev
;
504 skb
->queue_mapping
= qdisc_skb_cb(skb
)->slave_dev_queue_mapping
;
506 rc
= dev_queue_xmit(skb
);
507 if (likely(rc
== NET_XMIT_SUCCESS
|| rc
== NET_XMIT_CN
)) {
508 struct netvsc_vf_pcpu_stats
*pcpu_stats
509 = this_cpu_ptr(ndev_ctx
->vf_stats
);
511 u64_stats_update_begin(&pcpu_stats
->syncp
);
512 pcpu_stats
->tx_packets
++;
513 pcpu_stats
->tx_bytes
+= len
;
514 u64_stats_update_end(&pcpu_stats
->syncp
);
516 this_cpu_inc(ndev_ctx
->vf_stats
->tx_dropped
);
522 static int netvsc_start_xmit(struct sk_buff
*skb
, struct net_device
*net
)
524 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
525 struct hv_netvsc_packet
*packet
= NULL
;
527 unsigned int num_data_pgs
;
528 struct rndis_message
*rndis_msg
;
529 struct net_device
*vf_netdev
;
532 struct hv_page_buffer pb
[MAX_PAGE_BUFFER_COUNT
];
534 /* if VF is present and up then redirect packets
535 * already called with rcu_read_lock_bh
537 vf_netdev
= rcu_dereference_bh(net_device_ctx
->vf_netdev
);
538 if (vf_netdev
&& netif_running(vf_netdev
) &&
539 !netpoll_tx_running(net
))
540 return netvsc_vf_xmit(net
, vf_netdev
, skb
);
542 /* We will atmost need two pages to describe the rndis
543 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
544 * of pages in a single packet. If skb is scattered around
545 * more pages we try linearizing it.
548 num_data_pgs
= netvsc_get_slots(skb
) + 2;
550 if (unlikely(num_data_pgs
> MAX_PAGE_BUFFER_COUNT
)) {
551 ++net_device_ctx
->eth_stats
.tx_scattered
;
553 if (skb_linearize(skb
))
556 num_data_pgs
= netvsc_get_slots(skb
) + 2;
557 if (num_data_pgs
> MAX_PAGE_BUFFER_COUNT
) {
558 ++net_device_ctx
->eth_stats
.tx_too_big
;
564 * Place the rndis header in the skb head room and
565 * the skb->cb will be used for hv_netvsc_packet
568 ret
= skb_cow_head(skb
, RNDIS_AND_PPI_SIZE
);
572 /* Use the skb control buffer for building up the packet */
573 BUILD_BUG_ON(sizeof(struct hv_netvsc_packet
) >
574 FIELD_SIZEOF(struct sk_buff
, cb
));
575 packet
= (struct hv_netvsc_packet
*)skb
->cb
;
577 packet
->q_idx
= skb_get_queue_mapping(skb
);
579 packet
->total_data_buflen
= skb
->len
;
580 packet
->total_bytes
= skb
->len
;
581 packet
->total_packets
= 1;
583 rndis_msg
= (struct rndis_message
*)skb
->head
;
585 /* Add the rndis header */
586 rndis_msg
->ndis_msg_type
= RNDIS_MSG_PACKET
;
587 rndis_msg
->msg_len
= packet
->total_data_buflen
;
589 rndis_msg
->msg
.pkt
= (struct rndis_packet
) {
590 .data_offset
= sizeof(struct rndis_packet
),
591 .data_len
= packet
->total_data_buflen
,
592 .per_pkt_info_offset
= sizeof(struct rndis_packet
),
595 rndis_msg_size
= RNDIS_MESSAGE_SIZE(struct rndis_packet
);
597 hash
= skb_get_hash_raw(skb
);
598 if (hash
!= 0 && net
->real_num_tx_queues
> 1) {
601 rndis_msg_size
+= NDIS_HASH_PPI_SIZE
;
602 hash_info
= init_ppi_data(rndis_msg
, NDIS_HASH_PPI_SIZE
,
607 if (skb_vlan_tag_present(skb
)) {
608 struct ndis_pkt_8021q_info
*vlan
;
610 rndis_msg_size
+= NDIS_VLAN_PPI_SIZE
;
611 vlan
= init_ppi_data(rndis_msg
, NDIS_VLAN_PPI_SIZE
,
615 vlan
->vlanid
= skb_vlan_tag_get_id(skb
);
616 vlan
->cfi
= skb_vlan_tag_get_cfi(skb
);
617 vlan
->pri
= skb_vlan_tag_get_prio(skb
);
620 if (skb_is_gso(skb
)) {
621 struct ndis_tcp_lso_info
*lso_info
;
623 rndis_msg_size
+= NDIS_LSO_PPI_SIZE
;
624 lso_info
= init_ppi_data(rndis_msg
, NDIS_LSO_PPI_SIZE
,
625 TCP_LARGESEND_PKTINFO
);
628 lso_info
->lso_v2_transmit
.type
= NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE
;
629 if (skb
->protocol
== htons(ETH_P_IP
)) {
630 lso_info
->lso_v2_transmit
.ip_version
=
631 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4
;
632 ip_hdr(skb
)->tot_len
= 0;
633 ip_hdr(skb
)->check
= 0;
634 tcp_hdr(skb
)->check
=
635 ~csum_tcpudp_magic(ip_hdr(skb
)->saddr
,
636 ip_hdr(skb
)->daddr
, 0, IPPROTO_TCP
, 0);
638 lso_info
->lso_v2_transmit
.ip_version
=
639 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6
;
640 ipv6_hdr(skb
)->payload_len
= 0;
641 tcp_hdr(skb
)->check
=
642 ~csum_ipv6_magic(&ipv6_hdr(skb
)->saddr
,
643 &ipv6_hdr(skb
)->daddr
, 0, IPPROTO_TCP
, 0);
645 lso_info
->lso_v2_transmit
.tcp_header_offset
= skb_transport_offset(skb
);
646 lso_info
->lso_v2_transmit
.mss
= skb_shinfo(skb
)->gso_size
;
647 } else if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
648 if (net_checksum_info(skb
) & net_device_ctx
->tx_checksum_mask
) {
649 struct ndis_tcp_ip_checksum_info
*csum_info
;
651 rndis_msg_size
+= NDIS_CSUM_PPI_SIZE
;
652 csum_info
= init_ppi_data(rndis_msg
, NDIS_CSUM_PPI_SIZE
,
653 TCPIP_CHKSUM_PKTINFO
);
655 csum_info
->value
= 0;
656 csum_info
->transmit
.tcp_header_offset
= skb_transport_offset(skb
);
658 if (skb
->protocol
== htons(ETH_P_IP
)) {
659 csum_info
->transmit
.is_ipv4
= 1;
661 if (ip_hdr(skb
)->protocol
== IPPROTO_TCP
)
662 csum_info
->transmit
.tcp_checksum
= 1;
664 csum_info
->transmit
.udp_checksum
= 1;
666 csum_info
->transmit
.is_ipv6
= 1;
668 if (ipv6_hdr(skb
)->nexthdr
== IPPROTO_TCP
)
669 csum_info
->transmit
.tcp_checksum
= 1;
671 csum_info
->transmit
.udp_checksum
= 1;
674 /* Can't do offload of this type of checksum */
675 if (skb_checksum_help(skb
))
680 /* Start filling in the page buffers with the rndis hdr */
681 rndis_msg
->msg_len
+= rndis_msg_size
;
682 packet
->total_data_buflen
= rndis_msg
->msg_len
;
683 packet
->page_buf_cnt
= init_page_array(rndis_msg
, rndis_msg_size
,
686 /* timestamp packet in software */
687 skb_tx_timestamp(skb
);
689 ret
= netvsc_send(net
, packet
, rndis_msg
, pb
, skb
);
690 if (likely(ret
== 0))
693 if (ret
== -EAGAIN
) {
694 ++net_device_ctx
->eth_stats
.tx_busy
;
695 return NETDEV_TX_BUSY
;
699 ++net_device_ctx
->eth_stats
.tx_no_space
;
702 dev_kfree_skb_any(skb
);
703 net
->stats
.tx_dropped
++;
708 ++net_device_ctx
->eth_stats
.tx_no_memory
;
713 * netvsc_linkstatus_callback - Link up/down notification
715 void netvsc_linkstatus_callback(struct net_device
*net
,
716 struct rndis_message
*resp
)
718 struct rndis_indicate_status
*indicate
= &resp
->msg
.indicate_status
;
719 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
720 struct netvsc_reconfig
*event
;
723 /* Update the physical link speed when changing to another vSwitch */
724 if (indicate
->status
== RNDIS_STATUS_LINK_SPEED_CHANGE
) {
727 speed
= *(u32
*)((void *)indicate
728 + indicate
->status_buf_offset
) / 10000;
729 ndev_ctx
->speed
= speed
;
733 /* Handle these link change statuses below */
734 if (indicate
->status
!= RNDIS_STATUS_NETWORK_CHANGE
&&
735 indicate
->status
!= RNDIS_STATUS_MEDIA_CONNECT
&&
736 indicate
->status
!= RNDIS_STATUS_MEDIA_DISCONNECT
)
739 if (net
->reg_state
!= NETREG_REGISTERED
)
742 event
= kzalloc(sizeof(*event
), GFP_ATOMIC
);
745 event
->event
= indicate
->status
;
747 spin_lock_irqsave(&ndev_ctx
->lock
, flags
);
748 list_add_tail(&event
->list
, &ndev_ctx
->reconfig_events
);
749 spin_unlock_irqrestore(&ndev_ctx
->lock
, flags
);
751 schedule_delayed_work(&ndev_ctx
->dwork
, 0);
754 static void netvsc_comp_ipcsum(struct sk_buff
*skb
)
756 struct iphdr
*iph
= (struct iphdr
*)skb
->data
;
759 iph
->check
= ip_fast_csum(iph
, iph
->ihl
);
762 static struct sk_buff
*netvsc_alloc_recv_skb(struct net_device
*net
,
763 struct netvsc_channel
*nvchan
)
765 struct napi_struct
*napi
= &nvchan
->napi
;
766 const struct ndis_pkt_8021q_info
*vlan
= nvchan
->rsc
.vlan
;
767 const struct ndis_tcp_ip_checksum_info
*csum_info
=
768 nvchan
->rsc
.csum_info
;
769 const u32
*hash_info
= nvchan
->rsc
.hash_info
;
773 skb
= napi_alloc_skb(napi
, nvchan
->rsc
.pktlen
);
778 * Copy to skb. This copy is needed here since the memory pointed by
779 * hv_netvsc_packet cannot be deallocated
781 for (i
= 0; i
< nvchan
->rsc
.cnt
; i
++)
782 skb_put_data(skb
, nvchan
->rsc
.data
[i
], nvchan
->rsc
.len
[i
]);
784 skb
->protocol
= eth_type_trans(skb
, net
);
786 /* skb is already created with CHECKSUM_NONE */
787 skb_checksum_none_assert(skb
);
789 /* Incoming packets may have IP header checksum verified by the host.
790 * They may not have IP header checksum computed after coalescing.
791 * We compute it here if the flags are set, because on Linux, the IP
792 * checksum is always checked.
794 if (csum_info
&& csum_info
->receive
.ip_checksum_value_invalid
&&
795 csum_info
->receive
.ip_checksum_succeeded
&&
796 skb
->protocol
== htons(ETH_P_IP
))
797 netvsc_comp_ipcsum(skb
);
799 /* Do L4 checksum offload if enabled and present. */
800 if (csum_info
&& (net
->features
& NETIF_F_RXCSUM
)) {
801 if (csum_info
->receive
.tcp_checksum_succeeded
||
802 csum_info
->receive
.udp_checksum_succeeded
)
803 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
806 if (hash_info
&& (net
->features
& NETIF_F_RXHASH
))
807 skb_set_hash(skb
, *hash_info
, PKT_HASH_TYPE_L4
);
810 u16 vlan_tci
= vlan
->vlanid
| (vlan
->pri
<< VLAN_PRIO_SHIFT
) |
811 (vlan
->cfi
? VLAN_CFI_MASK
: 0);
813 __vlan_hwaccel_put_tag(skb
, htons(ETH_P_8021Q
),
821 * netvsc_recv_callback - Callback when we receive a packet from the
822 * "wire" on the specified device.
824 int netvsc_recv_callback(struct net_device
*net
,
825 struct netvsc_device
*net_device
,
826 struct netvsc_channel
*nvchan
)
828 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
829 struct vmbus_channel
*channel
= nvchan
->channel
;
830 u16 q_idx
= channel
->offermsg
.offer
.sub_channel_index
;
832 struct netvsc_stats
*rx_stats
;
834 if (net
->reg_state
!= NETREG_REGISTERED
)
835 return NVSP_STAT_FAIL
;
837 /* Allocate a skb - TODO direct I/O to pages? */
838 skb
= netvsc_alloc_recv_skb(net
, nvchan
);
840 if (unlikely(!skb
)) {
841 ++net_device_ctx
->eth_stats
.rx_no_memory
;
842 return NVSP_STAT_FAIL
;
845 skb_record_rx_queue(skb
, q_idx
);
848 * Even if injecting the packet, record the statistics
849 * on the synthetic device because modifying the VF device
850 * statistics will not work correctly.
852 rx_stats
= &nvchan
->rx_stats
;
853 u64_stats_update_begin(&rx_stats
->syncp
);
855 rx_stats
->bytes
+= nvchan
->rsc
.pktlen
;
857 if (skb
->pkt_type
== PACKET_BROADCAST
)
858 ++rx_stats
->broadcast
;
859 else if (skb
->pkt_type
== PACKET_MULTICAST
)
860 ++rx_stats
->multicast
;
861 u64_stats_update_end(&rx_stats
->syncp
);
863 napi_gro_receive(&nvchan
->napi
, skb
);
864 return NVSP_STAT_SUCCESS
;
867 static void netvsc_get_drvinfo(struct net_device
*net
,
868 struct ethtool_drvinfo
*info
)
870 strlcpy(info
->driver
, KBUILD_MODNAME
, sizeof(info
->driver
));
871 strlcpy(info
->fw_version
, "N/A", sizeof(info
->fw_version
));
874 static void netvsc_get_channels(struct net_device
*net
,
875 struct ethtool_channels
*channel
)
877 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
878 struct netvsc_device
*nvdev
= rtnl_dereference(net_device_ctx
->nvdev
);
881 channel
->max_combined
= nvdev
->max_chn
;
882 channel
->combined_count
= nvdev
->num_chn
;
886 /* Alloc struct netvsc_device_info, and initialize it from either existing
887 * struct netvsc_device, or from default values.
889 static struct netvsc_device_info
*netvsc_devinfo_get
890 (struct netvsc_device
*nvdev
)
892 struct netvsc_device_info
*dev_info
;
894 dev_info
= kzalloc(sizeof(*dev_info
), GFP_ATOMIC
);
900 dev_info
->num_chn
= nvdev
->num_chn
;
901 dev_info
->send_sections
= nvdev
->send_section_cnt
;
902 dev_info
->send_section_size
= nvdev
->send_section_size
;
903 dev_info
->recv_sections
= nvdev
->recv_section_cnt
;
904 dev_info
->recv_section_size
= nvdev
->recv_section_size
;
906 memcpy(dev_info
->rss_key
, nvdev
->extension
->rss_key
,
909 dev_info
->num_chn
= VRSS_CHANNEL_DEFAULT
;
910 dev_info
->send_sections
= NETVSC_DEFAULT_TX
;
911 dev_info
->send_section_size
= NETVSC_SEND_SECTION_SIZE
;
912 dev_info
->recv_sections
= NETVSC_DEFAULT_RX
;
913 dev_info
->recv_section_size
= NETVSC_RECV_SECTION_SIZE
;
919 static int netvsc_detach(struct net_device
*ndev
,
920 struct netvsc_device
*nvdev
)
922 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
923 struct hv_device
*hdev
= ndev_ctx
->device_ctx
;
926 /* Don't try continuing to try and setup sub channels */
927 if (cancel_work_sync(&nvdev
->subchan_work
))
930 /* If device was up (receiving) then shutdown */
931 if (netif_running(ndev
)) {
932 netvsc_tx_disable(nvdev
, ndev
);
934 ret
= rndis_filter_close(nvdev
);
937 "unable to close device (ret %d).\n", ret
);
941 ret
= netvsc_wait_until_empty(nvdev
);
944 "Ring buffer not empty after closing rndis\n");
949 netif_device_detach(ndev
);
951 rndis_filter_device_remove(hdev
, nvdev
);
956 static int netvsc_attach(struct net_device
*ndev
,
957 struct netvsc_device_info
*dev_info
)
959 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
960 struct hv_device
*hdev
= ndev_ctx
->device_ctx
;
961 struct netvsc_device
*nvdev
;
962 struct rndis_device
*rdev
;
965 nvdev
= rndis_filter_device_add(hdev
, dev_info
);
967 return PTR_ERR(nvdev
);
969 if (nvdev
->num_chn
> 1) {
970 ret
= rndis_set_subchannel(ndev
, nvdev
, dev_info
);
972 /* if unavailable, just proceed with one queue */
979 /* In any case device is now ready */
980 netif_device_attach(ndev
);
982 /* Note: enable and attach happen when sub-channels setup */
983 netif_carrier_off(ndev
);
985 if (netif_running(ndev
)) {
986 ret
= rndis_filter_open(nvdev
);
990 rdev
= nvdev
->extension
;
991 if (!rdev
->link_state
)
992 netif_carrier_on(ndev
);
998 netif_device_detach(ndev
);
1000 rndis_filter_device_remove(hdev
, nvdev
);
1005 static int netvsc_set_channels(struct net_device
*net
,
1006 struct ethtool_channels
*channels
)
1008 struct net_device_context
*net_device_ctx
= netdev_priv(net
);
1009 struct netvsc_device
*nvdev
= rtnl_dereference(net_device_ctx
->nvdev
);
1010 unsigned int orig
, count
= channels
->combined_count
;
1011 struct netvsc_device_info
*device_info
;
1014 /* We do not support separate count for rx, tx, or other */
1016 channels
->rx_count
|| channels
->tx_count
|| channels
->other_count
)
1019 if (!nvdev
|| nvdev
->destroy
)
1022 if (nvdev
->nvsp_version
< NVSP_PROTOCOL_VERSION_5
)
1025 if (count
> nvdev
->max_chn
)
1028 orig
= nvdev
->num_chn
;
1030 device_info
= netvsc_devinfo_get(nvdev
);
1035 device_info
->num_chn
= count
;
1037 ret
= netvsc_detach(net
, nvdev
);
1041 ret
= netvsc_attach(net
, device_info
);
1043 device_info
->num_chn
= orig
;
1044 if (netvsc_attach(net
, device_info
))
1045 netdev_err(net
, "restoring channel setting failed\n");
1054 netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings
*cmd
)
1056 struct ethtool_link_ksettings diff1
= *cmd
;
1057 struct ethtool_link_ksettings diff2
= {};
1059 diff1
.base
.speed
= 0;
1060 diff1
.base
.duplex
= 0;
1061 /* advertising and cmd are usually set */
1062 ethtool_link_ksettings_zero_link_mode(&diff1
, advertising
);
1064 /* We set port to PORT_OTHER */
1065 diff2
.base
.port
= PORT_OTHER
;
1067 return !memcmp(&diff1
, &diff2
, sizeof(diff1
));
1070 static void netvsc_init_settings(struct net_device
*dev
)
1072 struct net_device_context
*ndc
= netdev_priv(dev
);
1074 ndc
->l4_hash
= HV_DEFAULT_L4HASH
;
1076 ndc
->speed
= SPEED_UNKNOWN
;
1077 ndc
->duplex
= DUPLEX_FULL
;
1079 dev
->features
= NETIF_F_LRO
;
1082 static int netvsc_get_link_ksettings(struct net_device
*dev
,
1083 struct ethtool_link_ksettings
*cmd
)
1085 struct net_device_context
*ndc
= netdev_priv(dev
);
1087 cmd
->base
.speed
= ndc
->speed
;
1088 cmd
->base
.duplex
= ndc
->duplex
;
1089 cmd
->base
.port
= PORT_OTHER
;
1094 static int netvsc_set_link_ksettings(struct net_device
*dev
,
1095 const struct ethtool_link_ksettings
*cmd
)
1097 struct net_device_context
*ndc
= netdev_priv(dev
);
1100 speed
= cmd
->base
.speed
;
1101 if (!ethtool_validate_speed(speed
) ||
1102 !ethtool_validate_duplex(cmd
->base
.duplex
) ||
1103 !netvsc_validate_ethtool_ss_cmd(cmd
))
1107 ndc
->duplex
= cmd
->base
.duplex
;
1112 static int netvsc_change_mtu(struct net_device
*ndev
, int mtu
)
1114 struct net_device_context
*ndevctx
= netdev_priv(ndev
);
1115 struct net_device
*vf_netdev
= rtnl_dereference(ndevctx
->vf_netdev
);
1116 struct netvsc_device
*nvdev
= rtnl_dereference(ndevctx
->nvdev
);
1117 int orig_mtu
= ndev
->mtu
;
1118 struct netvsc_device_info
*device_info
;
1121 if (!nvdev
|| nvdev
->destroy
)
1124 device_info
= netvsc_devinfo_get(nvdev
);
1129 /* Change MTU of underlying VF netdev first. */
1131 ret
= dev_set_mtu(vf_netdev
, mtu
);
1136 ret
= netvsc_detach(ndev
, nvdev
);
1142 ret
= netvsc_attach(ndev
, device_info
);
1146 /* Attempt rollback to original MTU */
1147 ndev
->mtu
= orig_mtu
;
1149 if (netvsc_attach(ndev
, device_info
))
1150 netdev_err(ndev
, "restoring mtu failed\n");
1153 dev_set_mtu(vf_netdev
, orig_mtu
);
1160 static void netvsc_get_vf_stats(struct net_device
*net
,
1161 struct netvsc_vf_pcpu_stats
*tot
)
1163 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
1166 memset(tot
, 0, sizeof(*tot
));
1168 for_each_possible_cpu(i
) {
1169 const struct netvsc_vf_pcpu_stats
*stats
1170 = per_cpu_ptr(ndev_ctx
->vf_stats
, i
);
1171 u64 rx_packets
, rx_bytes
, tx_packets
, tx_bytes
;
1175 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1176 rx_packets
= stats
->rx_packets
;
1177 tx_packets
= stats
->tx_packets
;
1178 rx_bytes
= stats
->rx_bytes
;
1179 tx_bytes
= stats
->tx_bytes
;
1180 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1182 tot
->rx_packets
+= rx_packets
;
1183 tot
->tx_packets
+= tx_packets
;
1184 tot
->rx_bytes
+= rx_bytes
;
1185 tot
->tx_bytes
+= tx_bytes
;
1186 tot
->tx_dropped
+= stats
->tx_dropped
;
1190 static void netvsc_get_pcpu_stats(struct net_device
*net
,
1191 struct netvsc_ethtool_pcpu_stats
*pcpu_tot
)
1193 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
1194 struct netvsc_device
*nvdev
= rcu_dereference_rtnl(ndev_ctx
->nvdev
);
1197 /* fetch percpu stats of vf */
1198 for_each_possible_cpu(i
) {
1199 const struct netvsc_vf_pcpu_stats
*stats
=
1200 per_cpu_ptr(ndev_ctx
->vf_stats
, i
);
1201 struct netvsc_ethtool_pcpu_stats
*this_tot
= &pcpu_tot
[i
];
1205 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1206 this_tot
->vf_rx_packets
= stats
->rx_packets
;
1207 this_tot
->vf_tx_packets
= stats
->tx_packets
;
1208 this_tot
->vf_rx_bytes
= stats
->rx_bytes
;
1209 this_tot
->vf_tx_bytes
= stats
->tx_bytes
;
1210 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1211 this_tot
->rx_packets
= this_tot
->vf_rx_packets
;
1212 this_tot
->tx_packets
= this_tot
->vf_tx_packets
;
1213 this_tot
->rx_bytes
= this_tot
->vf_rx_bytes
;
1214 this_tot
->tx_bytes
= this_tot
->vf_tx_bytes
;
1217 /* fetch percpu stats of netvsc */
1218 for (i
= 0; i
< nvdev
->num_chn
; i
++) {
1219 const struct netvsc_channel
*nvchan
= &nvdev
->chan_table
[i
];
1220 const struct netvsc_stats
*stats
;
1221 struct netvsc_ethtool_pcpu_stats
*this_tot
=
1222 &pcpu_tot
[nvchan
->channel
->target_cpu
];
1226 stats
= &nvchan
->tx_stats
;
1228 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1229 packets
= stats
->packets
;
1230 bytes
= stats
->bytes
;
1231 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1233 this_tot
->tx_bytes
+= bytes
;
1234 this_tot
->tx_packets
+= packets
;
1236 stats
= &nvchan
->rx_stats
;
1238 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1239 packets
= stats
->packets
;
1240 bytes
= stats
->bytes
;
1241 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1243 this_tot
->rx_bytes
+= bytes
;
1244 this_tot
->rx_packets
+= packets
;
1248 static void netvsc_get_stats64(struct net_device
*net
,
1249 struct rtnl_link_stats64
*t
)
1251 struct net_device_context
*ndev_ctx
= netdev_priv(net
);
1252 struct netvsc_device
*nvdev
;
1253 struct netvsc_vf_pcpu_stats vf_tot
;
1258 nvdev
= rcu_dereference(ndev_ctx
->nvdev
);
1262 netdev_stats_to_stats64(t
, &net
->stats
);
1264 netvsc_get_vf_stats(net
, &vf_tot
);
1265 t
->rx_packets
+= vf_tot
.rx_packets
;
1266 t
->tx_packets
+= vf_tot
.tx_packets
;
1267 t
->rx_bytes
+= vf_tot
.rx_bytes
;
1268 t
->tx_bytes
+= vf_tot
.tx_bytes
;
1269 t
->tx_dropped
+= vf_tot
.tx_dropped
;
1271 for (i
= 0; i
< nvdev
->num_chn
; i
++) {
1272 const struct netvsc_channel
*nvchan
= &nvdev
->chan_table
[i
];
1273 const struct netvsc_stats
*stats
;
1274 u64 packets
, bytes
, multicast
;
1277 stats
= &nvchan
->tx_stats
;
1279 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1280 packets
= stats
->packets
;
1281 bytes
= stats
->bytes
;
1282 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1284 t
->tx_bytes
+= bytes
;
1285 t
->tx_packets
+= packets
;
1287 stats
= &nvchan
->rx_stats
;
1289 start
= u64_stats_fetch_begin_irq(&stats
->syncp
);
1290 packets
= stats
->packets
;
1291 bytes
= stats
->bytes
;
1292 multicast
= stats
->multicast
+ stats
->broadcast
;
1293 } while (u64_stats_fetch_retry_irq(&stats
->syncp
, start
));
1295 t
->rx_bytes
+= bytes
;
1296 t
->rx_packets
+= packets
;
1297 t
->multicast
+= multicast
;
1303 static int netvsc_set_mac_addr(struct net_device
*ndev
, void *p
)
1305 struct net_device_context
*ndc
= netdev_priv(ndev
);
1306 struct net_device
*vf_netdev
= rtnl_dereference(ndc
->vf_netdev
);
1307 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1308 struct sockaddr
*addr
= p
;
1311 err
= eth_prepare_mac_addr_change(ndev
, p
);
1319 err
= dev_set_mac_address(vf_netdev
, addr
, NULL
);
1324 err
= rndis_filter_set_device_mac(nvdev
, addr
->sa_data
);
1326 eth_commit_mac_addr_change(ndev
, p
);
1327 } else if (vf_netdev
) {
1328 /* rollback change on VF */
1329 memcpy(addr
->sa_data
, ndev
->dev_addr
, ETH_ALEN
);
1330 dev_set_mac_address(vf_netdev
, addr
, NULL
);
1336 static const struct {
1337 char name
[ETH_GSTRING_LEN
];
1339 } netvsc_stats
[] = {
1340 { "tx_scattered", offsetof(struct netvsc_ethtool_stats
, tx_scattered
) },
1341 { "tx_no_memory", offsetof(struct netvsc_ethtool_stats
, tx_no_memory
) },
1342 { "tx_no_space", offsetof(struct netvsc_ethtool_stats
, tx_no_space
) },
1343 { "tx_too_big", offsetof(struct netvsc_ethtool_stats
, tx_too_big
) },
1344 { "tx_busy", offsetof(struct netvsc_ethtool_stats
, tx_busy
) },
1345 { "tx_send_full", offsetof(struct netvsc_ethtool_stats
, tx_send_full
) },
1346 { "rx_comp_busy", offsetof(struct netvsc_ethtool_stats
, rx_comp_busy
) },
1347 { "rx_no_memory", offsetof(struct netvsc_ethtool_stats
, rx_no_memory
) },
1348 { "stop_queue", offsetof(struct netvsc_ethtool_stats
, stop_queue
) },
1349 { "wake_queue", offsetof(struct netvsc_ethtool_stats
, wake_queue
) },
1351 { "cpu%u_rx_packets",
1352 offsetof(struct netvsc_ethtool_pcpu_stats
, rx_packets
) },
1354 offsetof(struct netvsc_ethtool_pcpu_stats
, rx_bytes
) },
1355 { "cpu%u_tx_packets",
1356 offsetof(struct netvsc_ethtool_pcpu_stats
, tx_packets
) },
1358 offsetof(struct netvsc_ethtool_pcpu_stats
, tx_bytes
) },
1359 { "cpu%u_vf_rx_packets",
1360 offsetof(struct netvsc_ethtool_pcpu_stats
, vf_rx_packets
) },
1361 { "cpu%u_vf_rx_bytes",
1362 offsetof(struct netvsc_ethtool_pcpu_stats
, vf_rx_bytes
) },
1363 { "cpu%u_vf_tx_packets",
1364 offsetof(struct netvsc_ethtool_pcpu_stats
, vf_tx_packets
) },
1365 { "cpu%u_vf_tx_bytes",
1366 offsetof(struct netvsc_ethtool_pcpu_stats
, vf_tx_bytes
) },
1368 { "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats
, rx_packets
) },
1369 { "vf_rx_bytes", offsetof(struct netvsc_vf_pcpu_stats
, rx_bytes
) },
1370 { "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats
, tx_packets
) },
1371 { "vf_tx_bytes", offsetof(struct netvsc_vf_pcpu_stats
, tx_bytes
) },
1372 { "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats
, tx_dropped
) },
1375 #define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats)
1376 #define NETVSC_VF_STATS_LEN ARRAY_SIZE(vf_stats)
1378 /* statistics per queue (rx/tx packets/bytes) */
1379 #define NETVSC_PCPU_STATS_LEN (num_present_cpus() * ARRAY_SIZE(pcpu_stats))
1381 /* 4 statistics per queue (rx/tx packets/bytes) */
1382 #define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1384 static int netvsc_get_sset_count(struct net_device
*dev
, int string_set
)
1386 struct net_device_context
*ndc
= netdev_priv(dev
);
1387 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1392 switch (string_set
) {
1394 return NETVSC_GLOBAL_STATS_LEN
1395 + NETVSC_VF_STATS_LEN
1396 + NETVSC_QUEUE_STATS_LEN(nvdev
)
1397 + NETVSC_PCPU_STATS_LEN
;
1403 static void netvsc_get_ethtool_stats(struct net_device
*dev
,
1404 struct ethtool_stats
*stats
, u64
*data
)
1406 struct net_device_context
*ndc
= netdev_priv(dev
);
1407 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1408 const void *nds
= &ndc
->eth_stats
;
1409 const struct netvsc_stats
*qstats
;
1410 struct netvsc_vf_pcpu_stats sum
;
1411 struct netvsc_ethtool_pcpu_stats
*pcpu_sum
;
1419 for (i
= 0; i
< NETVSC_GLOBAL_STATS_LEN
; i
++)
1420 data
[i
] = *(unsigned long *)(nds
+ netvsc_stats
[i
].offset
);
1422 netvsc_get_vf_stats(dev
, &sum
);
1423 for (j
= 0; j
< NETVSC_VF_STATS_LEN
; j
++)
1424 data
[i
++] = *(u64
*)((void *)&sum
+ vf_stats
[j
].offset
);
1426 for (j
= 0; j
< nvdev
->num_chn
; j
++) {
1427 qstats
= &nvdev
->chan_table
[j
].tx_stats
;
1430 start
= u64_stats_fetch_begin_irq(&qstats
->syncp
);
1431 packets
= qstats
->packets
;
1432 bytes
= qstats
->bytes
;
1433 } while (u64_stats_fetch_retry_irq(&qstats
->syncp
, start
));
1434 data
[i
++] = packets
;
1437 qstats
= &nvdev
->chan_table
[j
].rx_stats
;
1439 start
= u64_stats_fetch_begin_irq(&qstats
->syncp
);
1440 packets
= qstats
->packets
;
1441 bytes
= qstats
->bytes
;
1442 } while (u64_stats_fetch_retry_irq(&qstats
->syncp
, start
));
1443 data
[i
++] = packets
;
1447 pcpu_sum
= kvmalloc_array(num_possible_cpus(),
1448 sizeof(struct netvsc_ethtool_pcpu_stats
),
1450 netvsc_get_pcpu_stats(dev
, pcpu_sum
);
1451 for_each_present_cpu(cpu
) {
1452 struct netvsc_ethtool_pcpu_stats
*this_sum
= &pcpu_sum
[cpu
];
1454 for (j
= 0; j
< ARRAY_SIZE(pcpu_stats
); j
++)
1455 data
[i
++] = *(u64
*)((void *)this_sum
1456 + pcpu_stats
[j
].offset
);
1461 static void netvsc_get_strings(struct net_device
*dev
, u32 stringset
, u8
*data
)
1463 struct net_device_context
*ndc
= netdev_priv(dev
);
1464 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1471 switch (stringset
) {
1473 for (i
= 0; i
< ARRAY_SIZE(netvsc_stats
); i
++) {
1474 memcpy(p
, netvsc_stats
[i
].name
, ETH_GSTRING_LEN
);
1475 p
+= ETH_GSTRING_LEN
;
1478 for (i
= 0; i
< ARRAY_SIZE(vf_stats
); i
++) {
1479 memcpy(p
, vf_stats
[i
].name
, ETH_GSTRING_LEN
);
1480 p
+= ETH_GSTRING_LEN
;
1483 for (i
= 0; i
< nvdev
->num_chn
; i
++) {
1484 sprintf(p
, "tx_queue_%u_packets", i
);
1485 p
+= ETH_GSTRING_LEN
;
1486 sprintf(p
, "tx_queue_%u_bytes", i
);
1487 p
+= ETH_GSTRING_LEN
;
1488 sprintf(p
, "rx_queue_%u_packets", i
);
1489 p
+= ETH_GSTRING_LEN
;
1490 sprintf(p
, "rx_queue_%u_bytes", i
);
1491 p
+= ETH_GSTRING_LEN
;
1494 for_each_present_cpu(cpu
) {
1495 for (i
= 0; i
< ARRAY_SIZE(pcpu_stats
); i
++) {
1496 sprintf(p
, pcpu_stats
[i
].name
, cpu
);
1497 p
+= ETH_GSTRING_LEN
;
1506 netvsc_get_rss_hash_opts(struct net_device_context
*ndc
,
1507 struct ethtool_rxnfc
*info
)
1509 const u32 l4_flag
= RXH_L4_B_0_1
| RXH_L4_B_2_3
;
1511 info
->data
= RXH_IP_SRC
| RXH_IP_DST
;
1513 switch (info
->flow_type
) {
1515 if (ndc
->l4_hash
& HV_TCP4_L4HASH
)
1516 info
->data
|= l4_flag
;
1521 if (ndc
->l4_hash
& HV_TCP6_L4HASH
)
1522 info
->data
|= l4_flag
;
1527 if (ndc
->l4_hash
& HV_UDP4_L4HASH
)
1528 info
->data
|= l4_flag
;
1533 if (ndc
->l4_hash
& HV_UDP6_L4HASH
)
1534 info
->data
|= l4_flag
;
1550 netvsc_get_rxnfc(struct net_device
*dev
, struct ethtool_rxnfc
*info
,
1553 struct net_device_context
*ndc
= netdev_priv(dev
);
1554 struct netvsc_device
*nvdev
= rtnl_dereference(ndc
->nvdev
);
1559 switch (info
->cmd
) {
1560 case ETHTOOL_GRXRINGS
:
1561 info
->data
= nvdev
->num_chn
;
1565 return netvsc_get_rss_hash_opts(ndc
, info
);
1570 static int netvsc_set_rss_hash_opts(struct net_device_context
*ndc
,
1571 struct ethtool_rxnfc
*info
)
1573 if (info
->data
== (RXH_IP_SRC
| RXH_IP_DST
|
1574 RXH_L4_B_0_1
| RXH_L4_B_2_3
)) {
1575 switch (info
->flow_type
) {
1577 ndc
->l4_hash
|= HV_TCP4_L4HASH
;
1581 ndc
->l4_hash
|= HV_TCP6_L4HASH
;
1585 ndc
->l4_hash
|= HV_UDP4_L4HASH
;
1589 ndc
->l4_hash
|= HV_UDP6_L4HASH
;
1599 if (info
->data
== (RXH_IP_SRC
| RXH_IP_DST
)) {
1600 switch (info
->flow_type
) {
1602 ndc
->l4_hash
&= ~HV_TCP4_L4HASH
;
1606 ndc
->l4_hash
&= ~HV_TCP6_L4HASH
;
1610 ndc
->l4_hash
&= ~HV_UDP4_L4HASH
;
1614 ndc
->l4_hash
&= ~HV_UDP6_L4HASH
;
1628 netvsc_set_rxnfc(struct net_device
*ndev
, struct ethtool_rxnfc
*info
)
1630 struct net_device_context
*ndc
= netdev_priv(ndev
);
1632 if (info
->cmd
== ETHTOOL_SRXFH
)
1633 return netvsc_set_rss_hash_opts(ndc
, info
);
1638 static u32
netvsc_get_rxfh_key_size(struct net_device
*dev
)
1640 return NETVSC_HASH_KEYLEN
;
1643 static u32
netvsc_rss_indir_size(struct net_device
*dev
)
1648 static int netvsc_get_rxfh(struct net_device
*dev
, u32
*indir
, u8
*key
,
1651 struct net_device_context
*ndc
= netdev_priv(dev
);
1652 struct netvsc_device
*ndev
= rtnl_dereference(ndc
->nvdev
);
1653 struct rndis_device
*rndis_dev
;
1660 *hfunc
= ETH_RSS_HASH_TOP
; /* Toeplitz */
1662 rndis_dev
= ndev
->extension
;
1664 for (i
= 0; i
< ITAB_NUM
; i
++)
1665 indir
[i
] = rndis_dev
->rx_table
[i
];
1669 memcpy(key
, rndis_dev
->rss_key
, NETVSC_HASH_KEYLEN
);
1674 static int netvsc_set_rxfh(struct net_device
*dev
, const u32
*indir
,
1675 const u8
*key
, const u8 hfunc
)
1677 struct net_device_context
*ndc
= netdev_priv(dev
);
1678 struct netvsc_device
*ndev
= rtnl_dereference(ndc
->nvdev
);
1679 struct rndis_device
*rndis_dev
;
1685 if (hfunc
!= ETH_RSS_HASH_NO_CHANGE
&& hfunc
!= ETH_RSS_HASH_TOP
)
1688 rndis_dev
= ndev
->extension
;
1690 for (i
= 0; i
< ITAB_NUM
; i
++)
1691 if (indir
[i
] >= ndev
->num_chn
)
1694 for (i
= 0; i
< ITAB_NUM
; i
++)
1695 rndis_dev
->rx_table
[i
] = indir
[i
];
1702 key
= rndis_dev
->rss_key
;
1705 return rndis_filter_set_rss_param(rndis_dev
, key
);
1708 /* Hyper-V RNDIS protocol does not have ring in the HW sense.
1709 * It does have pre-allocated receive area which is divided into sections.
1711 static void __netvsc_get_ringparam(struct netvsc_device
*nvdev
,
1712 struct ethtool_ringparam
*ring
)
1716 ring
->rx_pending
= nvdev
->recv_section_cnt
;
1717 ring
->tx_pending
= nvdev
->send_section_cnt
;
1719 if (nvdev
->nvsp_version
<= NVSP_PROTOCOL_VERSION_2
)
1720 max_buf_size
= NETVSC_RECEIVE_BUFFER_SIZE_LEGACY
;
1722 max_buf_size
= NETVSC_RECEIVE_BUFFER_SIZE
;
1724 ring
->rx_max_pending
= max_buf_size
/ nvdev
->recv_section_size
;
1725 ring
->tx_max_pending
= NETVSC_SEND_BUFFER_SIZE
1726 / nvdev
->send_section_size
;
1729 static void netvsc_get_ringparam(struct net_device
*ndev
,
1730 struct ethtool_ringparam
*ring
)
1732 struct net_device_context
*ndevctx
= netdev_priv(ndev
);
1733 struct netvsc_device
*nvdev
= rtnl_dereference(ndevctx
->nvdev
);
1738 __netvsc_get_ringparam(nvdev
, ring
);
1741 static int netvsc_set_ringparam(struct net_device
*ndev
,
1742 struct ethtool_ringparam
*ring
)
1744 struct net_device_context
*ndevctx
= netdev_priv(ndev
);
1745 struct netvsc_device
*nvdev
= rtnl_dereference(ndevctx
->nvdev
);
1746 struct netvsc_device_info
*device_info
;
1747 struct ethtool_ringparam orig
;
1751 if (!nvdev
|| nvdev
->destroy
)
1754 memset(&orig
, 0, sizeof(orig
));
1755 __netvsc_get_ringparam(nvdev
, &orig
);
1757 new_tx
= clamp_t(u32
, ring
->tx_pending
,
1758 NETVSC_MIN_TX_SECTIONS
, orig
.tx_max_pending
);
1759 new_rx
= clamp_t(u32
, ring
->rx_pending
,
1760 NETVSC_MIN_RX_SECTIONS
, orig
.rx_max_pending
);
1762 if (new_tx
== orig
.tx_pending
&&
1763 new_rx
== orig
.rx_pending
)
1764 return 0; /* no change */
1766 device_info
= netvsc_devinfo_get(nvdev
);
1771 device_info
->send_sections
= new_tx
;
1772 device_info
->recv_sections
= new_rx
;
1774 ret
= netvsc_detach(ndev
, nvdev
);
1778 ret
= netvsc_attach(ndev
, device_info
);
1780 device_info
->send_sections
= orig
.tx_pending
;
1781 device_info
->recv_sections
= orig
.rx_pending
;
1783 if (netvsc_attach(ndev
, device_info
))
1784 netdev_err(ndev
, "restoring ringparam failed");
1792 static int netvsc_set_features(struct net_device
*ndev
,
1793 netdev_features_t features
)
1795 netdev_features_t change
= features
^ ndev
->features
;
1796 struct net_device_context
*ndevctx
= netdev_priv(ndev
);
1797 struct netvsc_device
*nvdev
= rtnl_dereference(ndevctx
->nvdev
);
1798 struct net_device
*vf_netdev
= rtnl_dereference(ndevctx
->vf_netdev
);
1799 struct ndis_offload_params offloads
;
1802 if (!nvdev
|| nvdev
->destroy
)
1805 if (!(change
& NETIF_F_LRO
))
1808 memset(&offloads
, 0, sizeof(struct ndis_offload_params
));
1810 if (features
& NETIF_F_LRO
) {
1811 offloads
.rsc_ip_v4
= NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED
;
1812 offloads
.rsc_ip_v6
= NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED
;
1814 offloads
.rsc_ip_v4
= NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED
;
1815 offloads
.rsc_ip_v6
= NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED
;
1818 ret
= rndis_filter_set_offload_params(ndev
, nvdev
, &offloads
);
1821 features
^= NETIF_F_LRO
;
1822 ndev
->features
= features
;
1829 vf_netdev
->wanted_features
= features
;
1830 netdev_update_features(vf_netdev
);
1835 static u32
netvsc_get_msglevel(struct net_device
*ndev
)
1837 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
1839 return ndev_ctx
->msg_enable
;
1842 static void netvsc_set_msglevel(struct net_device
*ndev
, u32 val
)
1844 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
1846 ndev_ctx
->msg_enable
= val
;
1849 static const struct ethtool_ops ethtool_ops
= {
1850 .get_drvinfo
= netvsc_get_drvinfo
,
1851 .get_msglevel
= netvsc_get_msglevel
,
1852 .set_msglevel
= netvsc_set_msglevel
,
1853 .get_link
= ethtool_op_get_link
,
1854 .get_ethtool_stats
= netvsc_get_ethtool_stats
,
1855 .get_sset_count
= netvsc_get_sset_count
,
1856 .get_strings
= netvsc_get_strings
,
1857 .get_channels
= netvsc_get_channels
,
1858 .set_channels
= netvsc_set_channels
,
1859 .get_ts_info
= ethtool_op_get_ts_info
,
1860 .get_rxnfc
= netvsc_get_rxnfc
,
1861 .set_rxnfc
= netvsc_set_rxnfc
,
1862 .get_rxfh_key_size
= netvsc_get_rxfh_key_size
,
1863 .get_rxfh_indir_size
= netvsc_rss_indir_size
,
1864 .get_rxfh
= netvsc_get_rxfh
,
1865 .set_rxfh
= netvsc_set_rxfh
,
1866 .get_link_ksettings
= netvsc_get_link_ksettings
,
1867 .set_link_ksettings
= netvsc_set_link_ksettings
,
1868 .get_ringparam
= netvsc_get_ringparam
,
1869 .set_ringparam
= netvsc_set_ringparam
,
1872 static const struct net_device_ops device_ops
= {
1873 .ndo_open
= netvsc_open
,
1874 .ndo_stop
= netvsc_close
,
1875 .ndo_start_xmit
= netvsc_start_xmit
,
1876 .ndo_change_rx_flags
= netvsc_change_rx_flags
,
1877 .ndo_set_rx_mode
= netvsc_set_rx_mode
,
1878 .ndo_set_features
= netvsc_set_features
,
1879 .ndo_change_mtu
= netvsc_change_mtu
,
1880 .ndo_validate_addr
= eth_validate_addr
,
1881 .ndo_set_mac_address
= netvsc_set_mac_addr
,
1882 .ndo_select_queue
= netvsc_select_queue
,
1883 .ndo_get_stats64
= netvsc_get_stats64
,
1887 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1888 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1889 * present send GARP packet to network peers with netif_notify_peers().
1891 static void netvsc_link_change(struct work_struct
*w
)
1893 struct net_device_context
*ndev_ctx
=
1894 container_of(w
, struct net_device_context
, dwork
.work
);
1895 struct hv_device
*device_obj
= ndev_ctx
->device_ctx
;
1896 struct net_device
*net
= hv_get_drvdata(device_obj
);
1897 struct netvsc_device
*net_device
;
1898 struct rndis_device
*rdev
;
1899 struct netvsc_reconfig
*event
= NULL
;
1900 bool notify
= false, reschedule
= false;
1901 unsigned long flags
, next_reconfig
, delay
;
1903 /* if changes are happening, comeback later */
1904 if (!rtnl_trylock()) {
1905 schedule_delayed_work(&ndev_ctx
->dwork
, LINKCHANGE_INT
);
1909 net_device
= rtnl_dereference(ndev_ctx
->nvdev
);
1913 rdev
= net_device
->extension
;
1915 next_reconfig
= ndev_ctx
->last_reconfig
+ LINKCHANGE_INT
;
1916 if (time_is_after_jiffies(next_reconfig
)) {
1917 /* link_watch only sends one notification with current state
1918 * per second, avoid doing reconfig more frequently. Handle
1921 delay
= next_reconfig
- jiffies
;
1922 delay
= delay
< LINKCHANGE_INT
? delay
: LINKCHANGE_INT
;
1923 schedule_delayed_work(&ndev_ctx
->dwork
, delay
);
1926 ndev_ctx
->last_reconfig
= jiffies
;
1928 spin_lock_irqsave(&ndev_ctx
->lock
, flags
);
1929 if (!list_empty(&ndev_ctx
->reconfig_events
)) {
1930 event
= list_first_entry(&ndev_ctx
->reconfig_events
,
1931 struct netvsc_reconfig
, list
);
1932 list_del(&event
->list
);
1933 reschedule
= !list_empty(&ndev_ctx
->reconfig_events
);
1935 spin_unlock_irqrestore(&ndev_ctx
->lock
, flags
);
1940 switch (event
->event
) {
1941 /* Only the following events are possible due to the check in
1942 * netvsc_linkstatus_callback()
1944 case RNDIS_STATUS_MEDIA_CONNECT
:
1945 if (rdev
->link_state
) {
1946 rdev
->link_state
= false;
1947 netif_carrier_on(net
);
1948 netvsc_tx_enable(net_device
, net
);
1954 case RNDIS_STATUS_MEDIA_DISCONNECT
:
1955 if (!rdev
->link_state
) {
1956 rdev
->link_state
= true;
1957 netif_carrier_off(net
);
1958 netvsc_tx_disable(net_device
, net
);
1962 case RNDIS_STATUS_NETWORK_CHANGE
:
1963 /* Only makes sense if carrier is present */
1964 if (!rdev
->link_state
) {
1965 rdev
->link_state
= true;
1966 netif_carrier_off(net
);
1967 netvsc_tx_disable(net_device
, net
);
1968 event
->event
= RNDIS_STATUS_MEDIA_CONNECT
;
1969 spin_lock_irqsave(&ndev_ctx
->lock
, flags
);
1970 list_add(&event
->list
, &ndev_ctx
->reconfig_events
);
1971 spin_unlock_irqrestore(&ndev_ctx
->lock
, flags
);
1980 netdev_notify_peers(net
);
1982 /* link_watch only sends one notification with current state per
1983 * second, handle next reconfig event in 2 seconds.
1986 schedule_delayed_work(&ndev_ctx
->dwork
, LINKCHANGE_INT
);
1994 static struct net_device
*get_netvsc_byref(struct net_device
*vf_netdev
)
1996 struct net_device_context
*net_device_ctx
;
1997 struct net_device
*dev
;
1999 dev
= netdev_master_upper_dev_get(vf_netdev
);
2000 if (!dev
|| dev
->netdev_ops
!= &device_ops
)
2001 return NULL
; /* not a netvsc device */
2003 net_device_ctx
= netdev_priv(dev
);
2004 if (!rtnl_dereference(net_device_ctx
->nvdev
))
2005 return NULL
; /* device is removed */
2010 /* Called when VF is injecting data into network stack.
2011 * Change the associated network device from VF to netvsc.
2012 * note: already called with rcu_read_lock
2014 static rx_handler_result_t
netvsc_vf_handle_frame(struct sk_buff
**pskb
)
2016 struct sk_buff
*skb
= *pskb
;
2017 struct net_device
*ndev
= rcu_dereference(skb
->dev
->rx_handler_data
);
2018 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
2019 struct netvsc_vf_pcpu_stats
*pcpu_stats
2020 = this_cpu_ptr(ndev_ctx
->vf_stats
);
2022 skb
= skb_share_check(skb
, GFP_ATOMIC
);
2024 return RX_HANDLER_CONSUMED
;
2030 u64_stats_update_begin(&pcpu_stats
->syncp
);
2031 pcpu_stats
->rx_packets
++;
2032 pcpu_stats
->rx_bytes
+= skb
->len
;
2033 u64_stats_update_end(&pcpu_stats
->syncp
);
2035 return RX_HANDLER_ANOTHER
;
2038 static int netvsc_vf_join(struct net_device
*vf_netdev
,
2039 struct net_device
*ndev
)
2041 struct net_device_context
*ndev_ctx
= netdev_priv(ndev
);
2044 ret
= netdev_rx_handler_register(vf_netdev
,
2045 netvsc_vf_handle_frame
, ndev
);
2047 netdev_err(vf_netdev
,
2048 "can not register netvsc VF receive handler (err = %d)\n",
2050 goto rx_handler_failed
;
2053 ret
= netdev_master_upper_dev_link(vf_netdev
, ndev
,
2056 netdev_err(vf_netdev
,
2057 "can not set master device %s (err = %d)\n",
2059 goto upper_link_failed
;
2062 /* set slave flag before open to prevent IPv6 addrconf */
2063 vf_netdev
->flags
|= IFF_SLAVE
;
2065 schedule_delayed_work(&ndev_ctx
->vf_takeover
, VF_TAKEOVER_INT
);
2067 call_netdevice_notifiers(NETDEV_JOIN
, vf_netdev
);
2069 netdev_info(vf_netdev
, "joined to %s\n", ndev
->name
);
2073 netdev_rx_handler_unregister(vf_netdev
);
2078 static void __netvsc_vf_setup(struct net_device
*ndev
,
2079 struct net_device
*vf_netdev
)
2083 /* Align MTU of VF with master */
2084 ret
= dev_set_mtu(vf_netdev
, ndev
->mtu
);
2086 netdev_warn(vf_netdev
,
2087 "unable to change mtu to %u\n", ndev
->mtu
);
2089 /* set multicast etc flags on VF */
2090 dev_change_flags(vf_netdev
, ndev
->flags
| IFF_SLAVE
, NULL
);
2092 /* sync address list from ndev to VF */
2093 netif_addr_lock_bh(ndev
);
2094 dev_uc_sync(vf_netdev
, ndev
);
2095 dev_mc_sync(vf_netdev
, ndev
);
2096 netif_addr_unlock_bh(ndev
);
2098 if (netif_running(ndev
)) {
2099 ret
= dev_open(vf_netdev
, NULL
);
2101 netdev_warn(vf_netdev
,
2102 "unable to open: %d\n", ret
);
2106 /* Setup VF as slave of the synthetic device.
2107 * Runs in workqueue to avoid recursion in netlink callbacks.
2109 static void netvsc_vf_setup(struct work_struct
*w
)
2111 struct net_device_context
*ndev_ctx
2112 = container_of(w
, struct net_device_context
, vf_takeover
.work
);
2113 struct net_device
*ndev
= hv_get_drvdata(ndev_ctx
->device_ctx
);
2114 struct net_device
*vf_netdev
;
2116 if (!rtnl_trylock()) {
2117 schedule_delayed_work(&ndev_ctx
->vf_takeover
, 0);
2121 vf_netdev
= rtnl_dereference(ndev_ctx
->vf_netdev
);
2123 __netvsc_vf_setup(ndev
, vf_netdev
);
2128 /* Find netvsc by VF serial number.
2129 * The PCI hyperv controller records the serial number as the slot kobj name.
2131 static struct net_device
*get_netvsc_byslot(const struct net_device
*vf_netdev
)
2133 struct device
*parent
= vf_netdev
->dev
.parent
;
2134 struct net_device_context
*ndev_ctx
;
2135 struct pci_dev
*pdev
;
2138 if (!parent
|| !dev_is_pci(parent
))
2139 return NULL
; /* not a PCI device */
2141 pdev
= to_pci_dev(parent
);
2143 netdev_notice(vf_netdev
, "no PCI slot information\n");
2147 if (kstrtou32(pci_slot_name(pdev
->slot
), 10, &serial
)) {
2148 netdev_notice(vf_netdev
, "Invalid vf serial:%s\n",
2149 pci_slot_name(pdev
->slot
));
2153 list_for_each_entry(ndev_ctx
, &netvsc_dev_list
, list
) {
2154 if (!ndev_ctx
->vf_alloc
)
2157 if (ndev_ctx
->vf_serial
== serial
)
2158 return hv_get_drvdata(ndev_ctx
->device_ctx
);
2161 netdev_notice(vf_netdev
,
2162 "no netdev found for vf serial:%u\n", serial
);
2166 static int netvsc_register_vf(struct net_device
*vf_netdev
)
2168 struct net_device_context
*net_device_ctx
;
2169 struct netvsc_device
*netvsc_dev
;
2170 struct net_device
*ndev
;
2173 if (vf_netdev
->addr_len
!= ETH_ALEN
)
2176 ndev
= get_netvsc_byslot(vf_netdev
);
2180 net_device_ctx
= netdev_priv(ndev
);
2181 netvsc_dev
= rtnl_dereference(net_device_ctx
->nvdev
);
2182 if (!netvsc_dev
|| rtnl_dereference(net_device_ctx
->vf_netdev
))
2185 /* if synthetic interface is a different namespace,
2186 * then move the VF to that namespace; join will be
2187 * done again in that context.
2189 if (!net_eq(dev_net(ndev
), dev_net(vf_netdev
))) {
2190 ret
= dev_change_net_namespace(vf_netdev
,
2191 dev_net(ndev
), "eth%d");
2193 netdev_err(vf_netdev
,
2194 "could not move to same namespace as %s: %d\n",
2197 netdev_info(vf_netdev
,
2198 "VF moved to namespace with: %s\n",
2203 netdev_info(ndev
, "VF registering: %s\n", vf_netdev
->name
);
2205 if (netvsc_vf_join(vf_netdev
, ndev
) != 0)
2208 dev_hold(vf_netdev
);
2209 rcu_assign_pointer(net_device_ctx
->vf_netdev
, vf_netdev
);
2211 vf_netdev
->wanted_features
= ndev
->features
;
2212 netdev_update_features(vf_netdev
);
2217 /* VF up/down change detected, schedule to change data path */
2218 static int netvsc_vf_changed(struct net_device
*vf_netdev
)
2220 struct net_device_context
*net_device_ctx
;
2221 struct netvsc_device
*netvsc_dev
;
2222 struct net_device
*ndev
;
2223 bool vf_is_up
= netif_running(vf_netdev
);
2225 ndev
= get_netvsc_byref(vf_netdev
);
2229 net_device_ctx
= netdev_priv(ndev
);
2230 netvsc_dev
= rtnl_dereference(net_device_ctx
->nvdev
);
2234 netvsc_switch_datapath(ndev
, vf_is_up
);
2235 netdev_info(ndev
, "Data path switched %s VF: %s\n",
2236 vf_is_up
? "to" : "from", vf_netdev
->name
);
2241 static int netvsc_unregister_vf(struct net_device
*vf_netdev
)
2243 struct net_device
*ndev
;
2244 struct net_device_context
*net_device_ctx
;
2246 ndev
= get_netvsc_byref(vf_netdev
);
2250 net_device_ctx
= netdev_priv(ndev
);
2251 cancel_delayed_work_sync(&net_device_ctx
->vf_takeover
);
2253 netdev_info(ndev
, "VF unregistering: %s\n", vf_netdev
->name
);
2255 netdev_rx_handler_unregister(vf_netdev
);
2256 netdev_upper_dev_unlink(vf_netdev
, ndev
);
2257 RCU_INIT_POINTER(net_device_ctx
->vf_netdev
, NULL
);
2263 static int netvsc_probe(struct hv_device
*dev
,
2264 const struct hv_vmbus_device_id
*dev_id
)
2266 struct net_device
*net
= NULL
;
2267 struct net_device_context
*net_device_ctx
;
2268 struct netvsc_device_info
*device_info
= NULL
;
2269 struct netvsc_device
*nvdev
;
2272 net
= alloc_etherdev_mq(sizeof(struct net_device_context
),
2277 netif_carrier_off(net
);
2279 netvsc_init_settings(net
);
2281 net_device_ctx
= netdev_priv(net
);
2282 net_device_ctx
->device_ctx
= dev
;
2283 net_device_ctx
->msg_enable
= netif_msg_init(debug
, default_msg
);
2284 if (netif_msg_probe(net_device_ctx
))
2285 netdev_dbg(net
, "netvsc msg_enable: %d\n",
2286 net_device_ctx
->msg_enable
);
2288 hv_set_drvdata(dev
, net
);
2290 INIT_DELAYED_WORK(&net_device_ctx
->dwork
, netvsc_link_change
);
2292 spin_lock_init(&net_device_ctx
->lock
);
2293 INIT_LIST_HEAD(&net_device_ctx
->reconfig_events
);
2294 INIT_DELAYED_WORK(&net_device_ctx
->vf_takeover
, netvsc_vf_setup
);
2296 net_device_ctx
->vf_stats
2297 = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats
);
2298 if (!net_device_ctx
->vf_stats
)
2301 net
->netdev_ops
= &device_ops
;
2302 net
->ethtool_ops
= ðtool_ops
;
2303 SET_NETDEV_DEV(net
, &dev
->device
);
2305 /* We always need headroom for rndis header */
2306 net
->needed_headroom
= RNDIS_AND_PPI_SIZE
;
2308 /* Initialize the number of queues to be 1, we may change it if more
2309 * channels are offered later.
2311 netif_set_real_num_tx_queues(net
, 1);
2312 netif_set_real_num_rx_queues(net
, 1);
2314 /* Notify the netvsc driver of the new device */
2315 device_info
= netvsc_devinfo_get(NULL
);
2319 goto devinfo_failed
;
2322 nvdev
= rndis_filter_device_add(dev
, device_info
);
2323 if (IS_ERR(nvdev
)) {
2324 ret
= PTR_ERR(nvdev
);
2325 netdev_err(net
, "unable to add netvsc device (ret %d)\n", ret
);
2329 memcpy(net
->dev_addr
, device_info
->mac_adr
, ETH_ALEN
);
2331 /* We must get rtnl lock before scheduling nvdev->subchan_work,
2332 * otherwise netvsc_subchan_work() can get rtnl lock first and wait
2333 * all subchannels to show up, but that may not happen because
2334 * netvsc_probe() can't get rtnl lock and as a result vmbus_onoffer()
2335 * -> ... -> device_add() -> ... -> __device_attach() can't get
2336 * the device lock, so all the subchannels can't be processed --
2337 * finally netvsc_subchan_work() hangs forever.
2341 if (nvdev
->num_chn
> 1)
2342 schedule_work(&nvdev
->subchan_work
);
2344 /* hw_features computed in rndis_netdev_set_hwcaps() */
2345 net
->features
= net
->hw_features
|
2346 NETIF_F_HIGHDMA
| NETIF_F_HW_VLAN_CTAG_TX
|
2347 NETIF_F_HW_VLAN_CTAG_RX
;
2348 net
->vlan_features
= net
->features
;
2350 /* MTU range: 68 - 1500 or 65521 */
2351 net
->min_mtu
= NETVSC_MTU_MIN
;
2352 if (nvdev
->nvsp_version
>= NVSP_PROTOCOL_VERSION_2
)
2353 net
->max_mtu
= NETVSC_MTU
- ETH_HLEN
;
2355 net
->max_mtu
= ETH_DATA_LEN
;
2357 ret
= register_netdevice(net
);
2359 pr_err("Unable to register netdev.\n");
2360 goto register_failed
;
2363 list_add(&net_device_ctx
->list
, &netvsc_dev_list
);
2371 rndis_filter_device_remove(dev
, nvdev
);
2375 free_percpu(net_device_ctx
->vf_stats
);
2377 hv_set_drvdata(dev
, NULL
);
2383 static int netvsc_remove(struct hv_device
*dev
)
2385 struct net_device_context
*ndev_ctx
;
2386 struct net_device
*vf_netdev
, *net
;
2387 struct netvsc_device
*nvdev
;
2389 net
= hv_get_drvdata(dev
);
2391 dev_err(&dev
->device
, "No net device to remove\n");
2395 ndev_ctx
= netdev_priv(net
);
2397 cancel_delayed_work_sync(&ndev_ctx
->dwork
);
2400 nvdev
= rtnl_dereference(ndev_ctx
->nvdev
);
2402 cancel_work_sync(&nvdev
->subchan_work
);
2405 * Call to the vsc driver to let it know that the device is being
2406 * removed. Also blocks mtu and channel changes.
2408 vf_netdev
= rtnl_dereference(ndev_ctx
->vf_netdev
);
2410 netvsc_unregister_vf(vf_netdev
);
2413 rndis_filter_device_remove(dev
, nvdev
);
2415 unregister_netdevice(net
);
2416 list_del(&ndev_ctx
->list
);
2420 hv_set_drvdata(dev
, NULL
);
2422 free_percpu(ndev_ctx
->vf_stats
);
2427 static const struct hv_vmbus_device_id id_table
[] = {
2433 MODULE_DEVICE_TABLE(vmbus
, id_table
);
2435 /* The one and only one */
2436 static struct hv_driver netvsc_drv
= {
2437 .name
= KBUILD_MODNAME
,
2438 .id_table
= id_table
,
2439 .probe
= netvsc_probe
,
2440 .remove
= netvsc_remove
,
2442 .probe_type
= PROBE_FORCE_SYNCHRONOUS
,
2447 * On Hyper-V, every VF interface is matched with a corresponding
2448 * synthetic interface. The synthetic interface is presented first
2449 * to the guest. When the corresponding VF instance is registered,
2450 * we will take care of switching the data path.
2452 static int netvsc_netdev_event(struct notifier_block
*this,
2453 unsigned long event
, void *ptr
)
2455 struct net_device
*event_dev
= netdev_notifier_info_to_dev(ptr
);
2457 /* Skip our own events */
2458 if (event_dev
->netdev_ops
== &device_ops
)
2461 /* Avoid non-Ethernet type devices */
2462 if (event_dev
->type
!= ARPHRD_ETHER
)
2465 /* Avoid Vlan dev with same MAC registering as VF */
2466 if (is_vlan_dev(event_dev
))
2469 /* Avoid Bonding master dev with same MAC registering as VF */
2470 if ((event_dev
->priv_flags
& IFF_BONDING
) &&
2471 (event_dev
->flags
& IFF_MASTER
))
2475 case NETDEV_REGISTER
:
2476 return netvsc_register_vf(event_dev
);
2477 case NETDEV_UNREGISTER
:
2478 return netvsc_unregister_vf(event_dev
);
2481 return netvsc_vf_changed(event_dev
);
2487 static struct notifier_block netvsc_netdev_notifier
= {
2488 .notifier_call
= netvsc_netdev_event
,
2491 static void __exit
netvsc_drv_exit(void)
2493 unregister_netdevice_notifier(&netvsc_netdev_notifier
);
2494 vmbus_driver_unregister(&netvsc_drv
);
2497 static int __init
netvsc_drv_init(void)
2501 if (ring_size
< RING_SIZE_MIN
) {
2502 ring_size
= RING_SIZE_MIN
;
2503 pr_info("Increased ring_size to %u (min allowed)\n",
2506 netvsc_ring_bytes
= ring_size
* PAGE_SIZE
;
2508 ret
= vmbus_driver_register(&netvsc_drv
);
2512 register_netdevice_notifier(&netvsc_netdev_notifier
);
2516 MODULE_LICENSE("GPL");
2517 MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
2519 module_init(netvsc_drv_init
);
2520 module_exit(netvsc_drv_exit
);