2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/bpf.h>
35 #include <linux/etherdevice.h>
36 #include <linux/tcp.h>
37 #include <linux/if_vlan.h>
38 #include <linux/delay.h>
39 #include <linux/slab.h>
40 #include <linux/hash.h>
42 #include <net/busy_poll.h>
43 #include <net/vxlan.h>
44 #include <net/devlink.h>
46 #include <linux/mlx4/driver.h>
47 #include <linux/mlx4/device.h>
48 #include <linux/mlx4/cmd.h>
49 #include <linux/mlx4/cq.h>
54 #define MLX4_EN_MAX_XDP_MTU ((int)(PAGE_SIZE - ETH_HLEN - (2 * VLAN_HLEN) - \
57 int mlx4_en_setup_tc(struct net_device
*dev
, u8 up
)
59 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
61 unsigned int offset
= 0;
63 if (up
&& up
!= MLX4_EN_NUM_UP
)
66 netdev_set_num_tc(dev
, up
);
68 /* Partition Tx queues evenly amongst UP's */
69 for (i
= 0; i
< up
; i
++) {
70 netdev_set_tc_queue(dev
, i
, priv
->num_tx_rings_p_up
, offset
);
71 offset
+= priv
->num_tx_rings_p_up
;
74 #ifdef CONFIG_MLX4_EN_DCB
75 if (!mlx4_is_slave(priv
->mdev
->dev
)) {
78 priv
->flags
|= MLX4_EN_FLAG_DCB_ENABLED
;
80 priv
->flags
&= ~MLX4_EN_FLAG_DCB_ENABLED
;
81 priv
->cee_config
.pfc_state
= false;
84 #endif /* CONFIG_MLX4_EN_DCB */
89 static int __mlx4_en_setup_tc(struct net_device
*dev
, u32 handle
,
90 u32 chain_index
, __be16 proto
,
91 struct tc_to_netdev
*tc
)
93 if (tc
->type
!= TC_SETUP_MQPRIO
)
96 tc
->mqprio
->hw
= TC_MQPRIO_HW_OFFLOAD_TCS
;
98 return mlx4_en_setup_tc(dev
, tc
->mqprio
->num_tc
);
101 #ifdef CONFIG_RFS_ACCEL
103 struct mlx4_en_filter
{
104 struct list_head next
;
105 struct work_struct work
;
114 struct mlx4_en_priv
*priv
;
115 u32 flow_id
; /* RFS infrastructure id */
116 int id
; /* mlx4_en driver id */
117 u64 reg_id
; /* Flow steering API id */
118 u8 activated
; /* Used to prevent expiry before filter
121 struct hlist_node filter_chain
;
124 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv
*priv
);
126 static enum mlx4_net_trans_rule_id
mlx4_ip_proto_to_trans_rule_id(u8 ip_proto
)
130 return MLX4_NET_TRANS_RULE_ID_UDP
;
132 return MLX4_NET_TRANS_RULE_ID_TCP
;
134 return MLX4_NET_TRANS_RULE_NUM
;
138 /* Must not acquire state_lock, as its corresponding work_sync
141 static void mlx4_en_filter_work(struct work_struct
*work
)
143 struct mlx4_en_filter
*filter
= container_of(work
,
144 struct mlx4_en_filter
,
146 struct mlx4_en_priv
*priv
= filter
->priv
;
147 struct mlx4_spec_list spec_tcp_udp
= {
148 .id
= mlx4_ip_proto_to_trans_rule_id(filter
->ip_proto
),
151 .dst_port
= filter
->dst_port
,
152 .dst_port_msk
= (__force __be16
)-1,
153 .src_port
= filter
->src_port
,
154 .src_port_msk
= (__force __be16
)-1,
158 struct mlx4_spec_list spec_ip
= {
159 .id
= MLX4_NET_TRANS_RULE_ID_IPV4
,
162 .dst_ip
= filter
->dst_ip
,
163 .dst_ip_msk
= (__force __be32
)-1,
164 .src_ip
= filter
->src_ip
,
165 .src_ip_msk
= (__force __be32
)-1,
169 struct mlx4_spec_list spec_eth
= {
170 .id
= MLX4_NET_TRANS_RULE_ID_ETH
,
172 struct mlx4_net_trans_rule rule
= {
173 .list
= LIST_HEAD_INIT(rule
.list
),
174 .queue_mode
= MLX4_NET_TRANS_Q_LIFO
,
177 .promisc_mode
= MLX4_FS_REGULAR
,
179 .priority
= MLX4_DOMAIN_RFS
,
182 __be64 mac_mask
= cpu_to_be64(MLX4_MAC_MASK
<< 16);
184 if (spec_tcp_udp
.id
>= MLX4_NET_TRANS_RULE_NUM
) {
185 en_warn(priv
, "RFS: ignoring unsupported ip protocol (%d)\n",
189 list_add_tail(&spec_eth
.list
, &rule
.list
);
190 list_add_tail(&spec_ip
.list
, &rule
.list
);
191 list_add_tail(&spec_tcp_udp
.list
, &rule
.list
);
193 rule
.qpn
= priv
->rss_map
.qps
[filter
->rxq_index
].qpn
;
194 memcpy(spec_eth
.eth
.dst_mac
, priv
->dev
->dev_addr
, ETH_ALEN
);
195 memcpy(spec_eth
.eth
.dst_mac_msk
, &mac_mask
, ETH_ALEN
);
197 filter
->activated
= 0;
199 if (filter
->reg_id
) {
200 rc
= mlx4_flow_detach(priv
->mdev
->dev
, filter
->reg_id
);
201 if (rc
&& rc
!= -ENOENT
)
202 en_err(priv
, "Error detaching flow. rc = %d\n", rc
);
205 rc
= mlx4_flow_attach(priv
->mdev
->dev
, &rule
, &filter
->reg_id
);
207 en_err(priv
, "Error attaching flow. err = %d\n", rc
);
210 mlx4_en_filter_rfs_expire(priv
);
212 filter
->activated
= 1;
215 static inline struct hlist_head
*
216 filter_hash_bucket(struct mlx4_en_priv
*priv
, __be32 src_ip
, __be32 dst_ip
,
217 __be16 src_port
, __be16 dst_port
)
222 l
= (__force
unsigned long)src_port
|
223 ((__force
unsigned long)dst_port
<< 2);
224 l
^= (__force
unsigned long)(src_ip
^ dst_ip
);
226 bucket_idx
= hash_long(l
, MLX4_EN_FILTER_HASH_SHIFT
);
228 return &priv
->filter_hash
[bucket_idx
];
231 static struct mlx4_en_filter
*
232 mlx4_en_filter_alloc(struct mlx4_en_priv
*priv
, int rxq_index
, __be32 src_ip
,
233 __be32 dst_ip
, u8 ip_proto
, __be16 src_port
,
234 __be16 dst_port
, u32 flow_id
)
236 struct mlx4_en_filter
*filter
= NULL
;
238 filter
= kzalloc(sizeof(struct mlx4_en_filter
), GFP_ATOMIC
);
243 filter
->rxq_index
= rxq_index
;
244 INIT_WORK(&filter
->work
, mlx4_en_filter_work
);
246 filter
->src_ip
= src_ip
;
247 filter
->dst_ip
= dst_ip
;
248 filter
->ip_proto
= ip_proto
;
249 filter
->src_port
= src_port
;
250 filter
->dst_port
= dst_port
;
252 filter
->flow_id
= flow_id
;
254 filter
->id
= priv
->last_filter_id
++ % RPS_NO_FILTER
;
256 list_add_tail(&filter
->next
, &priv
->filters
);
257 hlist_add_head(&filter
->filter_chain
,
258 filter_hash_bucket(priv
, src_ip
, dst_ip
, src_port
,
264 static void mlx4_en_filter_free(struct mlx4_en_filter
*filter
)
266 struct mlx4_en_priv
*priv
= filter
->priv
;
269 list_del(&filter
->next
);
271 rc
= mlx4_flow_detach(priv
->mdev
->dev
, filter
->reg_id
);
272 if (rc
&& rc
!= -ENOENT
)
273 en_err(priv
, "Error detaching flow. rc = %d\n", rc
);
278 static inline struct mlx4_en_filter
*
279 mlx4_en_filter_find(struct mlx4_en_priv
*priv
, __be32 src_ip
, __be32 dst_ip
,
280 u8 ip_proto
, __be16 src_port
, __be16 dst_port
)
282 struct mlx4_en_filter
*filter
;
283 struct mlx4_en_filter
*ret
= NULL
;
285 hlist_for_each_entry(filter
,
286 filter_hash_bucket(priv
, src_ip
, dst_ip
,
289 if (filter
->src_ip
== src_ip
&&
290 filter
->dst_ip
== dst_ip
&&
291 filter
->ip_proto
== ip_proto
&&
292 filter
->src_port
== src_port
&&
293 filter
->dst_port
== dst_port
) {
303 mlx4_en_filter_rfs(struct net_device
*net_dev
, const struct sk_buff
*skb
,
304 u16 rxq_index
, u32 flow_id
)
306 struct mlx4_en_priv
*priv
= netdev_priv(net_dev
);
307 struct mlx4_en_filter
*filter
;
308 const struct iphdr
*ip
;
315 int nhoff
= skb_network_offset(skb
);
318 if (skb
->protocol
!= htons(ETH_P_IP
))
319 return -EPROTONOSUPPORT
;
321 ip
= (const struct iphdr
*)(skb
->data
+ nhoff
);
322 if (ip_is_fragment(ip
))
323 return -EPROTONOSUPPORT
;
325 if ((ip
->protocol
!= IPPROTO_TCP
) && (ip
->protocol
!= IPPROTO_UDP
))
326 return -EPROTONOSUPPORT
;
327 ports
= (const __be16
*)(skb
->data
+ nhoff
+ 4 * ip
->ihl
);
329 ip_proto
= ip
->protocol
;
335 spin_lock_bh(&priv
->filters_lock
);
336 filter
= mlx4_en_filter_find(priv
, src_ip
, dst_ip
, ip_proto
,
339 if (filter
->rxq_index
== rxq_index
)
342 filter
->rxq_index
= rxq_index
;
344 filter
= mlx4_en_filter_alloc(priv
, rxq_index
,
345 src_ip
, dst_ip
, ip_proto
,
346 src_port
, dst_port
, flow_id
);
353 queue_work(priv
->mdev
->workqueue
, &filter
->work
);
358 spin_unlock_bh(&priv
->filters_lock
);
363 void mlx4_en_cleanup_filters(struct mlx4_en_priv
*priv
)
365 struct mlx4_en_filter
*filter
, *tmp
;
368 spin_lock_bh(&priv
->filters_lock
);
369 list_for_each_entry_safe(filter
, tmp
, &priv
->filters
, next
) {
370 list_move(&filter
->next
, &del_list
);
371 hlist_del(&filter
->filter_chain
);
373 spin_unlock_bh(&priv
->filters_lock
);
375 list_for_each_entry_safe(filter
, tmp
, &del_list
, next
) {
376 cancel_work_sync(&filter
->work
);
377 mlx4_en_filter_free(filter
);
381 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv
*priv
)
383 struct mlx4_en_filter
*filter
= NULL
, *tmp
, *last_filter
= NULL
;
387 spin_lock_bh(&priv
->filters_lock
);
388 list_for_each_entry_safe(filter
, tmp
, &priv
->filters
, next
) {
389 if (i
> MLX4_EN_FILTER_EXPIRY_QUOTA
)
392 if (filter
->activated
&&
393 !work_pending(&filter
->work
) &&
394 rps_may_expire_flow(priv
->dev
,
395 filter
->rxq_index
, filter
->flow_id
,
397 list_move(&filter
->next
, &del_list
);
398 hlist_del(&filter
->filter_chain
);
400 last_filter
= filter
;
405 if (last_filter
&& (&last_filter
->next
!= priv
->filters
.next
))
406 list_move(&priv
->filters
, &last_filter
->next
);
408 spin_unlock_bh(&priv
->filters_lock
);
410 list_for_each_entry_safe(filter
, tmp
, &del_list
, next
)
411 mlx4_en_filter_free(filter
);
415 static int mlx4_en_vlan_rx_add_vid(struct net_device
*dev
,
416 __be16 proto
, u16 vid
)
418 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
419 struct mlx4_en_dev
*mdev
= priv
->mdev
;
423 en_dbg(HW
, priv
, "adding VLAN:%d\n", vid
);
425 set_bit(vid
, priv
->active_vlans
);
427 /* Add VID to port VLAN filter */
428 mutex_lock(&mdev
->state_lock
);
429 if (mdev
->device_up
&& priv
->port_up
) {
430 err
= mlx4_SET_VLAN_FLTR(mdev
->dev
, priv
);
432 en_err(priv
, "Failed configuring VLAN filter\n");
436 err
= mlx4_register_vlan(mdev
->dev
, priv
->port
, vid
, &idx
);
438 en_dbg(HW
, priv
, "Failed adding vlan %d\n", vid
);
441 mutex_unlock(&mdev
->state_lock
);
445 static int mlx4_en_vlan_rx_kill_vid(struct net_device
*dev
,
446 __be16 proto
, u16 vid
)
448 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
449 struct mlx4_en_dev
*mdev
= priv
->mdev
;
452 en_dbg(HW
, priv
, "Killing VID:%d\n", vid
);
454 clear_bit(vid
, priv
->active_vlans
);
456 /* Remove VID from port VLAN filter */
457 mutex_lock(&mdev
->state_lock
);
458 mlx4_unregister_vlan(mdev
->dev
, priv
->port
, vid
);
460 if (mdev
->device_up
&& priv
->port_up
) {
461 err
= mlx4_SET_VLAN_FLTR(mdev
->dev
, priv
);
463 en_err(priv
, "Failed configuring VLAN filter\n");
465 mutex_unlock(&mdev
->state_lock
);
470 static void mlx4_en_u64_to_mac(unsigned char dst_mac
[ETH_ALEN
+ 2], u64 src_mac
)
473 for (i
= ETH_ALEN
- 1; i
>= 0; --i
) {
474 dst_mac
[i
] = src_mac
& 0xff;
477 memset(&dst_mac
[ETH_ALEN
], 0, 2);
481 static int mlx4_en_tunnel_steer_add(struct mlx4_en_priv
*priv
, unsigned char *addr
,
482 int qpn
, u64
*reg_id
)
486 if (priv
->mdev
->dev
->caps
.tunnel_offload_mode
!= MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
||
487 priv
->mdev
->dev
->caps
.dmfs_high_steer_mode
== MLX4_STEERING_DMFS_A0_STATIC
)
488 return 0; /* do nothing */
490 err
= mlx4_tunnel_steer_add(priv
->mdev
->dev
, addr
, priv
->port
, qpn
,
491 MLX4_DOMAIN_NIC
, reg_id
);
493 en_err(priv
, "failed to add vxlan steering rule, err %d\n", err
);
496 en_dbg(DRV
, priv
, "added vxlan steering rule, mac %pM reg_id %llx\n", addr
, *reg_id
);
501 static int mlx4_en_uc_steer_add(struct mlx4_en_priv
*priv
,
502 unsigned char *mac
, int *qpn
, u64
*reg_id
)
504 struct mlx4_en_dev
*mdev
= priv
->mdev
;
505 struct mlx4_dev
*dev
= mdev
->dev
;
508 switch (dev
->caps
.steering_mode
) {
509 case MLX4_STEERING_MODE_B0
: {
514 memcpy(&gid
[10], mac
, ETH_ALEN
);
517 err
= mlx4_unicast_attach(dev
, &qp
, gid
, 0, MLX4_PROT_ETH
);
520 case MLX4_STEERING_MODE_DEVICE_MANAGED
: {
521 struct mlx4_spec_list spec_eth
= { {NULL
} };
522 __be64 mac_mask
= cpu_to_be64(MLX4_MAC_MASK
<< 16);
524 struct mlx4_net_trans_rule rule
= {
525 .queue_mode
= MLX4_NET_TRANS_Q_FIFO
,
528 .promisc_mode
= MLX4_FS_REGULAR
,
529 .priority
= MLX4_DOMAIN_NIC
,
532 rule
.port
= priv
->port
;
534 INIT_LIST_HEAD(&rule
.list
);
536 spec_eth
.id
= MLX4_NET_TRANS_RULE_ID_ETH
;
537 memcpy(spec_eth
.eth
.dst_mac
, mac
, ETH_ALEN
);
538 memcpy(spec_eth
.eth
.dst_mac_msk
, &mac_mask
, ETH_ALEN
);
539 list_add_tail(&spec_eth
.list
, &rule
.list
);
541 err
= mlx4_flow_attach(dev
, &rule
, reg_id
);
548 en_warn(priv
, "Failed Attaching Unicast\n");
553 static void mlx4_en_uc_steer_release(struct mlx4_en_priv
*priv
,
554 unsigned char *mac
, int qpn
, u64 reg_id
)
556 struct mlx4_en_dev
*mdev
= priv
->mdev
;
557 struct mlx4_dev
*dev
= mdev
->dev
;
559 switch (dev
->caps
.steering_mode
) {
560 case MLX4_STEERING_MODE_B0
: {
565 memcpy(&gid
[10], mac
, ETH_ALEN
);
568 mlx4_unicast_detach(dev
, &qp
, gid
, MLX4_PROT_ETH
);
571 case MLX4_STEERING_MODE_DEVICE_MANAGED
: {
572 mlx4_flow_detach(dev
, reg_id
);
576 en_err(priv
, "Invalid steering mode.\n");
580 static int mlx4_en_get_qp(struct mlx4_en_priv
*priv
)
582 struct mlx4_en_dev
*mdev
= priv
->mdev
;
583 struct mlx4_dev
*dev
= mdev
->dev
;
586 int *qpn
= &priv
->base_qpn
;
587 u64 mac
= mlx4_mac_to_u64(priv
->dev
->dev_addr
);
589 en_dbg(DRV
, priv
, "Registering MAC: %pM for adding\n",
590 priv
->dev
->dev_addr
);
591 index
= mlx4_register_mac(dev
, priv
->port
, mac
);
594 en_err(priv
, "Failed adding MAC: %pM\n",
595 priv
->dev
->dev_addr
);
599 en_info(priv
, "Steering Mode %d\n", dev
->caps
.steering_mode
);
601 if (dev
->caps
.steering_mode
== MLX4_STEERING_MODE_A0
) {
602 int base_qpn
= mlx4_get_base_qpn(dev
, priv
->port
);
603 *qpn
= base_qpn
+ index
;
607 err
= mlx4_qp_reserve_range(dev
, 1, 1, qpn
, MLX4_RESERVE_A0_QP
);
608 en_dbg(DRV
, priv
, "Reserved qp %d\n", *qpn
);
610 en_err(priv
, "Failed to reserve qp for mac registration\n");
611 mlx4_unregister_mac(dev
, priv
->port
, mac
);
618 static void mlx4_en_put_qp(struct mlx4_en_priv
*priv
)
620 struct mlx4_en_dev
*mdev
= priv
->mdev
;
621 struct mlx4_dev
*dev
= mdev
->dev
;
622 int qpn
= priv
->base_qpn
;
624 if (dev
->caps
.steering_mode
== MLX4_STEERING_MODE_A0
) {
625 u64 mac
= mlx4_mac_to_u64(priv
->dev
->dev_addr
);
626 en_dbg(DRV
, priv
, "Registering MAC: %pM for deleting\n",
627 priv
->dev
->dev_addr
);
628 mlx4_unregister_mac(dev
, priv
->port
, mac
);
630 en_dbg(DRV
, priv
, "Releasing qp: port %d, qpn %d\n",
632 mlx4_qp_release_range(dev
, qpn
, 1);
633 priv
->flags
&= ~MLX4_EN_FLAG_FORCE_PROMISC
;
637 static int mlx4_en_replace_mac(struct mlx4_en_priv
*priv
, int qpn
,
638 unsigned char *new_mac
, unsigned char *prev_mac
)
640 struct mlx4_en_dev
*mdev
= priv
->mdev
;
641 struct mlx4_dev
*dev
= mdev
->dev
;
643 u64 new_mac_u64
= mlx4_mac_to_u64(new_mac
);
645 if (dev
->caps
.steering_mode
!= MLX4_STEERING_MODE_A0
) {
646 struct hlist_head
*bucket
;
647 unsigned int mac_hash
;
648 struct mlx4_mac_entry
*entry
;
649 struct hlist_node
*tmp
;
650 u64 prev_mac_u64
= mlx4_mac_to_u64(prev_mac
);
652 bucket
= &priv
->mac_hash
[prev_mac
[MLX4_EN_MAC_HASH_IDX
]];
653 hlist_for_each_entry_safe(entry
, tmp
, bucket
, hlist
) {
654 if (ether_addr_equal_64bits(entry
->mac
, prev_mac
)) {
655 mlx4_en_uc_steer_release(priv
, entry
->mac
,
657 mlx4_unregister_mac(dev
, priv
->port
,
659 hlist_del_rcu(&entry
->hlist
);
661 memcpy(entry
->mac
, new_mac
, ETH_ALEN
);
663 mac_hash
= new_mac
[MLX4_EN_MAC_HASH_IDX
];
664 hlist_add_head_rcu(&entry
->hlist
,
665 &priv
->mac_hash
[mac_hash
]);
666 mlx4_register_mac(dev
, priv
->port
, new_mac_u64
);
667 err
= mlx4_en_uc_steer_add(priv
, new_mac
,
672 if (priv
->tunnel_reg_id
) {
673 mlx4_flow_detach(priv
->mdev
->dev
, priv
->tunnel_reg_id
);
674 priv
->tunnel_reg_id
= 0;
676 err
= mlx4_en_tunnel_steer_add(priv
, new_mac
, qpn
,
677 &priv
->tunnel_reg_id
);
684 return __mlx4_replace_mac(dev
, priv
->port
, qpn
, new_mac_u64
);
687 static int mlx4_en_do_set_mac(struct mlx4_en_priv
*priv
,
688 unsigned char new_mac
[ETH_ALEN
+ 2])
693 /* Remove old MAC and insert the new one */
694 err
= mlx4_en_replace_mac(priv
, priv
->base_qpn
,
695 new_mac
, priv
->current_mac
);
697 en_err(priv
, "Failed changing HW MAC address\n");
699 en_dbg(HW
, priv
, "Port is down while registering mac, exiting...\n");
702 memcpy(priv
->current_mac
, new_mac
, sizeof(priv
->current_mac
));
707 static int mlx4_en_set_mac(struct net_device
*dev
, void *addr
)
709 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
710 struct mlx4_en_dev
*mdev
= priv
->mdev
;
711 struct sockaddr
*saddr
= addr
;
712 unsigned char new_mac
[ETH_ALEN
+ 2];
715 if (!is_valid_ether_addr(saddr
->sa_data
))
716 return -EADDRNOTAVAIL
;
718 mutex_lock(&mdev
->state_lock
);
719 memcpy(new_mac
, saddr
->sa_data
, ETH_ALEN
);
720 err
= mlx4_en_do_set_mac(priv
, new_mac
);
722 memcpy(dev
->dev_addr
, saddr
->sa_data
, ETH_ALEN
);
723 mutex_unlock(&mdev
->state_lock
);
728 static void mlx4_en_clear_list(struct net_device
*dev
)
730 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
731 struct mlx4_en_mc_list
*tmp
, *mc_to_del
;
733 list_for_each_entry_safe(mc_to_del
, tmp
, &priv
->mc_list
, list
) {
734 list_del(&mc_to_del
->list
);
739 static void mlx4_en_cache_mclist(struct net_device
*dev
)
741 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
742 struct netdev_hw_addr
*ha
;
743 struct mlx4_en_mc_list
*tmp
;
745 mlx4_en_clear_list(dev
);
746 netdev_for_each_mc_addr(ha
, dev
) {
747 tmp
= kzalloc(sizeof(struct mlx4_en_mc_list
), GFP_ATOMIC
);
749 mlx4_en_clear_list(dev
);
752 memcpy(tmp
->addr
, ha
->addr
, ETH_ALEN
);
753 list_add_tail(&tmp
->list
, &priv
->mc_list
);
757 static void update_mclist_flags(struct mlx4_en_priv
*priv
,
758 struct list_head
*dst
,
759 struct list_head
*src
)
761 struct mlx4_en_mc_list
*dst_tmp
, *src_tmp
, *new_mc
;
764 /* Find all the entries that should be removed from dst,
765 * These are the entries that are not found in src
767 list_for_each_entry(dst_tmp
, dst
, list
) {
769 list_for_each_entry(src_tmp
, src
, list
) {
770 if (ether_addr_equal(dst_tmp
->addr
, src_tmp
->addr
)) {
776 dst_tmp
->action
= MCLIST_REM
;
779 /* Add entries that exist in src but not in dst
780 * mark them as need to add
782 list_for_each_entry(src_tmp
, src
, list
) {
784 list_for_each_entry(dst_tmp
, dst
, list
) {
785 if (ether_addr_equal(dst_tmp
->addr
, src_tmp
->addr
)) {
786 dst_tmp
->action
= MCLIST_NONE
;
792 new_mc
= kmemdup(src_tmp
,
793 sizeof(struct mlx4_en_mc_list
),
798 new_mc
->action
= MCLIST_ADD
;
799 list_add_tail(&new_mc
->list
, dst
);
804 static void mlx4_en_set_rx_mode(struct net_device
*dev
)
806 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
811 queue_work(priv
->mdev
->workqueue
, &priv
->rx_mode_task
);
814 static void mlx4_en_set_promisc_mode(struct mlx4_en_priv
*priv
,
815 struct mlx4_en_dev
*mdev
)
819 if (!(priv
->flags
& MLX4_EN_FLAG_PROMISC
)) {
820 if (netif_msg_rx_status(priv
))
821 en_warn(priv
, "Entering promiscuous mode\n");
822 priv
->flags
|= MLX4_EN_FLAG_PROMISC
;
824 /* Enable promiscouos mode */
825 switch (mdev
->dev
->caps
.steering_mode
) {
826 case MLX4_STEERING_MODE_DEVICE_MANAGED
:
827 err
= mlx4_flow_steer_promisc_add(mdev
->dev
,
830 MLX4_FS_ALL_DEFAULT
);
832 en_err(priv
, "Failed enabling promiscuous mode\n");
833 priv
->flags
|= MLX4_EN_FLAG_MC_PROMISC
;
836 case MLX4_STEERING_MODE_B0
:
837 err
= mlx4_unicast_promisc_add(mdev
->dev
,
841 en_err(priv
, "Failed enabling unicast promiscuous mode\n");
843 /* Add the default qp number as multicast
846 if (!(priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
)) {
847 err
= mlx4_multicast_promisc_add(mdev
->dev
,
851 en_err(priv
, "Failed enabling multicast promiscuous mode\n");
852 priv
->flags
|= MLX4_EN_FLAG_MC_PROMISC
;
856 case MLX4_STEERING_MODE_A0
:
857 err
= mlx4_SET_PORT_qpn_calc(mdev
->dev
,
862 en_err(priv
, "Failed enabling promiscuous mode\n");
866 /* Disable port multicast filter (unconditionally) */
867 err
= mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0,
868 0, MLX4_MCAST_DISABLE
);
870 en_err(priv
, "Failed disabling multicast filter\n");
874 static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv
*priv
,
875 struct mlx4_en_dev
*mdev
)
879 if (netif_msg_rx_status(priv
))
880 en_warn(priv
, "Leaving promiscuous mode\n");
881 priv
->flags
&= ~MLX4_EN_FLAG_PROMISC
;
883 /* Disable promiscouos mode */
884 switch (mdev
->dev
->caps
.steering_mode
) {
885 case MLX4_STEERING_MODE_DEVICE_MANAGED
:
886 err
= mlx4_flow_steer_promisc_remove(mdev
->dev
,
888 MLX4_FS_ALL_DEFAULT
);
890 en_err(priv
, "Failed disabling promiscuous mode\n");
891 priv
->flags
&= ~MLX4_EN_FLAG_MC_PROMISC
;
894 case MLX4_STEERING_MODE_B0
:
895 err
= mlx4_unicast_promisc_remove(mdev
->dev
,
899 en_err(priv
, "Failed disabling unicast promiscuous mode\n");
900 /* Disable Multicast promisc */
901 if (priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
) {
902 err
= mlx4_multicast_promisc_remove(mdev
->dev
,
906 en_err(priv
, "Failed disabling multicast promiscuous mode\n");
907 priv
->flags
&= ~MLX4_EN_FLAG_MC_PROMISC
;
911 case MLX4_STEERING_MODE_A0
:
912 err
= mlx4_SET_PORT_qpn_calc(mdev
->dev
,
916 en_err(priv
, "Failed disabling promiscuous mode\n");
921 static void mlx4_en_do_multicast(struct mlx4_en_priv
*priv
,
922 struct net_device
*dev
,
923 struct mlx4_en_dev
*mdev
)
925 struct mlx4_en_mc_list
*mclist
, *tmp
;
927 u8 mc_list
[16] = {0};
930 /* Enable/disable the multicast filter according to IFF_ALLMULTI */
931 if (dev
->flags
& IFF_ALLMULTI
) {
932 err
= mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0,
933 0, MLX4_MCAST_DISABLE
);
935 en_err(priv
, "Failed disabling multicast filter\n");
937 /* Add the default qp number as multicast promisc */
938 if (!(priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
)) {
939 switch (mdev
->dev
->caps
.steering_mode
) {
940 case MLX4_STEERING_MODE_DEVICE_MANAGED
:
941 err
= mlx4_flow_steer_promisc_add(mdev
->dev
,
947 case MLX4_STEERING_MODE_B0
:
948 err
= mlx4_multicast_promisc_add(mdev
->dev
,
953 case MLX4_STEERING_MODE_A0
:
957 en_err(priv
, "Failed entering multicast promisc mode\n");
958 priv
->flags
|= MLX4_EN_FLAG_MC_PROMISC
;
961 /* Disable Multicast promisc */
962 if (priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
) {
963 switch (mdev
->dev
->caps
.steering_mode
) {
964 case MLX4_STEERING_MODE_DEVICE_MANAGED
:
965 err
= mlx4_flow_steer_promisc_remove(mdev
->dev
,
970 case MLX4_STEERING_MODE_B0
:
971 err
= mlx4_multicast_promisc_remove(mdev
->dev
,
976 case MLX4_STEERING_MODE_A0
:
980 en_err(priv
, "Failed disabling multicast promiscuous mode\n");
981 priv
->flags
&= ~MLX4_EN_FLAG_MC_PROMISC
;
984 err
= mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0,
985 0, MLX4_MCAST_DISABLE
);
987 en_err(priv
, "Failed disabling multicast filter\n");
989 /* Flush mcast filter and init it with broadcast address */
990 mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, ETH_BCAST
,
991 1, MLX4_MCAST_CONFIG
);
993 /* Update multicast list - we cache all addresses so they won't
994 * change while HW is updated holding the command semaphor */
995 netif_addr_lock_bh(dev
);
996 mlx4_en_cache_mclist(dev
);
997 netif_addr_unlock_bh(dev
);
998 list_for_each_entry(mclist
, &priv
->mc_list
, list
) {
999 mcast_addr
= mlx4_mac_to_u64(mclist
->addr
);
1000 mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
,
1001 mcast_addr
, 0, MLX4_MCAST_CONFIG
);
1003 err
= mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0,
1004 0, MLX4_MCAST_ENABLE
);
1006 en_err(priv
, "Failed enabling multicast filter\n");
1008 update_mclist_flags(priv
, &priv
->curr_list
, &priv
->mc_list
);
1009 list_for_each_entry_safe(mclist
, tmp
, &priv
->curr_list
, list
) {
1010 if (mclist
->action
== MCLIST_REM
) {
1011 /* detach this address and delete from list */
1012 memcpy(&mc_list
[10], mclist
->addr
, ETH_ALEN
);
1013 mc_list
[5] = priv
->port
;
1014 err
= mlx4_multicast_detach(mdev
->dev
,
1015 priv
->rss_map
.indir_qp
,
1020 en_err(priv
, "Fail to detach multicast address\n");
1022 if (mclist
->tunnel_reg_id
) {
1023 err
= mlx4_flow_detach(priv
->mdev
->dev
, mclist
->tunnel_reg_id
);
1025 en_err(priv
, "Failed to detach multicast address\n");
1028 /* remove from list */
1029 list_del(&mclist
->list
);
1031 } else if (mclist
->action
== MCLIST_ADD
) {
1032 /* attach the address */
1033 memcpy(&mc_list
[10], mclist
->addr
, ETH_ALEN
);
1034 /* needed for B0 steering support */
1035 mc_list
[5] = priv
->port
;
1036 err
= mlx4_multicast_attach(mdev
->dev
,
1037 priv
->rss_map
.indir_qp
,
1043 en_err(priv
, "Fail to attach multicast address\n");
1045 err
= mlx4_en_tunnel_steer_add(priv
, &mc_list
[10], priv
->base_qpn
,
1046 &mclist
->tunnel_reg_id
);
1048 en_err(priv
, "Failed to attach multicast address\n");
1054 static void mlx4_en_do_uc_filter(struct mlx4_en_priv
*priv
,
1055 struct net_device
*dev
,
1056 struct mlx4_en_dev
*mdev
)
1058 struct netdev_hw_addr
*ha
;
1059 struct mlx4_mac_entry
*entry
;
1060 struct hlist_node
*tmp
;
1064 struct hlist_head
*bucket
;
1069 /* Note that we do not need to protect our mac_hash traversal with rcu,
1070 * since all modification code is protected by mdev->state_lock
1073 /* find what to remove */
1074 for (i
= 0; i
< MLX4_EN_MAC_HASH_SIZE
; ++i
) {
1075 bucket
= &priv
->mac_hash
[i
];
1076 hlist_for_each_entry_safe(entry
, tmp
, bucket
, hlist
) {
1078 netdev_for_each_uc_addr(ha
, dev
) {
1079 if (ether_addr_equal_64bits(entry
->mac
,
1086 /* MAC address of the port is not in uc list */
1087 if (ether_addr_equal_64bits(entry
->mac
,
1092 mac
= mlx4_mac_to_u64(entry
->mac
);
1093 mlx4_en_uc_steer_release(priv
, entry
->mac
,
1096 mlx4_unregister_mac(mdev
->dev
, priv
->port
, mac
);
1098 hlist_del_rcu(&entry
->hlist
);
1099 kfree_rcu(entry
, rcu
);
1100 en_dbg(DRV
, priv
, "Removed MAC %pM on port:%d\n",
1101 entry
->mac
, priv
->port
);
1107 /* if we didn't remove anything, there is no use in trying to add
1108 * again once we are in a forced promisc mode state
1110 if ((priv
->flags
& MLX4_EN_FLAG_FORCE_PROMISC
) && 0 == removed
)
1113 prev_flags
= priv
->flags
;
1114 priv
->flags
&= ~MLX4_EN_FLAG_FORCE_PROMISC
;
1116 /* find what to add */
1117 netdev_for_each_uc_addr(ha
, dev
) {
1119 bucket
= &priv
->mac_hash
[ha
->addr
[MLX4_EN_MAC_HASH_IDX
]];
1120 hlist_for_each_entry(entry
, bucket
, hlist
) {
1121 if (ether_addr_equal_64bits(entry
->mac
, ha
->addr
)) {
1128 entry
= kmalloc(sizeof(*entry
), GFP_KERNEL
);
1130 en_err(priv
, "Failed adding MAC %pM on port:%d (out of memory)\n",
1131 ha
->addr
, priv
->port
);
1132 priv
->flags
|= MLX4_EN_FLAG_FORCE_PROMISC
;
1135 mac
= mlx4_mac_to_u64(ha
->addr
);
1136 memcpy(entry
->mac
, ha
->addr
, ETH_ALEN
);
1137 err
= mlx4_register_mac(mdev
->dev
, priv
->port
, mac
);
1139 en_err(priv
, "Failed registering MAC %pM on port %d: %d\n",
1140 ha
->addr
, priv
->port
, err
);
1142 priv
->flags
|= MLX4_EN_FLAG_FORCE_PROMISC
;
1145 err
= mlx4_en_uc_steer_add(priv
, ha
->addr
,
1149 en_err(priv
, "Failed adding MAC %pM on port %d: %d\n",
1150 ha
->addr
, priv
->port
, err
);
1151 mlx4_unregister_mac(mdev
->dev
, priv
->port
, mac
);
1153 priv
->flags
|= MLX4_EN_FLAG_FORCE_PROMISC
;
1156 unsigned int mac_hash
;
1157 en_dbg(DRV
, priv
, "Added MAC %pM on port:%d\n",
1158 ha
->addr
, priv
->port
);
1159 mac_hash
= ha
->addr
[MLX4_EN_MAC_HASH_IDX
];
1160 bucket
= &priv
->mac_hash
[mac_hash
];
1161 hlist_add_head_rcu(&entry
->hlist
, bucket
);
1166 if (priv
->flags
& MLX4_EN_FLAG_FORCE_PROMISC
) {
1167 en_warn(priv
, "Forcing promiscuous mode on port:%d\n",
1169 } else if (prev_flags
& MLX4_EN_FLAG_FORCE_PROMISC
) {
1170 en_warn(priv
, "Stop forcing promiscuous mode on port:%d\n",
1175 static void mlx4_en_do_set_rx_mode(struct work_struct
*work
)
1177 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
1179 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1180 struct net_device
*dev
= priv
->dev
;
1182 mutex_lock(&mdev
->state_lock
);
1183 if (!mdev
->device_up
) {
1184 en_dbg(HW
, priv
, "Card is not up, ignoring rx mode change.\n");
1187 if (!priv
->port_up
) {
1188 en_dbg(HW
, priv
, "Port is down, ignoring rx mode change.\n");
1192 if (!netif_carrier_ok(dev
)) {
1193 if (!mlx4_en_QUERY_PORT(mdev
, priv
->port
)) {
1194 if (priv
->port_state
.link_state
) {
1195 priv
->last_link_state
= MLX4_DEV_EVENT_PORT_UP
;
1196 netif_carrier_on(dev
);
1197 en_dbg(LINK
, priv
, "Link Up\n");
1202 if (dev
->priv_flags
& IFF_UNICAST_FLT
)
1203 mlx4_en_do_uc_filter(priv
, dev
, mdev
);
1205 /* Promsicuous mode: disable all filters */
1206 if ((dev
->flags
& IFF_PROMISC
) ||
1207 (priv
->flags
& MLX4_EN_FLAG_FORCE_PROMISC
)) {
1208 mlx4_en_set_promisc_mode(priv
, mdev
);
1212 /* Not in promiscuous mode */
1213 if (priv
->flags
& MLX4_EN_FLAG_PROMISC
)
1214 mlx4_en_clear_promisc_mode(priv
, mdev
);
1216 mlx4_en_do_multicast(priv
, dev
, mdev
);
1218 mutex_unlock(&mdev
->state_lock
);
1221 #ifdef CONFIG_NET_POLL_CONTROLLER
1222 static void mlx4_en_netpoll(struct net_device
*dev
)
1224 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
1225 struct mlx4_en_cq
*cq
;
1228 for (i
= 0; i
< priv
->tx_ring_num
[TX
]; i
++) {
1229 cq
= priv
->tx_cq
[TX
][i
];
1230 napi_schedule(&cq
->napi
);
1235 static int mlx4_en_set_rss_steer_rules(struct mlx4_en_priv
*priv
)
1239 int *qpn
= &priv
->base_qpn
;
1240 struct mlx4_mac_entry
*entry
;
1242 err
= mlx4_en_uc_steer_add(priv
, priv
->dev
->dev_addr
, qpn
, ®_id
);
1246 err
= mlx4_en_tunnel_steer_add(priv
, priv
->dev
->dev_addr
, *qpn
,
1247 &priv
->tunnel_reg_id
);
1251 entry
= kmalloc(sizeof(*entry
), GFP_KERNEL
);
1257 memcpy(entry
->mac
, priv
->dev
->dev_addr
, sizeof(entry
->mac
));
1258 memcpy(priv
->current_mac
, entry
->mac
, sizeof(priv
->current_mac
));
1259 entry
->reg_id
= reg_id
;
1260 hlist_add_head_rcu(&entry
->hlist
,
1261 &priv
->mac_hash
[entry
->mac
[MLX4_EN_MAC_HASH_IDX
]]);
1266 if (priv
->tunnel_reg_id
)
1267 mlx4_flow_detach(priv
->mdev
->dev
, priv
->tunnel_reg_id
);
1270 mlx4_en_uc_steer_release(priv
, priv
->dev
->dev_addr
, *qpn
, reg_id
);
1274 static void mlx4_en_delete_rss_steer_rules(struct mlx4_en_priv
*priv
)
1278 int qpn
= priv
->base_qpn
;
1279 struct hlist_head
*bucket
;
1280 struct hlist_node
*tmp
;
1281 struct mlx4_mac_entry
*entry
;
1283 for (i
= 0; i
< MLX4_EN_MAC_HASH_SIZE
; ++i
) {
1284 bucket
= &priv
->mac_hash
[i
];
1285 hlist_for_each_entry_safe(entry
, tmp
, bucket
, hlist
) {
1286 mac
= mlx4_mac_to_u64(entry
->mac
);
1287 en_dbg(DRV
, priv
, "Registering MAC:%pM for deleting\n",
1289 mlx4_en_uc_steer_release(priv
, entry
->mac
,
1290 qpn
, entry
->reg_id
);
1292 mlx4_unregister_mac(priv
->mdev
->dev
, priv
->port
, mac
);
1293 hlist_del_rcu(&entry
->hlist
);
1294 kfree_rcu(entry
, rcu
);
1298 if (priv
->tunnel_reg_id
) {
1299 mlx4_flow_detach(priv
->mdev
->dev
, priv
->tunnel_reg_id
);
1300 priv
->tunnel_reg_id
= 0;
1304 static void mlx4_en_tx_timeout(struct net_device
*dev
)
1306 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
1307 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1310 if (netif_msg_timer(priv
))
1311 en_warn(priv
, "Tx timeout called on port:%d\n", priv
->port
);
1313 for (i
= 0; i
< priv
->tx_ring_num
[TX
]; i
++) {
1314 struct mlx4_en_tx_ring
*tx_ring
= priv
->tx_ring
[TX
][i
];
1316 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev
, i
)))
1318 en_warn(priv
, "TX timeout on queue: %d, QP: 0x%x, CQ: 0x%x, Cons: 0x%x, Prod: 0x%x\n",
1319 i
, tx_ring
->qpn
, tx_ring
->sp_cqn
,
1320 tx_ring
->cons
, tx_ring
->prod
);
1323 priv
->port_stats
.tx_timeout
++;
1324 en_dbg(DRV
, priv
, "Scheduling watchdog\n");
1325 queue_work(mdev
->workqueue
, &priv
->watchdog_task
);
1330 mlx4_en_get_stats64(struct net_device
*dev
, struct rtnl_link_stats64
*stats
)
1332 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
1334 spin_lock_bh(&priv
->stats_lock
);
1335 mlx4_en_fold_software_stats(dev
);
1336 netdev_stats_to_stats64(stats
, &dev
->stats
);
1337 spin_unlock_bh(&priv
->stats_lock
);
1340 static void mlx4_en_set_default_moderation(struct mlx4_en_priv
*priv
)
1342 struct mlx4_en_cq
*cq
;
1345 /* If we haven't received a specific coalescing setting
1346 * (module param), we set the moderation parameters as follows:
1347 * - moder_cnt is set to the number of mtu sized packets to
1348 * satisfy our coalescing target.
1349 * - moder_time is set to a fixed value.
1351 priv
->rx_frames
= MLX4_EN_RX_COAL_TARGET
;
1352 priv
->rx_usecs
= MLX4_EN_RX_COAL_TIME
;
1353 priv
->tx_frames
= MLX4_EN_TX_COAL_PKTS
;
1354 priv
->tx_usecs
= MLX4_EN_TX_COAL_TIME
;
1355 en_dbg(INTR
, priv
, "Default coalesing params for mtu:%d - rx_frames:%d rx_usecs:%d\n",
1356 priv
->dev
->mtu
, priv
->rx_frames
, priv
->rx_usecs
);
1358 /* Setup cq moderation params */
1359 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
1360 cq
= priv
->rx_cq
[i
];
1361 cq
->moder_cnt
= priv
->rx_frames
;
1362 cq
->moder_time
= priv
->rx_usecs
;
1363 priv
->last_moder_time
[i
] = MLX4_EN_AUTO_CONF
;
1364 priv
->last_moder_packets
[i
] = 0;
1365 priv
->last_moder_bytes
[i
] = 0;
1368 for (t
= 0 ; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
1369 for (i
= 0; i
< priv
->tx_ring_num
[t
]; i
++) {
1370 cq
= priv
->tx_cq
[t
][i
];
1371 cq
->moder_cnt
= priv
->tx_frames
;
1372 cq
->moder_time
= priv
->tx_usecs
;
1376 /* Reset auto-moderation params */
1377 priv
->pkt_rate_low
= MLX4_EN_RX_RATE_LOW
;
1378 priv
->rx_usecs_low
= MLX4_EN_RX_COAL_TIME_LOW
;
1379 priv
->pkt_rate_high
= MLX4_EN_RX_RATE_HIGH
;
1380 priv
->rx_usecs_high
= MLX4_EN_RX_COAL_TIME_HIGH
;
1381 priv
->sample_interval
= MLX4_EN_SAMPLE_INTERVAL
;
1382 priv
->adaptive_rx_coal
= 1;
1383 priv
->last_moder_jiffies
= 0;
1384 priv
->last_moder_tx_packets
= 0;
1387 static void mlx4_en_auto_moderation(struct mlx4_en_priv
*priv
)
1389 unsigned long period
= (unsigned long) (jiffies
- priv
->last_moder_jiffies
);
1390 u32 pkt_rate_high
, pkt_rate_low
;
1391 struct mlx4_en_cq
*cq
;
1392 unsigned long packets
;
1394 unsigned long avg_pkt_size
;
1395 unsigned long rx_packets
;
1396 unsigned long rx_bytes
;
1397 unsigned long rx_pkt_diff
;
1401 if (!priv
->adaptive_rx_coal
|| period
< priv
->sample_interval
* HZ
)
1404 pkt_rate_low
= READ_ONCE(priv
->pkt_rate_low
);
1405 pkt_rate_high
= READ_ONCE(priv
->pkt_rate_high
);
1407 for (ring
= 0; ring
< priv
->rx_ring_num
; ring
++) {
1408 rx_packets
= READ_ONCE(priv
->rx_ring
[ring
]->packets
);
1409 rx_bytes
= READ_ONCE(priv
->rx_ring
[ring
]->bytes
);
1411 rx_pkt_diff
= rx_packets
- priv
->last_moder_packets
[ring
];
1412 packets
= rx_pkt_diff
;
1413 rate
= packets
* HZ
/ period
;
1414 avg_pkt_size
= packets
? (rx_bytes
-
1415 priv
->last_moder_bytes
[ring
]) / packets
: 0;
1417 /* Apply auto-moderation only when packet rate
1418 * exceeds a rate that it matters */
1419 if (rate
> (MLX4_EN_RX_RATE_THRESH
/ priv
->rx_ring_num
) &&
1420 avg_pkt_size
> MLX4_EN_AVG_PKT_SMALL
) {
1421 if (rate
<= pkt_rate_low
)
1422 moder_time
= priv
->rx_usecs_low
;
1423 else if (rate
>= pkt_rate_high
)
1424 moder_time
= priv
->rx_usecs_high
;
1426 moder_time
= (rate
- pkt_rate_low
) *
1427 (priv
->rx_usecs_high
- priv
->rx_usecs_low
) /
1428 (pkt_rate_high
- pkt_rate_low
) +
1431 moder_time
= priv
->rx_usecs_low
;
1434 cq
= priv
->rx_cq
[ring
];
1435 if (moder_time
!= priv
->last_moder_time
[ring
] ||
1436 cq
->moder_cnt
!= priv
->rx_frames
) {
1437 priv
->last_moder_time
[ring
] = moder_time
;
1438 cq
->moder_time
= moder_time
;
1439 cq
->moder_cnt
= priv
->rx_frames
;
1440 err
= mlx4_en_set_cq_moder(priv
, cq
);
1442 en_err(priv
, "Failed modifying moderation for cq:%d\n",
1445 priv
->last_moder_packets
[ring
] = rx_packets
;
1446 priv
->last_moder_bytes
[ring
] = rx_bytes
;
1449 priv
->last_moder_jiffies
= jiffies
;
1452 static void mlx4_en_do_get_stats(struct work_struct
*work
)
1454 struct delayed_work
*delay
= to_delayed_work(work
);
1455 struct mlx4_en_priv
*priv
= container_of(delay
, struct mlx4_en_priv
,
1457 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1460 mutex_lock(&mdev
->state_lock
);
1461 if (mdev
->device_up
) {
1462 if (priv
->port_up
) {
1463 err
= mlx4_en_DUMP_ETH_STATS(mdev
, priv
->port
, 0);
1465 en_dbg(HW
, priv
, "Could not update stats\n");
1467 mlx4_en_auto_moderation(priv
);
1470 queue_delayed_work(mdev
->workqueue
, &priv
->stats_task
, STATS_DELAY
);
1472 if (mdev
->mac_removed
[MLX4_MAX_PORTS
+ 1 - priv
->port
]) {
1473 mlx4_en_do_set_mac(priv
, priv
->current_mac
);
1474 mdev
->mac_removed
[MLX4_MAX_PORTS
+ 1 - priv
->port
] = 0;
1476 mutex_unlock(&mdev
->state_lock
);
1479 /* mlx4_en_service_task - Run service task for tasks that needed to be done
1482 static void mlx4_en_service_task(struct work_struct
*work
)
1484 struct delayed_work
*delay
= to_delayed_work(work
);
1485 struct mlx4_en_priv
*priv
= container_of(delay
, struct mlx4_en_priv
,
1487 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1489 mutex_lock(&mdev
->state_lock
);
1490 if (mdev
->device_up
) {
1491 if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_TS
)
1492 mlx4_en_ptp_overflow_check(mdev
);
1494 mlx4_en_recover_from_oom(priv
);
1495 queue_delayed_work(mdev
->workqueue
, &priv
->service_task
,
1496 SERVICE_TASK_DELAY
);
1498 mutex_unlock(&mdev
->state_lock
);
1501 static void mlx4_en_linkstate(struct work_struct
*work
)
1503 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
1505 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1506 int linkstate
= priv
->link_state
;
1508 mutex_lock(&mdev
->state_lock
);
1509 /* If observable port state changed set carrier state and
1510 * report to system log */
1511 if (priv
->last_link_state
!= linkstate
) {
1512 if (linkstate
== MLX4_DEV_EVENT_PORT_DOWN
) {
1513 en_info(priv
, "Link Down\n");
1514 netif_carrier_off(priv
->dev
);
1516 en_info(priv
, "Link Up\n");
1517 netif_carrier_on(priv
->dev
);
1520 priv
->last_link_state
= linkstate
;
1521 mutex_unlock(&mdev
->state_lock
);
1524 static int mlx4_en_init_affinity_hint(struct mlx4_en_priv
*priv
, int ring_idx
)
1526 struct mlx4_en_rx_ring
*ring
= priv
->rx_ring
[ring_idx
];
1527 int numa_node
= priv
->mdev
->dev
->numa_node
;
1529 if (!zalloc_cpumask_var(&ring
->affinity_mask
, GFP_KERNEL
))
1532 cpumask_set_cpu(cpumask_local_spread(ring_idx
, numa_node
),
1533 ring
->affinity_mask
);
1537 static void mlx4_en_free_affinity_hint(struct mlx4_en_priv
*priv
, int ring_idx
)
1539 free_cpumask_var(priv
->rx_ring
[ring_idx
]->affinity_mask
);
1542 static void mlx4_en_init_recycle_ring(struct mlx4_en_priv
*priv
,
1545 struct mlx4_en_tx_ring
*tx_ring
= priv
->tx_ring
[TX_XDP
][tx_ring_idx
];
1546 int rr_index
= tx_ring_idx
;
1548 tx_ring
->free_tx_desc
= mlx4_en_recycle_tx_desc
;
1549 tx_ring
->recycle_ring
= priv
->rx_ring
[rr_index
];
1550 en_dbg(DRV
, priv
, "Set tx_ring[%d][%d]->recycle_ring = rx_ring[%d]\n",
1551 TX_XDP
, tx_ring_idx
, rr_index
);
1554 int mlx4_en_start_port(struct net_device
*dev
)
1556 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
1557 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1558 struct mlx4_en_cq
*cq
;
1559 struct mlx4_en_tx_ring
*tx_ring
;
1564 u8 mc_list
[16] = {0};
1566 if (priv
->port_up
) {
1567 en_dbg(DRV
, priv
, "start port called while port already up\n");
1571 INIT_LIST_HEAD(&priv
->mc_list
);
1572 INIT_LIST_HEAD(&priv
->curr_list
);
1573 INIT_LIST_HEAD(&priv
->ethtool_list
);
1574 memset(&priv
->ethtool_rules
[0], 0,
1575 sizeof(struct ethtool_flow_id
) * MAX_NUM_OF_FS_RULES
);
1577 /* Calculate Rx buf size */
1578 dev
->mtu
= min(dev
->mtu
, priv
->max_mtu
);
1579 mlx4_en_calc_rx_buf(dev
);
1580 en_dbg(DRV
, priv
, "Rx buf size:%d\n", priv
->rx_skb_size
);
1582 /* Configure rx cq's and rings */
1583 err
= mlx4_en_activate_rx_rings(priv
);
1585 en_err(priv
, "Failed to activate RX rings\n");
1588 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
1589 cq
= priv
->rx_cq
[i
];
1591 err
= mlx4_en_init_affinity_hint(priv
, i
);
1593 en_err(priv
, "Failed preparing IRQ affinity hint\n");
1597 err
= mlx4_en_activate_cq(priv
, cq
, i
);
1599 en_err(priv
, "Failed activating Rx CQ\n");
1600 mlx4_en_free_affinity_hint(priv
, i
);
1604 for (j
= 0; j
< cq
->size
; j
++) {
1605 struct mlx4_cqe
*cqe
= NULL
;
1607 cqe
= mlx4_en_get_cqe(cq
->buf
, j
, priv
->cqe_size
) +
1609 cqe
->owner_sr_opcode
= MLX4_CQE_OWNER_MASK
;
1612 err
= mlx4_en_set_cq_moder(priv
, cq
);
1614 en_err(priv
, "Failed setting cq moderation parameters\n");
1615 mlx4_en_deactivate_cq(priv
, cq
);
1616 mlx4_en_free_affinity_hint(priv
, i
);
1619 mlx4_en_arm_cq(priv
, cq
);
1620 priv
->rx_ring
[i
]->cqn
= cq
->mcq
.cqn
;
1625 en_dbg(DRV
, priv
, "Getting qp number for port %d\n", priv
->port
);
1626 err
= mlx4_en_get_qp(priv
);
1628 en_err(priv
, "Failed getting eth qp\n");
1631 mdev
->mac_removed
[priv
->port
] = 0;
1633 priv
->counter_index
=
1634 mlx4_get_default_counter_index(mdev
->dev
, priv
->port
);
1636 err
= mlx4_en_config_rss_steer(priv
);
1638 en_err(priv
, "Failed configuring rss steering\n");
1642 err
= mlx4_en_create_drop_qp(priv
);
1646 /* Configure tx cq's and rings */
1647 for (t
= 0 ; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
1648 u8 num_tx_rings_p_up
= t
== TX
?
1649 priv
->num_tx_rings_p_up
: priv
->tx_ring_num
[t
];
1651 for (i
= 0; i
< priv
->tx_ring_num
[t
]; i
++) {
1653 cq
= priv
->tx_cq
[t
][i
];
1654 err
= mlx4_en_activate_cq(priv
, cq
, i
);
1656 en_err(priv
, "Failed allocating Tx CQ\n");
1659 err
= mlx4_en_set_cq_moder(priv
, cq
);
1661 en_err(priv
, "Failed setting cq moderation parameters\n");
1662 mlx4_en_deactivate_cq(priv
, cq
);
1666 "Resetting index of collapsed CQ:%d to -1\n", i
);
1667 cq
->buf
->wqe_index
= cpu_to_be16(0xffff);
1669 /* Configure ring */
1670 tx_ring
= priv
->tx_ring
[t
][i
];
1671 err
= mlx4_en_activate_tx_ring(priv
, tx_ring
,
1673 i
/ num_tx_rings_p_up
);
1675 en_err(priv
, "Failed allocating Tx ring\n");
1676 mlx4_en_deactivate_cq(priv
, cq
);
1680 tx_ring
->tx_queue
= netdev_get_tx_queue(dev
, i
);
1681 tx_ring
->recycle_ring
= NULL
;
1683 mlx4_en_init_recycle_ring(priv
, i
);
1686 /* Arm CQ for TX completions */
1687 mlx4_en_arm_cq(priv
, cq
);
1689 /* Set initial ownership of all Tx TXBBs to SW (1) */
1690 for (j
= 0; j
< tx_ring
->buf_size
; j
+= STAMP_STRIDE
)
1691 *((u32
*)(tx_ring
->buf
+ j
)) = 0xffffffff;
1695 /* Configure port */
1696 err
= mlx4_SET_PORT_general(mdev
->dev
, priv
->port
,
1697 priv
->rx_skb_size
+ ETH_FCS_LEN
,
1698 priv
->prof
->tx_pause
,
1700 priv
->prof
->rx_pause
,
1701 priv
->prof
->rx_ppp
);
1703 en_err(priv
, "Failed setting port general configurations for port %d, with error %d\n",
1708 err
= mlx4_SET_PORT_user_mtu(mdev
->dev
, priv
->port
, dev
->mtu
);
1710 en_err(priv
, "Failed to pass user MTU(%d) to Firmware for port %d, with error %d\n",
1711 dev
->mtu
, priv
->port
, err
);
1715 /* Set default qp number */
1716 err
= mlx4_SET_PORT_qpn_calc(mdev
->dev
, priv
->port
, priv
->base_qpn
, 0);
1718 en_err(priv
, "Failed setting default qp numbers\n");
1722 if (mdev
->dev
->caps
.tunnel_offload_mode
== MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
) {
1723 err
= mlx4_SET_PORT_VXLAN(mdev
->dev
, priv
->port
, VXLAN_STEER_BY_OUTER_MAC
, 1);
1725 en_err(priv
, "Failed setting port L2 tunnel configuration, err %d\n",
1732 en_dbg(HW
, priv
, "Initializing port\n");
1733 err
= mlx4_INIT_PORT(mdev
->dev
, priv
->port
);
1735 en_err(priv
, "Failed Initializing port\n");
1739 /* Set Unicast and VXLAN steering rules */
1740 if (mdev
->dev
->caps
.steering_mode
!= MLX4_STEERING_MODE_A0
&&
1741 mlx4_en_set_rss_steer_rules(priv
))
1742 mlx4_warn(mdev
, "Failed setting steering rules\n");
1744 /* Attach rx QP to bradcast address */
1745 eth_broadcast_addr(&mc_list
[10]);
1746 mc_list
[5] = priv
->port
; /* needed for B0 steering support */
1747 if (mlx4_multicast_attach(mdev
->dev
, priv
->rss_map
.indir_qp
, mc_list
,
1748 priv
->port
, 0, MLX4_PROT_ETH
,
1749 &priv
->broadcast_id
))
1750 mlx4_warn(mdev
, "Failed Attaching Broadcast\n");
1752 /* Must redo promiscuous mode setup. */
1753 priv
->flags
&= ~(MLX4_EN_FLAG_PROMISC
| MLX4_EN_FLAG_MC_PROMISC
);
1755 /* Schedule multicast task to populate multicast list */
1756 queue_work(mdev
->workqueue
, &priv
->rx_mode_task
);
1758 if (priv
->mdev
->dev
->caps
.tunnel_offload_mode
== MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
)
1759 udp_tunnel_get_rx_info(dev
);
1761 priv
->port_up
= true;
1763 /* Process all completions if exist to prevent
1764 * the queues freezing if they are full
1766 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
1768 napi_schedule(&priv
->rx_cq
[i
]->napi
);
1772 netif_tx_start_all_queues(dev
);
1773 netif_device_attach(dev
);
1778 if (t
== MLX4_EN_NUM_TX_TYPES
) {
1780 i
= priv
->tx_ring_num
[t
];
1784 mlx4_en_deactivate_tx_ring(priv
, priv
->tx_ring
[t
][i
]);
1785 mlx4_en_deactivate_cq(priv
, priv
->tx_cq
[t
][i
]);
1789 i
= priv
->tx_ring_num
[t
];
1791 mlx4_en_destroy_drop_qp(priv
);
1793 mlx4_en_release_rss_steer(priv
);
1795 mlx4_en_put_qp(priv
);
1797 while (rx_index
--) {
1798 mlx4_en_deactivate_cq(priv
, priv
->rx_cq
[rx_index
]);
1799 mlx4_en_free_affinity_hint(priv
, rx_index
);
1801 for (i
= 0; i
< priv
->rx_ring_num
; i
++)
1802 mlx4_en_deactivate_rx_ring(priv
, priv
->rx_ring
[i
]);
1804 return err
; /* need to close devices */
1808 void mlx4_en_stop_port(struct net_device
*dev
, int detach
)
1810 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
1811 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1812 struct mlx4_en_mc_list
*mclist
, *tmp
;
1813 struct ethtool_flow_id
*flow
, *tmp_flow
;
1815 u8 mc_list
[16] = {0};
1817 if (!priv
->port_up
) {
1818 en_dbg(DRV
, priv
, "stop port called while port already down\n");
1823 mlx4_CLOSE_PORT(mdev
->dev
, priv
->port
);
1825 /* Synchronize with tx routine */
1826 netif_tx_lock_bh(dev
);
1828 netif_device_detach(dev
);
1829 netif_tx_stop_all_queues(dev
);
1830 netif_tx_unlock_bh(dev
);
1832 netif_tx_disable(dev
);
1834 spin_lock_bh(&priv
->stats_lock
);
1835 mlx4_en_fold_software_stats(dev
);
1836 /* Set port as not active */
1837 priv
->port_up
= false;
1838 spin_unlock_bh(&priv
->stats_lock
);
1840 priv
->counter_index
= MLX4_SINK_COUNTER_INDEX(mdev
->dev
);
1842 /* Promsicuous mode */
1843 if (mdev
->dev
->caps
.steering_mode
==
1844 MLX4_STEERING_MODE_DEVICE_MANAGED
) {
1845 priv
->flags
&= ~(MLX4_EN_FLAG_PROMISC
|
1846 MLX4_EN_FLAG_MC_PROMISC
);
1847 mlx4_flow_steer_promisc_remove(mdev
->dev
,
1849 MLX4_FS_ALL_DEFAULT
);
1850 mlx4_flow_steer_promisc_remove(mdev
->dev
,
1852 MLX4_FS_MC_DEFAULT
);
1853 } else if (priv
->flags
& MLX4_EN_FLAG_PROMISC
) {
1854 priv
->flags
&= ~MLX4_EN_FLAG_PROMISC
;
1856 /* Disable promiscouos mode */
1857 mlx4_unicast_promisc_remove(mdev
->dev
, priv
->base_qpn
,
1860 /* Disable Multicast promisc */
1861 if (priv
->flags
& MLX4_EN_FLAG_MC_PROMISC
) {
1862 mlx4_multicast_promisc_remove(mdev
->dev
, priv
->base_qpn
,
1864 priv
->flags
&= ~MLX4_EN_FLAG_MC_PROMISC
;
1868 /* Detach All multicasts */
1869 eth_broadcast_addr(&mc_list
[10]);
1870 mc_list
[5] = priv
->port
; /* needed for B0 steering support */
1871 mlx4_multicast_detach(mdev
->dev
, priv
->rss_map
.indir_qp
, mc_list
,
1872 MLX4_PROT_ETH
, priv
->broadcast_id
);
1873 list_for_each_entry(mclist
, &priv
->curr_list
, list
) {
1874 memcpy(&mc_list
[10], mclist
->addr
, ETH_ALEN
);
1875 mc_list
[5] = priv
->port
;
1876 mlx4_multicast_detach(mdev
->dev
, priv
->rss_map
.indir_qp
,
1877 mc_list
, MLX4_PROT_ETH
, mclist
->reg_id
);
1878 if (mclist
->tunnel_reg_id
)
1879 mlx4_flow_detach(mdev
->dev
, mclist
->tunnel_reg_id
);
1881 mlx4_en_clear_list(dev
);
1882 list_for_each_entry_safe(mclist
, tmp
, &priv
->curr_list
, list
) {
1883 list_del(&mclist
->list
);
1887 /* Flush multicast filter */
1888 mlx4_SET_MCAST_FLTR(mdev
->dev
, priv
->port
, 0, 1, MLX4_MCAST_CONFIG
);
1890 /* Remove flow steering rules for the port*/
1891 if (mdev
->dev
->caps
.steering_mode
==
1892 MLX4_STEERING_MODE_DEVICE_MANAGED
) {
1894 list_for_each_entry_safe(flow
, tmp_flow
,
1895 &priv
->ethtool_list
, list
) {
1896 mlx4_flow_detach(mdev
->dev
, flow
->id
);
1897 list_del(&flow
->list
);
1901 mlx4_en_destroy_drop_qp(priv
);
1904 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
1905 for (i
= 0; i
< priv
->tx_ring_num
[t
]; i
++) {
1906 mlx4_en_deactivate_tx_ring(priv
, priv
->tx_ring
[t
][i
]);
1907 mlx4_en_deactivate_cq(priv
, priv
->tx_cq
[t
][i
]);
1912 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++)
1913 for (i
= 0; i
< priv
->tx_ring_num
[t
]; i
++)
1914 mlx4_en_free_tx_buf(dev
, priv
->tx_ring
[t
][i
]);
1916 if (mdev
->dev
->caps
.steering_mode
!= MLX4_STEERING_MODE_A0
)
1917 mlx4_en_delete_rss_steer_rules(priv
);
1920 mlx4_en_release_rss_steer(priv
);
1922 /* Unregister Mac address for the port */
1923 mlx4_en_put_qp(priv
);
1924 if (!(mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN
))
1925 mdev
->mac_removed
[priv
->port
] = 1;
1928 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
1929 struct mlx4_en_cq
*cq
= priv
->rx_cq
[i
];
1931 napi_synchronize(&cq
->napi
);
1932 mlx4_en_deactivate_rx_ring(priv
, priv
->rx_ring
[i
]);
1933 mlx4_en_deactivate_cq(priv
, cq
);
1935 mlx4_en_free_affinity_hint(priv
, i
);
1939 static void mlx4_en_restart(struct work_struct
*work
)
1941 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
1943 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1944 struct net_device
*dev
= priv
->dev
;
1946 en_dbg(DRV
, priv
, "Watchdog task called for port %d\n", priv
->port
);
1949 mutex_lock(&mdev
->state_lock
);
1950 if (priv
->port_up
) {
1951 mlx4_en_stop_port(dev
, 1);
1952 if (mlx4_en_start_port(dev
))
1953 en_err(priv
, "Failed restarting port %d\n", priv
->port
);
1955 mutex_unlock(&mdev
->state_lock
);
1959 static void mlx4_en_clear_stats(struct net_device
*dev
)
1961 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
1962 struct mlx4_en_dev
*mdev
= priv
->mdev
;
1963 struct mlx4_en_tx_ring
**tx_ring
;
1966 if (!mlx4_is_slave(mdev
->dev
))
1967 if (mlx4_en_DUMP_ETH_STATS(mdev
, priv
->port
, 1))
1968 en_dbg(HW
, priv
, "Failed dumping statistics\n");
1970 memset(&priv
->pstats
, 0, sizeof(priv
->pstats
));
1971 memset(&priv
->pkstats
, 0, sizeof(priv
->pkstats
));
1972 memset(&priv
->port_stats
, 0, sizeof(priv
->port_stats
));
1973 memset(&priv
->rx_flowstats
, 0, sizeof(priv
->rx_flowstats
));
1974 memset(&priv
->tx_flowstats
, 0, sizeof(priv
->tx_flowstats
));
1975 memset(&priv
->rx_priority_flowstats
, 0,
1976 sizeof(priv
->rx_priority_flowstats
));
1977 memset(&priv
->tx_priority_flowstats
, 0,
1978 sizeof(priv
->tx_priority_flowstats
));
1979 memset(&priv
->pf_stats
, 0, sizeof(priv
->pf_stats
));
1981 tx_ring
= priv
->tx_ring
[TX
];
1982 for (i
= 0; i
< priv
->tx_ring_num
[TX
]; i
++) {
1983 tx_ring
[i
]->bytes
= 0;
1984 tx_ring
[i
]->packets
= 0;
1985 tx_ring
[i
]->tx_csum
= 0;
1986 tx_ring
[i
]->tx_dropped
= 0;
1987 tx_ring
[i
]->queue_stopped
= 0;
1988 tx_ring
[i
]->wake_queue
= 0;
1989 tx_ring
[i
]->tso_packets
= 0;
1990 tx_ring
[i
]->xmit_more
= 0;
1992 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
1993 priv
->rx_ring
[i
]->bytes
= 0;
1994 priv
->rx_ring
[i
]->packets
= 0;
1995 priv
->rx_ring
[i
]->csum_ok
= 0;
1996 priv
->rx_ring
[i
]->csum_none
= 0;
1997 priv
->rx_ring
[i
]->csum_complete
= 0;
2001 static int mlx4_en_open(struct net_device
*dev
)
2003 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2004 struct mlx4_en_dev
*mdev
= priv
->mdev
;
2007 mutex_lock(&mdev
->state_lock
);
2009 if (!mdev
->device_up
) {
2010 en_err(priv
, "Cannot open - device down/disabled\n");
2015 /* Reset HW statistics and SW counters */
2016 mlx4_en_clear_stats(dev
);
2018 err
= mlx4_en_start_port(dev
);
2020 en_err(priv
, "Failed starting port:%d\n", priv
->port
);
2023 mutex_unlock(&mdev
->state_lock
);
2028 static int mlx4_en_close(struct net_device
*dev
)
2030 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2031 struct mlx4_en_dev
*mdev
= priv
->mdev
;
2033 en_dbg(IFDOWN
, priv
, "Close port called\n");
2035 mutex_lock(&mdev
->state_lock
);
2037 mlx4_en_stop_port(dev
, 0);
2038 netif_carrier_off(dev
);
2040 mutex_unlock(&mdev
->state_lock
);
2044 static void mlx4_en_free_resources(struct mlx4_en_priv
*priv
)
2048 #ifdef CONFIG_RFS_ACCEL
2049 priv
->dev
->rx_cpu_rmap
= NULL
;
2052 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
2053 for (i
= 0; i
< priv
->tx_ring_num
[t
]; i
++) {
2054 if (priv
->tx_ring
[t
] && priv
->tx_ring
[t
][i
])
2055 mlx4_en_destroy_tx_ring(priv
,
2056 &priv
->tx_ring
[t
][i
]);
2057 if (priv
->tx_cq
[t
] && priv
->tx_cq
[t
][i
])
2058 mlx4_en_destroy_cq(priv
, &priv
->tx_cq
[t
][i
]);
2060 kfree(priv
->tx_ring
[t
]);
2061 kfree(priv
->tx_cq
[t
]);
2064 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
2065 if (priv
->rx_ring
[i
])
2066 mlx4_en_destroy_rx_ring(priv
, &priv
->rx_ring
[i
],
2067 priv
->prof
->rx_ring_size
, priv
->stride
);
2069 mlx4_en_destroy_cq(priv
, &priv
->rx_cq
[i
]);
2074 static int mlx4_en_alloc_resources(struct mlx4_en_priv
*priv
)
2076 struct mlx4_en_port_profile
*prof
= priv
->prof
;
2080 /* Create tx Rings */
2081 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
2082 for (i
= 0; i
< priv
->tx_ring_num
[t
]; i
++) {
2083 node
= cpu_to_node(i
% num_online_cpus());
2084 if (mlx4_en_create_cq(priv
, &priv
->tx_cq
[t
][i
],
2085 prof
->tx_ring_size
, i
, t
, node
))
2088 if (mlx4_en_create_tx_ring(priv
, &priv
->tx_ring
[t
][i
],
2090 TXBB_SIZE
, node
, i
))
2095 /* Create rx Rings */
2096 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
2097 node
= cpu_to_node(i
% num_online_cpus());
2098 if (mlx4_en_create_cq(priv
, &priv
->rx_cq
[i
],
2099 prof
->rx_ring_size
, i
, RX
, node
))
2102 if (mlx4_en_create_rx_ring(priv
, &priv
->rx_ring
[i
],
2103 prof
->rx_ring_size
, priv
->stride
,
2108 #ifdef CONFIG_RFS_ACCEL
2109 priv
->dev
->rx_cpu_rmap
= mlx4_get_cpu_rmap(priv
->mdev
->dev
, priv
->port
);
2115 en_err(priv
, "Failed to allocate NIC resources\n");
2116 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
2117 if (priv
->rx_ring
[i
])
2118 mlx4_en_destroy_rx_ring(priv
, &priv
->rx_ring
[i
],
2122 mlx4_en_destroy_cq(priv
, &priv
->rx_cq
[i
]);
2124 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
2125 for (i
= 0; i
< priv
->tx_ring_num
[t
]; i
++) {
2126 if (priv
->tx_ring
[t
][i
])
2127 mlx4_en_destroy_tx_ring(priv
,
2128 &priv
->tx_ring
[t
][i
]);
2129 if (priv
->tx_cq
[t
][i
])
2130 mlx4_en_destroy_cq(priv
, &priv
->tx_cq
[t
][i
]);
2137 static int mlx4_en_copy_priv(struct mlx4_en_priv
*dst
,
2138 struct mlx4_en_priv
*src
,
2139 struct mlx4_en_port_profile
*prof
)
2143 memcpy(&dst
->hwtstamp_config
, &prof
->hwtstamp_config
,
2144 sizeof(dst
->hwtstamp_config
));
2145 dst
->num_tx_rings_p_up
= src
->mdev
->profile
.num_tx_rings_p_up
;
2146 dst
->rx_ring_num
= prof
->rx_ring_num
;
2147 dst
->flags
= prof
->flags
;
2148 dst
->mdev
= src
->mdev
;
2149 dst
->port
= src
->port
;
2150 dst
->dev
= src
->dev
;
2152 dst
->stride
= roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc
) +
2153 DS_SIZE
* MLX4_EN_MAX_RX_FRAGS
);
2155 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
2156 dst
->tx_ring_num
[t
] = prof
->tx_ring_num
[t
];
2157 if (!dst
->tx_ring_num
[t
])
2160 dst
->tx_ring
[t
] = kzalloc(sizeof(struct mlx4_en_tx_ring
*) *
2161 MAX_TX_RINGS
, GFP_KERNEL
);
2162 if (!dst
->tx_ring
[t
])
2165 dst
->tx_cq
[t
] = kzalloc(sizeof(struct mlx4_en_cq
*) *
2166 MAX_TX_RINGS
, GFP_KERNEL
);
2167 if (!dst
->tx_cq
[t
]) {
2168 kfree(dst
->tx_ring
[t
]);
2177 kfree(dst
->tx_ring
[t
]);
2178 kfree(dst
->tx_cq
[t
]);
2183 static void mlx4_en_update_priv(struct mlx4_en_priv
*dst
,
2184 struct mlx4_en_priv
*src
)
2187 memcpy(dst
->rx_ring
, src
->rx_ring
,
2188 sizeof(struct mlx4_en_rx_ring
*) * src
->rx_ring_num
);
2189 memcpy(dst
->rx_cq
, src
->rx_cq
,
2190 sizeof(struct mlx4_en_cq
*) * src
->rx_ring_num
);
2191 memcpy(&dst
->hwtstamp_config
, &src
->hwtstamp_config
,
2192 sizeof(dst
->hwtstamp_config
));
2193 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
2194 dst
->tx_ring_num
[t
] = src
->tx_ring_num
[t
];
2195 dst
->tx_ring
[t
] = src
->tx_ring
[t
];
2196 dst
->tx_cq
[t
] = src
->tx_cq
[t
];
2198 dst
->rx_ring_num
= src
->rx_ring_num
;
2199 memcpy(dst
->prof
, src
->prof
, sizeof(struct mlx4_en_port_profile
));
2202 int mlx4_en_try_alloc_resources(struct mlx4_en_priv
*priv
,
2203 struct mlx4_en_priv
*tmp
,
2204 struct mlx4_en_port_profile
*prof
,
2205 bool carry_xdp_prog
)
2207 struct bpf_prog
*xdp_prog
;
2210 mlx4_en_copy_priv(tmp
, priv
, prof
);
2212 if (mlx4_en_alloc_resources(tmp
)) {
2214 "%s: Resource allocation failed, using previous configuration\n",
2216 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
2217 kfree(tmp
->tx_ring
[t
]);
2218 kfree(tmp
->tx_cq
[t
]);
2223 /* All rx_rings has the same xdp_prog. Pick the first one. */
2224 xdp_prog
= rcu_dereference_protected(
2225 priv
->rx_ring
[0]->xdp_prog
,
2226 lockdep_is_held(&priv
->mdev
->state_lock
));
2228 if (xdp_prog
&& carry_xdp_prog
) {
2229 xdp_prog
= bpf_prog_add(xdp_prog
, tmp
->rx_ring_num
);
2230 if (IS_ERR(xdp_prog
)) {
2231 mlx4_en_free_resources(tmp
);
2232 return PTR_ERR(xdp_prog
);
2234 for (i
= 0; i
< tmp
->rx_ring_num
; i
++)
2235 rcu_assign_pointer(tmp
->rx_ring
[i
]->xdp_prog
,
2242 void mlx4_en_safe_replace_resources(struct mlx4_en_priv
*priv
,
2243 struct mlx4_en_priv
*tmp
)
2245 mlx4_en_free_resources(priv
);
2246 mlx4_en_update_priv(priv
, tmp
);
2249 void mlx4_en_destroy_netdev(struct net_device
*dev
)
2251 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2252 struct mlx4_en_dev
*mdev
= priv
->mdev
;
2254 en_dbg(DRV
, priv
, "Destroying netdev on port:%d\n", priv
->port
);
2256 /* Unregister device - this will close the port if it was up */
2257 if (priv
->registered
) {
2258 devlink_port_type_clear(mlx4_get_devlink_port(mdev
->dev
,
2260 unregister_netdev(dev
);
2263 if (priv
->allocated
)
2264 mlx4_free_hwq_res(mdev
->dev
, &priv
->res
, MLX4_EN_PAGE_SIZE
);
2266 cancel_delayed_work(&priv
->stats_task
);
2267 cancel_delayed_work(&priv
->service_task
);
2268 /* flush any pending task for this netdev */
2269 flush_workqueue(mdev
->workqueue
);
2271 if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_TS
)
2272 mlx4_en_remove_timestamp(mdev
);
2274 /* Detach the netdev so tasks would not attempt to access it */
2275 mutex_lock(&mdev
->state_lock
);
2276 mdev
->pndev
[priv
->port
] = NULL
;
2277 mdev
->upper
[priv
->port
] = NULL
;
2279 #ifdef CONFIG_RFS_ACCEL
2280 mlx4_en_cleanup_filters(priv
);
2283 mlx4_en_free_resources(priv
);
2284 mutex_unlock(&mdev
->state_lock
);
2289 static bool mlx4_en_check_xdp_mtu(struct net_device
*dev
, int mtu
)
2291 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2293 if (mtu
> MLX4_EN_MAX_XDP_MTU
) {
2294 en_err(priv
, "mtu:%d > max:%d when XDP prog is attached\n",
2295 mtu
, MLX4_EN_MAX_XDP_MTU
);
2302 static int mlx4_en_change_mtu(struct net_device
*dev
, int new_mtu
)
2304 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2305 struct mlx4_en_dev
*mdev
= priv
->mdev
;
2308 en_dbg(DRV
, priv
, "Change MTU called - current:%d new:%d\n",
2311 if (priv
->tx_ring_num
[TX_XDP
] &&
2312 !mlx4_en_check_xdp_mtu(dev
, new_mtu
))
2317 if (netif_running(dev
)) {
2318 mutex_lock(&mdev
->state_lock
);
2319 if (!mdev
->device_up
) {
2320 /* NIC is probably restarting - let watchdog task reset
2322 en_dbg(DRV
, priv
, "Change MTU called with card down!?\n");
2324 mlx4_en_stop_port(dev
, 1);
2325 err
= mlx4_en_start_port(dev
);
2327 en_err(priv
, "Failed restarting port:%d\n",
2329 queue_work(mdev
->workqueue
, &priv
->watchdog_task
);
2332 mutex_unlock(&mdev
->state_lock
);
2337 static int mlx4_en_hwtstamp_set(struct net_device
*dev
, struct ifreq
*ifr
)
2339 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2340 struct mlx4_en_dev
*mdev
= priv
->mdev
;
2341 struct hwtstamp_config config
;
2343 if (copy_from_user(&config
, ifr
->ifr_data
, sizeof(config
)))
2346 /* reserved for future extensions */
2350 /* device doesn't support time stamping */
2351 if (!(mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_TS
))
2354 /* TX HW timestamp */
2355 switch (config
.tx_type
) {
2356 case HWTSTAMP_TX_OFF
:
2357 case HWTSTAMP_TX_ON
:
2363 /* RX HW timestamp */
2364 switch (config
.rx_filter
) {
2365 case HWTSTAMP_FILTER_NONE
:
2367 case HWTSTAMP_FILTER_ALL
:
2368 case HWTSTAMP_FILTER_SOME
:
2369 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT
:
2370 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC
:
2371 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ
:
2372 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT
:
2373 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC
:
2374 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ
:
2375 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT
:
2376 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC
:
2377 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ
:
2378 case HWTSTAMP_FILTER_PTP_V2_EVENT
:
2379 case HWTSTAMP_FILTER_PTP_V2_SYNC
:
2380 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ
:
2381 case HWTSTAMP_FILTER_NTP_ALL
:
2382 config
.rx_filter
= HWTSTAMP_FILTER_ALL
;
2388 if (mlx4_en_reset_config(dev
, config
, dev
->features
)) {
2389 config
.tx_type
= HWTSTAMP_TX_OFF
;
2390 config
.rx_filter
= HWTSTAMP_FILTER_NONE
;
2393 return copy_to_user(ifr
->ifr_data
, &config
,
2394 sizeof(config
)) ? -EFAULT
: 0;
2397 static int mlx4_en_hwtstamp_get(struct net_device
*dev
, struct ifreq
*ifr
)
2399 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2401 return copy_to_user(ifr
->ifr_data
, &priv
->hwtstamp_config
,
2402 sizeof(priv
->hwtstamp_config
)) ? -EFAULT
: 0;
2405 static int mlx4_en_ioctl(struct net_device
*dev
, struct ifreq
*ifr
, int cmd
)
2409 return mlx4_en_hwtstamp_set(dev
, ifr
);
2411 return mlx4_en_hwtstamp_get(dev
, ifr
);
2417 static netdev_features_t
mlx4_en_fix_features(struct net_device
*netdev
,
2418 netdev_features_t features
)
2420 struct mlx4_en_priv
*en_priv
= netdev_priv(netdev
);
2421 struct mlx4_en_dev
*mdev
= en_priv
->mdev
;
2423 /* Since there is no support for separate RX C-TAG/S-TAG vlan accel
2424 * enable/disable make sure S-TAG flag is always in same state as
2427 if (features
& NETIF_F_HW_VLAN_CTAG_RX
&&
2428 !(mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN
))
2429 features
|= NETIF_F_HW_VLAN_STAG_RX
;
2431 features
&= ~NETIF_F_HW_VLAN_STAG_RX
;
2436 static int mlx4_en_set_features(struct net_device
*netdev
,
2437 netdev_features_t features
)
2439 struct mlx4_en_priv
*priv
= netdev_priv(netdev
);
2443 if (DEV_FEATURE_CHANGED(netdev
, features
, NETIF_F_RXFCS
)) {
2444 en_info(priv
, "Turn %s RX-FCS\n",
2445 (features
& NETIF_F_RXFCS
) ? "ON" : "OFF");
2449 if (DEV_FEATURE_CHANGED(netdev
, features
, NETIF_F_RXALL
)) {
2450 u8 ignore_fcs_value
= (features
& NETIF_F_RXALL
) ? 1 : 0;
2452 en_info(priv
, "Turn %s RX-ALL\n",
2453 ignore_fcs_value
? "ON" : "OFF");
2454 ret
= mlx4_SET_PORT_fcs_check(priv
->mdev
->dev
,
2455 priv
->port
, ignore_fcs_value
);
2460 if (DEV_FEATURE_CHANGED(netdev
, features
, NETIF_F_HW_VLAN_CTAG_RX
)) {
2461 en_info(priv
, "Turn %s RX vlan strip offload\n",
2462 (features
& NETIF_F_HW_VLAN_CTAG_RX
) ? "ON" : "OFF");
2466 if (DEV_FEATURE_CHANGED(netdev
, features
, NETIF_F_HW_VLAN_CTAG_TX
))
2467 en_info(priv
, "Turn %s TX vlan strip offload\n",
2468 (features
& NETIF_F_HW_VLAN_CTAG_TX
) ? "ON" : "OFF");
2470 if (DEV_FEATURE_CHANGED(netdev
, features
, NETIF_F_HW_VLAN_STAG_TX
))
2471 en_info(priv
, "Turn %s TX S-VLAN strip offload\n",
2472 (features
& NETIF_F_HW_VLAN_STAG_TX
) ? "ON" : "OFF");
2474 if (DEV_FEATURE_CHANGED(netdev
, features
, NETIF_F_LOOPBACK
)) {
2475 en_info(priv
, "Turn %s loopback\n",
2476 (features
& NETIF_F_LOOPBACK
) ? "ON" : "OFF");
2477 mlx4_en_update_loopback_state(netdev
, features
);
2481 ret
= mlx4_en_reset_config(netdev
, priv
->hwtstamp_config
,
2490 static int mlx4_en_set_vf_mac(struct net_device
*dev
, int queue
, u8
*mac
)
2492 struct mlx4_en_priv
*en_priv
= netdev_priv(dev
);
2493 struct mlx4_en_dev
*mdev
= en_priv
->mdev
;
2495 return mlx4_set_vf_mac(mdev
->dev
, en_priv
->port
, queue
, mac
);
2498 static int mlx4_en_set_vf_vlan(struct net_device
*dev
, int vf
, u16 vlan
, u8 qos
,
2501 struct mlx4_en_priv
*en_priv
= netdev_priv(dev
);
2502 struct mlx4_en_dev
*mdev
= en_priv
->mdev
;
2504 return mlx4_set_vf_vlan(mdev
->dev
, en_priv
->port
, vf
, vlan
, qos
,
2508 static int mlx4_en_set_vf_rate(struct net_device
*dev
, int vf
, int min_tx_rate
,
2511 struct mlx4_en_priv
*en_priv
= netdev_priv(dev
);
2512 struct mlx4_en_dev
*mdev
= en_priv
->mdev
;
2514 return mlx4_set_vf_rate(mdev
->dev
, en_priv
->port
, vf
, min_tx_rate
,
2518 static int mlx4_en_set_vf_spoofchk(struct net_device
*dev
, int vf
, bool setting
)
2520 struct mlx4_en_priv
*en_priv
= netdev_priv(dev
);
2521 struct mlx4_en_dev
*mdev
= en_priv
->mdev
;
2523 return mlx4_set_vf_spoofchk(mdev
->dev
, en_priv
->port
, vf
, setting
);
2526 static int mlx4_en_get_vf_config(struct net_device
*dev
, int vf
, struct ifla_vf_info
*ivf
)
2528 struct mlx4_en_priv
*en_priv
= netdev_priv(dev
);
2529 struct mlx4_en_dev
*mdev
= en_priv
->mdev
;
2531 return mlx4_get_vf_config(mdev
->dev
, en_priv
->port
, vf
, ivf
);
2534 static int mlx4_en_set_vf_link_state(struct net_device
*dev
, int vf
, int link_state
)
2536 struct mlx4_en_priv
*en_priv
= netdev_priv(dev
);
2537 struct mlx4_en_dev
*mdev
= en_priv
->mdev
;
2539 return mlx4_set_vf_link_state(mdev
->dev
, en_priv
->port
, vf
, link_state
);
2542 static int mlx4_en_get_vf_stats(struct net_device
*dev
, int vf
,
2543 struct ifla_vf_stats
*vf_stats
)
2545 struct mlx4_en_priv
*en_priv
= netdev_priv(dev
);
2546 struct mlx4_en_dev
*mdev
= en_priv
->mdev
;
2548 return mlx4_get_vf_stats(mdev
->dev
, en_priv
->port
, vf
, vf_stats
);
2551 #define PORT_ID_BYTE_LEN 8
2552 static int mlx4_en_get_phys_port_id(struct net_device
*dev
,
2553 struct netdev_phys_item_id
*ppid
)
2555 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2556 struct mlx4_dev
*mdev
= priv
->mdev
->dev
;
2558 u64 phys_port_id
= mdev
->caps
.phys_port_id
[priv
->port
];
2563 ppid
->id_len
= sizeof(phys_port_id
);
2564 for (i
= PORT_ID_BYTE_LEN
- 1; i
>= 0; --i
) {
2565 ppid
->id
[i
] = phys_port_id
& 0xff;
2571 static void mlx4_en_add_vxlan_offloads(struct work_struct
*work
)
2574 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
2577 ret
= mlx4_config_vxlan_port(priv
->mdev
->dev
, priv
->vxlan_port
);
2581 ret
= mlx4_SET_PORT_VXLAN(priv
->mdev
->dev
, priv
->port
,
2582 VXLAN_STEER_BY_OUTER_MAC
, 1);
2585 en_err(priv
, "failed setting L2 tunnel configuration ret %d\n", ret
);
2590 priv
->dev
->hw_enc_features
|= NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
2592 NETIF_F_TSO
| NETIF_F_TSO6
|
2593 NETIF_F_GSO_UDP_TUNNEL
|
2594 NETIF_F_GSO_UDP_TUNNEL_CSUM
|
2595 NETIF_F_GSO_PARTIAL
;
2598 static void mlx4_en_del_vxlan_offloads(struct work_struct
*work
)
2601 struct mlx4_en_priv
*priv
= container_of(work
, struct mlx4_en_priv
,
2603 /* unset offloads */
2604 priv
->dev
->hw_enc_features
&= ~(NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
|
2606 NETIF_F_TSO
| NETIF_F_TSO6
|
2607 NETIF_F_GSO_UDP_TUNNEL
|
2608 NETIF_F_GSO_UDP_TUNNEL_CSUM
|
2609 NETIF_F_GSO_PARTIAL
);
2611 ret
= mlx4_SET_PORT_VXLAN(priv
->mdev
->dev
, priv
->port
,
2612 VXLAN_STEER_BY_OUTER_MAC
, 0);
2614 en_err(priv
, "failed setting L2 tunnel configuration ret %d\n", ret
);
2616 priv
->vxlan_port
= 0;
2619 static void mlx4_en_add_vxlan_port(struct net_device
*dev
,
2620 struct udp_tunnel_info
*ti
)
2622 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2623 __be16 port
= ti
->port
;
2624 __be16 current_port
;
2626 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
2629 if (ti
->sa_family
!= AF_INET
)
2632 if (priv
->mdev
->dev
->caps
.tunnel_offload_mode
!= MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
)
2635 current_port
= priv
->vxlan_port
;
2636 if (current_port
&& current_port
!= port
) {
2637 en_warn(priv
, "vxlan port %d configured, can't add port %d\n",
2638 ntohs(current_port
), ntohs(port
));
2642 priv
->vxlan_port
= port
;
2643 queue_work(priv
->mdev
->workqueue
, &priv
->vxlan_add_task
);
2646 static void mlx4_en_del_vxlan_port(struct net_device
*dev
,
2647 struct udp_tunnel_info
*ti
)
2649 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2650 __be16 port
= ti
->port
;
2651 __be16 current_port
;
2653 if (ti
->type
!= UDP_TUNNEL_TYPE_VXLAN
)
2656 if (ti
->sa_family
!= AF_INET
)
2659 if (priv
->mdev
->dev
->caps
.tunnel_offload_mode
!= MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
)
2662 current_port
= priv
->vxlan_port
;
2663 if (current_port
!= port
) {
2664 en_dbg(DRV
, priv
, "vxlan port %d isn't configured, ignoring\n", ntohs(port
));
2668 queue_work(priv
->mdev
->workqueue
, &priv
->vxlan_del_task
);
2671 static netdev_features_t
mlx4_en_features_check(struct sk_buff
*skb
,
2672 struct net_device
*dev
,
2673 netdev_features_t features
)
2675 features
= vlan_features_check(skb
, features
);
2676 features
= vxlan_features_check(skb
, features
);
2678 /* The ConnectX-3 doesn't support outer IPv6 checksums but it does
2679 * support inner IPv6 checksums and segmentation so we need to
2680 * strip that feature if this is an IPv6 encapsulated frame.
2682 if (skb
->encapsulation
&&
2683 (skb
->ip_summed
== CHECKSUM_PARTIAL
)) {
2684 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2686 if (!priv
->vxlan_port
||
2687 (ip_hdr(skb
)->version
!= 4) ||
2688 (udp_hdr(skb
)->dest
!= priv
->vxlan_port
))
2689 features
&= ~(NETIF_F_CSUM_MASK
| NETIF_F_GSO_MASK
);
2695 static int mlx4_en_set_tx_maxrate(struct net_device
*dev
, int queue_index
, u32 maxrate
)
2697 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2698 struct mlx4_en_tx_ring
*tx_ring
= priv
->tx_ring
[TX
][queue_index
];
2699 struct mlx4_update_qp_params params
;
2702 if (!(priv
->mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT
))
2705 /* rate provided to us in Mbs, check if it fits into 12 bits, if not use Gbs */
2706 if (maxrate
>> 12) {
2707 params
.rate_unit
= MLX4_QP_RATE_LIMIT_GBS
;
2708 params
.rate_val
= maxrate
/ 1000;
2709 } else if (maxrate
) {
2710 params
.rate_unit
= MLX4_QP_RATE_LIMIT_MBS
;
2711 params
.rate_val
= maxrate
;
2712 } else { /* zero serves to revoke the QP rate-limitation */
2713 params
.rate_unit
= 0;
2714 params
.rate_val
= 0;
2717 err
= mlx4_update_qp(priv
->mdev
->dev
, tx_ring
->qpn
, MLX4_UPDATE_QP_RATE_LIMIT
,
2722 static int mlx4_xdp_set(struct net_device
*dev
, struct bpf_prog
*prog
)
2724 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2725 struct mlx4_en_dev
*mdev
= priv
->mdev
;
2726 struct mlx4_en_port_profile new_prof
;
2727 struct bpf_prog
*old_prog
;
2728 struct mlx4_en_priv
*tmp
;
2735 xdp_ring_num
= prog
? priv
->rx_ring_num
: 0;
2737 /* No need to reconfigure buffers when simply swapping the
2738 * program for a new one.
2740 if (priv
->tx_ring_num
[TX_XDP
] == xdp_ring_num
) {
2742 prog
= bpf_prog_add(prog
, priv
->rx_ring_num
- 1);
2744 return PTR_ERR(prog
);
2746 mutex_lock(&mdev
->state_lock
);
2747 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
2748 old_prog
= rcu_dereference_protected(
2749 priv
->rx_ring
[i
]->xdp_prog
,
2750 lockdep_is_held(&mdev
->state_lock
));
2751 rcu_assign_pointer(priv
->rx_ring
[i
]->xdp_prog
, prog
);
2753 bpf_prog_put(old_prog
);
2755 mutex_unlock(&mdev
->state_lock
);
2759 if (!mlx4_en_check_xdp_mtu(dev
, dev
->mtu
))
2762 tmp
= kzalloc(sizeof(*tmp
), GFP_KERNEL
);
2767 prog
= bpf_prog_add(prog
, priv
->rx_ring_num
- 1);
2769 err
= PTR_ERR(prog
);
2774 mutex_lock(&mdev
->state_lock
);
2775 memcpy(&new_prof
, priv
->prof
, sizeof(struct mlx4_en_port_profile
));
2776 new_prof
.tx_ring_num
[TX_XDP
] = xdp_ring_num
;
2778 if (priv
->tx_ring_num
[TX
] + xdp_ring_num
> MAX_TX_RINGS
) {
2780 new_prof
.tx_ring_num
[TX
] =
2781 MAX_TX_RINGS
- ALIGN(xdp_ring_num
, MLX4_EN_NUM_UP
);
2782 en_warn(priv
, "Reducing the number of TX rings, to not exceed the max total rings number.\n");
2785 err
= mlx4_en_try_alloc_resources(priv
, tmp
, &new_prof
, false);
2788 bpf_prog_sub(prog
, priv
->rx_ring_num
- 1);
2792 if (priv
->port_up
) {
2794 mlx4_en_stop_port(dev
, 1);
2797 mlx4_en_safe_replace_resources(priv
, tmp
);
2799 netif_set_real_num_tx_queues(dev
, priv
->tx_ring_num
[TX
]);
2801 for (i
= 0; i
< priv
->rx_ring_num
; i
++) {
2802 old_prog
= rcu_dereference_protected(
2803 priv
->rx_ring
[i
]->xdp_prog
,
2804 lockdep_is_held(&mdev
->state_lock
));
2805 rcu_assign_pointer(priv
->rx_ring
[i
]->xdp_prog
, prog
);
2807 bpf_prog_put(old_prog
);
2811 err
= mlx4_en_start_port(dev
);
2813 en_err(priv
, "Failed starting port %d for XDP change\n",
2815 queue_work(mdev
->workqueue
, &priv
->watchdog_task
);
2820 mutex_unlock(&mdev
->state_lock
);
2826 static bool mlx4_xdp_attached(struct net_device
*dev
)
2828 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
2830 return !!priv
->tx_ring_num
[TX_XDP
];
2833 static int mlx4_xdp(struct net_device
*dev
, struct netdev_xdp
*xdp
)
2835 switch (xdp
->command
) {
2836 case XDP_SETUP_PROG
:
2837 return mlx4_xdp_set(dev
, xdp
->prog
);
2838 case XDP_QUERY_PROG
:
2839 xdp
->prog_attached
= mlx4_xdp_attached(dev
);
2846 static const struct net_device_ops mlx4_netdev_ops
= {
2847 .ndo_open
= mlx4_en_open
,
2848 .ndo_stop
= mlx4_en_close
,
2849 .ndo_start_xmit
= mlx4_en_xmit
,
2850 .ndo_select_queue
= mlx4_en_select_queue
,
2851 .ndo_get_stats64
= mlx4_en_get_stats64
,
2852 .ndo_set_rx_mode
= mlx4_en_set_rx_mode
,
2853 .ndo_set_mac_address
= mlx4_en_set_mac
,
2854 .ndo_validate_addr
= eth_validate_addr
,
2855 .ndo_change_mtu
= mlx4_en_change_mtu
,
2856 .ndo_do_ioctl
= mlx4_en_ioctl
,
2857 .ndo_tx_timeout
= mlx4_en_tx_timeout
,
2858 .ndo_vlan_rx_add_vid
= mlx4_en_vlan_rx_add_vid
,
2859 .ndo_vlan_rx_kill_vid
= mlx4_en_vlan_rx_kill_vid
,
2860 #ifdef CONFIG_NET_POLL_CONTROLLER
2861 .ndo_poll_controller
= mlx4_en_netpoll
,
2863 .ndo_set_features
= mlx4_en_set_features
,
2864 .ndo_fix_features
= mlx4_en_fix_features
,
2865 .ndo_setup_tc
= __mlx4_en_setup_tc
,
2866 #ifdef CONFIG_RFS_ACCEL
2867 .ndo_rx_flow_steer
= mlx4_en_filter_rfs
,
2869 .ndo_get_phys_port_id
= mlx4_en_get_phys_port_id
,
2870 .ndo_udp_tunnel_add
= mlx4_en_add_vxlan_port
,
2871 .ndo_udp_tunnel_del
= mlx4_en_del_vxlan_port
,
2872 .ndo_features_check
= mlx4_en_features_check
,
2873 .ndo_set_tx_maxrate
= mlx4_en_set_tx_maxrate
,
2874 .ndo_xdp
= mlx4_xdp
,
2877 static const struct net_device_ops mlx4_netdev_ops_master
= {
2878 .ndo_open
= mlx4_en_open
,
2879 .ndo_stop
= mlx4_en_close
,
2880 .ndo_start_xmit
= mlx4_en_xmit
,
2881 .ndo_select_queue
= mlx4_en_select_queue
,
2882 .ndo_get_stats64
= mlx4_en_get_stats64
,
2883 .ndo_set_rx_mode
= mlx4_en_set_rx_mode
,
2884 .ndo_set_mac_address
= mlx4_en_set_mac
,
2885 .ndo_validate_addr
= eth_validate_addr
,
2886 .ndo_change_mtu
= mlx4_en_change_mtu
,
2887 .ndo_tx_timeout
= mlx4_en_tx_timeout
,
2888 .ndo_vlan_rx_add_vid
= mlx4_en_vlan_rx_add_vid
,
2889 .ndo_vlan_rx_kill_vid
= mlx4_en_vlan_rx_kill_vid
,
2890 .ndo_set_vf_mac
= mlx4_en_set_vf_mac
,
2891 .ndo_set_vf_vlan
= mlx4_en_set_vf_vlan
,
2892 .ndo_set_vf_rate
= mlx4_en_set_vf_rate
,
2893 .ndo_set_vf_spoofchk
= mlx4_en_set_vf_spoofchk
,
2894 .ndo_set_vf_link_state
= mlx4_en_set_vf_link_state
,
2895 .ndo_get_vf_stats
= mlx4_en_get_vf_stats
,
2896 .ndo_get_vf_config
= mlx4_en_get_vf_config
,
2897 #ifdef CONFIG_NET_POLL_CONTROLLER
2898 .ndo_poll_controller
= mlx4_en_netpoll
,
2900 .ndo_set_features
= mlx4_en_set_features
,
2901 .ndo_fix_features
= mlx4_en_fix_features
,
2902 .ndo_setup_tc
= __mlx4_en_setup_tc
,
2903 #ifdef CONFIG_RFS_ACCEL
2904 .ndo_rx_flow_steer
= mlx4_en_filter_rfs
,
2906 .ndo_get_phys_port_id
= mlx4_en_get_phys_port_id
,
2907 .ndo_udp_tunnel_add
= mlx4_en_add_vxlan_port
,
2908 .ndo_udp_tunnel_del
= mlx4_en_del_vxlan_port
,
2909 .ndo_features_check
= mlx4_en_features_check
,
2910 .ndo_set_tx_maxrate
= mlx4_en_set_tx_maxrate
,
2911 .ndo_xdp
= mlx4_xdp
,
2914 struct mlx4_en_bond
{
2915 struct work_struct work
;
2916 struct mlx4_en_priv
*priv
;
2918 struct mlx4_port_map port_map
;
2921 static void mlx4_en_bond_work(struct work_struct
*work
)
2923 struct mlx4_en_bond
*bond
= container_of(work
,
2924 struct mlx4_en_bond
,
2927 struct mlx4_dev
*dev
= bond
->priv
->mdev
->dev
;
2929 if (bond
->is_bonded
) {
2930 if (!mlx4_is_bonded(dev
)) {
2931 err
= mlx4_bond(dev
);
2933 en_err(bond
->priv
, "Fail to bond device\n");
2936 err
= mlx4_port_map_set(dev
, &bond
->port_map
);
2938 en_err(bond
->priv
, "Fail to set port map [%d][%d]: %d\n",
2939 bond
->port_map
.port1
,
2940 bond
->port_map
.port2
,
2943 } else if (mlx4_is_bonded(dev
)) {
2944 err
= mlx4_unbond(dev
);
2946 en_err(bond
->priv
, "Fail to unbond device\n");
2948 dev_put(bond
->priv
->dev
);
2952 static int mlx4_en_queue_bond_work(struct mlx4_en_priv
*priv
, int is_bonded
,
2953 u8 v2p_p1
, u8 v2p_p2
)
2955 struct mlx4_en_bond
*bond
= NULL
;
2957 bond
= kzalloc(sizeof(*bond
), GFP_ATOMIC
);
2961 INIT_WORK(&bond
->work
, mlx4_en_bond_work
);
2963 bond
->is_bonded
= is_bonded
;
2964 bond
->port_map
.port1
= v2p_p1
;
2965 bond
->port_map
.port2
= v2p_p2
;
2966 dev_hold(priv
->dev
);
2967 queue_work(priv
->mdev
->workqueue
, &bond
->work
);
2971 int mlx4_en_netdev_event(struct notifier_block
*this,
2972 unsigned long event
, void *ptr
)
2974 struct net_device
*ndev
= netdev_notifier_info_to_dev(ptr
);
2976 struct mlx4_en_dev
*mdev
;
2977 struct mlx4_dev
*dev
;
2978 int i
, num_eth_ports
= 0;
2979 bool do_bond
= true;
2980 struct mlx4_en_priv
*priv
;
2984 if (!net_eq(dev_net(ndev
), &init_net
))
2987 mdev
= container_of(this, struct mlx4_en_dev
, nb
);
2990 /* Go into this mode only when two network devices set on two ports
2991 * of the same mlx4 device are slaves of the same bonding master
2993 mlx4_foreach_port(i
, dev
, MLX4_PORT_TYPE_ETH
) {
2995 if (!port
&& (mdev
->pndev
[i
] == ndev
))
2997 mdev
->upper
[i
] = mdev
->pndev
[i
] ?
2998 netdev_master_upper_dev_get(mdev
->pndev
[i
]) : NULL
;
2999 /* condition not met: network device is a slave */
3000 if (!mdev
->upper
[i
])
3002 if (num_eth_ports
< 2)
3004 /* condition not met: same master */
3005 if (mdev
->upper
[i
] != mdev
->upper
[i
-1])
3008 /* condition not met: 2 salves */
3009 do_bond
= (num_eth_ports
== 2) ? do_bond
: false;
3011 /* handle only events that come with enough info */
3012 if ((do_bond
&& (event
!= NETDEV_BONDING_INFO
)) || !port
)
3015 priv
= netdev_priv(ndev
);
3017 struct netdev_notifier_bonding_info
*notifier_info
= ptr
;
3018 struct netdev_bonding_info
*bonding_info
=
3019 ¬ifier_info
->bonding_info
;
3021 /* required mode 1, 2 or 4 */
3022 if ((bonding_info
->master
.bond_mode
!= BOND_MODE_ACTIVEBACKUP
) &&
3023 (bonding_info
->master
.bond_mode
!= BOND_MODE_XOR
) &&
3024 (bonding_info
->master
.bond_mode
!= BOND_MODE_8023AD
))
3027 /* require exactly 2 slaves */
3028 if (bonding_info
->master
.num_slaves
!= 2)
3033 if (bonding_info
->master
.bond_mode
==
3034 BOND_MODE_ACTIVEBACKUP
) {
3035 /* in active-backup mode virtual ports are
3036 * mapped to the physical port of the active
3038 if (bonding_info
->slave
.state
==
3039 BOND_STATE_BACKUP
) {
3047 } else { /* BOND_STATE_ACTIVE */
3056 } else { /* Active-Active */
3057 /* in active-active mode a virtual port is
3058 * mapped to the native physical port if and only
3059 * if the physical port is up */
3060 __s8 link
= bonding_info
->slave
.link
;
3066 if ((link
== BOND_LINK_UP
) ||
3067 (link
== BOND_LINK_FAIL
)) {
3072 } else { /* BOND_LINK_DOWN || BOND_LINK_BACK */
3082 mlx4_en_queue_bond_work(priv
, do_bond
,
3083 v2p_port1
, v2p_port2
);
3088 void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev
*dev
,
3089 struct mlx4_en_stats_bitmap
*stats_bitmap
,
3090 u8 rx_ppp
, u8 rx_pause
,
3091 u8 tx_ppp
, u8 tx_pause
)
3093 int last_i
= NUM_MAIN_STATS
+ NUM_PORT_STATS
+ NUM_PF_STATS
;
3095 if (!mlx4_is_slave(dev
) &&
3096 (dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN
)) {
3097 mutex_lock(&stats_bitmap
->mutex
);
3098 bitmap_clear(stats_bitmap
->bitmap
, last_i
, NUM_FLOW_STATS
);
3101 bitmap_set(stats_bitmap
->bitmap
, last_i
,
3102 NUM_FLOW_PRIORITY_STATS_RX
);
3103 last_i
+= NUM_FLOW_PRIORITY_STATS_RX
;
3105 if (rx_pause
&& !(rx_ppp
))
3106 bitmap_set(stats_bitmap
->bitmap
, last_i
,
3108 last_i
+= NUM_FLOW_STATS_RX
;
3111 bitmap_set(stats_bitmap
->bitmap
, last_i
,
3112 NUM_FLOW_PRIORITY_STATS_TX
);
3113 last_i
+= NUM_FLOW_PRIORITY_STATS_TX
;
3115 if (tx_pause
&& !(tx_ppp
))
3116 bitmap_set(stats_bitmap
->bitmap
, last_i
,
3118 last_i
+= NUM_FLOW_STATS_TX
;
3120 mutex_unlock(&stats_bitmap
->mutex
);
3124 void mlx4_en_set_stats_bitmap(struct mlx4_dev
*dev
,
3125 struct mlx4_en_stats_bitmap
*stats_bitmap
,
3126 u8 rx_ppp
, u8 rx_pause
,
3127 u8 tx_ppp
, u8 tx_pause
)
3131 mutex_init(&stats_bitmap
->mutex
);
3132 bitmap_zero(stats_bitmap
->bitmap
, NUM_ALL_STATS
);
3134 if (mlx4_is_slave(dev
)) {
3135 bitmap_set(stats_bitmap
->bitmap
, last_i
+
3136 MLX4_FIND_NETDEV_STAT(rx_packets
), 1);
3137 bitmap_set(stats_bitmap
->bitmap
, last_i
+
3138 MLX4_FIND_NETDEV_STAT(tx_packets
), 1);
3139 bitmap_set(stats_bitmap
->bitmap
, last_i
+
3140 MLX4_FIND_NETDEV_STAT(rx_bytes
), 1);
3141 bitmap_set(stats_bitmap
->bitmap
, last_i
+
3142 MLX4_FIND_NETDEV_STAT(tx_bytes
), 1);
3143 bitmap_set(stats_bitmap
->bitmap
, last_i
+
3144 MLX4_FIND_NETDEV_STAT(rx_dropped
), 1);
3145 bitmap_set(stats_bitmap
->bitmap
, last_i
+
3146 MLX4_FIND_NETDEV_STAT(tx_dropped
), 1);
3148 bitmap_set(stats_bitmap
->bitmap
, last_i
, NUM_MAIN_STATS
);
3150 last_i
+= NUM_MAIN_STATS
;
3152 bitmap_set(stats_bitmap
->bitmap
, last_i
, NUM_PORT_STATS
);
3153 last_i
+= NUM_PORT_STATS
;
3155 if (mlx4_is_master(dev
))
3156 bitmap_set(stats_bitmap
->bitmap
, last_i
,
3158 last_i
+= NUM_PF_STATS
;
3160 mlx4_en_update_pfc_stats_bitmap(dev
, stats_bitmap
,
3163 last_i
+= NUM_FLOW_STATS
;
3165 if (!mlx4_is_slave(dev
))
3166 bitmap_set(stats_bitmap
->bitmap
, last_i
, NUM_PKT_STATS
);
3167 last_i
+= NUM_PKT_STATS
;
3169 bitmap_set(stats_bitmap
->bitmap
, last_i
, NUM_XDP_STATS
);
3170 last_i
+= NUM_XDP_STATS
;
3173 int mlx4_en_init_netdev(struct mlx4_en_dev
*mdev
, int port
,
3174 struct mlx4_en_port_profile
*prof
)
3176 struct net_device
*dev
;
3177 struct mlx4_en_priv
*priv
;
3181 dev
= alloc_etherdev_mqs(sizeof(struct mlx4_en_priv
),
3182 MAX_TX_RINGS
, MAX_RX_RINGS
);
3186 netif_set_real_num_tx_queues(dev
, prof
->tx_ring_num
[TX
]);
3187 netif_set_real_num_rx_queues(dev
, prof
->rx_ring_num
);
3189 SET_NETDEV_DEV(dev
, &mdev
->dev
->persist
->pdev
->dev
);
3190 dev
->dev_port
= port
- 1;
3193 * Initialize driver private data
3196 priv
= netdev_priv(dev
);
3197 memset(priv
, 0, sizeof(struct mlx4_en_priv
));
3198 priv
->counter_index
= MLX4_SINK_COUNTER_INDEX(mdev
->dev
);
3199 spin_lock_init(&priv
->stats_lock
);
3200 INIT_WORK(&priv
->rx_mode_task
, mlx4_en_do_set_rx_mode
);
3201 INIT_WORK(&priv
->watchdog_task
, mlx4_en_restart
);
3202 INIT_WORK(&priv
->linkstate_task
, mlx4_en_linkstate
);
3203 INIT_DELAYED_WORK(&priv
->stats_task
, mlx4_en_do_get_stats
);
3204 INIT_DELAYED_WORK(&priv
->service_task
, mlx4_en_service_task
);
3205 INIT_WORK(&priv
->vxlan_add_task
, mlx4_en_add_vxlan_offloads
);
3206 INIT_WORK(&priv
->vxlan_del_task
, mlx4_en_del_vxlan_offloads
);
3207 #ifdef CONFIG_RFS_ACCEL
3208 INIT_LIST_HEAD(&priv
->filters
);
3209 spin_lock_init(&priv
->filters_lock
);
3214 priv
->ddev
= &mdev
->pdev
->dev
;
3217 priv
->port_up
= false;
3218 priv
->flags
= prof
->flags
;
3219 priv
->pflags
= MLX4_EN_PRIV_FLAGS_BLUEFLAME
;
3220 priv
->ctrl_flags
= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE
|
3221 MLX4_WQE_CTRL_SOLICITED
);
3222 priv
->num_tx_rings_p_up
= mdev
->profile
.num_tx_rings_p_up
;
3223 priv
->tx_work_limit
= MLX4_EN_DEFAULT_TX_WORK
;
3224 netdev_rss_key_fill(priv
->rss_key
, sizeof(priv
->rss_key
));
3226 for (t
= 0; t
< MLX4_EN_NUM_TX_TYPES
; t
++) {
3227 priv
->tx_ring_num
[t
] = prof
->tx_ring_num
[t
];
3228 if (!priv
->tx_ring_num
[t
])
3231 priv
->tx_ring
[t
] = kzalloc(sizeof(struct mlx4_en_tx_ring
*) *
3232 MAX_TX_RINGS
, GFP_KERNEL
);
3233 if (!priv
->tx_ring
[t
]) {
3237 priv
->tx_cq
[t
] = kzalloc(sizeof(struct mlx4_en_cq
*) *
3238 MAX_TX_RINGS
, GFP_KERNEL
);
3239 if (!priv
->tx_cq
[t
]) {
3240 kfree(priv
->tx_ring
[t
]);
3245 priv
->rx_ring_num
= prof
->rx_ring_num
;
3246 priv
->cqe_factor
= (mdev
->dev
->caps
.cqe_size
== 64) ? 1 : 0;
3247 priv
->cqe_size
= mdev
->dev
->caps
.cqe_size
;
3248 priv
->mac_index
= -1;
3249 priv
->msg_enable
= MLX4_EN_MSG_LEVEL
;
3250 #ifdef CONFIG_MLX4_EN_DCB
3251 if (!mlx4_is_slave(priv
->mdev
->dev
)) {
3252 priv
->dcbx_cap
= DCB_CAP_DCBX_VER_CEE
| DCB_CAP_DCBX_HOST
|
3253 DCB_CAP_DCBX_VER_IEEE
;
3254 priv
->flags
|= MLX4_EN_DCB_ENABLED
;
3255 priv
->cee_config
.pfc_state
= false;
3257 for (i
= 0; i
< MLX4_EN_NUM_UP
; i
++)
3258 priv
->cee_config
.dcb_pfc
[i
] = pfc_disabled
;
3260 if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_ETS_CFG
) {
3261 dev
->dcbnl_ops
= &mlx4_en_dcbnl_ops
;
3263 en_info(priv
, "enabling only PFC DCB ops\n");
3264 dev
->dcbnl_ops
= &mlx4_en_dcbnl_pfc_ops
;
3269 for (i
= 0; i
< MLX4_EN_MAC_HASH_SIZE
; ++i
)
3270 INIT_HLIST_HEAD(&priv
->mac_hash
[i
]);
3272 /* Query for default mac and max mtu */
3273 priv
->max_mtu
= mdev
->dev
->caps
.eth_mtu_cap
[priv
->port
];
3275 if (mdev
->dev
->caps
.rx_checksum_flags_port
[priv
->port
] &
3276 MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP
)
3277 priv
->flags
|= MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP
;
3279 /* Set default MAC */
3280 dev
->addr_len
= ETH_ALEN
;
3281 mlx4_en_u64_to_mac(dev
->dev_addr
, mdev
->dev
->caps
.def_mac
[priv
->port
]);
3282 if (!is_valid_ether_addr(dev
->dev_addr
)) {
3283 en_err(priv
, "Port: %d, invalid mac burned: %pM, quiting\n",
3284 priv
->port
, dev
->dev_addr
);
3287 } else if (mlx4_is_slave(priv
->mdev
->dev
) &&
3288 (priv
->mdev
->dev
->port_random_macs
& 1 << priv
->port
)) {
3289 /* Random MAC was assigned in mlx4_slave_cap
3290 * in mlx4_core module
3292 dev
->addr_assign_type
|= NET_ADDR_RANDOM
;
3293 en_warn(priv
, "Assigned random MAC address %pM\n", dev
->dev_addr
);
3296 memcpy(priv
->current_mac
, dev
->dev_addr
, sizeof(priv
->current_mac
));
3298 priv
->stride
= roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc
) +
3299 DS_SIZE
* MLX4_EN_MAX_RX_FRAGS
);
3300 err
= mlx4_en_alloc_resources(priv
);
3304 /* Initialize time stamping config */
3305 priv
->hwtstamp_config
.flags
= 0;
3306 priv
->hwtstamp_config
.tx_type
= HWTSTAMP_TX_OFF
;
3307 priv
->hwtstamp_config
.rx_filter
= HWTSTAMP_FILTER_NONE
;
3309 /* Allocate page for receive rings */
3310 err
= mlx4_alloc_hwq_res(mdev
->dev
, &priv
->res
,
3313 en_err(priv
, "Failed to allocate page for rx qps\n");
3316 priv
->allocated
= 1;
3319 * Initialize netdev entry points
3321 if (mlx4_is_master(priv
->mdev
->dev
))
3322 dev
->netdev_ops
= &mlx4_netdev_ops_master
;
3324 dev
->netdev_ops
= &mlx4_netdev_ops
;
3325 dev
->watchdog_timeo
= MLX4_EN_WATCHDOG_TIMEOUT
;
3326 netif_set_real_num_tx_queues(dev
, priv
->tx_ring_num
[TX
]);
3327 netif_set_real_num_rx_queues(dev
, priv
->rx_ring_num
);
3329 dev
->ethtool_ops
= &mlx4_en_ethtool_ops
;
3332 * Set driver features
3334 dev
->hw_features
= NETIF_F_SG
| NETIF_F_IP_CSUM
| NETIF_F_IPV6_CSUM
;
3335 if (mdev
->LSO_support
)
3336 dev
->hw_features
|= NETIF_F_TSO
| NETIF_F_TSO6
;
3338 dev
->vlan_features
= dev
->hw_features
;
3340 dev
->hw_features
|= NETIF_F_RXCSUM
| NETIF_F_RXHASH
;
3341 dev
->features
= dev
->hw_features
| NETIF_F_HIGHDMA
|
3342 NETIF_F_HW_VLAN_CTAG_TX
| NETIF_F_HW_VLAN_CTAG_RX
|
3343 NETIF_F_HW_VLAN_CTAG_FILTER
;
3344 dev
->hw_features
|= NETIF_F_LOOPBACK
|
3345 NETIF_F_HW_VLAN_CTAG_TX
| NETIF_F_HW_VLAN_CTAG_RX
;
3347 if (!(mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN
)) {
3348 dev
->features
|= NETIF_F_HW_VLAN_STAG_RX
|
3349 NETIF_F_HW_VLAN_STAG_FILTER
;
3350 dev
->hw_features
|= NETIF_F_HW_VLAN_STAG_RX
;
3353 if (mlx4_is_slave(mdev
->dev
)) {
3354 bool vlan_offload_disabled
;
3357 err
= get_phv_bit(mdev
->dev
, port
, &phv
);
3359 dev
->hw_features
|= NETIF_F_HW_VLAN_STAG_TX
;
3360 priv
->pflags
|= MLX4_EN_PRIV_FLAGS_PHV
;
3362 err
= mlx4_get_is_vlan_offload_disabled(mdev
->dev
, port
,
3363 &vlan_offload_disabled
);
3364 if (!err
&& vlan_offload_disabled
) {
3365 dev
->hw_features
&= ~(NETIF_F_HW_VLAN_CTAG_TX
|
3366 NETIF_F_HW_VLAN_CTAG_RX
|
3367 NETIF_F_HW_VLAN_STAG_TX
|
3368 NETIF_F_HW_VLAN_STAG_RX
);
3369 dev
->features
&= ~(NETIF_F_HW_VLAN_CTAG_TX
|
3370 NETIF_F_HW_VLAN_CTAG_RX
|
3371 NETIF_F_HW_VLAN_STAG_TX
|
3372 NETIF_F_HW_VLAN_STAG_RX
);
3375 if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_PHV_EN
&&
3376 !(mdev
->dev
->caps
.flags2
&
3377 MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN
))
3378 dev
->hw_features
|= NETIF_F_HW_VLAN_STAG_TX
;
3381 if (mdev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_FCS_KEEP
)
3382 dev
->hw_features
|= NETIF_F_RXFCS
;
3384 if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_IGNORE_FCS
)
3385 dev
->hw_features
|= NETIF_F_RXALL
;
3387 if (mdev
->dev
->caps
.steering_mode
==
3388 MLX4_STEERING_MODE_DEVICE_MANAGED
&&
3389 mdev
->dev
->caps
.dmfs_high_steer_mode
!= MLX4_STEERING_DMFS_A0_STATIC
)
3390 dev
->hw_features
|= NETIF_F_NTUPLE
;
3392 if (mdev
->dev
->caps
.steering_mode
!= MLX4_STEERING_MODE_A0
)
3393 dev
->priv_flags
|= IFF_UNICAST_FLT
;
3395 /* Setting a default hash function value */
3396 if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_RSS_TOP
) {
3397 priv
->rss_hash_fn
= ETH_RSS_HASH_TOP
;
3398 } else if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_RSS_XOR
) {
3399 priv
->rss_hash_fn
= ETH_RSS_HASH_XOR
;
3402 "No RSS hash capabilities exposed, using Toeplitz\n");
3403 priv
->rss_hash_fn
= ETH_RSS_HASH_TOP
;
3406 if (mdev
->dev
->caps
.tunnel_offload_mode
== MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
) {
3407 dev
->hw_features
|= NETIF_F_GSO_UDP_TUNNEL
|
3408 NETIF_F_GSO_UDP_TUNNEL_CSUM
|
3409 NETIF_F_GSO_PARTIAL
;
3410 dev
->features
|= NETIF_F_GSO_UDP_TUNNEL
|
3411 NETIF_F_GSO_UDP_TUNNEL_CSUM
|
3412 NETIF_F_GSO_PARTIAL
;
3413 dev
->gso_partial_features
= NETIF_F_GSO_UDP_TUNNEL_CSUM
;
3416 /* MTU range: 46 - hw-specific max */
3417 dev
->min_mtu
= MLX4_EN_MIN_MTU
;
3418 dev
->max_mtu
= priv
->max_mtu
;
3420 mdev
->pndev
[port
] = dev
;
3421 mdev
->upper
[port
] = NULL
;
3423 netif_carrier_off(dev
);
3424 mlx4_en_set_default_moderation(priv
);
3426 en_warn(priv
, "Using %d TX rings\n", prof
->tx_ring_num
[TX
]);
3427 en_warn(priv
, "Using %d RX rings\n", prof
->rx_ring_num
);
3429 mlx4_en_update_loopback_state(priv
->dev
, priv
->dev
->features
);
3431 /* Configure port */
3432 mlx4_en_calc_rx_buf(dev
);
3433 err
= mlx4_SET_PORT_general(mdev
->dev
, priv
->port
,
3434 priv
->rx_skb_size
+ ETH_FCS_LEN
,
3435 prof
->tx_pause
, prof
->tx_ppp
,
3436 prof
->rx_pause
, prof
->rx_ppp
);
3438 en_err(priv
, "Failed setting port general configurations for port %d, with error %d\n",
3443 if (mdev
->dev
->caps
.tunnel_offload_mode
== MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
) {
3444 err
= mlx4_SET_PORT_VXLAN(mdev
->dev
, priv
->port
, VXLAN_STEER_BY_OUTER_MAC
, 1);
3446 en_err(priv
, "Failed setting port L2 tunnel configuration, err %d\n",
3453 en_warn(priv
, "Initializing port\n");
3454 err
= mlx4_INIT_PORT(mdev
->dev
, priv
->port
);
3456 en_err(priv
, "Failed Initializing port\n");
3459 queue_delayed_work(mdev
->workqueue
, &priv
->stats_task
, STATS_DELAY
);
3461 /* Initialize time stamp mechanism */
3462 if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_TS
)
3463 mlx4_en_init_timestamp(mdev
);
3465 queue_delayed_work(mdev
->workqueue
, &priv
->service_task
,
3466 SERVICE_TASK_DELAY
);
3468 mlx4_en_set_stats_bitmap(mdev
->dev
, &priv
->stats_bitmap
,
3469 mdev
->profile
.prof
[priv
->port
].rx_ppp
,
3470 mdev
->profile
.prof
[priv
->port
].rx_pause
,
3471 mdev
->profile
.prof
[priv
->port
].tx_ppp
,
3472 mdev
->profile
.prof
[priv
->port
].tx_pause
);
3474 err
= register_netdev(dev
);
3476 en_err(priv
, "Netdev registration failed for port %d\n", port
);
3480 priv
->registered
= 1;
3481 devlink_port_type_eth_set(mlx4_get_devlink_port(mdev
->dev
, priv
->port
),
3488 kfree(priv
->tx_ring
[t
]);
3489 kfree(priv
->tx_cq
[t
]);
3492 mlx4_en_destroy_netdev(dev
);
3496 int mlx4_en_reset_config(struct net_device
*dev
,
3497 struct hwtstamp_config ts_config
,
3498 netdev_features_t features
)
3500 struct mlx4_en_priv
*priv
= netdev_priv(dev
);
3501 struct mlx4_en_dev
*mdev
= priv
->mdev
;
3502 struct mlx4_en_port_profile new_prof
;
3503 struct mlx4_en_priv
*tmp
;
3507 if (priv
->hwtstamp_config
.tx_type
== ts_config
.tx_type
&&
3508 priv
->hwtstamp_config
.rx_filter
== ts_config
.rx_filter
&&
3509 !DEV_FEATURE_CHANGED(dev
, features
, NETIF_F_HW_VLAN_CTAG_RX
) &&
3510 !DEV_FEATURE_CHANGED(dev
, features
, NETIF_F_RXFCS
))
3511 return 0; /* Nothing to change */
3513 if (DEV_FEATURE_CHANGED(dev
, features
, NETIF_F_HW_VLAN_CTAG_RX
) &&
3514 (features
& NETIF_F_HW_VLAN_CTAG_RX
) &&
3515 (priv
->hwtstamp_config
.rx_filter
!= HWTSTAMP_FILTER_NONE
)) {
3516 en_warn(priv
, "Can't turn ON rx vlan offload while time-stamping rx filter is ON\n");
3520 tmp
= kzalloc(sizeof(*tmp
), GFP_KERNEL
);
3524 mutex_lock(&mdev
->state_lock
);
3526 memcpy(&new_prof
, priv
->prof
, sizeof(struct mlx4_en_port_profile
));
3527 memcpy(&new_prof
.hwtstamp_config
, &ts_config
, sizeof(ts_config
));
3529 err
= mlx4_en_try_alloc_resources(priv
, tmp
, &new_prof
, true);
3533 if (priv
->port_up
) {
3535 mlx4_en_stop_port(dev
, 1);
3538 en_warn(priv
, "Changing device configuration rx filter(%x) rx vlan(%x)\n",
3539 ts_config
.rx_filter
,
3540 !!(features
& NETIF_F_HW_VLAN_CTAG_RX
));
3542 mlx4_en_safe_replace_resources(priv
, tmp
);
3544 if (DEV_FEATURE_CHANGED(dev
, features
, NETIF_F_HW_VLAN_CTAG_RX
)) {
3545 if (features
& NETIF_F_HW_VLAN_CTAG_RX
)
3546 dev
->features
|= NETIF_F_HW_VLAN_CTAG_RX
;
3548 dev
->features
&= ~NETIF_F_HW_VLAN_CTAG_RX
;
3549 } else if (ts_config
.rx_filter
== HWTSTAMP_FILTER_NONE
) {
3550 /* RX time-stamping is OFF, update the RX vlan offload
3551 * to the latest wanted state
3553 if (dev
->wanted_features
& NETIF_F_HW_VLAN_CTAG_RX
)
3554 dev
->features
|= NETIF_F_HW_VLAN_CTAG_RX
;
3556 dev
->features
&= ~NETIF_F_HW_VLAN_CTAG_RX
;
3559 if (DEV_FEATURE_CHANGED(dev
, features
, NETIF_F_RXFCS
)) {
3560 if (features
& NETIF_F_RXFCS
)
3561 dev
->features
|= NETIF_F_RXFCS
;
3563 dev
->features
&= ~NETIF_F_RXFCS
;
3566 /* RX vlan offload and RX time-stamping can't co-exist !
3567 * Regardless of the caller's choice,
3568 * Turn Off RX vlan offload in case of time-stamping is ON
3570 if (ts_config
.rx_filter
!= HWTSTAMP_FILTER_NONE
) {
3571 if (dev
->features
& NETIF_F_HW_VLAN_CTAG_RX
)
3572 en_warn(priv
, "Turning off RX vlan offload since RX time-stamping is ON\n");
3573 dev
->features
&= ~NETIF_F_HW_VLAN_CTAG_RX
;
3577 err
= mlx4_en_start_port(dev
);
3579 en_err(priv
, "Failed starting port\n");
3583 mutex_unlock(&mdev
->state_lock
);
3586 netdev_features_change(dev
);