[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / mellanox / mlx5 / core / lag.c

/*
 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/netdevice.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/vport.h>
#include "mlx5_core.h"

enum {
	MLX5_LAG_FLAG_BONDED = 1 << 0,
};

struct lag_func {
	struct mlx5_core_dev *dev;
	struct net_device    *netdev;
};

/* Used for collection of netdev event info. */
struct lag_tracker {
	enum   netdev_lag_tx_type           tx_type;
	struct netdev_lag_lower_state_info  netdev_state[MLX5_MAX_PORTS];
	bool is_bonded;
};

/* LAG data of a ConnectX card.
 * It serves both its phys functions.
 */
struct mlx5_lag {
	u8                        flags;
	u8                        v2p_map[MLX5_MAX_PORTS];
	struct lag_func           pf[MLX5_MAX_PORTS];
	struct lag_tracker        tracker;
	struct delayed_work       bond_work;
	struct notifier_block     nb;

	/* Admin state. Allow lag only if allowed is true
	 * even if network conditions for lag were met
	 */
	bool                      allowed;
};

/* General purpose, use for short periods of time.
 * Beware of lock dependencies (preferably, no locks should be acquired
 * under it).
 */
static DEFINE_MUTEX(lag_mutex);

static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, u8 remap_port1,
			       u8 remap_port2)
{
	u32   in[MLX5_ST_SZ_DW(create_lag_in)]   = {0};
	u32   out[MLX5_ST_SZ_DW(create_lag_out)] = {0};
	void *lag_ctx = MLX5_ADDR_OF(create_lag_in, in, ctx);

	MLX5_SET(create_lag_in, in, opcode, MLX5_CMD_OP_CREATE_LAG);

	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, remap_port1);
	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, remap_port2);

	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}

static int mlx5_cmd_modify_lag(struct mlx5_core_dev *dev, u8 remap_port1,
			       u8 remap_port2)
{
	u32   in[MLX5_ST_SZ_DW(modify_lag_in)]   = {0};
	u32   out[MLX5_ST_SZ_DW(modify_lag_out)] = {0};
	void *lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);

	MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
	MLX5_SET(modify_lag_in, in, field_select, 0x1);

	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, remap_port1);
	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, remap_port2);

	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}

static int mlx5_cmd_destroy_lag(struct mlx5_core_dev *dev)
{
	u32  in[MLX5_ST_SZ_DW(destroy_lag_in)]  = {0};
	u32 out[MLX5_ST_SZ_DW(destroy_lag_out)] = {0};

	MLX5_SET(destroy_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_LAG);

	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}

int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev)
{
	u32  in[MLX5_ST_SZ_DW(create_vport_lag_in)]  = {0};
	u32 out[MLX5_ST_SZ_DW(create_vport_lag_out)] = {0};

	MLX5_SET(create_vport_lag_in, in, opcode, MLX5_CMD_OP_CREATE_VPORT_LAG);

	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_cmd_create_vport_lag);

int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev)
{
	u32  in[MLX5_ST_SZ_DW(destroy_vport_lag_in)]  = {0};
	u32 out[MLX5_ST_SZ_DW(destroy_vport_lag_out)] = {0};

	MLX5_SET(destroy_vport_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_VPORT_LAG);

	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_cmd_destroy_vport_lag);

static int mlx5_cmd_query_cong_counter(struct mlx5_core_dev *dev,
				       bool reset, void *out, int out_size)
{
	u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)] = { };

	MLX5_SET(query_cong_statistics_in, in, opcode,
		 MLX5_CMD_OP_QUERY_CONG_STATISTICS);
	MLX5_SET(query_cong_statistics_in, in, clear, reset);
	return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size);
}

static struct mlx5_lag *mlx5_lag_dev_get(struct mlx5_core_dev *dev)
{
	return dev->priv.lag;
}

static int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev,
				       struct net_device *ndev)
{
	int i;

	for (i = 0; i < MLX5_MAX_PORTS; i++)
		if (ldev->pf[i].netdev == ndev)
			return i;

	return -1;
}

static bool mlx5_lag_is_bonded(struct mlx5_lag *ldev)
{
	return !!(ldev->flags & MLX5_LAG_FLAG_BONDED);
}

static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
					   u8 *port1, u8 *port2)
{
	*port1 = 1;
	*port2 = 2;
	if (!tracker->netdev_state[0].tx_enabled ||
	    !tracker->netdev_state[0].link_up) {
		*port1 = 2;
		return;
	}

	if (!tracker->netdev_state[1].tx_enabled ||
	    !tracker->netdev_state[1].link_up)
		*port2 = 1;
}

static void mlx5_activate_lag(struct mlx5_lag *ldev,
			      struct lag_tracker *tracker)
{
	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
	int err;

	ldev->flags |= MLX5_LAG_FLAG_BONDED;

	mlx5_infer_tx_affinity_mapping(tracker, &ldev->v2p_map[0],
				       &ldev->v2p_map[1]);

	err = mlx5_cmd_create_lag(dev0, ldev->v2p_map[0], ldev->v2p_map[1]);
	if (err)
		mlx5_core_err(dev0,
			      "Failed to create LAG (%d)\n",
			      err);
}

static void mlx5_deactivate_lag(struct mlx5_lag *ldev)
{
	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
	int err;

	ldev->flags &= ~MLX5_LAG_FLAG_BONDED;

	err = mlx5_cmd_destroy_lag(dev0);
	if (err)
		mlx5_core_err(dev0,
			      "Failed to destroy LAG (%d)\n",
			      err);
}

static void mlx5_do_bond(struct mlx5_lag *ldev)
{
	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
	struct mlx5_core_dev *dev1 = ldev->pf[1].dev;
	struct lag_tracker tracker;
	u8 v2p_port1, v2p_port2;
	int i, err;
	bool do_bond;

	if (!dev0 || !dev1)
		return;

	mutex_lock(&lag_mutex);
	tracker = ldev->tracker;
	mutex_unlock(&lag_mutex);

	do_bond = tracker.is_bonded && ldev->allowed;

	if (do_bond && !mlx5_lag_is_bonded(ldev)) {
		for (i = 0; i < MLX5_MAX_PORTS; i++)
			mlx5_remove_dev_by_protocol(ldev->pf[i].dev,
						    MLX5_INTERFACE_PROTOCOL_IB);

		mlx5_activate_lag(ldev, &tracker);

		mlx5_add_dev_by_protocol(dev0, MLX5_INTERFACE_PROTOCOL_IB);
		mlx5_nic_vport_enable_roce(dev1);
	} else if (do_bond && mlx5_lag_is_bonded(ldev)) {
		mlx5_infer_tx_affinity_mapping(&tracker, &v2p_port1,
					       &v2p_port2);

		if ((v2p_port1 != ldev->v2p_map[0]) ||
		    (v2p_port2 != ldev->v2p_map[1])) {
			ldev->v2p_map[0] = v2p_port1;
			ldev->v2p_map[1] = v2p_port2;

			err = mlx5_cmd_modify_lag(dev0, v2p_port1, v2p_port2);
			if (err)
				mlx5_core_err(dev0,
					      "Failed to modify LAG (%d)\n",
					      err);
		}
	} else if (!do_bond && mlx5_lag_is_bonded(ldev)) {
		mlx5_remove_dev_by_protocol(dev0, MLX5_INTERFACE_PROTOCOL_IB);
		mlx5_nic_vport_disable_roce(dev1);

		mlx5_deactivate_lag(ldev);

		for (i = 0; i < MLX5_MAX_PORTS; i++)
			if (ldev->pf[i].dev)
				mlx5_add_dev_by_protocol(ldev->pf[i].dev,
							 MLX5_INTERFACE_PROTOCOL_IB);
	}
}

static void mlx5_queue_bond_work(struct mlx5_lag *ldev, unsigned long delay)
{
	schedule_delayed_work(&ldev->bond_work, delay);
}

static void mlx5_do_bond_work(struct work_struct *work)
{
	struct delayed_work *delayed_work = to_delayed_work(work);
	struct mlx5_lag *ldev = container_of(delayed_work, struct mlx5_lag,
					     bond_work);
	int status;

	status = mlx5_dev_list_trylock();
	if (!status) {
		/* 1 sec delay. */
		mlx5_queue_bond_work(ldev, HZ);
		return;
	}

	mlx5_do_bond(ldev);
	mlx5_dev_list_unlock();
}

static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
					 struct lag_tracker *tracker,
					 struct net_device *ndev,
					 struct netdev_notifier_changeupper_info *info)
{
	struct net_device *upper = info->upper_dev, *ndev_tmp;
	struct netdev_lag_upper_info *lag_upper_info = NULL;
	bool is_bonded;
	int bond_status = 0;
	int num_slaves = 0;
	int idx;

	if (!netif_is_lag_master(upper))
		return 0;

	if (info->linking)
		lag_upper_info = info->upper_info;

	/* The event may still be of interest if the slave does not belong to
	 * us, but is enslaved to a master which has one or more of our netdevs
	 * as slaves (e.g., if a new slave is added to a master that bonds two
	 * of our netdevs, we should unbond).
	 */
	rcu_read_lock();
	for_each_netdev_in_bond_rcu(upper, ndev_tmp) {
		idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev_tmp);
		if (idx > -1)
			bond_status |= (1 << idx);

		num_slaves++;
	}
	rcu_read_unlock();

	/* None of this lagdev's netdevs are slaves of this master. */
	if (!(bond_status & 0x3))
		return 0;

	if (lag_upper_info)
		tracker->tx_type = lag_upper_info->tx_type;

	/* Determine bonding status:
	 * A device is considered bonded if both its physical ports are slaves
	 * of the same lag master, and only them.
	 * Lag mode must be activebackup or hash.
	 */
	is_bonded = (num_slaves == MLX5_MAX_PORTS) &&
		    (bond_status == 0x3) &&
		    ((tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) ||
		     (tracker->tx_type == NETDEV_LAG_TX_TYPE_HASH));

	if (tracker->is_bonded != is_bonded) {
		tracker->is_bonded = is_bonded;
		return 1;
	}

	return 0;
}

static int mlx5_handle_changelowerstate_event(struct mlx5_lag *ldev,
					      struct lag_tracker *tracker,
					      struct net_device *ndev,
					      struct netdev_notifier_changelowerstate_info *info)
{
	struct netdev_lag_lower_state_info *lag_lower_info;
	int idx;

	if (!netif_is_lag_port(ndev))
		return 0;

	idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev);
	if (idx == -1)
		return 0;

	/* This information is used to determine virtual to physical
	 * port mapping.
	 */
	lag_lower_info = info->lower_state_info;
	if (!lag_lower_info)
		return 0;

	tracker->netdev_state[idx] = *lag_lower_info;

	return 1;
}

static int mlx5_lag_netdev_event(struct notifier_block *this,
				 unsigned long event, void *ptr)
{
	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
	struct lag_tracker tracker;
	struct mlx5_lag *ldev;
	int changed = 0;

	if (!net_eq(dev_net(ndev), &init_net))
		return NOTIFY_DONE;

	if ((event != NETDEV_CHANGEUPPER) && (event != NETDEV_CHANGELOWERSTATE))
		return NOTIFY_DONE;

	ldev    = container_of(this, struct mlx5_lag, nb);
	tracker = ldev->tracker;

	switch (event) {
	case NETDEV_CHANGEUPPER:
		changed = mlx5_handle_changeupper_event(ldev, &tracker, ndev,
							ptr);
		break;
	case NETDEV_CHANGELOWERSTATE:
		changed = mlx5_handle_changelowerstate_event(ldev, &tracker,
							     ndev, ptr);
		break;
	}

	mutex_lock(&lag_mutex);
	ldev->tracker = tracker;
	mutex_unlock(&lag_mutex);

	if (changed)
		mlx5_queue_bond_work(ldev, 0);

	return NOTIFY_DONE;
}

static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
{
	if ((ldev->pf[0].dev && mlx5_sriov_is_enabled(ldev->pf[0].dev)) ||
	    (ldev->pf[1].dev && mlx5_sriov_is_enabled(ldev->pf[1].dev)))
		return false;
	else
		return true;
}

static struct mlx5_lag *mlx5_lag_dev_alloc(void)
{
	struct mlx5_lag *ldev;

	ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
	if (!ldev)
		return NULL;

	INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work);
	ldev->allowed = mlx5_lag_check_prereq(ldev);

	return ldev;
}

static void mlx5_lag_dev_free(struct mlx5_lag *ldev)
{
	kfree(ldev);
}

static void mlx5_lag_dev_add_pf(struct mlx5_lag *ldev,
				struct mlx5_core_dev *dev,
				struct net_device *netdev)
{
	unsigned int fn = PCI_FUNC(dev->pdev->devfn);

	if (fn >= MLX5_MAX_PORTS)
		return;

	mutex_lock(&lag_mutex);
	ldev->pf[fn].dev    = dev;
	ldev->pf[fn].netdev = netdev;
	ldev->tracker.netdev_state[fn].link_up = 0;
	ldev->tracker.netdev_state[fn].tx_enabled = 0;

	ldev->allowed = mlx5_lag_check_prereq(ldev);
	dev->priv.lag = ldev;

	mutex_unlock(&lag_mutex);
}

static void mlx5_lag_dev_remove_pf(struct mlx5_lag *ldev,
				   struct mlx5_core_dev *dev)
{
	int i;

	for (i = 0; i < MLX5_MAX_PORTS; i++)
		if (ldev->pf[i].dev == dev)
			break;

	if (i == MLX5_MAX_PORTS)
		return;

	mutex_lock(&lag_mutex);
	memset(&ldev->pf[i], 0, sizeof(*ldev->pf));

	dev->priv.lag = NULL;
	ldev->allowed = mlx5_lag_check_prereq(ldev);
	mutex_unlock(&lag_mutex);
}

/* Must be called with intf_mutex held */
void mlx5_lag_add(struct mlx5_core_dev *dev, struct net_device *netdev)
{
	struct mlx5_lag *ldev = NULL;
	struct mlx5_core_dev *tmp_dev;

	if (!MLX5_CAP_GEN(dev, vport_group_manager) ||
	    !MLX5_CAP_GEN(dev, lag_master) ||
	    (MLX5_CAP_GEN(dev, num_lag_ports) != MLX5_MAX_PORTS))
		return;

	tmp_dev = mlx5_get_next_phys_dev(dev);
	if (tmp_dev)
		ldev = tmp_dev->priv.lag;

	if (!ldev) {
		ldev = mlx5_lag_dev_alloc();
		if (!ldev) {
			mlx5_core_err(dev, "Failed to alloc lag dev\n");
			return;
		}
	}

	mlx5_lag_dev_add_pf(ldev, dev, netdev);

	if (!ldev->nb.notifier_call) {
		ldev->nb.notifier_call = mlx5_lag_netdev_event;
		if (register_netdevice_notifier(&ldev->nb)) {
			ldev->nb.notifier_call = NULL;
			mlx5_core_err(dev, "Failed to register LAG netdev notifier\n");
		}
	}
}

/* Must be called with intf_mutex held */
void mlx5_lag_remove(struct mlx5_core_dev *dev)
{
	struct mlx5_lag *ldev;
	int i;

	ldev = mlx5_lag_dev_get(dev);
	if (!ldev)
		return;

	if (mlx5_lag_is_bonded(ldev))
		mlx5_deactivate_lag(ldev);

	mlx5_lag_dev_remove_pf(ldev, dev);

	for (i = 0; i < MLX5_MAX_PORTS; i++)
		if (ldev->pf[i].dev)
			break;

	if (i == MLX5_MAX_PORTS) {
		if (ldev->nb.notifier_call)
			unregister_netdevice_notifier(&ldev->nb);
		cancel_delayed_work_sync(&ldev->bond_work);
		mlx5_lag_dev_free(ldev);
	}
}

bool mlx5_lag_is_active(struct mlx5_core_dev *dev)
{
	struct mlx5_lag *ldev;
	bool res;

	mutex_lock(&lag_mutex);
	ldev = mlx5_lag_dev_get(dev);
	res  = ldev && mlx5_lag_is_bonded(ldev);
	mutex_unlock(&lag_mutex);

	return res;
}
EXPORT_SYMBOL(mlx5_lag_is_active);

static int mlx5_lag_set_state(struct mlx5_core_dev *dev, bool allow)
{
	struct mlx5_lag *ldev;
	int ret = 0;
	bool lag_active;

	mlx5_dev_list_lock();

	ldev = mlx5_lag_dev_get(dev);
	if (!ldev) {
		ret = -ENODEV;
		goto unlock;
	}
	lag_active = mlx5_lag_is_bonded(ldev);
	if (!mlx5_lag_check_prereq(ldev) && allow) {
		ret = -EINVAL;
		goto unlock;
	}
	if (ldev->allowed == allow)
		goto unlock;
	ldev->allowed = allow;
	if ((lag_active && !allow) || allow)
		mlx5_do_bond(ldev);
unlock:
	mlx5_dev_list_unlock();
	return ret;
}

int mlx5_lag_forbid(struct mlx5_core_dev *dev)
{
	return mlx5_lag_set_state(dev, false);
}

int mlx5_lag_allow(struct mlx5_core_dev *dev)
{
	return mlx5_lag_set_state(dev, true);
}

struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev)
{
	struct net_device *ndev = NULL;
	struct mlx5_lag *ldev;

	mutex_lock(&lag_mutex);
	ldev = mlx5_lag_dev_get(dev);

	if (!(ldev && mlx5_lag_is_bonded(ldev)))
		goto unlock;

	if (ldev->tracker.tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
		ndev = ldev->tracker.netdev_state[0].tx_enabled ?
		       ldev->pf[0].netdev : ldev->pf[1].netdev;
	} else {
		ndev = ldev->pf[0].netdev;
	}
	if (ndev)
		dev_hold(ndev);

unlock:
	mutex_unlock(&lag_mutex);

	return ndev;
}
EXPORT_SYMBOL(mlx5_lag_get_roce_netdev);

bool mlx5_lag_intf_add(struct mlx5_interface *intf, struct mlx5_priv *priv)
{
	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev,
						 priv);
	struct mlx5_lag *ldev;

	if (intf->protocol != MLX5_INTERFACE_PROTOCOL_IB)
		return true;

	ldev = mlx5_lag_dev_get(dev);
	if (!ldev || !mlx5_lag_is_bonded(ldev) || ldev->pf[0].dev == dev)
		return true;

	/* If bonded, we do not add an IB device for PF1. */
	return false;
}

int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
				 u64 *values,
				 int num_counters,
				 size_t *offsets)
{
	int outlen = MLX5_ST_SZ_BYTES(query_cong_statistics_out);
	struct mlx5_core_dev *mdev[MLX5_MAX_PORTS];
	struct mlx5_lag *ldev;
	int num_ports;
	int ret, i, j;
	void *out;

	out = kvzalloc(outlen, GFP_KERNEL);
	if (!out)
		return -ENOMEM;

	memset(values, 0, sizeof(*values) * num_counters);

	mutex_lock(&lag_mutex);
	ldev = mlx5_lag_dev_get(dev);
	if (ldev && mlx5_lag_is_bonded(ldev)) {
		num_ports = MLX5_MAX_PORTS;
		mdev[0] = ldev->pf[0].dev;
		mdev[1] = ldev->pf[1].dev;
	} else {
		num_ports = 1;
		mdev[0] = dev;
	}

	for (i = 0; i < num_ports; ++i) {
		ret = mlx5_cmd_query_cong_counter(mdev[i], false, out, outlen);
		if (ret)
			goto unlock;

		for (j = 0; j < num_counters; ++j)
			values[j] += be64_to_cpup((__be64 *)(out + offsets[j]));
	}

unlock:
	mutex_unlock(&lag_mutex);
	kvfree(out);
	return ret;
}
EXPORT_SYMBOL(mlx5_lag_query_cong_counters);
Commit	Line	Data
7907f23a AH	1	/*
	2	* Copyright (c) 2016, Mellanox Technologies. All rights reserved.
	3	*
	4	* This software is available to you under a choice of one of two
	5	* licenses. You may choose to be licensed under the terms of the GNU
	6	* General Public License (GPL) Version 2, available from the file
	7	* COPYING in the main directory of this source tree, or the
	8	* OpenIB.org BSD license below:
	9	*
	10	* Redistribution and use in source and binary forms, with or
	11	* without modification, are permitted provided that the following
	12	* conditions are met:
	13	*
	14	* - Redistributions of source code must retain the above
	15	* copyright notice, this list of conditions and the following
	16	* disclaimer.
	17	*
	18	* - Redistributions in binary form must reproduce the above
	19	* copyright notice, this list of conditions and the following
	20	* disclaimer in the documentation and/or other materials
	21	* provided with the distribution.
	22	*
	23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	30	* SOFTWARE.
	31	*/
	32
	33	#include <linux/netdevice.h>
	34	#include <linux/mlx5/driver.h>
	35	#include <linux/mlx5/vport.h>
	36	#include "mlx5_core.h"
	37
	38	enum {
	39	MLX5_LAG_FLAG_BONDED = 1 << 0,
	40	};
	41
	42	struct lag_func {
	43	struct mlx5_core_dev *dev;
	44	struct net_device *netdev;
	45	};
	46
	47	/* Used for collection of netdev event info. */
	48	struct lag_tracker {
	49	enum netdev_lag_tx_type tx_type;
	50	struct netdev_lag_lower_state_info netdev_state[MLX5_MAX_PORTS];
	51	bool is_bonded;
	52	};
	53
	54	/* LAG data of a ConnectX card.
	55	* It serves both its phys functions.
	56	*/
	57	struct mlx5_lag {
	58	u8 flags;
	59	u8 v2p_map[MLX5_MAX_PORTS];
	60	struct lag_func pf[MLX5_MAX_PORTS];
	61	struct lag_tracker tracker;
	62	struct delayed_work bond_work;
	63	struct notifier_block nb;
552db7bc MS	64
	65	/* Admin state. Allow lag only if allowed is true
	66	* even if network conditions for lag were met
	67	*/
	68	bool allowed;
7907f23a AH	69	};
	70
	71	/* General purpose, use for short periods of time.
	72	* Beware of lock dependencies (preferably, no locks should be acquired
	73	* under it).
	74	*/
	75	static DEFINE_MUTEX(lag_mutex);
	76
	77	static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, u8 remap_port1,
	78	u8 remap_port2)
	79	{
	80	u32 in[MLX5_ST_SZ_DW(create_lag_in)] = {0};
	81	u32 out[MLX5_ST_SZ_DW(create_lag_out)] = {0};
	82	void *lag_ctx = MLX5_ADDR_OF(create_lag_in, in, ctx);
	83
	84	MLX5_SET(create_lag_in, in, opcode, MLX5_CMD_OP_CREATE_LAG);
	85
	86	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, remap_port1);
	87	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, remap_port2);
	88
	89	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	90	}
	91
	92	static int mlx5_cmd_modify_lag(struct mlx5_core_dev *dev, u8 remap_port1,
	93	u8 remap_port2)
	94	{
	95	u32 in[MLX5_ST_SZ_DW(modify_lag_in)] = {0};
	96	u32 out[MLX5_ST_SZ_DW(modify_lag_out)] = {0};
	97	void *lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);
	98
	99	MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
	100	MLX5_SET(modify_lag_in, in, field_select, 0x1);
	101
	102	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, remap_port1);
	103	MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, remap_port2);
	104
	105	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	106	}
	107
	108	static int mlx5_cmd_destroy_lag(struct mlx5_core_dev *dev)
	109	{
	110	u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {0};
	111	u32 out[MLX5_ST_SZ_DW(destroy_lag_out)] = {0};
	112
	113	MLX5_SET(destroy_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_LAG);
	114
	115	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	116	}
	117
3bc34f3b AH	118	int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev)
	119	{
	120	u32 in[MLX5_ST_SZ_DW(create_vport_lag_in)] = {0};
	121	u32 out[MLX5_ST_SZ_DW(create_vport_lag_out)] = {0};
	122
	123	MLX5_SET(create_vport_lag_in, in, opcode, MLX5_CMD_OP_CREATE_VPORT_LAG);
	124
	125	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	126	}
	127	EXPORT_SYMBOL(mlx5_cmd_create_vport_lag);
	128
	129	int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev)
	130	{
	131	u32 in[MLX5_ST_SZ_DW(destroy_vport_lag_in)] = {0};
	132	u32 out[MLX5_ST_SZ_DW(destroy_vport_lag_out)] = {0};
	133
	134	MLX5_SET(destroy_vport_lag_in, in, opcode, MLX5_CMD_OP_DESTROY_VPORT_LAG);
	135
	136	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
	137	}
	138	EXPORT_SYMBOL(mlx5_cmd_destroy_vport_lag);
	139
71a0ff65 MD	140	static int mlx5_cmd_query_cong_counter(struct mlx5_core_dev *dev,
	141	bool reset, void *out, int out_size)
	142	{
	143	u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)] = { };
	144
	145	MLX5_SET(query_cong_statistics_in, in, opcode,
	146	MLX5_CMD_OP_QUERY_CONG_STATISTICS);
	147	MLX5_SET(query_cong_statistics_in, in, clear, reset);
	148	return mlx5_cmd_exec(dev, in, sizeof(in), out, out_size);
	149	}
	150
7907f23a AH	151	static struct mlx5_lag mlx5_lag_dev_get(struct mlx5_core_dev dev)
	152	{
	153	return dev->priv.lag;
	154	}
	155
	156	static int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev,
	157	struct net_device *ndev)
	158	{
	159	int i;
	160
	161	for (i = 0; i < MLX5_MAX_PORTS; i++)
	162	if (ldev->pf[i].netdev == ndev)
	163	return i;
	164
	165	return -1;
	166	}
	167
	168	static bool mlx5_lag_is_bonded(struct mlx5_lag *ldev)
	169	{
	170	return !!(ldev->flags & MLX5_LAG_FLAG_BONDED);
	171	}
	172
	173	static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
	174	u8 port1, u8 port2)
	175	{
dc798b4c AH	176	*port1 = 1;
	177	*port2 = 2;
	178	if (!tracker->netdev_state[0].tx_enabled \|\|
	179	!tracker->netdev_state[0].link_up) {
	180	*port1 = 2;
	181	return;
7907f23a	182	}
dc798b4c AH	183
	184	if (!tracker->netdev_state[1].tx_enabled \|\|
	185	!tracker->netdev_state[1].link_up)
	186	*port2 = 1;
7907f23a AH	187	}
	188
	189	static void mlx5_activate_lag(struct mlx5_lag *ldev,
	190	struct lag_tracker *tracker)
	191	{
	192	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
	193	int err;
	194
	195	ldev->flags \|= MLX5_LAG_FLAG_BONDED;
	196
	197	mlx5_infer_tx_affinity_mapping(tracker, &ldev->v2p_map[0],
	198	&ldev->v2p_map[1]);
	199
	200	err = mlx5_cmd_create_lag(dev0, ldev->v2p_map[0], ldev->v2p_map[1]);
	201	if (err)
	202	mlx5_core_err(dev0,
	203	"Failed to create LAG (%d)\n",
	204	err);
	205	}
	206
	207	static void mlx5_deactivate_lag(struct mlx5_lag *ldev)
	208	{
	209	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
	210	int err;
	211
	212	ldev->flags &= ~MLX5_LAG_FLAG_BONDED;
	213
	214	err = mlx5_cmd_destroy_lag(dev0);
	215	if (err)
	216	mlx5_core_err(dev0,
	217	"Failed to destroy LAG (%d)\n",
	218	err);
	219	}
	220
	221	static void mlx5_do_bond(struct mlx5_lag *ldev)
	222	{
	223	struct mlx5_core_dev *dev0 = ldev->pf[0].dev;
	224	struct mlx5_core_dev *dev1 = ldev->pf[1].dev;
	225	struct lag_tracker tracker;
	226	u8 v2p_port1, v2p_port2;
	227	int i, err;
552db7bc	228	bool do_bond;
7907f23a AH	229
	230	if (!dev0 \|\| !dev1)
	231	return;
	232
	233	mutex_lock(&lag_mutex);
	234	tracker = ldev->tracker;
	235	mutex_unlock(&lag_mutex);
	236
552db7bc	237	do_bond = tracker.is_bonded && ldev->allowed;
edb31b16	238
552db7bc	239	if (do_bond && !mlx5_lag_is_bonded(ldev)) {
7907f23a AH	240	for (i = 0; i < MLX5_MAX_PORTS; i++)
	241	mlx5_remove_dev_by_protocol(ldev->pf[i].dev,
	242	MLX5_INTERFACE_PROTOCOL_IB);
	243
	244	mlx5_activate_lag(ldev, &tracker);
	245
	246	mlx5_add_dev_by_protocol(dev0, MLX5_INTERFACE_PROTOCOL_IB);
	247	mlx5_nic_vport_enable_roce(dev1);
552db7bc	248	} else if (do_bond && mlx5_lag_is_bonded(ldev)) {
7907f23a AH	249	mlx5_infer_tx_affinity_mapping(&tracker, &v2p_port1,
	250	&v2p_port2);
	251
	252	if ((v2p_port1 != ldev->v2p_map[0]) \|\|
	253	(v2p_port2 != ldev->v2p_map[1])) {
	254	ldev->v2p_map[0] = v2p_port1;
	255	ldev->v2p_map[1] = v2p_port2;
	256
	257	err = mlx5_cmd_modify_lag(dev0, v2p_port1, v2p_port2);
	258	if (err)
	259	mlx5_core_err(dev0,
	260	"Failed to modify LAG (%d)\n",
	261	err);
	262	}
552db7bc	263	} else if (!do_bond && mlx5_lag_is_bonded(ldev)) {
7907f23a AH	264	mlx5_remove_dev_by_protocol(dev0, MLX5_INTERFACE_PROTOCOL_IB);
	265	mlx5_nic_vport_disable_roce(dev1);
	266
	267	mlx5_deactivate_lag(ldev);
	268
	269	for (i = 0; i < MLX5_MAX_PORTS; i++)
	270	if (ldev->pf[i].dev)
	271	mlx5_add_dev_by_protocol(ldev->pf[i].dev,
	272	MLX5_INTERFACE_PROTOCOL_IB);
	273	}
	274	}
	275
	276	static void mlx5_queue_bond_work(struct mlx5_lag *ldev, unsigned long delay)
	277	{
	278	schedule_delayed_work(&ldev->bond_work, delay);
	279	}
	280
	281	static void mlx5_do_bond_work(struct work_struct *work)
	282	{
	283	struct delayed_work *delayed_work = to_delayed_work(work);
	284	struct mlx5_lag *ldev = container_of(delayed_work, struct mlx5_lag,
	285	bond_work);
	286	int status;
	287
f1ee87fe	288	status = mlx5_dev_list_trylock();
7907f23a AH	289	if (!status) {
	290	/* 1 sec delay. */
	291	mlx5_queue_bond_work(ldev, HZ);
	292	return;
	293	}
	294
	295	mlx5_do_bond(ldev);
f1ee87fe	296	mlx5_dev_list_unlock();
7907f23a AH	297	}
	298
	299	static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev,
	300	struct lag_tracker *tracker,
	301	struct net_device *ndev,
	302	struct netdev_notifier_changeupper_info *info)
	303	{
	304	struct net_device upper = info->upper_dev, ndev_tmp;
e497ec68	305	struct netdev_lag_upper_info *lag_upper_info = NULL;
7907f23a AH	306	bool is_bonded;
	307	int bond_status = 0;
	308	int num_slaves = 0;
	309	int idx;
	310
	311	if (!netif_is_lag_master(upper))
	312	return 0;
	313
e497ec68 TB	314	if (info->linking)
e497ec68 TB	315	lag_upper_info = info->upper_info;
7907f23a AH	316
	317	/* The event may still be of interest if the slave does not belong to
	318	* us, but is enslaved to a master which has one or more of our netdevs
	319	* as slaves (e.g., if a new slave is added to a master that bonds two
	320	* of our netdevs, we should unbond).
	321	*/
	322	rcu_read_lock();
	323	for_each_netdev_in_bond_rcu(upper, ndev_tmp) {
	324	idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev_tmp);
	325	if (idx > -1)
	326	bond_status \|= (1 << idx);
	327
	328	num_slaves++;
	329	}
	330	rcu_read_unlock();
	331
	332	/* None of this lagdev's netdevs are slaves of this master. */
	333	if (!(bond_status & 0x3))
	334	return 0;
	335
	336	if (lag_upper_info)
	337	tracker->tx_type = lag_upper_info->tx_type;
	338
	339	/* Determine bonding status:
	340	* A device is considered bonded if both its physical ports are slaves
	341	* of the same lag master, and only them.
	342	* Lag mode must be activebackup or hash.
	343	*/
	344	is_bonded = (num_slaves == MLX5_MAX_PORTS) &&
	345	(bond_status == 0x3) &&
	346	((tracker->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) \|\|
	347	(tracker->tx_type == NETDEV_LAG_TX_TYPE_HASH));
	348
	349	if (tracker->is_bonded != is_bonded) {
	350	tracker->is_bonded = is_bonded;
	351	return 1;
	352	}
	353
	354	return 0;
	355	}
	356
	357	static int mlx5_handle_changelowerstate_event(struct mlx5_lag *ldev,
	358	struct lag_tracker *tracker,
	359	struct net_device *ndev,
	360	struct netdev_notifier_changelowerstate_info *info)
	361	{
	362	struct netdev_lag_lower_state_info *lag_lower_info;
	363	int idx;
	364
	365	if (!netif_is_lag_port(ndev))
	366	return 0;
	367
	368	idx = mlx5_lag_dev_get_netdev_idx(ldev, ndev);
	369	if (idx == -1)
	370	return 0;
	371
	372	/* This information is used to determine virtual to physical
	373	* port mapping.
	374	*/
	375	lag_lower_info = info->lower_state_info;
	376	if (!lag_lower_info)
	377	return 0;
	378
	379	tracker->netdev_state[idx] = *lag_lower_info;
380
381	return 1;
382	}
383
384	static int mlx5_lag_netdev_event(struct notifier_block *this,
385	unsigned long event, void *ptr)
386	{
387	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
388	struct lag_tracker tracker;
389	struct mlx5_lag *ldev;
390	int changed = 0;
391
392	if (!net_eq(dev_net(ndev), &init_net))
393	return NOTIFY_DONE;
394
395	if ((event != NETDEV_CHANGEUPPER) && (event != NETDEV_CHANGELOWERSTATE))
396	return NOTIFY_DONE;
397
398	ldev = container_of(this, struct mlx5_lag, nb);
399	tracker = ldev->tracker;
400
401	switch (event) {
402	case NETDEV_CHANGEUPPER:
403	changed = mlx5_handle_changeupper_event(ldev, &tracker, ndev,
404	ptr);
405	break;
406	case NETDEV_CHANGELOWERSTATE:
407	changed = mlx5_handle_changelowerstate_event(ldev, &tracker,
408	ndev, ptr);
409	break;
410	}
411
412	mutex_lock(&lag_mutex);
413	ldev->tracker = tracker;
414	mutex_unlock(&lag_mutex);
415
416	if (changed)
417	mlx5_queue_bond_work(ldev, 0);
418
419	return NOTIFY_DONE;
420	}
421
552db7bc MS	422	static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
	423	{
	424	if ((ldev->pf[0].dev && mlx5_sriov_is_enabled(ldev->pf[0].dev)) \|\|
	425	(ldev->pf[1].dev && mlx5_sriov_is_enabled(ldev->pf[1].dev)))
	426	return false;
	427	else
	428	return true;
	429	}
	430
7907f23a AH	431	static struct mlx5_lag *mlx5_lag_dev_alloc(void)
	432	{
	433	struct mlx5_lag *ldev;
	434
	435	ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
	436	if (!ldev)
	437	return NULL;
	438
	439	INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work);
552db7bc	440	ldev->allowed = mlx5_lag_check_prereq(ldev);
7907f23a AH	441
	442	return ldev;
	443	}
	444
	445	static void mlx5_lag_dev_free(struct mlx5_lag *ldev)
	446	{
	447	kfree(ldev);
	448	}
	449
	450	static void mlx5_lag_dev_add_pf(struct mlx5_lag *ldev,
	451	struct mlx5_core_dev *dev,
	452	struct net_device *netdev)
	453	{
	454	unsigned int fn = PCI_FUNC(dev->pdev->devfn);
	455
	456	if (fn >= MLX5_MAX_PORTS)
	457	return;
	458
	459	mutex_lock(&lag_mutex);
	460	ldev->pf[fn].dev = dev;
	461	ldev->pf[fn].netdev = netdev;
	462	ldev->tracker.netdev_state[fn].link_up = 0;
	463	ldev->tracker.netdev_state[fn].tx_enabled = 0;
	464
552db7bc	465	ldev->allowed = mlx5_lag_check_prereq(ldev);
7907f23a	466	dev->priv.lag = ldev;
552db7bc	467
7907f23a AH	468	mutex_unlock(&lag_mutex);
	469	}
	470
	471	static void mlx5_lag_dev_remove_pf(struct mlx5_lag *ldev,
	472	struct mlx5_core_dev *dev)
	473	{
	474	int i;
	475
	476	for (i = 0; i < MLX5_MAX_PORTS; i++)
	477	if (ldev->pf[i].dev == dev)
	478	break;
	479
	480	if (i == MLX5_MAX_PORTS)
	481	return;
	482
	483	mutex_lock(&lag_mutex);
	484	memset(&ldev->pf[i], 0, sizeof(*ldev->pf));
	485
	486	dev->priv.lag = NULL;
552db7bc	487	ldev->allowed = mlx5_lag_check_prereq(ldev);
7907f23a AH	488	mutex_unlock(&lag_mutex);
	489	}
	490
7907f23a AH	491	/* Must be called with intf_mutex held */
	492	void mlx5_lag_add(struct mlx5_core_dev dev, struct net_device netdev)
	493	{
	494	struct mlx5_lag *ldev = NULL;
	495	struct mlx5_core_dev *tmp_dev;
7907f23a AH	496
	497	if (!MLX5_CAP_GEN(dev, vport_group_manager) \|\|
	498	!MLX5_CAP_GEN(dev, lag_master) \|\|
	499	(MLX5_CAP_GEN(dev, num_lag_ports) != MLX5_MAX_PORTS))
	500	return;
	501
f1ee87fe MHY	502	tmp_dev = mlx5_get_next_phys_dev(dev);
	503	if (tmp_dev)
	504	ldev = tmp_dev->priv.lag;
7907f23a AH	505
	506	if (!ldev) {
	507	ldev = mlx5_lag_dev_alloc();
	508	if (!ldev) {
	509	mlx5_core_err(dev, "Failed to alloc lag dev\n");
	510	return;
	511	}
	512	}
	513
	514	mlx5_lag_dev_add_pf(ldev, dev, netdev);
	515
	516	if (!ldev->nb.notifier_call) {
	517	ldev->nb.notifier_call = mlx5_lag_netdev_event;
	518	if (register_netdevice_notifier(&ldev->nb)) {
	519	ldev->nb.notifier_call = NULL;
	520	mlx5_core_err(dev, "Failed to register LAG netdev notifier\n");
	521	}
	522	}
	523	}
	524
	525	/* Must be called with intf_mutex held */
	526	void mlx5_lag_remove(struct mlx5_core_dev *dev)
	527	{
	528	struct mlx5_lag *ldev;
	529	int i;
	530
	531	ldev = mlx5_lag_dev_get(dev);
	532	if (!ldev)
	533	return;
	534
	535	if (mlx5_lag_is_bonded(ldev))
	536	mlx5_deactivate_lag(ldev);
	537
	538	mlx5_lag_dev_remove_pf(ldev, dev);
	539
	540	for (i = 0; i < MLX5_MAX_PORTS; i++)
	541	if (ldev->pf[i].dev)
	542	break;
	543
	544	if (i == MLX5_MAX_PORTS) {
	545	if (ldev->nb.notifier_call)
	546	unregister_netdevice_notifier(&ldev->nb);
	547	cancel_delayed_work_sync(&ldev->bond_work);
	548	mlx5_lag_dev_free(ldev);
	549	}
	550	}
	551
	552	bool mlx5_lag_is_active(struct mlx5_core_dev *dev)
	553	{
	554	struct mlx5_lag *ldev;
	555	bool res;
	556
	557	mutex_lock(&lag_mutex);
	558	ldev = mlx5_lag_dev_get(dev);
	559	res = ldev && mlx5_lag_is_bonded(ldev);
	560	mutex_unlock(&lag_mutex);
	561
	562	return res;
	563	}
	564	EXPORT_SYMBOL(mlx5_lag_is_active);
	565
552db7bc MS	566	static int mlx5_lag_set_state(struct mlx5_core_dev *dev, bool allow)
	567	{
	568	struct mlx5_lag *ldev;
	569	int ret = 0;
	570	bool lag_active;
	571
	572	mlx5_dev_list_lock();
	573
	574	ldev = mlx5_lag_dev_get(dev);
	575	if (!ldev) {
	576	ret = -ENODEV;
	577	goto unlock;
	578	}
	579	lag_active = mlx5_lag_is_bonded(ldev);
	580	if (!mlx5_lag_check_prereq(ldev) && allow) {
	581	ret = -EINVAL;
	582	goto unlock;
	583	}
	584	if (ldev->allowed == allow)
	585	goto unlock;
	586	ldev->allowed = allow;
	587	if ((lag_active && !allow) \|\| allow)
	588	mlx5_do_bond(ldev);
	589	unlock:
	590	mlx5_dev_list_unlock();
	591	return ret;
	592	}
	593
	594	int mlx5_lag_forbid(struct mlx5_core_dev *dev)
	595	{
	596	return mlx5_lag_set_state(dev, false);
	597	}
	598
	599	int mlx5_lag_allow(struct mlx5_core_dev *dev)
	600	{
	601	return mlx5_lag_set_state(dev, true);
	602	}
	603
6a32047a AH	604	struct net_device mlx5_lag_get_roce_netdev(struct mlx5_core_dev dev)
	605	{
	606	struct net_device *ndev = NULL;
	607	struct mlx5_lag *ldev;
	608
	609	mutex_lock(&lag_mutex);
	610	ldev = mlx5_lag_dev_get(dev);
	611
	612	if (!(ldev && mlx5_lag_is_bonded(ldev)))
	613	goto unlock;
	614
	615	if (ldev->tracker.tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
	616	ndev = ldev->tracker.netdev_state[0].tx_enabled ?
	617	ldev->pf[0].netdev : ldev->pf[1].netdev;
	618	} else {
	619	ndev = ldev->pf[0].netdev;
	620	}
	621	if (ndev)
	622	dev_hold(ndev);
	623
	624	unlock:
	625	mutex_unlock(&lag_mutex);
	626
	627	return ndev;
	628	}
	629	EXPORT_SYMBOL(mlx5_lag_get_roce_netdev);
	630
917b41aa AH	631	bool mlx5_lag_intf_add(struct mlx5_interface intf, struct mlx5_priv priv)
	632	{
	633	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev,
	634	priv);
	635	struct mlx5_lag *ldev;
	636
	637	if (intf->protocol != MLX5_INTERFACE_PROTOCOL_IB)
	638	return true;
	639
	640	ldev = mlx5_lag_dev_get(dev);
	641	if (!ldev \|\| !mlx5_lag_is_bonded(ldev) \|\| ldev->pf[0].dev == dev)
	642	return true;
	643
	644	/* If bonded, we do not add an IB device for PF1. */
	645	return false;
	646	}
71a0ff65 MD	647
	648	int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
	649	u64 *values,
	650	int num_counters,
	651	size_t *offsets)
	652	{
	653	int outlen = MLX5_ST_SZ_BYTES(query_cong_statistics_out);
	654	struct mlx5_core_dev *mdev[MLX5_MAX_PORTS];
	655	struct mlx5_lag *ldev;
	656	int num_ports;
	657	int ret, i, j;
	658	void *out;
	659
	660	out = kvzalloc(outlen, GFP_KERNEL);
	661	if (!out)
	662	return -ENOMEM;
	663
	664	memset(values, 0, sizeof(values) num_counters);
	665
	666	mutex_lock(&lag_mutex);
	667	ldev = mlx5_lag_dev_get(dev);
	668	if (ldev && mlx5_lag_is_bonded(ldev)) {
	669	num_ports = MLX5_MAX_PORTS;
	670	mdev[0] = ldev->pf[0].dev;
	671	mdev[1] = ldev->pf[1].dev;
	672	} else {
	673	num_ports = 1;
	674	mdev[0] = dev;
	675	}
	676
	677	for (i = 0; i < num_ports; ++i) {
	678	ret = mlx5_cmd_query_cong_counter(mdev[i], false, out, outlen);
	679	if (ret)
	680	goto unlock;
	681
	682	for (j = 0; j < num_counters; ++j)
	683	values[j] += be64_to_cpup((__be64 *)(out + offsets[j]));
	684	}
	685
	686	unlock:
	687	mutex_unlock(&lag_mutex);
	688	kvfree(out);
	689	return ret;
	690	}
	691	EXPORT_SYMBOL(mlx5_lag_query_cong_counters);