[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / mellanox / mlx5 / core / fw_reset.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved. */

#include "fw_reset.h"
#include "diag/fw_tracer.h"

enum {
	MLX5_FW_RESET_FLAGS_RESET_REQUESTED,
	MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
	MLX5_FW_RESET_FLAGS_PENDING_COMP
};

struct mlx5_fw_reset {
	struct mlx5_core_dev *dev;
	struct mlx5_nb nb;
	struct workqueue_struct *wq;
	struct work_struct fw_live_patch_work;
	struct work_struct reset_request_work;
	struct work_struct reset_reload_work;
	struct work_struct reset_now_work;
	struct work_struct reset_abort_work;
	unsigned long reset_flags;
	struct timer_list timer;
	struct completion done;
	int ret;
};

void mlx5_fw_reset_enable_remote_dev_reset_set(struct mlx5_core_dev *dev, bool enable)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	if (enable)
		clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
	else
		set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
}

bool mlx5_fw_reset_enable_remote_dev_reset_get(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	return !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
}

static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level,
			     u8 reset_type_sel, u8 sync_resp, bool sync_start)
{
	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};

	MLX5_SET(mfrl_reg, in, reset_level, reset_level);
	MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp);
	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start);

	return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1);
}

static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type)
{
	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	int err;

	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0);
	if (err)
		return err;

	if (reset_level)
		*reset_level = MLX5_GET(mfrl_reg, out, reset_level);
	if (reset_type)
		*reset_type = MLX5_GET(mfrl_reg, out, reset_type);

	return 0;
}

int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type)
{
	return mlx5_reg_mfrl_query(dev, reset_level, reset_type);
}

int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	int err;

	set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
	err = mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, reset_type_sel, 0, true);
	if (err)
		clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
	return err;
}

int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev)
{
	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false);
}

static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	/* if this is the driver that initiated the fw reset, devlink completed the reload */
	if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) {
		complete(&fw_reset->done);
	} else {
		mlx5_load_one(dev);
		devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0,
							BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
							BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
	}
}

static void mlx5_sync_reset_reload_work(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      reset_reload_work);
	struct mlx5_core_dev *dev = fw_reset->dev;
	int err;

	mlx5_enter_error_state(dev, true);
	mlx5_unload_one(dev);
	err = mlx5_health_wait_pci_up(dev);
	if (err)
		mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
	fw_reset->ret = err;
	mlx5_fw_reset_complete_reload(dev);
}

static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	del_timer_sync(&fw_reset->timer);
}

static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
		mlx5_core_warn(dev, "Reset request was already cleared\n");
		return -EALREADY;
	}

	mlx5_stop_sync_reset_poll(dev);
	if (poll_health)
		mlx5_start_health_poll(dev);
	return 0;
}

#define MLX5_RESET_POLL_INTERVAL	(HZ / 10)
static void poll_sync_reset(struct timer_list *t)
{
	struct mlx5_fw_reset *fw_reset = from_timer(fw_reset, t, timer);
	struct mlx5_core_dev *dev = fw_reset->dev;
	u32 fatal_error;

	if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
		return;

	fatal_error = mlx5_health_check_fatal_sensors(dev);

	if (fatal_error) {
		mlx5_core_warn(dev, "Got Device Reset\n");
		mlx5_sync_reset_clear_reset_requested(dev, false);
		queue_work(fw_reset->wq, &fw_reset->reset_reload_work);
		return;
	}

	mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL));
}

static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	timer_setup(&fw_reset->timer, poll_sync_reset, 0);
	fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL);
	add_timer(&fw_reset->timer);
}

static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev)
{
	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false);
}

static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev)
{
	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false);
}

static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
		mlx5_core_warn(dev, "Reset request was already set\n");
		return -EALREADY;
	}
	mlx5_stop_health_poll(dev, true);
	mlx5_start_sync_reset_poll(dev);
	return 0;
}

static void mlx5_fw_live_patch_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      fw_live_patch_work);
	struct mlx5_core_dev *dev = fw_reset->dev;

	mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev),
		       fw_rev_min(dev), fw_rev_sub(dev));

	if (mlx5_fw_tracer_reload(dev->tracer))
		mlx5_core_err(dev, "Failed to reload FW tracer\n");
}

static void mlx5_sync_reset_request_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      reset_request_work);
	struct mlx5_core_dev *dev = fw_reset->dev;
	int err;

	if (test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags)) {
		err = mlx5_fw_reset_set_reset_sync_nack(dev);
		mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s",
			       err ? "Failed" : "Sent");
		return;
	}
	if (mlx5_sync_reset_set_reset_requested(dev))
		return;

	err = mlx5_fw_reset_set_reset_sync_ack(dev);
	if (err)
		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
	else
		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n");
}

#define MLX5_PCI_LINK_UP_TIMEOUT 2000

static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev)
{
	struct pci_bus *bridge_bus = dev->pdev->bus;
	struct pci_dev *bridge = bridge_bus->self;
	u16 reg16, dev_id, sdev_id;
	unsigned long timeout;
	struct pci_dev *sdev;
	int cap, err;
	u32 reg32;

	/* Check that all functions under the pci bridge are PFs of
	 * this device otherwise fail this function.
	 */
	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
	if (err)
		return err;
	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
		err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id);
		if (err)
			return err;
		if (sdev_id != dev_id)
			return -EPERM;
	}

	cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
	if (!cap)
		return -EOPNOTSUPP;

	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
		pci_save_state(sdev);
		pci_cfg_access_lock(sdev);
	}
	/* PCI link toggle */
	err = pci_read_config_word(bridge, cap + PCI_EXP_LNKCTL, &reg16);
	if (err)
		return err;
	reg16 |= PCI_EXP_LNKCTL_LD;
	err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16);
	if (err)
		return err;
	msleep(500);
	reg16 &= ~PCI_EXP_LNKCTL_LD;
	err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16);
	if (err)
		return err;

	/* Check link */
	err = pci_read_config_dword(bridge, cap + PCI_EXP_LNKCAP, &reg32);
	if (err)
		return err;
	if (!(reg32 & PCI_EXP_LNKCAP_DLLLARC)) {
		mlx5_core_warn(dev, "No PCI link reporting capability (0x%08x)\n", reg32);
		msleep(1000);
		goto restore;
	}

	timeout = jiffies + msecs_to_jiffies(MLX5_PCI_LINK_UP_TIMEOUT);
	do {
		err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, &reg16);
		if (err)
			return err;
		if (reg16 & PCI_EXP_LNKSTA_DLLLA)
			break;
		msleep(20);
	} while (!time_after(jiffies, timeout));

	if (reg16 & PCI_EXP_LNKSTA_DLLLA) {
		mlx5_core_info(dev, "PCI Link up\n");
	} else {
		mlx5_core_err(dev, "PCI link not ready (0x%04x) after %d ms\n",
			      reg16, MLX5_PCI_LINK_UP_TIMEOUT);
		err = -ETIMEDOUT;
	}

restore:
	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
		pci_cfg_access_unlock(sdev);
		pci_restore_state(sdev);
	}

	return err;
}

static void mlx5_sync_reset_now_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      reset_now_work);
	struct mlx5_core_dev *dev = fw_reset->dev;
	int err;

	if (mlx5_sync_reset_clear_reset_requested(dev, false))
		return;

	mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n");

	err = mlx5_cmd_fast_teardown_hca(dev);
	if (err) {
		mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err);
		goto done;
	}

	err = mlx5_pci_link_toggle(dev);
	if (err) {
		mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, no reset done, err %d\n", err);
		goto done;
	}

	mlx5_enter_error_state(dev, true);
	mlx5_unload_one(dev);
done:
	fw_reset->ret = err;
	mlx5_fw_reset_complete_reload(dev);
}

static void mlx5_sync_reset_abort_event(struct work_struct *work)
{
	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
						      reset_abort_work);
	struct mlx5_core_dev *dev = fw_reset->dev;

	if (mlx5_sync_reset_clear_reset_requested(dev, true))
		return;
	mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
}

static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct mlx5_eqe *eqe)
{
	struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe;
	u8 sync_event_rst_type;

	sync_fw_update_eqe = &eqe->data.sync_fw_update;
	sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK;
	switch (sync_event_rst_type) {
	case MLX5_SYNC_RST_STATE_RESET_REQUEST:
		queue_work(fw_reset->wq, &fw_reset->reset_request_work);
		break;
	case MLX5_SYNC_RST_STATE_RESET_NOW:
		queue_work(fw_reset->wq, &fw_reset->reset_now_work);
		break;
	case MLX5_SYNC_RST_STATE_RESET_ABORT:
		queue_work(fw_reset->wq, &fw_reset->reset_abort_work);
		break;
	}
}

static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data)
{
	struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb);
	struct mlx5_eqe *eqe = data;

	switch (eqe->sub_type) {
	case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT:
			queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work);
		break;
	case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT:
		mlx5_sync_reset_events_handle(fw_reset, eqe);
		break;
	default:
		return NOTIFY_DONE;
	}

	return NOTIFY_OK;
}

#define MLX5_FW_RESET_TIMEOUT_MSEC 5000
int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev)
{
	unsigned long timeout = msecs_to_jiffies(MLX5_FW_RESET_TIMEOUT_MSEC);
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	int err;

	if (!wait_for_completion_timeout(&fw_reset->done, timeout)) {
		mlx5_core_warn(dev, "FW sync reset timeout after %d seconds\n",
			       MLX5_FW_RESET_TIMEOUT_MSEC / 1000);
		err = -ETIMEDOUT;
		goto out;
	}
	err = fw_reset->ret;
out:
	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
	return err;
}

void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT);
	mlx5_eq_notifier_register(dev, &fw_reset->nb);
}

void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev)
{
	mlx5_eq_notifier_unregister(dev, &dev->priv.fw_reset->nb);
}

int mlx5_fw_reset_init(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = kzalloc(sizeof(*fw_reset), GFP_KERNEL);

	if (!fw_reset)
		return -ENOMEM;
	fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events");
	if (!fw_reset->wq) {
		kfree(fw_reset);
		return -ENOMEM;
	}

	fw_reset->dev = dev;
	dev->priv.fw_reset = fw_reset;

	INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event);
	INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event);
	INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work);
	INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event);
	INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event);

	init_completion(&fw_reset->done);
	return 0;
}

void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev)
{
	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;

	destroy_workqueue(fw_reset->wq);
	kfree(dev->priv.fw_reset);
}
Commit	Line	Data
3180472f MS	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
	3
	4	#include "fw_reset.h"
2d693567	5	#include "diag/fw_tracer.h"
3180472f	6
38b9f903 MS	7	enum {
38b9f903 MS	8	MLX5_FW_RESET_FLAGS_RESET_REQUESTED,
b4f7cbb3	9	MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
5ec69744	10	MLX5_FW_RESET_FLAGS_PENDING_COMP
38b9f903 MS	11	};
	12
	13	struct mlx5_fw_reset {
	14	struct mlx5_core_dev *dev;
	15	struct mlx5_nb nb;
	16	struct workqueue_struct *wq;
2d693567	17	struct work_struct fw_live_patch_work;
38b9f903 MS	18	struct work_struct reset_request_work;
38b9f903 MS	19	struct work_struct reset_reload_work;
eabe8e5e	20	struct work_struct reset_now_work;
7dd6df32	21	struct work_struct reset_abort_work;
38b9f903 MS	22	unsigned long reset_flags;
38b9f903 MS	23	struct timer_list timer;
5ec69744 MS	24	struct completion done;
5ec69744 MS	25	int ret;
38b9f903 MS	26	};
38b9f903 MS	27
b4f7cbb3 MS	28	void mlx5_fw_reset_enable_remote_dev_reset_set(struct mlx5_core_dev *dev, bool enable)
	29	{
	30	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	31
	32	if (enable)
	33	clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
	34	else
	35	set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
	36	}
	37
	38	bool mlx5_fw_reset_enable_remote_dev_reset_get(struct mlx5_core_dev *dev)
	39	{
	40	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	41
	42	return !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
	43	}
	44
3180472f MS	45	static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level,
	46	u8 reset_type_sel, u8 sync_resp, bool sync_start)
	47	{
	48	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	49	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	50
	51	MLX5_SET(mfrl_reg, in, reset_level, reset_level);
	52	MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
	53	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp);
	54	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start);
	55
	56	return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1);
	57	}
	58
	59	static int mlx5_reg_mfrl_query(struct mlx5_core_dev dev, u8 reset_level, u8 *reset_type)
	60	{
	61	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	62	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
	63	int err;
	64
	65	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0);
	66	if (err)
	67	return err;
	68
	69	if (reset_level)
	70	*reset_level = MLX5_GET(mfrl_reg, out, reset_level);
	71	if (reset_type)
	72	*reset_type = MLX5_GET(mfrl_reg, out, reset_type);
	73
	74	return 0;
	75	}
	76
	77	int mlx5_fw_reset_query(struct mlx5_core_dev dev, u8 reset_level, u8 *reset_type)
	78	{
	79	return mlx5_reg_mfrl_query(dev, reset_level, reset_type);
	80	}
	81
	82	int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel)
	83	{
5ec69744 MS	84	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	85	int err;
	86
	87	set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
	88	err = mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, reset_type_sel, 0, true);
	89	if (err)
	90	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
	91	return err;
3180472f MS	92	}
	93
	94	int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev)
	95	{
	96	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false);
	97	}
38b9f903	98
5ec69744 MS	99	static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev)
	100	{
	101	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	102
	103	/* if this is the driver that initiated the fw reset, devlink completed the reload */
	104	if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) {
	105	complete(&fw_reset->done);
	106	} else {
6dea2f7e	107	mlx5_load_one(dev);
5ec69744 MS	108	devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0,
	109	BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) \|
	110	BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
	111	}
	112	}
	113
38b9f903 MS	114	static void mlx5_sync_reset_reload_work(struct work_struct *work)
	115	{
	116	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
	117	reset_reload_work);
	118	struct mlx5_core_dev *dev = fw_reset->dev;
5ec69744	119	int err;
38b9f903 MS	120
38b9f903 MS	121	mlx5_enter_error_state(dev, true);
6dea2f7e	122	mlx5_unload_one(dev);
5ec69744 MS	123	err = mlx5_health_wait_pci_up(dev);
5ec69744 MS	124	if (err)
38b9f903	125	mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
5ec69744 MS	126	fw_reset->ret = err;
5ec69744 MS	127	mlx5_fw_reset_complete_reload(dev);
38b9f903 MS	128	}
	129
	130	static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
	131	{
	132	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	133
73787daf	134	del_timer_sync(&fw_reset->timer);
38b9f903 MS	135	}
38b9f903 MS	136
de299086	137	static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
38b9f903 MS	138	{
	139	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	140
de299086 MS	141	if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
	142	mlx5_core_warn(dev, "Reset request was already cleared\n");
	143	return -EALREADY;
	144	}
	145
38b9f903	146	mlx5_stop_sync_reset_poll(dev);
38b9f903 MS	147	if (poll_health)
38b9f903 MS	148	mlx5_start_health_poll(dev);
de299086	149	return 0;
38b9f903 MS	150	}
	151
	152	#define MLX5_RESET_POLL_INTERVAL (HZ / 10)
	153	static void poll_sync_reset(struct timer_list *t)
	154	{
	155	struct mlx5_fw_reset *fw_reset = from_timer(fw_reset, t, timer);
	156	struct mlx5_core_dev *dev = fw_reset->dev;
	157	u32 fatal_error;
	158
	159	if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
	160	return;
	161
	162	fatal_error = mlx5_health_check_fatal_sensors(dev);
	163
	164	if (fatal_error) {
	165	mlx5_core_warn(dev, "Got Device Reset\n");
	166	mlx5_sync_reset_clear_reset_requested(dev, false);
	167	queue_work(fw_reset->wq, &fw_reset->reset_reload_work);
	168	return;
	169	}
	170
	171	mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL));
	172	}
	173
	174	static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev)
	175	{
	176	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	177
	178	timer_setup(&fw_reset->timer, poll_sync_reset, 0);
	179	fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL);
	180	add_timer(&fw_reset->timer);
	181	}
	182
	183	static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev)
	184	{
	185	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false);
	186	}
	187
b4f7cbb3 MS	188	static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev)
	189	{
	190	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false);
	191	}
	192
de299086	193	static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
38b9f903 MS	194	{
	195	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	196
de299086 MS	197	if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
	198	mlx5_core_warn(dev, "Reset request was already set\n");
	199	return -EALREADY;
	200	}
38b9f903	201	mlx5_stop_health_poll(dev, true);
38b9f903	202	mlx5_start_sync_reset_poll(dev);
de299086	203	return 0;
38b9f903 MS	204	}
38b9f903 MS	205
2d693567 MS	206	static void mlx5_fw_live_patch_event(struct work_struct *work)
	207	{
	208	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
	209	fw_live_patch_work);
	210	struct mlx5_core_dev *dev = fw_reset->dev;
2d693567 MS	211
	212	mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev),
	213	fw_rev_min(dev), fw_rev_sub(dev));
	214
7e615b99	215	if (mlx5_fw_tracer_reload(dev->tracer))
2d693567 MS	216	mlx5_core_err(dev, "Failed to reload FW tracer\n");
	217	}
	218
38b9f903 MS	219	static void mlx5_sync_reset_request_event(struct work_struct *work)
	220	{
	221	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
	222	reset_request_work);
	223	struct mlx5_core_dev *dev = fw_reset->dev;
	224	int err;
	225
b4f7cbb3 MS	226	if (test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags)) {
	227	err = mlx5_fw_reset_set_reset_sync_nack(dev);
	228	mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s",
	229	err ? "Failed" : "Sent");
	230	return;
	231	}
de299086 MS	232	if (mlx5_sync_reset_set_reset_requested(dev))
	233	return;
	234
38b9f903 MS	235	err = mlx5_fw_reset_set_reset_sync_ack(dev);
	236	if (err)
	237	mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
	238	else
	239	mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n");
	240	}
	241
eabe8e5e MS	242	#define MLX5_PCI_LINK_UP_TIMEOUT 2000
	243
	244	static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev)
	245	{
	246	struct pci_bus *bridge_bus = dev->pdev->bus;
	247	struct pci_dev *bridge = bridge_bus->self;
	248	u16 reg16, dev_id, sdev_id;
	249	unsigned long timeout;
	250	struct pci_dev *sdev;
	251	int cap, err;
	252	u32 reg32;
	253
	254	/* Check that all functions under the pci bridge are PFs of
	255	* this device otherwise fail this function.
	256	*/
	257	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
	258	if (err)
	259	return err;
	260	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
	261	err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id);
	262	if (err)
	263	return err;
	264	if (sdev_id != dev_id)
	265	return -EPERM;
	266	}
	267
	268	cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
	269	if (!cap)
	270	return -EOPNOTSUPP;
	271
	272	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
	273	pci_save_state(sdev);
	274	pci_cfg_access_lock(sdev);
	275	}
	276	/* PCI link toggle */
	277	err = pci_read_config_word(bridge, cap + PCI_EXP_LNKCTL, &reg16);
	278	if (err)
	279	return err;
	280	reg16 \|= PCI_EXP_LNKCTL_LD;
	281	err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16);
	282	if (err)
	283	return err;
	284	msleep(500);
	285	reg16 &= ~PCI_EXP_LNKCTL_LD;
	286	err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16);
	287	if (err)
	288	return err;
	289
	290	/* Check link */
	291	err = pci_read_config_dword(bridge, cap + PCI_EXP_LNKCAP, &reg32);
	292	if (err)
	293	return err;
	294	if (!(reg32 & PCI_EXP_LNKCAP_DLLLARC)) {
	295	mlx5_core_warn(dev, "No PCI link reporting capability (0x%08x)\n", reg32);
	296	msleep(1000);
	297	goto restore;
	298	}
	299
	300	timeout = jiffies + msecs_to_jiffies(MLX5_PCI_LINK_UP_TIMEOUT);
	301	do {
	302	err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, &reg16);
	303	if (err)
	304	return err;
	305	if (reg16 & PCI_EXP_LNKSTA_DLLLA)
306	break;
307	msleep(20);
308	} while (!time_after(jiffies, timeout));
309
310	if (reg16 & PCI_EXP_LNKSTA_DLLLA) {
311	mlx5_core_info(dev, "PCI Link up\n");
312	} else {
313	mlx5_core_err(dev, "PCI link not ready (0x%04x) after %d ms\n",
314	reg16, MLX5_PCI_LINK_UP_TIMEOUT);
315	err = -ETIMEDOUT;
316	}
317
318	restore:
319	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
320	pci_cfg_access_unlock(sdev);
321	pci_restore_state(sdev);
322	}
323
324	return err;
325	}
326
327	static void mlx5_sync_reset_now_event(struct work_struct *work)
328	{
329	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
330	reset_now_work);
331	struct mlx5_core_dev *dev = fw_reset->dev;
332	int err;
333
de299086 MS	334	if (mlx5_sync_reset_clear_reset_requested(dev, false))
de299086 MS	335	return;
eabe8e5e MS	336
	337	mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n");
	338
	339	err = mlx5_cmd_fast_teardown_hca(dev);
	340	if (err) {
	341	mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err);
	342	goto done;
	343	}
	344
	345	err = mlx5_pci_link_toggle(dev);
	346	if (err) {
	347	mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, no reset done, err %d\n", err);
	348	goto done;
	349	}
	350
	351	mlx5_enter_error_state(dev, true);
6dea2f7e	352	mlx5_unload_one(dev);
eabe8e5e	353	done:
5ec69744 MS	354	fw_reset->ret = err;
5ec69744 MS	355	mlx5_fw_reset_complete_reload(dev);
eabe8e5e MS	356	}
eabe8e5e MS	357
7dd6df32 MS	358	static void mlx5_sync_reset_abort_event(struct work_struct *work)
	359	{
	360	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
	361	reset_abort_work);
	362	struct mlx5_core_dev *dev = fw_reset->dev;
	363
de299086	364	if (mlx5_sync_reset_clear_reset_requested(dev, true))
5940e642	365	return;
7dd6df32 MS	366	mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
	367	}
	368
38b9f903 MS	369	static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset fw_reset, struct mlx5_eqe eqe)
	370	{
	371	struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe;
	372	u8 sync_event_rst_type;
	373
	374	sync_fw_update_eqe = &eqe->data.sync_fw_update;
	375	sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK;
	376	switch (sync_event_rst_type) {
	377	case MLX5_SYNC_RST_STATE_RESET_REQUEST:
	378	queue_work(fw_reset->wq, &fw_reset->reset_request_work);
	379	break;
eabe8e5e MS	380	case MLX5_SYNC_RST_STATE_RESET_NOW:
	381	queue_work(fw_reset->wq, &fw_reset->reset_now_work);
	382	break;
7dd6df32 MS	383	case MLX5_SYNC_RST_STATE_RESET_ABORT:
	384	queue_work(fw_reset->wq, &fw_reset->reset_abort_work);
	385	break;
38b9f903 MS	386	}
	387	}
	388
	389	static int fw_reset_event_notifier(struct notifier_block nb, unsigned long action, void data)
	390	{
	391	struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb);
	392	struct mlx5_eqe *eqe = data;
	393
	394	switch (eqe->sub_type) {
2d693567 MS	395	case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT:
	396	queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work);
	397	break;
38b9f903 MS	398	case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT:
	399	mlx5_sync_reset_events_handle(fw_reset, eqe);
	400	break;
	401	default:
	402	return NOTIFY_DONE;
	403	}
	404
	405	return NOTIFY_OK;
	406	}
	407
5ec69744 MS	408	#define MLX5_FW_RESET_TIMEOUT_MSEC 5000
	409	int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev)
	410	{
	411	unsigned long timeout = msecs_to_jiffies(MLX5_FW_RESET_TIMEOUT_MSEC);
	412	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	413	int err;
	414
	415	if (!wait_for_completion_timeout(&fw_reset->done, timeout)) {
	416	mlx5_core_warn(dev, "FW sync reset timeout after %d seconds\n",
	417	MLX5_FW_RESET_TIMEOUT_MSEC / 1000);
	418	err = -ETIMEDOUT;
	419	goto out;
	420	}
	421	err = fw_reset->ret;
	422	out:
	423	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
	424	return err;
	425	}
	426
38b9f903 MS	427	void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev)
	428	{
	429	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	430
	431	MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT);
	432	mlx5_eq_notifier_register(dev, &fw_reset->nb);
	433	}
	434
	435	void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev)
	436	{
	437	mlx5_eq_notifier_unregister(dev, &dev->priv.fw_reset->nb);
	438	}
	439
	440	int mlx5_fw_reset_init(struct mlx5_core_dev *dev)
	441	{
	442	struct mlx5_fw_reset fw_reset = kzalloc(sizeof(fw_reset), GFP_KERNEL);
	443
	444	if (!fw_reset)
	445	return -ENOMEM;
	446	fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events");
	447	if (!fw_reset->wq) {
	448	kfree(fw_reset);
	449	return -ENOMEM;
	450	}
	451
	452	fw_reset->dev = dev;
	453	dev->priv.fw_reset = fw_reset;
	454
2d693567	455	INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event);
38b9f903 MS	456	INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event);
38b9f903 MS	457	INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work);
eabe8e5e	458	INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event);
7dd6df32	459	INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event);
38b9f903	460
5ec69744	461	init_completion(&fw_reset->done);
38b9f903 MS	462	return 0;
	463	}
	464
	465	void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev)
	466	{
	467	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
	468
	469	destroy_workqueue(fw_reset->wq);
	470	kfree(dev->priv.fw_reset);
	471	}