[mirror_ubuntu-eoan-kernel.git] / drivers / net / ethernet / mellanox / mlx5 / core / health.c

/*
 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/vmalloc.h>
#include <linux/hardirq.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/cmd.h>
#include "mlx5_core.h"
#include "lib/eq.h"
#include "lib/mlx5.h"

enum {
	MLX5_HEALTH_POLL_INTERVAL	= 2 * HZ,
	MAX_MISSES			= 3,
};

enum {
	MLX5_HEALTH_SYNDR_FW_ERR		= 0x1,
	MLX5_HEALTH_SYNDR_IRISC_ERR		= 0x7,
	MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR	= 0x8,
	MLX5_HEALTH_SYNDR_CRC_ERR		= 0x9,
	MLX5_HEALTH_SYNDR_FETCH_PCI_ERR		= 0xa,
	MLX5_HEALTH_SYNDR_HW_FTL_ERR		= 0xb,
	MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR	= 0xc,
	MLX5_HEALTH_SYNDR_EQ_ERR		= 0xd,
	MLX5_HEALTH_SYNDR_EQ_INV		= 0xe,
	MLX5_HEALTH_SYNDR_FFSER_ERR		= 0xf,
	MLX5_HEALTH_SYNDR_HIGH_TEMP		= 0x10
};

enum {
	MLX5_DROP_NEW_HEALTH_WORK,
	MLX5_DROP_NEW_RECOVERY_WORK,
};

u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)
{
	return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
}

void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
{
	u32 cur_cmdq_addr_l_sz;

	cur_cmdq_addr_l_sz = ioread32be(&dev->iseg->cmdq_addr_l_sz);
	iowrite32be((cur_cmdq_addr_l_sz & 0xFFFFF000) |
		    state << MLX5_NIC_IFC_OFFSET,
		    &dev->iseg->cmdq_addr_l_sz);
}

static int in_fatal(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;
	struct health_buffer __iomem *h = health->health;

	if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
		return 1;

	if (ioread32be(&h->fw_ver) == 0xffffffff)
		return 1;

	return 0;
}

void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
{
	mutex_lock(&dev->intf_state_mutex);
	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
		goto unlock;

	mlx5_core_err(dev, "start\n");
	if (pci_channel_offline(dev->pdev) || in_fatal(dev) || force) {
		dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
		mlx5_cmd_flush(dev);
	}

	mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
	mlx5_core_err(dev, "end\n");

unlock:
	mutex_unlock(&dev->intf_state_mutex);
}

static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
{
	u8 nic_interface = mlx5_get_nic_state(dev);

	switch (nic_interface) {
	case MLX5_NIC_IFC_FULL:
		mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n");
		break;

	case MLX5_NIC_IFC_DISABLED:
		mlx5_core_warn(dev, "starting teardown\n");
		break;

	case MLX5_NIC_IFC_NO_DRAM_NIC:
		mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n");
		break;
	default:
		mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n",
			       nic_interface);
	}

	mlx5_disable_device(dev);
}

static void health_recover(struct work_struct *work)
{
	struct mlx5_core_health *health;
	struct delayed_work *dwork;
	struct mlx5_core_dev *dev;
	struct mlx5_priv *priv;
	u8 nic_state;

	dwork = container_of(work, struct delayed_work, work);
	health = container_of(dwork, struct mlx5_core_health, recover_work);
	priv = container_of(health, struct mlx5_priv, health);
	dev = container_of(priv, struct mlx5_core_dev, priv);

	nic_state = mlx5_get_nic_state(dev);
	if (nic_state == MLX5_NIC_IFC_INVALID) {
		dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n");
		return;
	}

	dev_err(&dev->pdev->dev, "starting health recovery flow\n");
	mlx5_recover_device(dev);
}

/* How much time to wait until health resetting the driver (in msecs) */
#define MLX5_RECOVERY_DELAY_MSECS 60000
static void health_care(struct work_struct *work)
{
	unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS);
	struct mlx5_core_health *health;
	struct mlx5_core_dev *dev;
	struct mlx5_priv *priv;
	unsigned long flags;

	health = container_of(work, struct mlx5_core_health, work);
	priv = container_of(health, struct mlx5_priv, health);
	dev = container_of(priv, struct mlx5_core_dev, priv);
	mlx5_core_warn(dev, "handling bad device here\n");
	mlx5_handle_bad_state(dev);

	spin_lock_irqsave(&health->wq_lock, flags);
	if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags))
		schedule_delayed_work(&health->recover_work, recover_delay);
	else
		dev_err(&dev->pdev->dev,
			"new health works are not permitted at this stage\n");
	spin_unlock_irqrestore(&health->wq_lock, flags);
}

static const char *hsynd_str(u8 synd)
{
	switch (synd) {
	case MLX5_HEALTH_SYNDR_FW_ERR:
		return "firmware internal error";
	case MLX5_HEALTH_SYNDR_IRISC_ERR:
		return "irisc not responding";
	case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR:
		return "unrecoverable hardware error";
	case MLX5_HEALTH_SYNDR_CRC_ERR:
		return "firmware CRC error";
	case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR:
		return "ICM fetch PCI error";
	case MLX5_HEALTH_SYNDR_HW_FTL_ERR:
		return "HW fatal error\n";
	case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR:
		return "async EQ buffer overrun";
	case MLX5_HEALTH_SYNDR_EQ_ERR:
		return "EQ error";
	case MLX5_HEALTH_SYNDR_EQ_INV:
		return "Invalid EQ referenced";
	case MLX5_HEALTH_SYNDR_FFSER_ERR:
		return "FFSER error";
	case MLX5_HEALTH_SYNDR_HIGH_TEMP:
		return "High temperature";
	default:
		return "unrecognized error";
	}
}

static void print_health_info(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;
	struct health_buffer __iomem *h = health->health;
	char fw_str[18];
	u32 fw;
	int i;

	/* If the syndrome is 0, the device is OK and no need to print buffer */
	if (!ioread8(&h->synd))
		return;

	for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
		dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i));

	dev_err(&dev->pdev->dev, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr));
	dev_err(&dev->pdev->dev, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra));
	sprintf(fw_str, "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev));
	dev_err(&dev->pdev->dev, "fw_ver %s\n", fw_str);
	dev_err(&dev->pdev->dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id));
	dev_err(&dev->pdev->dev, "irisc_index %d\n", ioread8(&h->irisc_index));
	dev_err(&dev->pdev->dev, "synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd)));
	dev_err(&dev->pdev->dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd));
	fw = ioread32be(&h->fw_ver);
	dev_err(&dev->pdev->dev, "raw fw_ver 0x%08x\n", fw);
}

static unsigned long get_next_poll_jiffies(void)
{
	unsigned long next;

	get_random_bytes(&next, sizeof(next));
	next %= HZ;
	next += jiffies + MLX5_HEALTH_POLL_INTERVAL;

	return next;
}

void mlx5_trigger_health_work(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;
	unsigned long flags;

	spin_lock_irqsave(&health->wq_lock, flags);
	if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
		queue_work(health->wq, &health->work);
	else
		dev_err(&dev->pdev->dev,
			"new health works are not permitted at this stage\n");
	spin_unlock_irqrestore(&health->wq_lock, flags);
}

static void poll_health(struct timer_list *t)
{
	struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
	struct mlx5_core_health *health = &dev->priv.health;
	u32 count;

	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
		goto out;

	count = ioread32be(health->health_counter);
	if (count == health->prev)
		++health->miss_counter;
	else
		health->miss_counter = 0;

	health->prev = count;
	if (health->miss_counter == MAX_MISSES) {
		dev_err(&dev->pdev->dev, "device's health compromised - reached miss count\n");
		print_health_info(dev);
	}

	if (in_fatal(dev) && !health->sick) {
		health->sick = true;
		print_health_info(dev);
		mlx5_trigger_health_work(dev);
	}

out:
	mod_timer(&health->timer, get_next_poll_jiffies());
}

void mlx5_start_health_poll(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;

	timer_setup(&health->timer, poll_health, 0);
	health->sick = 0;
	clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
	clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
	health->health = &dev->iseg->health;
	health->health_counter = &dev->iseg->health_counter;

	health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL);
	add_timer(&health->timer);
}

void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health)
{
	struct mlx5_core_health *health = &dev->priv.health;
	unsigned long flags;

	if (disable_health) {
		spin_lock_irqsave(&health->wq_lock, flags);
		set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
		set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
		spin_unlock_irqrestore(&health->wq_lock, flags);
	}

	del_timer_sync(&health->timer);
}

void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;
	unsigned long flags;

	spin_lock_irqsave(&health->wq_lock, flags);
	set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
	set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
	spin_unlock_irqrestore(&health->wq_lock, flags);
	cancel_delayed_work_sync(&health->recover_work);
	cancel_work_sync(&health->work);
}

void mlx5_drain_health_recovery(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;
	unsigned long flags;

	spin_lock_irqsave(&health->wq_lock, flags);
	set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
	spin_unlock_irqrestore(&health->wq_lock, flags);
	cancel_delayed_work_sync(&dev->priv.health.recover_work);
}

void mlx5_health_cleanup(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health = &dev->priv.health;

	destroy_workqueue(health->wq);
}

int mlx5_health_init(struct mlx5_core_dev *dev)
{
	struct mlx5_core_health *health;
	char *name;

	health = &dev->priv.health;
	name = kmalloc(64, GFP_KERNEL);
	if (!name)
		return -ENOMEM;

	strcpy(name, "mlx5_health");
	strcat(name, dev_name(&dev->pdev->dev));
	health->wq = create_singlethread_workqueue(name);
	kfree(name);
	if (!health->wq)
		return -ENOMEM;
	spin_lock_init(&health->wq_lock);
	INIT_WORK(&health->work, health_care);
	INIT_DELAYED_WORK(&health->recover_work, health_recover);

	return 0;
}
Commit	Line	Data
e126ba97	1	/*
302bdf68	2	* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97 EC	3	*
	4	* This software is available to you under a choice of one of two
	5	* licenses. You may choose to be licensed under the terms of the GNU
	6	* General Public License (GPL) Version 2, available from the file
	7	* COPYING in the main directory of this source tree, or the
	8	* OpenIB.org BSD license below:
	9	*
	10	* Redistribution and use in source and binary forms, with or
	11	* without modification, are permitted provided that the following
	12	* conditions are met:
	13	*
	14	* - Redistributions of source code must retain the above
	15	* copyright notice, this list of conditions and the following
	16	* disclaimer.
	17	*
	18	* - Redistributions in binary form must reproduce the above
	19	* copyright notice, this list of conditions and the following
	20	* disclaimer in the documentation and/or other materials
	21	* provided with the distribution.
	22	*
	23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	30	* SOFTWARE.
	31	*/
	32
	33	#include <linux/kernel.h>
	34	#include <linux/module.h>
	35	#include <linux/random.h>
	36	#include <linux/vmalloc.h>
89d44f0a	37	#include <linux/hardirq.h>
e126ba97 EC	38	#include <linux/mlx5/driver.h>
	39	#include <linux/mlx5/cmd.h>
	40	#include "mlx5_core.h"
f2f3df55	41	#include "lib/eq.h"
02039fb6	42	#include "lib/mlx5.h"
e126ba97 EC	43
	44	enum {
	45	MLX5_HEALTH_POLL_INTERVAL = 2 * HZ,
	46	MAX_MISSES = 3,
	47	};
	48
	49	enum {
	50	MLX5_HEALTH_SYNDR_FW_ERR = 0x1,
	51	MLX5_HEALTH_SYNDR_IRISC_ERR = 0x7,
171bb2c5	52	MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR = 0x8,
e126ba97 EC	53	MLX5_HEALTH_SYNDR_CRC_ERR = 0x9,
	54	MLX5_HEALTH_SYNDR_FETCH_PCI_ERR = 0xa,
	55	MLX5_HEALTH_SYNDR_HW_FTL_ERR = 0xb,
	56	MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR = 0xc,
	57	MLX5_HEALTH_SYNDR_EQ_ERR = 0xd,
171bb2c5	58	MLX5_HEALTH_SYNDR_EQ_INV = 0xe,
e126ba97	59	MLX5_HEALTH_SYNDR_FFSER_ERR = 0xf,
171bb2c5	60	MLX5_HEALTH_SYNDR_HIGH_TEMP = 0x10
e126ba97 EC	61	};
e126ba97 EC	62
05ac2c0b MHY	63	enum {
05ac2c0b MHY	64	MLX5_DROP_NEW_HEALTH_WORK,
2a0165a0	65	MLX5_DROP_NEW_RECOVERY_WORK,
05ac2c0b MHY	66	};
05ac2c0b MHY	67
fcd29ad1	68	u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)
fd76ee4d EC	69	{
	70	return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
	71	}
	72
fcd29ad1 FD	73	void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
	74	{
	75	u32 cur_cmdq_addr_l_sz;
	76
	77	cur_cmdq_addr_l_sz = ioread32be(&dev->iseg->cmdq_addr_l_sz);
	78	iowrite32be((cur_cmdq_addr_l_sz & 0xFFFFF000) \|
	79	state << MLX5_NIC_IFC_OFFSET,
	80	&dev->iseg->cmdq_addr_l_sz);
	81	}
	82
fd76ee4d EC	83	static int in_fatal(struct mlx5_core_dev *dev)
	84	{
	85	struct mlx5_core_health *health = &dev->priv.health;
	86	struct health_buffer __iomem *h = health->health;
	87
fcd29ad1	88	if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
fd76ee4d EC	89	return 1;
	90
	91	if (ioread32be(&h->fw_ver) == 0xffffffff)
	92	return 1;
	93
	94	return 0;
	95	}
	96
8812c24d	97	void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
89d44f0a	98	{
c1d4d2e9	99	mutex_lock(&dev->intf_state_mutex);
89d44f0a	100	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
c1d4d2e9	101	goto unlock;
89d44f0a MD	102
89d44f0a MD	103	mlx5_core_err(dev, "start\n");
8812c24d	104	if (pci_channel_offline(dev->pdev) \|\| in_fatal(dev) \|\| force) {
89d44f0a	105	dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
4cab346b	106	mlx5_cmd_flush(dev);
c1d4d2e9	107	}
89d44f0a	108
02039fb6	109	mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
89d44f0a	110	mlx5_core_err(dev, "end\n");
c1d4d2e9 MHY	111
	112	unlock:
	113	mutex_unlock(&dev->intf_state_mutex);
89d44f0a MD	114	}
	115
	116	static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
	117	{
fcd29ad1	118	u8 nic_interface = mlx5_get_nic_state(dev);
89d44f0a MD	119
	120	switch (nic_interface) {
	121	case MLX5_NIC_IFC_FULL:
	122	mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n");
	123	break;
	124
	125	case MLX5_NIC_IFC_DISABLED:
	126	mlx5_core_warn(dev, "starting teardown\n");
	127	break;
	128
	129	case MLX5_NIC_IFC_NO_DRAM_NIC:
	130	mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n");
	131	break;
	132	default:
	133	mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n",
	134	nic_interface);
	135	}
	136
	137	mlx5_disable_device(dev);
	138	}
	139
04c0c1ab MHY	140	static void health_recover(struct work_struct *work)
	141	{
	142	struct mlx5_core_health *health;
	143	struct delayed_work *dwork;
	144	struct mlx5_core_dev *dev;
	145	struct mlx5_priv *priv;
	146	u8 nic_state;
	147
	148	dwork = container_of(work, struct delayed_work, work);
	149	health = container_of(dwork, struct mlx5_core_health, recover_work);
	150	priv = container_of(health, struct mlx5_priv, health);
	151	dev = container_of(priv, struct mlx5_core_dev, priv);
	152
fcd29ad1	153	nic_state = mlx5_get_nic_state(dev);
04c0c1ab MHY	154	if (nic_state == MLX5_NIC_IFC_INVALID) {
	155	dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n");
	156	return;
	157	}
	158
	159	dev_err(&dev->pdev->dev, "starting health recovery flow\n");
	160	mlx5_recover_device(dev);
	161	}
	162
	163	/* How much time to wait until health resetting the driver (in msecs) */
	164	#define MLX5_RECOVERY_DELAY_MSECS 60000
e126ba97 EC	165	static void health_care(struct work_struct *work)
e126ba97 EC	166	{
04c0c1ab	167	unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS);
ac6ea6e8	168	struct mlx5_core_health *health;
e126ba97 EC	169	struct mlx5_core_dev *dev;
e126ba97 EC	170	struct mlx5_priv *priv;
0179720d	171	unsigned long flags;
e126ba97	172
ac6ea6e8 EC	173	health = container_of(work, struct mlx5_core_health, work);
	174	priv = container_of(health, struct mlx5_priv, health);
	175	dev = container_of(priv, struct mlx5_core_dev, priv);
	176	mlx5_core_warn(dev, "handling bad device here\n");
89d44f0a	177	mlx5_handle_bad_state(dev);
04c0c1ab	178
0179720d	179	spin_lock_irqsave(&health->wq_lock, flags);
2a0165a0	180	if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags))
04c0c1ab MHY	181	schedule_delayed_work(&health->recover_work, recover_delay);
	182	else
	183	dev_err(&dev->pdev->dev,
	184	"new health works are not permitted at this stage\n");
0179720d	185	spin_unlock_irqrestore(&health->wq_lock, flags);
e126ba97 EC	186	}
	187
	188	static const char *hsynd_str(u8 synd)
	189	{
	190	switch (synd) {
	191	case MLX5_HEALTH_SYNDR_FW_ERR:
	192	return "firmware internal error";
	193	case MLX5_HEALTH_SYNDR_IRISC_ERR:
	194	return "irisc not responding";
171bb2c5 EC	195	case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR:
171bb2c5 EC	196	return "unrecoverable hardware error";
e126ba97 EC	197	case MLX5_HEALTH_SYNDR_CRC_ERR:
	198	return "firmware CRC error";
	199	case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR:
	200	return "ICM fetch PCI error";
	201	case MLX5_HEALTH_SYNDR_HW_FTL_ERR:
	202	return "HW fatal error\n";
	203	case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR:
	204	return "async EQ buffer overrun";
	205	case MLX5_HEALTH_SYNDR_EQ_ERR:
	206	return "EQ error";
171bb2c5	207	case MLX5_HEALTH_SYNDR_EQ_INV:
c01e0159	208	return "Invalid EQ referenced";
e126ba97 EC	209	case MLX5_HEALTH_SYNDR_FFSER_ERR:
e126ba97 EC	210	return "FFSER error";
171bb2c5	211	case MLX5_HEALTH_SYNDR_HIGH_TEMP:
c01e0159	212	return "High temperature";
e126ba97 EC	213	default:
	214	return "unrecognized error";
	215	}
	216	}
	217
	218	static void print_health_info(struct mlx5_core_dev *dev)
	219	{
	220	struct mlx5_core_health *health = &dev->priv.health;
	221	struct health_buffer __iomem *h = health->health;
0144a95e EC	222	char fw_str[18];
0144a95e EC	223	u32 fw;
e126ba97 EC	224	int i;
e126ba97 EC	225
777ec2b2	226	/* If the syndrome is 0, the device is OK and no need to print buffer */
89d44f0a MD	227	if (!ioread8(&h->synd))
	228	return;
	229
e126ba97	230	for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
0144a95e EC	231	dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i));
	232
	233	dev_err(&dev->pdev->dev, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr));
	234	dev_err(&dev->pdev->dev, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra));
712bfef6	235	sprintf(fw_str, "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev));
0144a95e EC	236	dev_err(&dev->pdev->dev, "fw_ver %s\n", fw_str);
	237	dev_err(&dev->pdev->dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id));
	238	dev_err(&dev->pdev->dev, "irisc_index %d\n", ioread8(&h->irisc_index));
	239	dev_err(&dev->pdev->dev, "synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd)));
	240	dev_err(&dev->pdev->dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd));
712bfef6 EC	241	fw = ioread32be(&h->fw_ver);
712bfef6 EC	242	dev_err(&dev->pdev->dev, "raw fw_ver 0x%08x\n", fw);
e126ba97 EC	243	}
e126ba97 EC	244
fd76ee4d EC	245	static unsigned long get_next_poll_jiffies(void)
	246	{
	247	unsigned long next;
	248
	249	get_random_bytes(&next, sizeof(next));
	250	next %= HZ;
	251	next += jiffies + MLX5_HEALTH_POLL_INTERVAL;
	252
	253	return next;
	254	}
	255
0179720d IT	256	void mlx5_trigger_health_work(struct mlx5_core_dev *dev)
	257	{
	258	struct mlx5_core_health *health = &dev->priv.health;
	259	unsigned long flags;
	260
	261	spin_lock_irqsave(&health->wq_lock, flags);
	262	if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
	263	queue_work(health->wq, &health->work);
	264	else
	265	dev_err(&dev->pdev->dev,
	266	"new health works are not permitted at this stage\n");
	267	spin_unlock_irqrestore(&health->wq_lock, flags);
	268	}
	269
0365b047	270	static void poll_health(struct timer_list *t)
e126ba97	271	{
0365b047	272	struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
e126ba97	273	struct mlx5_core_health *health = &dev->priv.health;
e126ba97 EC	274	u32 count;
e126ba97 EC	275
3fece5d6 MHY	276	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
3fece5d6 MHY	277	goto out;
89d44f0a	278
e126ba97 EC	279	count = ioread32be(health->health_counter);
	280	if (count == health->prev)
	281	++health->miss_counter;
	282	else
	283	health->miss_counter = 0;
	284
	285	health->prev = count;
	286	if (health->miss_counter == MAX_MISSES) {
fd76ee4d	287	dev_err(&dev->pdev->dev, "device's health compromised - reached miss count\n");
e126ba97	288	print_health_info(dev);
fd76ee4d EC	289	}
	290
	291	if (in_fatal(dev) && !health->sick) {
	292	health->sick = true;
	293	print_health_info(dev);
0179720d	294	mlx5_trigger_health_work(dev);
e126ba97	295	}
3fece5d6 MHY	296
	297	out:
	298	mod_timer(&health->timer, get_next_poll_jiffies());
e126ba97 EC	299	}
	300
	301	void mlx5_start_health_poll(struct mlx5_core_dev *dev)
	302	{
	303	struct mlx5_core_health *health = &dev->priv.health;
	304
0365b047	305	timer_setup(&health->timer, poll_health, 0);
2241007b	306	health->sick = 0;
05ac2c0b	307	clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
2a0165a0	308	clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
e126ba97 EC	309	health->health = &dev->iseg->health;
	310	health->health_counter = &dev->iseg->health_counter;
	311
e126ba97 EC	312	health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL);
	313	add_timer(&health->timer);
	314	}
	315
76d5581c	316	void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health)
e126ba97 EC	317	{
e126ba97 EC	318	struct mlx5_core_health *health = &dev->priv.health;
76d5581c JM	319	unsigned long flags;
	320
	321	if (disable_health) {
	322	spin_lock_irqsave(&health->wq_lock, flags);
	323	set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
	324	set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
	325	spin_unlock_irqrestore(&health->wq_lock, flags);
	326	}
e126ba97 EC	327
e126ba97 EC	328	del_timer_sync(&health->timer);
e126ba97 EC	329	}
e126ba97 EC	330
05ac2c0b MHY	331	void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
	332	{
	333	struct mlx5_core_health *health = &dev->priv.health;
0179720d	334	unsigned long flags;
05ac2c0b	335
0179720d	336	spin_lock_irqsave(&health->wq_lock, flags);
05ac2c0b	337	set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
2a0165a0	338	set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
0179720d	339	spin_unlock_irqrestore(&health->wq_lock, flags);
04c0c1ab	340	cancel_delayed_work_sync(&health->recover_work);
05ac2c0b MHY	341	cancel_work_sync(&health->work);
	342	}
	343
2a0165a0 MHY	344	void mlx5_drain_health_recovery(struct mlx5_core_dev *dev)
	345	{
	346	struct mlx5_core_health *health = &dev->priv.health;
6377ed0b	347	unsigned long flags;
2a0165a0	348
6377ed0b	349	spin_lock_irqsave(&health->wq_lock, flags);
2a0165a0	350	set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
6377ed0b	351	spin_unlock_irqrestore(&health->wq_lock, flags);
2a0165a0 MHY	352	cancel_delayed_work_sync(&dev->priv.health.recover_work);
	353	}
	354
ac6ea6e8	355	void mlx5_health_cleanup(struct mlx5_core_dev *dev)
e126ba97	356	{
ac6ea6e8 EC	357	struct mlx5_core_health *health = &dev->priv.health;
ac6ea6e8 EC	358
05ac2c0b	359	destroy_workqueue(health->wq);
e126ba97 EC	360	}
e126ba97 EC	361
ac6ea6e8	362	int mlx5_health_init(struct mlx5_core_dev *dev)
e126ba97	363	{
ac6ea6e8 EC	364	struct mlx5_core_health *health;
	365	char *name;
	366
	367	health = &dev->priv.health;
	368	name = kmalloc(64, GFP_KERNEL);
	369	if (!name)
	370	return -ENOMEM;
	371
	372	strcpy(name, "mlx5_health");
	373	strcat(name, dev_name(&dev->pdev->dev));
05ac2c0b	374	health->wq = create_singlethread_workqueue(name);
ac6ea6e8	375	kfree(name);
05ac2c0b MHY	376	if (!health->wq)
	377	return -ENOMEM;
	378	spin_lock_init(&health->wq_lock);
ac6ea6e8	379	INIT_WORK(&health->work, health_care);
04c0c1ab	380	INIT_DELAYED_WORK(&health->recover_work, health_recover);
ac6ea6e8 EC	381
ac6ea6e8 EC	382	return 0;
e126ba97	383	}