]>
Commit | Line | Data |
---|---|---|
e126ba97 | 1 | /* |
302bdf68 | 2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
e126ba97 EC |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | ||
33 | #include <linux/kernel.h> | |
34 | #include <linux/module.h> | |
35 | #include <linux/random.h> | |
36 | #include <linux/vmalloc.h> | |
89d44f0a | 37 | #include <linux/hardirq.h> |
e126ba97 EC |
38 | #include <linux/mlx5/driver.h> |
39 | #include <linux/mlx5/cmd.h> | |
40 | #include "mlx5_core.h" | |
41 | ||
42 | enum { | |
43 | MLX5_HEALTH_POLL_INTERVAL = 2 * HZ, | |
44 | MAX_MISSES = 3, | |
45 | }; | |
46 | ||
47 | enum { | |
48 | MLX5_HEALTH_SYNDR_FW_ERR = 0x1, | |
49 | MLX5_HEALTH_SYNDR_IRISC_ERR = 0x7, | |
171bb2c5 | 50 | MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR = 0x8, |
e126ba97 EC |
51 | MLX5_HEALTH_SYNDR_CRC_ERR = 0x9, |
52 | MLX5_HEALTH_SYNDR_FETCH_PCI_ERR = 0xa, | |
53 | MLX5_HEALTH_SYNDR_HW_FTL_ERR = 0xb, | |
54 | MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR = 0xc, | |
55 | MLX5_HEALTH_SYNDR_EQ_ERR = 0xd, | |
171bb2c5 | 56 | MLX5_HEALTH_SYNDR_EQ_INV = 0xe, |
e126ba97 | 57 | MLX5_HEALTH_SYNDR_FFSER_ERR = 0xf, |
171bb2c5 | 58 | MLX5_HEALTH_SYNDR_HIGH_TEMP = 0x10 |
e126ba97 EC |
59 | }; |
60 | ||
fd76ee4d EC |
61 | enum { |
62 | MLX5_NIC_IFC_FULL = 0, | |
63 | MLX5_NIC_IFC_DISABLED = 1, | |
64 | MLX5_NIC_IFC_NO_DRAM_NIC = 2 | |
65 | }; | |
66 | ||
67 | static u8 get_nic_interface(struct mlx5_core_dev *dev) | |
68 | { | |
69 | return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; | |
70 | } | |
71 | ||
89d44f0a MD |
72 | static void trigger_cmd_completions(struct mlx5_core_dev *dev) |
73 | { | |
74 | unsigned long flags; | |
75 | u64 vector; | |
76 | ||
77 | /* wait for pending handlers to complete */ | |
78 | synchronize_irq(dev->priv.msix_arr[MLX5_EQ_VEC_CMD].vector); | |
79 | spin_lock_irqsave(&dev->cmd.alloc_lock, flags); | |
80 | vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1); | |
81 | if (!vector) | |
82 | goto no_trig; | |
83 | ||
84 | vector |= MLX5_TRIGGERED_CMD_COMP; | |
85 | spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); | |
86 | ||
87 | mlx5_core_dbg(dev, "vector 0x%llx\n", vector); | |
88 | mlx5_cmd_comp_handler(dev, vector); | |
89 | return; | |
90 | ||
91 | no_trig: | |
92 | spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); | |
93 | } | |
94 | ||
fd76ee4d EC |
95 | static int in_fatal(struct mlx5_core_dev *dev) |
96 | { | |
97 | struct mlx5_core_health *health = &dev->priv.health; | |
98 | struct health_buffer __iomem *h = health->health; | |
99 | ||
100 | if (get_nic_interface(dev) == MLX5_NIC_IFC_DISABLED) | |
101 | return 1; | |
102 | ||
103 | if (ioread32be(&h->fw_ver) == 0xffffffff) | |
104 | return 1; | |
105 | ||
106 | return 0; | |
107 | } | |
108 | ||
89d44f0a MD |
109 | void mlx5_enter_error_state(struct mlx5_core_dev *dev) |
110 | { | |
c1d4d2e9 | 111 | mutex_lock(&dev->intf_state_mutex); |
89d44f0a | 112 | if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) |
c1d4d2e9 | 113 | goto unlock; |
89d44f0a MD |
114 | |
115 | mlx5_core_err(dev, "start\n"); | |
c1d4d2e9 | 116 | if (pci_channel_offline(dev->pdev) || in_fatal(dev)) { |
89d44f0a | 117 | dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; |
c1d4d2e9 MHY |
118 | trigger_cmd_completions(dev); |
119 | } | |
89d44f0a MD |
120 | |
121 | mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0); | |
122 | mlx5_core_err(dev, "end\n"); | |
c1d4d2e9 MHY |
123 | |
124 | unlock: | |
125 | mutex_unlock(&dev->intf_state_mutex); | |
89d44f0a MD |
126 | } |
127 | ||
128 | static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) | |
129 | { | |
130 | u8 nic_interface = get_nic_interface(dev); | |
131 | ||
132 | switch (nic_interface) { | |
133 | case MLX5_NIC_IFC_FULL: | |
134 | mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); | |
135 | break; | |
136 | ||
137 | case MLX5_NIC_IFC_DISABLED: | |
138 | mlx5_core_warn(dev, "starting teardown\n"); | |
139 | break; | |
140 | ||
141 | case MLX5_NIC_IFC_NO_DRAM_NIC: | |
142 | mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n"); | |
143 | break; | |
144 | default: | |
145 | mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n", | |
146 | nic_interface); | |
147 | } | |
148 | ||
149 | mlx5_disable_device(dev); | |
150 | } | |
151 | ||
e126ba97 EC |
152 | static void health_care(struct work_struct *work) |
153 | { | |
ac6ea6e8 | 154 | struct mlx5_core_health *health; |
e126ba97 EC |
155 | struct mlx5_core_dev *dev; |
156 | struct mlx5_priv *priv; | |
e126ba97 | 157 | |
ac6ea6e8 EC |
158 | health = container_of(work, struct mlx5_core_health, work); |
159 | priv = container_of(health, struct mlx5_priv, health); | |
160 | dev = container_of(priv, struct mlx5_core_dev, priv); | |
161 | mlx5_core_warn(dev, "handling bad device here\n"); | |
89d44f0a | 162 | mlx5_handle_bad_state(dev); |
e126ba97 EC |
163 | } |
164 | ||
165 | static const char *hsynd_str(u8 synd) | |
166 | { | |
167 | switch (synd) { | |
168 | case MLX5_HEALTH_SYNDR_FW_ERR: | |
169 | return "firmware internal error"; | |
170 | case MLX5_HEALTH_SYNDR_IRISC_ERR: | |
171 | return "irisc not responding"; | |
171bb2c5 EC |
172 | case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR: |
173 | return "unrecoverable hardware error"; | |
e126ba97 EC |
174 | case MLX5_HEALTH_SYNDR_CRC_ERR: |
175 | return "firmware CRC error"; | |
176 | case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR: | |
177 | return "ICM fetch PCI error"; | |
178 | case MLX5_HEALTH_SYNDR_HW_FTL_ERR: | |
179 | return "HW fatal error\n"; | |
180 | case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR: | |
181 | return "async EQ buffer overrun"; | |
182 | case MLX5_HEALTH_SYNDR_EQ_ERR: | |
183 | return "EQ error"; | |
171bb2c5 | 184 | case MLX5_HEALTH_SYNDR_EQ_INV: |
c01e0159 | 185 | return "Invalid EQ referenced"; |
e126ba97 EC |
186 | case MLX5_HEALTH_SYNDR_FFSER_ERR: |
187 | return "FFSER error"; | |
171bb2c5 | 188 | case MLX5_HEALTH_SYNDR_HIGH_TEMP: |
c01e0159 | 189 | return "High temperature"; |
e126ba97 EC |
190 | default: |
191 | return "unrecognized error"; | |
192 | } | |
193 | } | |
194 | ||
0144a95e | 195 | static u16 get_maj(u32 fw) |
582c016e | 196 | { |
0144a95e | 197 | return fw >> 28; |
582c016e RD |
198 | } |
199 | ||
0144a95e | 200 | static u16 get_min(u32 fw) |
582c016e | 201 | { |
0144a95e EC |
202 | return fw >> 16 & 0xfff; |
203 | } | |
204 | ||
205 | static u16 get_sub(u32 fw) | |
206 | { | |
207 | return fw & 0xffff; | |
582c016e RD |
208 | } |
209 | ||
e126ba97 EC |
210 | static void print_health_info(struct mlx5_core_dev *dev) |
211 | { | |
212 | struct mlx5_core_health *health = &dev->priv.health; | |
213 | struct health_buffer __iomem *h = health->health; | |
0144a95e EC |
214 | char fw_str[18]; |
215 | u32 fw; | |
e126ba97 EC |
216 | int i; |
217 | ||
89d44f0a MD |
218 | /* If the syndrom is 0, the device is OK and no need to print buffer */ |
219 | if (!ioread8(&h->synd)) | |
220 | return; | |
221 | ||
e126ba97 | 222 | for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) |
0144a95e EC |
223 | dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i)); |
224 | ||
225 | dev_err(&dev->pdev->dev, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr)); | |
226 | dev_err(&dev->pdev->dev, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra)); | |
227 | fw = ioread32be(&h->fw_ver); | |
228 | sprintf(fw_str, "%d.%d.%d", get_maj(fw), get_min(fw), get_sub(fw)); | |
229 | dev_err(&dev->pdev->dev, "fw_ver %s\n", fw_str); | |
230 | dev_err(&dev->pdev->dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id)); | |
231 | dev_err(&dev->pdev->dev, "irisc_index %d\n", ioread8(&h->irisc_index)); | |
232 | dev_err(&dev->pdev->dev, "synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd))); | |
233 | dev_err(&dev->pdev->dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); | |
e126ba97 EC |
234 | } |
235 | ||
fd76ee4d EC |
236 | static unsigned long get_next_poll_jiffies(void) |
237 | { | |
238 | unsigned long next; | |
239 | ||
240 | get_random_bytes(&next, sizeof(next)); | |
241 | next %= HZ; | |
242 | next += jiffies + MLX5_HEALTH_POLL_INTERVAL; | |
243 | ||
244 | return next; | |
245 | } | |
246 | ||
e126ba97 EC |
247 | static void poll_health(unsigned long data) |
248 | { | |
249 | struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data; | |
250 | struct mlx5_core_health *health = &dev->priv.health; | |
e126ba97 EC |
251 | u32 count; |
252 | ||
89d44f0a | 253 | if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { |
89d44f0a MD |
254 | mod_timer(&health->timer, get_next_poll_jiffies()); |
255 | return; | |
256 | } | |
257 | ||
e126ba97 EC |
258 | count = ioread32be(health->health_counter); |
259 | if (count == health->prev) | |
260 | ++health->miss_counter; | |
261 | else | |
262 | health->miss_counter = 0; | |
263 | ||
264 | health->prev = count; | |
265 | if (health->miss_counter == MAX_MISSES) { | |
fd76ee4d | 266 | dev_err(&dev->pdev->dev, "device's health compromised - reached miss count\n"); |
e126ba97 | 267 | print_health_info(dev); |
e126ba97 | 268 | } else { |
fd76ee4d EC |
269 | mod_timer(&health->timer, get_next_poll_jiffies()); |
270 | } | |
271 | ||
272 | if (in_fatal(dev) && !health->sick) { | |
273 | health->sick = true; | |
274 | print_health_info(dev); | |
0a91605c | 275 | schedule_work(&health->work); |
e126ba97 EC |
276 | } |
277 | } | |
278 | ||
279 | void mlx5_start_health_poll(struct mlx5_core_dev *dev) | |
280 | { | |
281 | struct mlx5_core_health *health = &dev->priv.health; | |
282 | ||
e126ba97 EC |
283 | init_timer(&health->timer); |
284 | health->health = &dev->iseg->health; | |
285 | health->health_counter = &dev->iseg->health_counter; | |
286 | ||
287 | health->timer.data = (unsigned long)dev; | |
288 | health->timer.function = poll_health; | |
289 | health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL); | |
290 | add_timer(&health->timer); | |
291 | } | |
292 | ||
293 | void mlx5_stop_health_poll(struct mlx5_core_dev *dev) | |
294 | { | |
295 | struct mlx5_core_health *health = &dev->priv.health; | |
296 | ||
297 | del_timer_sync(&health->timer); | |
e126ba97 EC |
298 | } |
299 | ||
ac6ea6e8 | 300 | void mlx5_health_cleanup(struct mlx5_core_dev *dev) |
e126ba97 | 301 | { |
ac6ea6e8 EC |
302 | struct mlx5_core_health *health = &dev->priv.health; |
303 | ||
0a91605c | 304 | flush_work(&health->work); |
e126ba97 EC |
305 | } |
306 | ||
ac6ea6e8 | 307 | int mlx5_health_init(struct mlx5_core_dev *dev) |
e126ba97 | 308 | { |
ac6ea6e8 EC |
309 | struct mlx5_core_health *health; |
310 | char *name; | |
311 | ||
312 | health = &dev->priv.health; | |
313 | name = kmalloc(64, GFP_KERNEL); | |
314 | if (!name) | |
315 | return -ENOMEM; | |
316 | ||
317 | strcpy(name, "mlx5_health"); | |
318 | strcat(name, dev_name(&dev->pdev->dev)); | |
ac6ea6e8 | 319 | kfree(name); |
ac6ea6e8 EC |
320 | |
321 | INIT_WORK(&health->work, health_care); | |
322 | ||
323 | return 0; | |
e126ba97 | 324 | } |