]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/net/ethernet/mellanox/mlx4/eq.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
net/mlx4: Add EQ pool
[mirror_ubuntu-artful-kernel.git] / drivers / net / ethernet / mellanox / mlx4 / eq.c
1 /*
2 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <linux/interrupt.h>
35 #include <linux/slab.h>
36 #include <linux/export.h>
37 #include <linux/mm.h>
38 #include <linux/dma-mapping.h>
39
40 #include <linux/mlx4/cmd.h>
41 #include <linux/cpu_rmap.h>
42
43 #include "mlx4.h"
44 #include "fw.h"
45
46 enum {
47 MLX4_IRQNAME_SIZE = 32
48 };
49
50 enum {
51 MLX4_NUM_ASYNC_EQE = 0x100,
52 MLX4_NUM_SPARE_EQE = 0x80,
53 MLX4_EQ_ENTRY_SIZE = 0x20
54 };
55
56 #define MLX4_EQ_STATUS_OK ( 0 << 28)
57 #define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28)
58 #define MLX4_EQ_OWNER_SW ( 0 << 24)
59 #define MLX4_EQ_OWNER_HW ( 1 << 24)
60 #define MLX4_EQ_FLAG_EC ( 1 << 18)
61 #define MLX4_EQ_FLAG_OI ( 1 << 17)
62 #define MLX4_EQ_STATE_ARMED ( 9 << 8)
63 #define MLX4_EQ_STATE_FIRED (10 << 8)
64 #define MLX4_EQ_STATE_ALWAYS_ARMED (11 << 8)
65
66 #define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG) | \
67 (1ull << MLX4_EVENT_TYPE_COMM_EST) | \
68 (1ull << MLX4_EVENT_TYPE_SQ_DRAINED) | \
69 (1ull << MLX4_EVENT_TYPE_CQ_ERROR) | \
70 (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR) | \
71 (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR) | \
72 (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED) | \
73 (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
74 (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR) | \
75 (1ull << MLX4_EVENT_TYPE_PORT_CHANGE) | \
76 (1ull << MLX4_EVENT_TYPE_ECC_DETECT) | \
77 (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR) | \
78 (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE) | \
79 (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT) | \
80 (1ull << MLX4_EVENT_TYPE_CMD) | \
81 (1ull << MLX4_EVENT_TYPE_OP_REQUIRED) | \
82 (1ull << MLX4_EVENT_TYPE_COMM_CHANNEL) | \
83 (1ull << MLX4_EVENT_TYPE_FLR_EVENT) | \
84 (1ull << MLX4_EVENT_TYPE_FATAL_WARNING))
85
86 static u64 get_async_ev_mask(struct mlx4_dev *dev)
87 {
88 u64 async_ev_mask = MLX4_ASYNC_EVENT_MASK;
89 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)
90 async_ev_mask |= (1ull << MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT);
91 if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT)
92 async_ev_mask |= (1ull << MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT);
93
94 return async_ev_mask;
95 }
96
97 static void eq_set_ci(struct mlx4_eq *eq, int req_not)
98 {
99 __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) |
100 req_not << 31),
101 eq->doorbell);
102 /* We still want ordering, just not swabbing, so add a barrier */
103 mb();
104 }
105
106 static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry, u8 eqe_factor,
107 u8 eqe_size)
108 {
109 /* (entry & (eq->nent - 1)) gives us a cyclic array */
110 unsigned long offset = (entry & (eq->nent - 1)) * eqe_size;
111 /* CX3 is capable of extending the EQE from 32 to 64 bytes with
112 * strides of 64B,128B and 256B.
113 * When 64B EQE is used, the first (in the lower addresses)
114 * 32 bytes in the 64 byte EQE are reserved and the next 32 bytes
115 * contain the legacy EQE information.
116 * In all other cases, the first 32B contains the legacy EQE info.
117 */
118 return eq->page_list[offset / PAGE_SIZE].buf + (offset + (eqe_factor ? MLX4_EQ_ENTRY_SIZE : 0)) % PAGE_SIZE;
119 }
120
121 static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq, u8 eqe_factor, u8 size)
122 {
123 struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index, eqe_factor, size);
124 return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe;
125 }
126
127 static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_eq)
128 {
129 struct mlx4_eqe *eqe =
130 &slave_eq->event_eqe[slave_eq->cons & (SLAVE_EVENT_EQ_SIZE - 1)];
131 return (!!(eqe->owner & 0x80) ^
132 !!(slave_eq->cons & SLAVE_EVENT_EQ_SIZE)) ?
133 eqe : NULL;
134 }
135
136 void mlx4_gen_slave_eqe(struct work_struct *work)
137 {
138 struct mlx4_mfunc_master_ctx *master =
139 container_of(work, struct mlx4_mfunc_master_ctx,
140 slave_event_work);
141 struct mlx4_mfunc *mfunc =
142 container_of(master, struct mlx4_mfunc, master);
143 struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc);
144 struct mlx4_dev *dev = &priv->dev;
145 struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq;
146 struct mlx4_eqe *eqe;
147 u8 slave;
148 int i, phys_port, slave_port;
149
150 for (eqe = next_slave_event_eqe(slave_eq); eqe;
151 eqe = next_slave_event_eqe(slave_eq)) {
152 slave = eqe->slave_id;
153
154 /* All active slaves need to receive the event */
155 if (slave == ALL_SLAVES) {
156 for (i = 0; i <= dev->persist->num_vfs; i++) {
157 phys_port = 0;
158 if (eqe->type == MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT &&
159 eqe->subtype == MLX4_DEV_PMC_SUBTYPE_PORT_INFO) {
160 phys_port = eqe->event.port_mgmt_change.port;
161 slave_port = mlx4_phys_to_slave_port(dev, i, phys_port);
162 if (slave_port < 0) /* VF doesn't have this port */
163 continue;
164 eqe->event.port_mgmt_change.port = slave_port;
165 }
166 if (mlx4_GEN_EQE(dev, i, eqe))
167 mlx4_warn(dev, "Failed to generate event for slave %d\n",
168 i);
169 if (phys_port)
170 eqe->event.port_mgmt_change.port = phys_port;
171 }
172 } else {
173 if (mlx4_GEN_EQE(dev, slave, eqe))
174 mlx4_warn(dev, "Failed to generate event for slave %d\n",
175 slave);
176 }
177 ++slave_eq->cons;
178 }
179 }
180
181
182 static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe)
183 {
184 struct mlx4_priv *priv = mlx4_priv(dev);
185 struct mlx4_slave_event_eq *slave_eq = &priv->mfunc.master.slave_eq;
186 struct mlx4_eqe *s_eqe;
187 unsigned long flags;
188
189 spin_lock_irqsave(&slave_eq->event_lock, flags);
190 s_eqe = &slave_eq->event_eqe[slave_eq->prod & (SLAVE_EVENT_EQ_SIZE - 1)];
191 if ((!!(s_eqe->owner & 0x80)) ^
192 (!!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE))) {
193 mlx4_warn(dev, "Master failed to generate an EQE for slave: %d. No free EQE on slave events queue\n",
194 slave);
195 spin_unlock_irqrestore(&slave_eq->event_lock, flags);
196 return;
197 }
198
199 memcpy(s_eqe, eqe, dev->caps.eqe_size - 1);
200 s_eqe->slave_id = slave;
201 /* ensure all information is written before setting the ownersip bit */
202 dma_wmb();
203 s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80;
204 ++slave_eq->prod;
205
206 queue_work(priv->mfunc.master.comm_wq,
207 &priv->mfunc.master.slave_event_work);
208 spin_unlock_irqrestore(&slave_eq->event_lock, flags);
209 }
210
211 static void mlx4_slave_event(struct mlx4_dev *dev, int slave,
212 struct mlx4_eqe *eqe)
213 {
214 struct mlx4_priv *priv = mlx4_priv(dev);
215
216 if (slave < 0 || slave > dev->persist->num_vfs ||
217 slave == dev->caps.function ||
218 !priv->mfunc.master.slave_state[slave].active)
219 return;
220
221 slave_event(dev, slave, eqe);
222 }
223
224 int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port)
225 {
226 struct mlx4_eqe eqe;
227
228 struct mlx4_priv *priv = mlx4_priv(dev);
229 struct mlx4_slave_state *s_slave = &priv->mfunc.master.slave_state[slave];
230
231 if (!s_slave->active)
232 return 0;
233
234 memset(&eqe, 0, sizeof eqe);
235
236 eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
237 eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE;
238 eqe.event.port_mgmt_change.port = mlx4_phys_to_slave_port(dev, slave, port);
239
240 return mlx4_GEN_EQE(dev, slave, &eqe);
241 }
242 EXPORT_SYMBOL(mlx4_gen_pkey_eqe);
243
244 int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port)
245 {
246 struct mlx4_eqe eqe;
247
248 /*don't send if we don't have the that slave */
249 if (dev->persist->num_vfs < slave)
250 return 0;
251 memset(&eqe, 0, sizeof eqe);
252
253 eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
254 eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO;
255 eqe.event.port_mgmt_change.port = mlx4_phys_to_slave_port(dev, slave, port);
256
257 return mlx4_GEN_EQE(dev, slave, &eqe);
258 }
259 EXPORT_SYMBOL(mlx4_gen_guid_change_eqe);
260
261 int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port,
262 u8 port_subtype_change)
263 {
264 struct mlx4_eqe eqe;
265 u8 slave_port = mlx4_phys_to_slave_port(dev, slave, port);
266
267 /*don't send if we don't have the that slave */
268 if (dev->persist->num_vfs < slave)
269 return 0;
270 memset(&eqe, 0, sizeof eqe);
271
272 eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE;
273 eqe.subtype = port_subtype_change;
274 eqe.event.port_change.port = cpu_to_be32(slave_port << 28);
275
276 mlx4_dbg(dev, "%s: sending: %d to slave: %d on port: %d\n", __func__,
277 port_subtype_change, slave, port);
278 return mlx4_GEN_EQE(dev, slave, &eqe);
279 }
280 EXPORT_SYMBOL(mlx4_gen_port_state_change_eqe);
281
282 enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port)
283 {
284 struct mlx4_priv *priv = mlx4_priv(dev);
285 struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state;
286 struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave);
287
288 if (slave >= dev->num_slaves || port > dev->caps.num_ports ||
289 port <= 0 || !test_bit(port - 1, actv_ports.ports)) {
290 pr_err("%s: Error: asking for slave:%d, port:%d\n",
291 __func__, slave, port);
292 return SLAVE_PORT_DOWN;
293 }
294 return s_state[slave].port_state[port];
295 }
296 EXPORT_SYMBOL(mlx4_get_slave_port_state);
297
298 static int mlx4_set_slave_port_state(struct mlx4_dev *dev, int slave, u8 port,
299 enum slave_port_state state)
300 {
301 struct mlx4_priv *priv = mlx4_priv(dev);
302 struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state;
303 struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave);
304
305 if (slave >= dev->num_slaves || port > dev->caps.num_ports ||
306 port <= 0 || !test_bit(port - 1, actv_ports.ports)) {
307 pr_err("%s: Error: asking for slave:%d, port:%d\n",
308 __func__, slave, port);
309 return -1;
310 }
311 s_state[slave].port_state[port] = state;
312
313 return 0;
314 }
315
316 static void set_all_slave_state(struct mlx4_dev *dev, u8 port, int event)
317 {
318 int i;
319 enum slave_port_gen_event gen_event;
320 struct mlx4_slaves_pport slaves_pport = mlx4_phys_to_slaves_pport(dev,
321 port);
322
323 for (i = 0; i < dev->persist->num_vfs + 1; i++)
324 if (test_bit(i, slaves_pport.slaves))
325 set_and_calc_slave_port_state(dev, i, port,
326 event, &gen_event);
327 }
328 /**************************************************************************
329 The function get as input the new event to that port,
330 and according to the prev state change the slave's port state.
331 The events are:
332 MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
333 MLX4_PORT_STATE_DEV_EVENT_PORT_UP
334 MLX4_PORT_STATE_IB_EVENT_GID_VALID
335 MLX4_PORT_STATE_IB_EVENT_GID_INVALID
336 ***************************************************************************/
337 int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave,
338 u8 port, int event,
339 enum slave_port_gen_event *gen_event)
340 {
341 struct mlx4_priv *priv = mlx4_priv(dev);
342 struct mlx4_slave_state *ctx = NULL;
343 unsigned long flags;
344 int ret = -1;
345 struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave);
346 enum slave_port_state cur_state =
347 mlx4_get_slave_port_state(dev, slave, port);
348
349 *gen_event = SLAVE_PORT_GEN_EVENT_NONE;
350
351 if (slave >= dev->num_slaves || port > dev->caps.num_ports ||
352 port <= 0 || !test_bit(port - 1, actv_ports.ports)) {
353 pr_err("%s: Error: asking for slave:%d, port:%d\n",
354 __func__, slave, port);
355 return ret;
356 }
357
358 ctx = &priv->mfunc.master.slave_state[slave];
359 spin_lock_irqsave(&ctx->lock, flags);
360
361 switch (cur_state) {
362 case SLAVE_PORT_DOWN:
363 if (MLX4_PORT_STATE_DEV_EVENT_PORT_UP == event)
364 mlx4_set_slave_port_state(dev, slave, port,
365 SLAVE_PENDING_UP);
366 break;
367 case SLAVE_PENDING_UP:
368 if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event)
369 mlx4_set_slave_port_state(dev, slave, port,
370 SLAVE_PORT_DOWN);
371 else if (MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID == event) {
372 mlx4_set_slave_port_state(dev, slave, port,
373 SLAVE_PORT_UP);
374 *gen_event = SLAVE_PORT_GEN_EVENT_UP;
375 }
376 break;
377 case SLAVE_PORT_UP:
378 if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) {
379 mlx4_set_slave_port_state(dev, slave, port,
380 SLAVE_PORT_DOWN);
381 *gen_event = SLAVE_PORT_GEN_EVENT_DOWN;
382 } else if (MLX4_PORT_STATE_IB_EVENT_GID_INVALID ==
383 event) {
384 mlx4_set_slave_port_state(dev, slave, port,
385 SLAVE_PENDING_UP);
386 *gen_event = SLAVE_PORT_GEN_EVENT_DOWN;
387 }
388 break;
389 default:
390 pr_err("%s: BUG!!! UNKNOWN state: slave:%d, port:%d\n",
391 __func__, slave, port);
392 goto out;
393 }
394 ret = mlx4_get_slave_port_state(dev, slave, port);
395
396 out:
397 spin_unlock_irqrestore(&ctx->lock, flags);
398 return ret;
399 }
400
401 EXPORT_SYMBOL(set_and_calc_slave_port_state);
402
403 int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr)
404 {
405 struct mlx4_eqe eqe;
406
407 memset(&eqe, 0, sizeof eqe);
408
409 eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
410 eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PORT_INFO;
411 eqe.event.port_mgmt_change.port = port;
412 eqe.event.port_mgmt_change.params.port_info.changed_attr =
413 cpu_to_be32((u32) attr);
414
415 slave_event(dev, ALL_SLAVES, &eqe);
416 return 0;
417 }
418 EXPORT_SYMBOL(mlx4_gen_slaves_port_mgt_ev);
419
420 void mlx4_master_handle_slave_flr(struct work_struct *work)
421 {
422 struct mlx4_mfunc_master_ctx *master =
423 container_of(work, struct mlx4_mfunc_master_ctx,
424 slave_flr_event_work);
425 struct mlx4_mfunc *mfunc =
426 container_of(master, struct mlx4_mfunc, master);
427 struct mlx4_priv *priv =
428 container_of(mfunc, struct mlx4_priv, mfunc);
429 struct mlx4_dev *dev = &priv->dev;
430 struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state;
431 int i;
432 int err;
433 unsigned long flags;
434
435 mlx4_dbg(dev, "mlx4_handle_slave_flr\n");
436
437 for (i = 0 ; i < dev->num_slaves; i++) {
438
439 if (MLX4_COMM_CMD_FLR == slave_state[i].last_cmd) {
440 mlx4_dbg(dev, "mlx4_handle_slave_flr: clean slave: %d\n",
441 i);
442 /* In case of 'Reset flow' FLR can be generated for
443 * a slave before mlx4_load_one is done.
444 * make sure interface is up before trying to delete
445 * slave resources which weren't allocated yet.
446 */
447 if (dev->persist->interface_state &
448 MLX4_INTERFACE_STATE_UP)
449 mlx4_delete_all_resources_for_slave(dev, i);
450 /*return the slave to running mode*/
451 spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags);
452 slave_state[i].last_cmd = MLX4_COMM_CMD_RESET;
453 slave_state[i].is_slave_going_down = 0;
454 spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags);
455 /*notify the FW:*/
456 err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE,
457 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
458 if (err)
459 mlx4_warn(dev, "Failed to notify FW on FLR done (slave:%d)\n",
460 i);
461 }
462 }
463 }
464
465 static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
466 {
467 struct mlx4_priv *priv = mlx4_priv(dev);
468 struct mlx4_eqe *eqe;
469 int cqn = -1;
470 int eqes_found = 0;
471 int set_ci = 0;
472 int port;
473 int slave = 0;
474 int ret;
475 u32 flr_slave;
476 u8 update_slave_state;
477 int i;
478 enum slave_port_gen_event gen_event;
479 unsigned long flags;
480 struct mlx4_vport_state *s_info;
481 int eqe_size = dev->caps.eqe_size;
482
483 while ((eqe = next_eqe_sw(eq, dev->caps.eqe_factor, eqe_size))) {
484 /*
485 * Make sure we read EQ entry contents after we've
486 * checked the ownership bit.
487 */
488 dma_rmb();
489
490 switch (eqe->type) {
491 case MLX4_EVENT_TYPE_COMP:
492 cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff;
493 mlx4_cq_completion(dev, cqn);
494 break;
495
496 case MLX4_EVENT_TYPE_PATH_MIG:
497 case MLX4_EVENT_TYPE_COMM_EST:
498 case MLX4_EVENT_TYPE_SQ_DRAINED:
499 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
500 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
501 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
502 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
503 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
504 mlx4_dbg(dev, "event %d arrived\n", eqe->type);
505 if (mlx4_is_master(dev)) {
506 /* forward only to slave owning the QP */
507 ret = mlx4_get_slave_from_resource_id(dev,
508 RES_QP,
509 be32_to_cpu(eqe->event.qp.qpn)
510 & 0xffffff, &slave);
511 if (ret && ret != -ENOENT) {
512 mlx4_dbg(dev, "QP event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n",
513 eqe->type, eqe->subtype,
514 eq->eqn, eq->cons_index, ret);
515 break;
516 }
517
518 if (!ret && slave != dev->caps.function) {
519 mlx4_slave_event(dev, slave, eqe);
520 break;
521 }
522
523 }
524 mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) &
525 0xffffff, eqe->type);
526 break;
527
528 case MLX4_EVENT_TYPE_SRQ_LIMIT:
529 mlx4_dbg(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT\n",
530 __func__);
531 case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:
532 if (mlx4_is_master(dev)) {
533 /* forward only to slave owning the SRQ */
534 ret = mlx4_get_slave_from_resource_id(dev,
535 RES_SRQ,
536 be32_to_cpu(eqe->event.srq.srqn)
537 & 0xffffff,
538 &slave);
539 if (ret && ret != -ENOENT) {
540 mlx4_warn(dev, "SRQ event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n",
541 eqe->type, eqe->subtype,
542 eq->eqn, eq->cons_index, ret);
543 break;
544 }
545 mlx4_warn(dev, "%s: slave:%d, srq_no:0x%x, event: %02x(%02x)\n",
546 __func__, slave,
547 be32_to_cpu(eqe->event.srq.srqn),
548 eqe->type, eqe->subtype);
549
550 if (!ret && slave != dev->caps.function) {
551 mlx4_warn(dev, "%s: sending event %02x(%02x) to slave:%d\n",
552 __func__, eqe->type,
553 eqe->subtype, slave);
554 mlx4_slave_event(dev, slave, eqe);
555 break;
556 }
557 }
558 mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) &
559 0xffffff, eqe->type);
560 break;
561
562 case MLX4_EVENT_TYPE_CMD:
563 mlx4_cmd_event(dev,
564 be16_to_cpu(eqe->event.cmd.token),
565 eqe->event.cmd.status,
566 be64_to_cpu(eqe->event.cmd.out_param));
567 break;
568
569 case MLX4_EVENT_TYPE_PORT_CHANGE: {
570 struct mlx4_slaves_pport slaves_port;
571 port = be32_to_cpu(eqe->event.port_change.port) >> 28;
572 slaves_port = mlx4_phys_to_slaves_pport(dev, port);
573 if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
574 mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN,
575 port);
576 mlx4_priv(dev)->sense.do_sense_port[port] = 1;
577 if (!mlx4_is_master(dev))
578 break;
579 for (i = 0; i < dev->persist->num_vfs + 1;
580 i++) {
581 if (!test_bit(i, slaves_port.slaves))
582 continue;
583 if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) {
584 if (i == mlx4_master_func_num(dev))
585 continue;
586 mlx4_dbg(dev, "%s: Sending MLX4_PORT_CHANGE_SUBTYPE_DOWN to slave: %d, port:%d\n",
587 __func__, i, port);
588 s_info = &priv->mfunc.master.vf_oper[slave].vport[port].state;
589 if (IFLA_VF_LINK_STATE_AUTO == s_info->link_state) {
590 eqe->event.port_change.port =
591 cpu_to_be32(
592 (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF)
593 | (mlx4_phys_to_slave_port(dev, i, port) << 28));
594 mlx4_slave_event(dev, i, eqe);
595 }
596 } else { /* IB port */
597 set_and_calc_slave_port_state(dev, i, port,
598 MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
599 &gen_event);
600 /*we can be in pending state, then do not send port_down event*/
601 if (SLAVE_PORT_GEN_EVENT_DOWN == gen_event) {
602 if (i == mlx4_master_func_num(dev))
603 continue;
604 eqe->event.port_change.port =
605 cpu_to_be32(
606 (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF)
607 | (mlx4_phys_to_slave_port(dev, i, port) << 28));
608 mlx4_slave_event(dev, i, eqe);
609 }
610 }
611 }
612 } else {
613 mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP, port);
614
615 mlx4_priv(dev)->sense.do_sense_port[port] = 0;
616
617 if (!mlx4_is_master(dev))
618 break;
619 if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH)
620 for (i = 0;
621 i < dev->persist->num_vfs + 1;
622 i++) {
623 if (!test_bit(i, slaves_port.slaves))
624 continue;
625 if (i == mlx4_master_func_num(dev))
626 continue;
627 s_info = &priv->mfunc.master.vf_oper[slave].vport[port].state;
628 if (IFLA_VF_LINK_STATE_AUTO == s_info->link_state) {
629 eqe->event.port_change.port =
630 cpu_to_be32(
631 (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF)
632 | (mlx4_phys_to_slave_port(dev, i, port) << 28));
633 mlx4_slave_event(dev, i, eqe);
634 }
635 }
636 else /* IB port */
637 /* port-up event will be sent to a slave when the
638 * slave's alias-guid is set. This is done in alias_GUID.c
639 */
640 set_all_slave_state(dev, port, MLX4_DEV_EVENT_PORT_UP);
641 }
642 break;
643 }
644
645 case MLX4_EVENT_TYPE_CQ_ERROR:
646 mlx4_warn(dev, "CQ %s on CQN %06x\n",
647 eqe->event.cq_err.syndrome == 1 ?
648 "overrun" : "access violation",
649 be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff);
650 if (mlx4_is_master(dev)) {
651 ret = mlx4_get_slave_from_resource_id(dev,
652 RES_CQ,
653 be32_to_cpu(eqe->event.cq_err.cqn)
654 & 0xffffff, &slave);
655 if (ret && ret != -ENOENT) {
656 mlx4_dbg(dev, "CQ event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n",
657 eqe->type, eqe->subtype,
658 eq->eqn, eq->cons_index, ret);
659 break;
660 }
661
662 if (!ret && slave != dev->caps.function) {
663 mlx4_slave_event(dev, slave, eqe);
664 break;
665 }
666 }
667 mlx4_cq_event(dev,
668 be32_to_cpu(eqe->event.cq_err.cqn)
669 & 0xffffff,
670 eqe->type);
671 break;
672
673 case MLX4_EVENT_TYPE_EQ_OVERFLOW:
674 mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn);
675 break;
676
677 case MLX4_EVENT_TYPE_OP_REQUIRED:
678 atomic_inc(&priv->opreq_count);
679 /* FW commands can't be executed from interrupt context
680 * working in deferred task
681 */
682 queue_work(mlx4_wq, &priv->opreq_task);
683 break;
684
685 case MLX4_EVENT_TYPE_COMM_CHANNEL:
686 if (!mlx4_is_master(dev)) {
687 mlx4_warn(dev, "Received comm channel event for non master device\n");
688 break;
689 }
690 memcpy(&priv->mfunc.master.comm_arm_bit_vector,
691 eqe->event.comm_channel_arm.bit_vec,
692 sizeof eqe->event.comm_channel_arm.bit_vec);
693 queue_work(priv->mfunc.master.comm_wq,
694 &priv->mfunc.master.comm_work);
695 break;
696
697 case MLX4_EVENT_TYPE_FLR_EVENT:
698 flr_slave = be32_to_cpu(eqe->event.flr_event.slave_id);
699 if (!mlx4_is_master(dev)) {
700 mlx4_warn(dev, "Non-master function received FLR event\n");
701 break;
702 }
703
704 mlx4_dbg(dev, "FLR event for slave: %d\n", flr_slave);
705
706 if (flr_slave >= dev->num_slaves) {
707 mlx4_warn(dev,
708 "Got FLR for unknown function: %d\n",
709 flr_slave);
710 update_slave_state = 0;
711 } else
712 update_slave_state = 1;
713
714 spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags);
715 if (update_slave_state) {
716 priv->mfunc.master.slave_state[flr_slave].active = false;
717 priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR;
718 priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1;
719 }
720 spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags);
721 mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_SHUTDOWN,
722 flr_slave);
723 queue_work(priv->mfunc.master.comm_wq,
724 &priv->mfunc.master.slave_flr_event_work);
725 break;
726
727 case MLX4_EVENT_TYPE_FATAL_WARNING:
728 if (eqe->subtype == MLX4_FATAL_WARNING_SUBTYPE_WARMING) {
729 if (mlx4_is_master(dev))
730 for (i = 0; i < dev->num_slaves; i++) {
731 mlx4_dbg(dev, "%s: Sending MLX4_FATAL_WARNING_SUBTYPE_WARMING to slave: %d\n",
732 __func__, i);
733 if (i == dev->caps.function)
734 continue;
735 mlx4_slave_event(dev, i, eqe);
736 }
737 mlx4_err(dev, "Temperature Threshold was reached! Threshold: %d celsius degrees; Current Temperature: %d\n",
738 be16_to_cpu(eqe->event.warming.warning_threshold),
739 be16_to_cpu(eqe->event.warming.current_temperature));
740 } else
741 mlx4_warn(dev, "Unhandled event FATAL WARNING (%02x), subtype %02x on EQ %d at index %u. owner=%x, nent=0x%x, slave=%x, ownership=%s\n",
742 eqe->type, eqe->subtype, eq->eqn,
743 eq->cons_index, eqe->owner, eq->nent,
744 eqe->slave_id,
745 !!(eqe->owner & 0x80) ^
746 !!(eq->cons_index & eq->nent) ? "HW" : "SW");
747
748 break;
749
750 case MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT:
751 mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_MGMT_CHANGE,
752 (unsigned long) eqe);
753 break;
754
755 case MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT:
756 switch (eqe->subtype) {
757 case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_BAD_CABLE:
758 mlx4_warn(dev, "Bad cable detected on port %u\n",
759 eqe->event.bad_cable.port);
760 break;
761 case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_UNSUPPORTED_CABLE:
762 mlx4_warn(dev, "Unsupported cable detected\n");
763 break;
764 default:
765 mlx4_dbg(dev,
766 "Unhandled recoverable error event detected: %02x(%02x) on EQ %d at index %u. owner=%x, nent=0x%x, ownership=%s\n",
767 eqe->type, eqe->subtype, eq->eqn,
768 eq->cons_index, eqe->owner, eq->nent,
769 !!(eqe->owner & 0x80) ^
770 !!(eq->cons_index & eq->nent) ? "HW" : "SW");
771 break;
772 }
773 break;
774
775 case MLX4_EVENT_TYPE_EEC_CATAS_ERROR:
776 case MLX4_EVENT_TYPE_ECC_DETECT:
777 default:
778 mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at index %u. owner=%x, nent=0x%x, slave=%x, ownership=%s\n",
779 eqe->type, eqe->subtype, eq->eqn,
780 eq->cons_index, eqe->owner, eq->nent,
781 eqe->slave_id,
782 !!(eqe->owner & 0x80) ^
783 !!(eq->cons_index & eq->nent) ? "HW" : "SW");
784 break;
785 };
786
787 ++eq->cons_index;
788 eqes_found = 1;
789 ++set_ci;
790
791 /*
792 * The HCA will think the queue has overflowed if we
793 * don't tell it we've been processing events. We
794 * create our EQs with MLX4_NUM_SPARE_EQE extra
795 * entries, so we must update our consumer index at
796 * least that often.
797 */
798 if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) {
799 eq_set_ci(eq, 0);
800 set_ci = 0;
801 }
802 }
803
804 eq_set_ci(eq, 1);
805
806 /* cqn is 24bit wide but is initialized such that its higher bits
807 * are ones too. Thus, if we got any event, cqn's high bits should be off
808 * and we need to schedule the tasklet.
809 */
810 if (!(cqn & ~0xffffff))
811 tasklet_schedule(&eq->tasklet_ctx.task);
812
813 return eqes_found;
814 }
815
816 static irqreturn_t mlx4_interrupt(int irq, void *dev_ptr)
817 {
818 struct mlx4_dev *dev = dev_ptr;
819 struct mlx4_priv *priv = mlx4_priv(dev);
820 int work = 0;
821 int i;
822
823 writel(priv->eq_table.clr_mask, priv->eq_table.clr_int);
824
825 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
826 work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]);
827
828 return IRQ_RETVAL(work);
829 }
830
831 static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr)
832 {
833 struct mlx4_eq *eq = eq_ptr;
834 struct mlx4_dev *dev = eq->dev;
835
836 mlx4_eq_int(dev, eq);
837
838 /* MSI-X vectors always belong to us */
839 return IRQ_HANDLED;
840 }
841
842 int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave,
843 struct mlx4_vhcr *vhcr,
844 struct mlx4_cmd_mailbox *inbox,
845 struct mlx4_cmd_mailbox *outbox,
846 struct mlx4_cmd_info *cmd)
847 {
848 struct mlx4_priv *priv = mlx4_priv(dev);
849 struct mlx4_slave_event_eq_info *event_eq =
850 priv->mfunc.master.slave_state[slave].event_eq;
851 u32 in_modifier = vhcr->in_modifier;
852 u32 eqn = in_modifier & 0x3FF;
853 u64 in_param = vhcr->in_param;
854 int err = 0;
855 int i;
856
857 if (slave == dev->caps.function)
858 err = mlx4_cmd(dev, in_param, (in_modifier & 0x80000000) | eqn,
859 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B,
860 MLX4_CMD_NATIVE);
861 if (!err)
862 for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i)
863 if (in_param & (1LL << i))
864 event_eq[i].eqn = in_modifier >> 31 ? -1 : eqn;
865
866 return err;
867 }
868
869 static int mlx4_MAP_EQ(struct mlx4_dev *dev, u64 event_mask, int unmap,
870 int eq_num)
871 {
872 return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num,
873 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B,
874 MLX4_CMD_WRAPPED);
875 }
876
877 static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
878 int eq_num)
879 {
880 return mlx4_cmd(dev, mailbox->dma, eq_num, 0,
881 MLX4_CMD_SW2HW_EQ, MLX4_CMD_TIME_CLASS_A,
882 MLX4_CMD_WRAPPED);
883 }
884
885 static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, int eq_num)
886 {
887 return mlx4_cmd(dev, 0, eq_num, 1, MLX4_CMD_HW2SW_EQ,
888 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
889 }
890
891 static int mlx4_num_eq_uar(struct mlx4_dev *dev)
892 {
893 /*
894 * Each UAR holds 4 EQ doorbells. To figure out how many UARs
895 * we need to map, take the difference of highest index and
896 * the lowest index we'll use and add 1.
897 */
898 return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs) / 4 -
899 dev->caps.reserved_eqs / 4 + 1;
900 }
901
902 static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq)
903 {
904 struct mlx4_priv *priv = mlx4_priv(dev);
905 int index;
906
907 index = eq->eqn / 4 - dev->caps.reserved_eqs / 4;
908
909 if (!priv->eq_table.uar_map[index]) {
910 priv->eq_table.uar_map[index] =
911 ioremap(pci_resource_start(dev->persist->pdev, 2) +
912 ((eq->eqn / 4) << PAGE_SHIFT),
913 PAGE_SIZE);
914 if (!priv->eq_table.uar_map[index]) {
915 mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n",
916 eq->eqn);
917 return NULL;
918 }
919 }
920
921 return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4);
922 }
923
924 static void mlx4_unmap_uar(struct mlx4_dev *dev)
925 {
926 struct mlx4_priv *priv = mlx4_priv(dev);
927 int i;
928
929 for (i = 0; i < mlx4_num_eq_uar(dev); ++i)
930 if (priv->eq_table.uar_map[i]) {
931 iounmap(priv->eq_table.uar_map[i]);
932 priv->eq_table.uar_map[i] = NULL;
933 }
934 }
935
936 static int mlx4_create_eq(struct mlx4_dev *dev, int nent,
937 u8 intr, struct mlx4_eq *eq)
938 {
939 struct mlx4_priv *priv = mlx4_priv(dev);
940 struct mlx4_cmd_mailbox *mailbox;
941 struct mlx4_eq_context *eq_context;
942 int npages;
943 u64 *dma_list = NULL;
944 dma_addr_t t;
945 u64 mtt_addr;
946 int err = -ENOMEM;
947 int i;
948
949 eq->dev = dev;
950 eq->nent = roundup_pow_of_two(max(nent, 2));
951 /* CX3 is capable of extending the CQE/EQE from 32 to 64 bytes, with
952 * strides of 64B,128B and 256B.
953 */
954 npages = PAGE_ALIGN(eq->nent * dev->caps.eqe_size) / PAGE_SIZE;
955
956 eq->page_list = kmalloc(npages * sizeof *eq->page_list,
957 GFP_KERNEL);
958 if (!eq->page_list)
959 goto err_out;
960
961 for (i = 0; i < npages; ++i)
962 eq->page_list[i].buf = NULL;
963
964 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
965 if (!dma_list)
966 goto err_out_free;
967
968 mailbox = mlx4_alloc_cmd_mailbox(dev);
969 if (IS_ERR(mailbox))
970 goto err_out_free;
971 eq_context = mailbox->buf;
972
973 for (i = 0; i < npages; ++i) {
974 eq->page_list[i].buf = dma_alloc_coherent(&dev->persist->
975 pdev->dev,
976 PAGE_SIZE, &t,
977 GFP_KERNEL);
978 if (!eq->page_list[i].buf)
979 goto err_out_free_pages;
980
981 dma_list[i] = t;
982 eq->page_list[i].map = t;
983
984 memset(eq->page_list[i].buf, 0, PAGE_SIZE);
985 }
986
987 eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap);
988 if (eq->eqn == -1)
989 goto err_out_free_pages;
990
991 eq->doorbell = mlx4_get_eq_uar(dev, eq);
992 if (!eq->doorbell) {
993 err = -ENOMEM;
994 goto err_out_free_eq;
995 }
996
997 err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt);
998 if (err)
999 goto err_out_free_eq;
1000
1001 err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list);
1002 if (err)
1003 goto err_out_free_mtt;
1004
1005 eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK |
1006 MLX4_EQ_STATE_ARMED);
1007 eq_context->log_eq_size = ilog2(eq->nent);
1008 eq_context->intr = intr;
1009 eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT;
1010
1011 mtt_addr = mlx4_mtt_addr(dev, &eq->mtt);
1012 eq_context->mtt_base_addr_h = mtt_addr >> 32;
1013 eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
1014
1015 err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn);
1016 if (err) {
1017 mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err);
1018 goto err_out_free_mtt;
1019 }
1020
1021 kfree(dma_list);
1022 mlx4_free_cmd_mailbox(dev, mailbox);
1023
1024 eq->cons_index = 0;
1025
1026 INIT_LIST_HEAD(&eq->tasklet_ctx.list);
1027 INIT_LIST_HEAD(&eq->tasklet_ctx.process_list);
1028 spin_lock_init(&eq->tasklet_ctx.lock);
1029 tasklet_init(&eq->tasklet_ctx.task, mlx4_cq_tasklet_cb,
1030 (unsigned long)&eq->tasklet_ctx);
1031
1032 return err;
1033
1034 err_out_free_mtt:
1035 mlx4_mtt_cleanup(dev, &eq->mtt);
1036
1037 err_out_free_eq:
1038 mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR);
1039
1040 err_out_free_pages:
1041 for (i = 0; i < npages; ++i)
1042 if (eq->page_list[i].buf)
1043 dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE,
1044 eq->page_list[i].buf,
1045 eq->page_list[i].map);
1046
1047 mlx4_free_cmd_mailbox(dev, mailbox);
1048
1049 err_out_free:
1050 kfree(eq->page_list);
1051 kfree(dma_list);
1052
1053 err_out:
1054 return err;
1055 }
1056
1057 static void mlx4_free_eq(struct mlx4_dev *dev,
1058 struct mlx4_eq *eq)
1059 {
1060 struct mlx4_priv *priv = mlx4_priv(dev);
1061 int err;
1062 int i;
1063 /* CX3 is capable of extending the CQE/EQE from 32 to 64 bytes, with
1064 * strides of 64B,128B and 256B
1065 */
1066 int npages = PAGE_ALIGN(dev->caps.eqe_size * eq->nent) / PAGE_SIZE;
1067
1068 err = mlx4_HW2SW_EQ(dev, eq->eqn);
1069 if (err)
1070 mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err);
1071
1072 synchronize_irq(eq->irq);
1073 tasklet_disable(&eq->tasklet_ctx.task);
1074
1075 mlx4_mtt_cleanup(dev, &eq->mtt);
1076 for (i = 0; i < npages; ++i)
1077 dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE,
1078 eq->page_list[i].buf,
1079 eq->page_list[i].map);
1080
1081 kfree(eq->page_list);
1082 mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR);
1083 }
1084
1085 static void mlx4_free_irqs(struct mlx4_dev *dev)
1086 {
1087 struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table;
1088 int i;
1089
1090 if (eq_table->have_irq)
1091 free_irq(dev->persist->pdev->irq, dev);
1092
1093 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
1094 if (eq_table->eq[i].have_irq) {
1095 free_irq(eq_table->eq[i].irq, eq_table->eq + i);
1096 eq_table->eq[i].have_irq = 0;
1097 }
1098
1099 kfree(eq_table->irq_names);
1100 }
1101
1102 static int mlx4_map_clr_int(struct mlx4_dev *dev)
1103 {
1104 struct mlx4_priv *priv = mlx4_priv(dev);
1105
1106 priv->clr_base = ioremap(pci_resource_start(dev->persist->pdev,
1107 priv->fw.clr_int_bar) +
1108 priv->fw.clr_int_base, MLX4_CLR_INT_SIZE);
1109 if (!priv->clr_base) {
1110 mlx4_err(dev, "Couldn't map interrupt clear register, aborting\n");
1111 return -ENOMEM;
1112 }
1113
1114 return 0;
1115 }
1116
1117 static void mlx4_unmap_clr_int(struct mlx4_dev *dev)
1118 {
1119 struct mlx4_priv *priv = mlx4_priv(dev);
1120
1121 iounmap(priv->clr_base);
1122 }
1123
1124 int mlx4_alloc_eq_table(struct mlx4_dev *dev)
1125 {
1126 struct mlx4_priv *priv = mlx4_priv(dev);
1127
1128 priv->eq_table.eq = kcalloc(dev->caps.num_eqs - dev->caps.reserved_eqs,
1129 sizeof *priv->eq_table.eq, GFP_KERNEL);
1130 if (!priv->eq_table.eq)
1131 return -ENOMEM;
1132
1133 return 0;
1134 }
1135
1136 void mlx4_free_eq_table(struct mlx4_dev *dev)
1137 {
1138 kfree(mlx4_priv(dev)->eq_table.eq);
1139 }
1140
1141 int mlx4_init_eq_table(struct mlx4_dev *dev)
1142 {
1143 struct mlx4_priv *priv = mlx4_priv(dev);
1144 int err;
1145 int i;
1146
1147 priv->eq_table.uar_map = kcalloc(mlx4_num_eq_uar(dev),
1148 sizeof *priv->eq_table.uar_map,
1149 GFP_KERNEL);
1150 if (!priv->eq_table.uar_map) {
1151 err = -ENOMEM;
1152 goto err_out_free;
1153 }
1154
1155 err = mlx4_bitmap_init(&priv->eq_table.bitmap,
1156 roundup_pow_of_two(dev->caps.num_eqs),
1157 dev->caps.num_eqs - 1,
1158 dev->caps.reserved_eqs,
1159 roundup_pow_of_two(dev->caps.num_eqs) -
1160 dev->caps.num_eqs);
1161 if (err)
1162 goto err_out_free;
1163
1164 for (i = 0; i < mlx4_num_eq_uar(dev); ++i)
1165 priv->eq_table.uar_map[i] = NULL;
1166
1167 if (!mlx4_is_slave(dev)) {
1168 err = mlx4_map_clr_int(dev);
1169 if (err)
1170 goto err_out_bitmap;
1171
1172 priv->eq_table.clr_mask =
1173 swab32(1 << (priv->eq_table.inta_pin & 31));
1174 priv->eq_table.clr_int = priv->clr_base +
1175 (priv->eq_table.inta_pin < 32 ? 4 : 0);
1176 }
1177
1178 priv->eq_table.irq_names =
1179 kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1),
1180 GFP_KERNEL);
1181 if (!priv->eq_table.irq_names) {
1182 err = -ENOMEM;
1183 goto err_out_clr_int;
1184 }
1185
1186 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) {
1187 if (i == MLX4_EQ_ASYNC) {
1188 err = mlx4_create_eq(dev,
1189 MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE,
1190 0, &priv->eq_table.eq[MLX4_EQ_ASYNC]);
1191 } else {
1192 #ifdef CONFIG_RFS_ACCEL
1193 struct mlx4_eq *eq = &priv->eq_table.eq[i];
1194 int port = find_first_bit(eq->actv_ports.ports,
1195 dev->caps.num_ports) + 1;
1196
1197 if (port <= dev->caps.num_ports) {
1198 struct mlx4_port_info *info =
1199 &mlx4_priv(dev)->port[port];
1200
1201 if (!info->rmap) {
1202 info->rmap = alloc_irq_cpu_rmap(
1203 mlx4_get_eqs_per_port(dev, port));
1204 if (!info->rmap) {
1205 mlx4_warn(dev, "Failed to allocate cpu rmap\n");
1206 err = -ENOMEM;
1207 goto err_out_unmap;
1208 }
1209 }
1210
1211 err = irq_cpu_rmap_add(
1212 info->rmap, eq->irq);
1213 if (err)
1214 mlx4_warn(dev, "Failed adding irq rmap\n");
1215 }
1216 #endif
1217 err = mlx4_create_eq(dev, dev->caps.num_cqs -
1218 dev->caps.reserved_cqs +
1219 MLX4_NUM_SPARE_EQE,
1220 (dev->flags & MLX4_FLAG_MSI_X) ?
1221 i + 1 - !!(i > MLX4_EQ_ASYNC) : 0,
1222 eq);
1223 }
1224 if (err)
1225 goto err_out_unmap;
1226 }
1227
1228 if (dev->flags & MLX4_FLAG_MSI_X) {
1229 const char *eq_name;
1230
1231 snprintf(priv->eq_table.irq_names +
1232 MLX4_EQ_ASYNC * MLX4_IRQNAME_SIZE,
1233 MLX4_IRQNAME_SIZE,
1234 "mlx4-async@pci:%s",
1235 pci_name(dev->persist->pdev));
1236 eq_name = priv->eq_table.irq_names +
1237 MLX4_EQ_ASYNC * MLX4_IRQNAME_SIZE;
1238
1239 err = request_irq(priv->eq_table.eq[MLX4_EQ_ASYNC].irq,
1240 mlx4_msi_x_interrupt, 0, eq_name,
1241 priv->eq_table.eq + MLX4_EQ_ASYNC);
1242 if (err)
1243 goto err_out_unmap;
1244
1245 priv->eq_table.eq[MLX4_EQ_ASYNC].have_irq = 1;
1246 } else {
1247 snprintf(priv->eq_table.irq_names,
1248 MLX4_IRQNAME_SIZE,
1249 DRV_NAME "@pci:%s",
1250 pci_name(dev->persist->pdev));
1251 err = request_irq(dev->persist->pdev->irq, mlx4_interrupt,
1252 IRQF_SHARED, priv->eq_table.irq_names, dev);
1253 if (err)
1254 goto err_out_unmap;
1255
1256 priv->eq_table.have_irq = 1;
1257 }
1258
1259 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1260 priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
1261 if (err)
1262 mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
1263 priv->eq_table.eq[MLX4_EQ_ASYNC].eqn, err);
1264
1265 /* arm ASYNC eq */
1266 eq_set_ci(&priv->eq_table.eq[MLX4_EQ_ASYNC], 1);
1267
1268 return 0;
1269
1270 err_out_unmap:
1271 while (i >= 0)
1272 mlx4_free_eq(dev, &priv->eq_table.eq[i--]);
1273 #ifdef CONFIG_RFS_ACCEL
1274 for (i = 1; i <= dev->caps.num_ports; i++) {
1275 if (mlx4_priv(dev)->port[i].rmap) {
1276 free_irq_cpu_rmap(mlx4_priv(dev)->port[i].rmap);
1277 mlx4_priv(dev)->port[i].rmap = NULL;
1278 }
1279 }
1280 #endif
1281 mlx4_free_irqs(dev);
1282
1283 err_out_clr_int:
1284 if (!mlx4_is_slave(dev))
1285 mlx4_unmap_clr_int(dev);
1286
1287 err_out_bitmap:
1288 mlx4_unmap_uar(dev);
1289 mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
1290
1291 err_out_free:
1292 kfree(priv->eq_table.uar_map);
1293
1294 return err;
1295 }
1296
1297 void mlx4_cleanup_eq_table(struct mlx4_dev *dev)
1298 {
1299 struct mlx4_priv *priv = mlx4_priv(dev);
1300 int i;
1301
1302 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1,
1303 priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
1304
1305 #ifdef CONFIG_RFS_ACCEL
1306 for (i = 1; i <= dev->caps.num_ports; i++) {
1307 if (mlx4_priv(dev)->port[i].rmap) {
1308 free_irq_cpu_rmap(mlx4_priv(dev)->port[i].rmap);
1309 mlx4_priv(dev)->port[i].rmap = NULL;
1310 }
1311 }
1312 #endif
1313 mlx4_free_irqs(dev);
1314
1315 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
1316 mlx4_free_eq(dev, &priv->eq_table.eq[i]);
1317
1318 if (!mlx4_is_slave(dev))
1319 mlx4_unmap_clr_int(dev);
1320
1321 mlx4_unmap_uar(dev);
1322 mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
1323
1324 kfree(priv->eq_table.uar_map);
1325 }
1326
1327 /* A test that verifies that we can accept interrupts on all
1328 * the irq vectors of the device.
1329 * Interrupts are checked using the NOP command.
1330 */
1331 int mlx4_test_interrupts(struct mlx4_dev *dev)
1332 {
1333 struct mlx4_priv *priv = mlx4_priv(dev);
1334 int i;
1335 int err;
1336
1337 err = mlx4_NOP(dev);
1338 /* When not in MSI_X, there is only one irq to check */
1339 if (!(dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(dev))
1340 return err;
1341
1342 /* A loop over all completion vectors, for each vector we will check
1343 * whether it works by mapping command completions to that vector
1344 * and performing a NOP command
1345 */
1346 for(i = 0; !err && (i < dev->caps.num_comp_vectors); ++i) {
1347 /* Temporary use polling for command completions */
1348 mlx4_cmd_use_polling(dev);
1349
1350 /* Map the new eq to handle all asynchronous events */
1351 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1352 priv->eq_table.eq[i].eqn);
1353 if (err) {
1354 mlx4_warn(dev, "Failed mapping eq for interrupt test\n");
1355 mlx4_cmd_use_events(dev);
1356 break;
1357 }
1358
1359 /* Go back to using events */
1360 mlx4_cmd_use_events(dev);
1361 err = mlx4_NOP(dev);
1362 }
1363
1364 /* Return to default */
1365 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1366 priv->eq_table.eq[MLX4_EQ_ASYNC].eqn);
1367 return err;
1368 }
1369 EXPORT_SYMBOL(mlx4_test_interrupts);
1370
1371 bool mlx4_is_eq_vector_valid(struct mlx4_dev *dev, u8 port, int vector)
1372 {
1373 struct mlx4_priv *priv = mlx4_priv(dev);
1374
1375 vector = MLX4_CQ_TO_EQ_VECTOR(vector);
1376 if (vector < 0 || (vector >= dev->caps.num_comp_vectors + 1) ||
1377 (vector == MLX4_EQ_ASYNC))
1378 return false;
1379
1380 return test_bit(port - 1, priv->eq_table.eq[vector].actv_ports.ports);
1381 }
1382 EXPORT_SYMBOL(mlx4_is_eq_vector_valid);
1383
1384 u32 mlx4_get_eqs_per_port(struct mlx4_dev *dev, u8 port)
1385 {
1386 struct mlx4_priv *priv = mlx4_priv(dev);
1387 unsigned int i;
1388 unsigned int sum = 0;
1389
1390 for (i = 0; i < dev->caps.num_comp_vectors + 1; i++)
1391 sum += !!test_bit(port - 1,
1392 priv->eq_table.eq[i].actv_ports.ports);
1393
1394 return sum;
1395 }
1396 EXPORT_SYMBOL(mlx4_get_eqs_per_port);
1397
1398 int mlx4_is_eq_shared(struct mlx4_dev *dev, int vector)
1399 {
1400 struct mlx4_priv *priv = mlx4_priv(dev);
1401
1402 vector = MLX4_CQ_TO_EQ_VECTOR(vector);
1403 if (vector <= 0 || (vector >= dev->caps.num_comp_vectors + 1))
1404 return -EINVAL;
1405
1406 return !!(bitmap_weight(priv->eq_table.eq[vector].actv_ports.ports,
1407 dev->caps.num_ports) > 1);
1408 }
1409 EXPORT_SYMBOL(mlx4_is_eq_shared);
1410
1411 struct cpu_rmap *mlx4_get_cpu_rmap(struct mlx4_dev *dev, int port)
1412 {
1413 return mlx4_priv(dev)->port[port].rmap;
1414 }
1415 EXPORT_SYMBOL(mlx4_get_cpu_rmap);
1416
1417 int mlx4_assign_eq(struct mlx4_dev *dev, u8 port, int *vector)
1418 {
1419 struct mlx4_priv *priv = mlx4_priv(dev);
1420 int err = 0, i = 0;
1421 u32 min_ref_count_val = (u32)-1;
1422 int requested_vector = MLX4_CQ_TO_EQ_VECTOR(*vector);
1423 int *prequested_vector = NULL;
1424
1425
1426 mutex_lock(&priv->msix_ctl.pool_lock);
1427 if (requested_vector < (dev->caps.num_comp_vectors + 1) &&
1428 (requested_vector >= 0) &&
1429 (requested_vector != MLX4_EQ_ASYNC)) {
1430 if (test_bit(port - 1,
1431 priv->eq_table.eq[requested_vector].actv_ports.ports)) {
1432 prequested_vector = &requested_vector;
1433 } else {
1434 struct mlx4_eq *eq;
1435
1436 for (i = 1; i < port;
1437 requested_vector += mlx4_get_eqs_per_port(dev, i++))
1438 ;
1439
1440 eq = &priv->eq_table.eq[requested_vector];
1441 if (requested_vector < dev->caps.num_comp_vectors + 1 &&
1442 test_bit(port - 1, eq->actv_ports.ports)) {
1443 prequested_vector = &requested_vector;
1444 }
1445 }
1446 }
1447
1448 if (!prequested_vector) {
1449 requested_vector = -1;
1450 for (i = 0; min_ref_count_val && i < dev->caps.num_comp_vectors + 1;
1451 i++) {
1452 struct mlx4_eq *eq = &priv->eq_table.eq[i];
1453
1454 if (min_ref_count_val > eq->ref_count &&
1455 test_bit(port - 1, eq->actv_ports.ports)) {
1456 min_ref_count_val = eq->ref_count;
1457 requested_vector = i;
1458 }
1459 }
1460
1461 if (requested_vector < 0) {
1462 err = -ENOSPC;
1463 goto err_unlock;
1464 }
1465
1466 prequested_vector = &requested_vector;
1467 }
1468
1469 if (!test_bit(*prequested_vector, priv->msix_ctl.pool_bm) &&
1470 dev->flags & MLX4_FLAG_MSI_X) {
1471 set_bit(*prequested_vector, priv->msix_ctl.pool_bm);
1472 snprintf(priv->eq_table.irq_names +
1473 *prequested_vector * MLX4_IRQNAME_SIZE,
1474 MLX4_IRQNAME_SIZE, "mlx4-%d@%s",
1475 *prequested_vector, dev_name(&dev->persist->pdev->dev));
1476
1477 err = request_irq(priv->eq_table.eq[*prequested_vector].irq,
1478 mlx4_msi_x_interrupt, 0,
1479 &priv->eq_table.irq_names[*prequested_vector << 5],
1480 priv->eq_table.eq + *prequested_vector);
1481
1482 if (err) {
1483 clear_bit(*prequested_vector, priv->msix_ctl.pool_bm);
1484 *prequested_vector = -1;
1485 } else {
1486 eq_set_ci(&priv->eq_table.eq[*prequested_vector], 1);
1487 priv->eq_table.eq[*prequested_vector].have_irq = 1;
1488 }
1489 }
1490
1491 if (!err && *prequested_vector >= 0)
1492 priv->eq_table.eq[*prequested_vector].ref_count++;
1493
1494 err_unlock:
1495 mutex_unlock(&priv->msix_ctl.pool_lock);
1496
1497 if (!err && *prequested_vector >= 0)
1498 *vector = MLX4_EQ_TO_CQ_VECTOR(*prequested_vector);
1499 else
1500 *vector = 0;
1501
1502 return err;
1503 }
1504 EXPORT_SYMBOL(mlx4_assign_eq);
1505
1506 int mlx4_eq_get_irq(struct mlx4_dev *dev, int cq_vec)
1507 {
1508 struct mlx4_priv *priv = mlx4_priv(dev);
1509
1510 return priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(cq_vec)].irq;
1511 }
1512 EXPORT_SYMBOL(mlx4_eq_get_irq);
1513
1514 void mlx4_release_eq(struct mlx4_dev *dev, int vec)
1515 {
1516 struct mlx4_priv *priv = mlx4_priv(dev);
1517 int eq_vec = MLX4_CQ_TO_EQ_VECTOR(vec);
1518
1519 mutex_lock(&priv->msix_ctl.pool_lock);
1520 priv->eq_table.eq[eq_vec].ref_count--;
1521
1522 /* once we allocated EQ, we don't release it because it might be binded
1523 * to cpu_rmap.
1524 */
1525 mutex_unlock(&priv->msix_ctl.pool_lock);
1526 }
1527 EXPORT_SYMBOL(mlx4_release_eq);
1528
Ubuntu Artful kernel mirror