]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - drivers/net/ethernet/mellanox/mlx4/eq.c
projects / mirror_ubuntu-zesty-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'cleanup' into for-linus
[mirror_ubuntu-zesty-kernel.git] / drivers / net / ethernet / mellanox / mlx4 / eq.c
1 /*
2 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <linux/init.h>
35 #include <linux/interrupt.h>
36 #include <linux/slab.h>
37 #include <linux/export.h>
38 #include <linux/mm.h>
39 #include <linux/dma-mapping.h>
40
41 #include <linux/mlx4/cmd.h>
42 #include <linux/cpu_rmap.h>
43
44 #include "mlx4.h"
45 #include "fw.h"
46
47 enum {
48 MLX4_IRQNAME_SIZE = 32
49 };
50
51 enum {
52 MLX4_NUM_ASYNC_EQE = 0x100,
53 MLX4_NUM_SPARE_EQE = 0x80,
54 MLX4_EQ_ENTRY_SIZE = 0x20
55 };
56
57 #define MLX4_EQ_STATUS_OK ( 0 << 28)
58 #define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28)
59 #define MLX4_EQ_OWNER_SW ( 0 << 24)
60 #define MLX4_EQ_OWNER_HW ( 1 << 24)
61 #define MLX4_EQ_FLAG_EC ( 1 << 18)
62 #define MLX4_EQ_FLAG_OI ( 1 << 17)
63 #define MLX4_EQ_STATE_ARMED ( 9 << 8)
64 #define MLX4_EQ_STATE_FIRED (10 << 8)
65 #define MLX4_EQ_STATE_ALWAYS_ARMED (11 << 8)
66
67 #define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG) | \
68 (1ull << MLX4_EVENT_TYPE_COMM_EST) | \
69 (1ull << MLX4_EVENT_TYPE_SQ_DRAINED) | \
70 (1ull << MLX4_EVENT_TYPE_CQ_ERROR) | \
71 (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR) | \
72 (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR) | \
73 (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED) | \
74 (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
75 (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR) | \
76 (1ull << MLX4_EVENT_TYPE_PORT_CHANGE) | \
77 (1ull << MLX4_EVENT_TYPE_ECC_DETECT) | \
78 (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR) | \
79 (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE) | \
80 (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT) | \
81 (1ull << MLX4_EVENT_TYPE_CMD) | \
82 (1ull << MLX4_EVENT_TYPE_COMM_CHANNEL) | \
83 (1ull << MLX4_EVENT_TYPE_FLR_EVENT) | \
84 (1ull << MLX4_EVENT_TYPE_FATAL_WARNING))
85
86 static u64 get_async_ev_mask(struct mlx4_dev *dev)
87 {
88 u64 async_ev_mask = MLX4_ASYNC_EVENT_MASK;
89 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)
90 async_ev_mask |= (1ull << MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT);
91
92 return async_ev_mask;
93 }
94
95 static void eq_set_ci(struct mlx4_eq *eq, int req_not)
96 {
97 __raw_writel((__force u32) cpu_to_be32((eq->cons_index & 0xffffff) |
98 req_not << 31),
99 eq->doorbell);
100 /* We still want ordering, just not swabbing, so add a barrier */
101 mb();
102 }
103
104 static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry)
105 {
106 unsigned long off = (entry & (eq->nent - 1)) * MLX4_EQ_ENTRY_SIZE;
107 return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE;
108 }
109
110 static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq)
111 {
112 struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index);
113 return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe;
114 }
115
116 static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_eq)
117 {
118 struct mlx4_eqe *eqe =
119 &slave_eq->event_eqe[slave_eq->cons & (SLAVE_EVENT_EQ_SIZE - 1)];
120 return (!!(eqe->owner & 0x80) ^
121 !!(slave_eq->cons & SLAVE_EVENT_EQ_SIZE)) ?
122 eqe : NULL;
123 }
124
125 void mlx4_gen_slave_eqe(struct work_struct *work)
126 {
127 struct mlx4_mfunc_master_ctx *master =
128 container_of(work, struct mlx4_mfunc_master_ctx,
129 slave_event_work);
130 struct mlx4_mfunc *mfunc =
131 container_of(master, struct mlx4_mfunc, master);
132 struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc);
133 struct mlx4_dev *dev = &priv->dev;
134 struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq;
135 struct mlx4_eqe *eqe;
136 u8 slave;
137 int i;
138
139 for (eqe = next_slave_event_eqe(slave_eq); eqe;
140 eqe = next_slave_event_eqe(slave_eq)) {
141 slave = eqe->slave_id;
142
143 /* All active slaves need to receive the event */
144 if (slave == ALL_SLAVES) {
145 for (i = 0; i < dev->num_slaves; i++) {
146 if (i != dev->caps.function &&
147 master->slave_state[i].active)
148 if (mlx4_GEN_EQE(dev, i, eqe))
149 mlx4_warn(dev, "Failed to "
150 " generate event "
151 "for slave %d\n", i);
152 }
153 } else {
154 if (mlx4_GEN_EQE(dev, slave, eqe))
155 mlx4_warn(dev, "Failed to generate event "
156 "for slave %d\n", slave);
157 }
158 ++slave_eq->cons;
159 }
160 }
161
162
163 static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe)
164 {
165 struct mlx4_priv *priv = mlx4_priv(dev);
166 struct mlx4_slave_event_eq *slave_eq = &priv->mfunc.master.slave_eq;
167 struct mlx4_eqe *s_eqe =
168 &slave_eq->event_eqe[slave_eq->prod & (SLAVE_EVENT_EQ_SIZE - 1)];
169
170 if ((!!(s_eqe->owner & 0x80)) ^
171 (!!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE))) {
172 mlx4_warn(dev, "Master failed to generate an EQE for slave: %d. "
173 "No free EQE on slave events queue\n", slave);
174 return;
175 }
176
177 memcpy(s_eqe, eqe, sizeof(struct mlx4_eqe) - 1);
178 s_eqe->slave_id = slave;
179 /* ensure all information is written before setting the ownersip bit */
180 wmb();
181 s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80;
182 ++slave_eq->prod;
183
184 queue_work(priv->mfunc.master.comm_wq,
185 &priv->mfunc.master.slave_event_work);
186 }
187
188 static void mlx4_slave_event(struct mlx4_dev *dev, int slave,
189 struct mlx4_eqe *eqe)
190 {
191 struct mlx4_priv *priv = mlx4_priv(dev);
192 struct mlx4_slave_state *s_slave =
193 &priv->mfunc.master.slave_state[slave];
194
195 if (!s_slave->active) {
196 /*mlx4_warn(dev, "Trying to pass event to inactive slave\n");*/
197 return;
198 }
199
200 slave_event(dev, slave, eqe);
201 }
202
203 void mlx4_master_handle_slave_flr(struct work_struct *work)
204 {
205 struct mlx4_mfunc_master_ctx *master =
206 container_of(work, struct mlx4_mfunc_master_ctx,
207 slave_flr_event_work);
208 struct mlx4_mfunc *mfunc =
209 container_of(master, struct mlx4_mfunc, master);
210 struct mlx4_priv *priv =
211 container_of(mfunc, struct mlx4_priv, mfunc);
212 struct mlx4_dev *dev = &priv->dev;
213 struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state;
214 int i;
215 int err;
216
217 mlx4_dbg(dev, "mlx4_handle_slave_flr\n");
218
219 for (i = 0 ; i < dev->num_slaves; i++) {
220
221 if (MLX4_COMM_CMD_FLR == slave_state[i].last_cmd) {
222 mlx4_dbg(dev, "mlx4_handle_slave_flr: "
223 "clean slave: %d\n", i);
224
225 mlx4_delete_all_resources_for_slave(dev, i);
226 /*return the slave to running mode*/
227 spin_lock(&priv->mfunc.master.slave_state_lock);
228 slave_state[i].last_cmd = MLX4_COMM_CMD_RESET;
229 slave_state[i].is_slave_going_down = 0;
230 spin_unlock(&priv->mfunc.master.slave_state_lock);
231 /*notify the FW:*/
232 err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE,
233 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
234 if (err)
235 mlx4_warn(dev, "Failed to notify FW on "
236 "FLR done (slave:%d)\n", i);
237 }
238 }
239 }
240
241 static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
242 {
243 struct mlx4_priv *priv = mlx4_priv(dev);
244 struct mlx4_eqe *eqe;
245 int cqn;
246 int eqes_found = 0;
247 int set_ci = 0;
248 int port;
249 int slave = 0;
250 int ret;
251 u32 flr_slave;
252 u8 update_slave_state;
253 int i;
254
255 while ((eqe = next_eqe_sw(eq))) {
256 /*
257 * Make sure we read EQ entry contents after we've
258 * checked the ownership bit.
259 */
260 rmb();
261
262 switch (eqe->type) {
263 case MLX4_EVENT_TYPE_COMP:
264 cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff;
265 mlx4_cq_completion(dev, cqn);
266 break;
267
268 case MLX4_EVENT_TYPE_PATH_MIG:
269 case MLX4_EVENT_TYPE_COMM_EST:
270 case MLX4_EVENT_TYPE_SQ_DRAINED:
271 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
272 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
273 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
274 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
275 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
276 mlx4_dbg(dev, "event %d arrived\n", eqe->type);
277 if (mlx4_is_master(dev)) {
278 /* forward only to slave owning the QP */
279 ret = mlx4_get_slave_from_resource_id(dev,
280 RES_QP,
281 be32_to_cpu(eqe->event.qp.qpn)
282 & 0xffffff, &slave);
283 if (ret && ret != -ENOENT) {
284 mlx4_dbg(dev, "QP event %02x(%02x) on "
285 "EQ %d at index %u: could "
286 "not get slave id (%d)\n",
287 eqe->type, eqe->subtype,
288 eq->eqn, eq->cons_index, ret);
289 break;
290 }
291
292 if (!ret && slave != dev->caps.function) {
293 mlx4_slave_event(dev, slave, eqe);
294 break;
295 }
296
297 }
298 mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) &
299 0xffffff, eqe->type);
300 break;
301
302 case MLX4_EVENT_TYPE_SRQ_LIMIT:
303 mlx4_warn(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT\n",
304 __func__);
305 case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:
306 if (mlx4_is_master(dev)) {
307 /* forward only to slave owning the SRQ */
308 ret = mlx4_get_slave_from_resource_id(dev,
309 RES_SRQ,
310 be32_to_cpu(eqe->event.srq.srqn)
311 & 0xffffff,
312 &slave);
313 if (ret && ret != -ENOENT) {
314 mlx4_warn(dev, "SRQ event %02x(%02x) "
315 "on EQ %d at index %u: could"
316 " not get slave id (%d)\n",
317 eqe->type, eqe->subtype,
318 eq->eqn, eq->cons_index, ret);
319 break;
320 }
321 mlx4_warn(dev, "%s: slave:%d, srq_no:0x%x,"
322 " event: %02x(%02x)\n", __func__,
323 slave,
324 be32_to_cpu(eqe->event.srq.srqn),
325 eqe->type, eqe->subtype);
326
327 if (!ret && slave != dev->caps.function) {
328 mlx4_warn(dev, "%s: sending event "
329 "%02x(%02x) to slave:%d\n",
330 __func__, eqe->type,
331 eqe->subtype, slave);
332 mlx4_slave_event(dev, slave, eqe);
333 break;
334 }
335 }
336 mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) &
337 0xffffff, eqe->type);
338 break;
339
340 case MLX4_EVENT_TYPE_CMD:
341 mlx4_cmd_event(dev,
342 be16_to_cpu(eqe->event.cmd.token),
343 eqe->event.cmd.status,
344 be64_to_cpu(eqe->event.cmd.out_param));
345 break;
346
347 case MLX4_EVENT_TYPE_PORT_CHANGE:
348 port = be32_to_cpu(eqe->event.port_change.port) >> 28;
349 if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
350 mlx4_dispatch_event(dev,
351 MLX4_DEV_EVENT_PORT_DOWN,
352 port);
353 mlx4_priv(dev)->sense.do_sense_port[port] = 1;
354 if (mlx4_is_master(dev))
355 /*change the state of all slave's port
356 * to down:*/
357 for (i = 0; i < dev->num_slaves; i++) {
358 mlx4_dbg(dev, "%s: Sending "
359 "MLX4_PORT_CHANGE_SUBTYPE_DOWN"
360 " to slave: %d, port:%d\n",
361 __func__, i, port);
362 if (i == dev->caps.function)
363 continue;
364 mlx4_slave_event(dev, i, eqe);
365 }
366 } else {
367 mlx4_dispatch_event(dev,
368 MLX4_DEV_EVENT_PORT_UP,
369 port);
370 mlx4_priv(dev)->sense.do_sense_port[port] = 0;
371
372 if (mlx4_is_master(dev)) {
373 for (i = 0; i < dev->num_slaves; i++) {
374 if (i == dev->caps.function)
375 continue;
376 mlx4_slave_event(dev, i, eqe);
377 }
378 }
379 }
380 break;
381
382 case MLX4_EVENT_TYPE_CQ_ERROR:
383 mlx4_warn(dev, "CQ %s on CQN %06x\n",
384 eqe->event.cq_err.syndrome == 1 ?
385 "overrun" : "access violation",
386 be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff);
387 if (mlx4_is_master(dev)) {
388 ret = mlx4_get_slave_from_resource_id(dev,
389 RES_CQ,
390 be32_to_cpu(eqe->event.cq_err.cqn)
391 & 0xffffff, &slave);
392 if (ret && ret != -ENOENT) {
393 mlx4_dbg(dev, "CQ event %02x(%02x) on "
394 "EQ %d at index %u: could "
395 "not get slave id (%d)\n",
396 eqe->type, eqe->subtype,
397 eq->eqn, eq->cons_index, ret);
398 break;
399 }
400
401 if (!ret && slave != dev->caps.function) {
402 mlx4_slave_event(dev, slave, eqe);
403 break;
404 }
405 }
406 mlx4_cq_event(dev,
407 be32_to_cpu(eqe->event.cq_err.cqn)
408 & 0xffffff,
409 eqe->type);
410 break;
411
412 case MLX4_EVENT_TYPE_EQ_OVERFLOW:
413 mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn);
414 break;
415
416 case MLX4_EVENT_TYPE_COMM_CHANNEL:
417 if (!mlx4_is_master(dev)) {
418 mlx4_warn(dev, "Received comm channel event "
419 "for non master device\n");
420 break;
421 }
422 memcpy(&priv->mfunc.master.comm_arm_bit_vector,
423 eqe->event.comm_channel_arm.bit_vec,
424 sizeof eqe->event.comm_channel_arm.bit_vec);
425 queue_work(priv->mfunc.master.comm_wq,
426 &priv->mfunc.master.comm_work);
427 break;
428
429 case MLX4_EVENT_TYPE_FLR_EVENT:
430 flr_slave = be32_to_cpu(eqe->event.flr_event.slave_id);
431 if (!mlx4_is_master(dev)) {
432 mlx4_warn(dev, "Non-master function received"
433 "FLR event\n");
434 break;
435 }
436
437 mlx4_dbg(dev, "FLR event for slave: %d\n", flr_slave);
438
439 if (flr_slave >= dev->num_slaves) {
440 mlx4_warn(dev,
441 "Got FLR for unknown function: %d\n",
442 flr_slave);
443 update_slave_state = 0;
444 } else
445 update_slave_state = 1;
446
447 spin_lock(&priv->mfunc.master.slave_state_lock);
448 if (update_slave_state) {
449 priv->mfunc.master.slave_state[flr_slave].active = false;
450 priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR;
451 priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1;
452 }
453 spin_unlock(&priv->mfunc.master.slave_state_lock);
454 queue_work(priv->mfunc.master.comm_wq,
455 &priv->mfunc.master.slave_flr_event_work);
456 break;
457
458 case MLX4_EVENT_TYPE_FATAL_WARNING:
459 if (eqe->subtype == MLX4_FATAL_WARNING_SUBTYPE_WARMING) {
460 if (mlx4_is_master(dev))
461 for (i = 0; i < dev->num_slaves; i++) {
462 mlx4_dbg(dev, "%s: Sending "
463 "MLX4_FATAL_WARNING_SUBTYPE_WARMING"
464 " to slave: %d\n", __func__, i);
465 if (i == dev->caps.function)
466 continue;
467 mlx4_slave_event(dev, i, eqe);
468 }
469 mlx4_err(dev, "Temperature Threshold was reached! "
470 "Threshold: %d celsius degrees; "
471 "Current Temperature: %d\n",
472 be16_to_cpu(eqe->event.warming.warning_threshold),
473 be16_to_cpu(eqe->event.warming.current_temperature));
474 } else
475 mlx4_warn(dev, "Unhandled event FATAL WARNING (%02x), "
476 "subtype %02x on EQ %d at index %u. owner=%x, "
477 "nent=0x%x, slave=%x, ownership=%s\n",
478 eqe->type, eqe->subtype, eq->eqn,
479 eq->cons_index, eqe->owner, eq->nent,
480 eqe->slave_id,
481 !!(eqe->owner & 0x80) ^
482 !!(eq->cons_index & eq->nent) ? "HW" : "SW");
483
484 break;
485
486 case MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT:
487 mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_MGMT_CHANGE,
488 (unsigned long) eqe);
489 break;
490
491 case MLX4_EVENT_TYPE_EEC_CATAS_ERROR:
492 case MLX4_EVENT_TYPE_ECC_DETECT:
493 default:
494 mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at "
495 "index %u. owner=%x, nent=0x%x, slave=%x, "
496 "ownership=%s\n",
497 eqe->type, eqe->subtype, eq->eqn,
498 eq->cons_index, eqe->owner, eq->nent,
499 eqe->slave_id,
500 !!(eqe->owner & 0x80) ^
501 !!(eq->cons_index & eq->nent) ? "HW" : "SW");
502 break;
503 };
504
505 ++eq->cons_index;
506 eqes_found = 1;
507 ++set_ci;
508
509 /*
510 * The HCA will think the queue has overflowed if we
511 * don't tell it we've been processing events. We
512 * create our EQs with MLX4_NUM_SPARE_EQE extra
513 * entries, so we must update our consumer index at
514 * least that often.
515 */
516 if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) {
517 eq_set_ci(eq, 0);
518 set_ci = 0;
519 }
520 }
521
522 eq_set_ci(eq, 1);
523
524 return eqes_found;
525 }
526
527 static irqreturn_t mlx4_interrupt(int irq, void *dev_ptr)
528 {
529 struct mlx4_dev *dev = dev_ptr;
530 struct mlx4_priv *priv = mlx4_priv(dev);
531 int work = 0;
532 int i;
533
534 writel(priv->eq_table.clr_mask, priv->eq_table.clr_int);
535
536 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
537 work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]);
538
539 return IRQ_RETVAL(work);
540 }
541
542 static irqreturn_t mlx4_msi_x_interrupt(int irq, void *eq_ptr)
543 {
544 struct mlx4_eq *eq = eq_ptr;
545 struct mlx4_dev *dev = eq->dev;
546
547 mlx4_eq_int(dev, eq);
548
549 /* MSI-X vectors always belong to us */
550 return IRQ_HANDLED;
551 }
552
553 int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave,
554 struct mlx4_vhcr *vhcr,
555 struct mlx4_cmd_mailbox *inbox,
556 struct mlx4_cmd_mailbox *outbox,
557 struct mlx4_cmd_info *cmd)
558 {
559 struct mlx4_priv *priv = mlx4_priv(dev);
560 struct mlx4_slave_event_eq_info *event_eq =
561 priv->mfunc.master.slave_state[slave].event_eq;
562 u32 in_modifier = vhcr->in_modifier;
563 u32 eqn = in_modifier & 0x1FF;
564 u64 in_param = vhcr->in_param;
565 int err = 0;
566 int i;
567
568 if (slave == dev->caps.function)
569 err = mlx4_cmd(dev, in_param, (in_modifier & 0x80000000) | eqn,
570 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B,
571 MLX4_CMD_NATIVE);
572 if (!err)
573 for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i)
574 if (in_param & (1LL << i))
575 event_eq[i].eqn = in_modifier >> 31 ? -1 : eqn;
576
577 return err;
578 }
579
580 static int mlx4_MAP_EQ(struct mlx4_dev *dev, u64 event_mask, int unmap,
581 int eq_num)
582 {
583 return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num,
584 0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B,
585 MLX4_CMD_WRAPPED);
586 }
587
588 static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
589 int eq_num)
590 {
591 return mlx4_cmd(dev, mailbox->dma, eq_num, 0,
592 MLX4_CMD_SW2HW_EQ, MLX4_CMD_TIME_CLASS_A,
593 MLX4_CMD_WRAPPED);
594 }
595
596 static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
597 int eq_num)
598 {
599 return mlx4_cmd_box(dev, 0, mailbox->dma, eq_num,
600 0, MLX4_CMD_HW2SW_EQ, MLX4_CMD_TIME_CLASS_A,
601 MLX4_CMD_WRAPPED);
602 }
603
604 static int mlx4_num_eq_uar(struct mlx4_dev *dev)
605 {
606 /*
607 * Each UAR holds 4 EQ doorbells. To figure out how many UARs
608 * we need to map, take the difference of highest index and
609 * the lowest index we'll use and add 1.
610 */
611 return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs +
612 dev->caps.comp_pool)/4 - dev->caps.reserved_eqs/4 + 1;
613 }
614
615 static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq)
616 {
617 struct mlx4_priv *priv = mlx4_priv(dev);
618 int index;
619
620 index = eq->eqn / 4 - dev->caps.reserved_eqs / 4;
621
622 if (!priv->eq_table.uar_map[index]) {
623 priv->eq_table.uar_map[index] =
624 ioremap(pci_resource_start(dev->pdev, 2) +
625 ((eq->eqn / 4) << PAGE_SHIFT),
626 PAGE_SIZE);
627 if (!priv->eq_table.uar_map[index]) {
628 mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n",
629 eq->eqn);
630 return NULL;
631 }
632 }
633
634 return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4);
635 }
636
637 static int mlx4_create_eq(struct mlx4_dev *dev, int nent,
638 u8 intr, struct mlx4_eq *eq)
639 {
640 struct mlx4_priv *priv = mlx4_priv(dev);
641 struct mlx4_cmd_mailbox *mailbox;
642 struct mlx4_eq_context *eq_context;
643 int npages;
644 u64 *dma_list = NULL;
645 dma_addr_t t;
646 u64 mtt_addr;
647 int err = -ENOMEM;
648 int i;
649
650 eq->dev = dev;
651 eq->nent = roundup_pow_of_two(max(nent, 2));
652 npages = PAGE_ALIGN(eq->nent * MLX4_EQ_ENTRY_SIZE) / PAGE_SIZE;
653
654 eq->page_list = kmalloc(npages * sizeof *eq->page_list,
655 GFP_KERNEL);
656 if (!eq->page_list)
657 goto err_out;
658
659 for (i = 0; i < npages; ++i)
660 eq->page_list[i].buf = NULL;
661
662 dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
663 if (!dma_list)
664 goto err_out_free;
665
666 mailbox = mlx4_alloc_cmd_mailbox(dev);
667 if (IS_ERR(mailbox))
668 goto err_out_free;
669 eq_context = mailbox->buf;
670
671 for (i = 0; i < npages; ++i) {
672 eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev,
673 PAGE_SIZE, &t, GFP_KERNEL);
674 if (!eq->page_list[i].buf)
675 goto err_out_free_pages;
676
677 dma_list[i] = t;
678 eq->page_list[i].map = t;
679
680 memset(eq->page_list[i].buf, 0, PAGE_SIZE);
681 }
682
683 eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap);
684 if (eq->eqn == -1)
685 goto err_out_free_pages;
686
687 eq->doorbell = mlx4_get_eq_uar(dev, eq);
688 if (!eq->doorbell) {
689 err = -ENOMEM;
690 goto err_out_free_eq;
691 }
692
693 err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt);
694 if (err)
695 goto err_out_free_eq;
696
697 err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list);
698 if (err)
699 goto err_out_free_mtt;
700
701 memset(eq_context, 0, sizeof *eq_context);
702 eq_context->flags = cpu_to_be32(MLX4_EQ_STATUS_OK |
703 MLX4_EQ_STATE_ARMED);
704 eq_context->log_eq_size = ilog2(eq->nent);
705 eq_context->intr = intr;
706 eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT;
707
708 mtt_addr = mlx4_mtt_addr(dev, &eq->mtt);
709 eq_context->mtt_base_addr_h = mtt_addr >> 32;
710 eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
711
712 err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn);
713 if (err) {
714 mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err);
715 goto err_out_free_mtt;
716 }
717
718 kfree(dma_list);
719 mlx4_free_cmd_mailbox(dev, mailbox);
720
721 eq->cons_index = 0;
722
723 return err;
724
725 err_out_free_mtt:
726 mlx4_mtt_cleanup(dev, &eq->mtt);
727
728 err_out_free_eq:
729 mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn);
730
731 err_out_free_pages:
732 for (i = 0; i < npages; ++i)
733 if (eq->page_list[i].buf)
734 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
735 eq->page_list[i].buf,
736 eq->page_list[i].map);
737
738 mlx4_free_cmd_mailbox(dev, mailbox);
739
740 err_out_free:
741 kfree(eq->page_list);
742 kfree(dma_list);
743
744 err_out:
745 return err;
746 }
747
748 static void mlx4_free_eq(struct mlx4_dev *dev,
749 struct mlx4_eq *eq)
750 {
751 struct mlx4_priv *priv = mlx4_priv(dev);
752 struct mlx4_cmd_mailbox *mailbox;
753 int err;
754 int npages = PAGE_ALIGN(MLX4_EQ_ENTRY_SIZE * eq->nent) / PAGE_SIZE;
755 int i;
756
757 mailbox = mlx4_alloc_cmd_mailbox(dev);
758 if (IS_ERR(mailbox))
759 return;
760
761 err = mlx4_HW2SW_EQ(dev, mailbox, eq->eqn);
762 if (err)
763 mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err);
764
765 if (0) {
766 mlx4_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn);
767 for (i = 0; i < sizeof (struct mlx4_eq_context) / 4; ++i) {
768 if (i % 4 == 0)
769 pr_cont("[%02x] ", i * 4);
770 pr_cont(" %08x", be32_to_cpup(mailbox->buf + i * 4));
771 if ((i + 1) % 4 == 0)
772 pr_cont("\n");
773 }
774 }
775
776 mlx4_mtt_cleanup(dev, &eq->mtt);
777 for (i = 0; i < npages; ++i)
778 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
779 eq->page_list[i].buf,
780 eq->page_list[i].map);
781
782 kfree(eq->page_list);
783 mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn);
784 mlx4_free_cmd_mailbox(dev, mailbox);
785 }
786
787 static void mlx4_free_irqs(struct mlx4_dev *dev)
788 {
789 struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table;
790 struct mlx4_priv *priv = mlx4_priv(dev);
791 int i, vec;
792
793 if (eq_table->have_irq)
794 free_irq(dev->pdev->irq, dev);
795
796 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
797 if (eq_table->eq[i].have_irq) {
798 free_irq(eq_table->eq[i].irq, eq_table->eq + i);
799 eq_table->eq[i].have_irq = 0;
800 }
801
802 for (i = 0; i < dev->caps.comp_pool; i++) {
803 /*
804 * Freeing the assigned irq's
805 * all bits should be 0, but we need to validate
806 */
807 if (priv->msix_ctl.pool_bm & 1ULL << i) {
808 /* NO need protecting*/
809 vec = dev->caps.num_comp_vectors + 1 + i;
810 free_irq(priv->eq_table.eq[vec].irq,
811 &priv->eq_table.eq[vec]);
812 }
813 }
814
815
816 kfree(eq_table->irq_names);
817 }
818
819 static int mlx4_map_clr_int(struct mlx4_dev *dev)
820 {
821 struct mlx4_priv *priv = mlx4_priv(dev);
822
823 priv->clr_base = ioremap(pci_resource_start(dev->pdev, priv->fw.clr_int_bar) +
824 priv->fw.clr_int_base, MLX4_CLR_INT_SIZE);
825 if (!priv->clr_base) {
826 mlx4_err(dev, "Couldn't map interrupt clear register, aborting.\n");
827 return -ENOMEM;
828 }
829
830 return 0;
831 }
832
833 static void mlx4_unmap_clr_int(struct mlx4_dev *dev)
834 {
835 struct mlx4_priv *priv = mlx4_priv(dev);
836
837 iounmap(priv->clr_base);
838 }
839
840 int mlx4_alloc_eq_table(struct mlx4_dev *dev)
841 {
842 struct mlx4_priv *priv = mlx4_priv(dev);
843
844 priv->eq_table.eq = kcalloc(dev->caps.num_eqs - dev->caps.reserved_eqs,
845 sizeof *priv->eq_table.eq, GFP_KERNEL);
846 if (!priv->eq_table.eq)
847 return -ENOMEM;
848
849 return 0;
850 }
851
852 void mlx4_free_eq_table(struct mlx4_dev *dev)
853 {
854 kfree(mlx4_priv(dev)->eq_table.eq);
855 }
856
857 int mlx4_init_eq_table(struct mlx4_dev *dev)
858 {
859 struct mlx4_priv *priv = mlx4_priv(dev);
860 int err;
861 int i;
862
863 priv->eq_table.uar_map = kcalloc(mlx4_num_eq_uar(dev),
864 sizeof *priv->eq_table.uar_map,
865 GFP_KERNEL);
866 if (!priv->eq_table.uar_map) {
867 err = -ENOMEM;
868 goto err_out_free;
869 }
870
871 err = mlx4_bitmap_init(&priv->eq_table.bitmap, dev->caps.num_eqs,
872 dev->caps.num_eqs - 1, dev->caps.reserved_eqs, 0);
873 if (err)
874 goto err_out_free;
875
876 for (i = 0; i < mlx4_num_eq_uar(dev); ++i)
877 priv->eq_table.uar_map[i] = NULL;
878
879 if (!mlx4_is_slave(dev)) {
880 err = mlx4_map_clr_int(dev);
881 if (err)
882 goto err_out_bitmap;
883
884 priv->eq_table.clr_mask =
885 swab32(1 << (priv->eq_table.inta_pin & 31));
886 priv->eq_table.clr_int = priv->clr_base +
887 (priv->eq_table.inta_pin < 32 ? 4 : 0);
888 }
889
890 priv->eq_table.irq_names =
891 kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1 +
892 dev->caps.comp_pool),
893 GFP_KERNEL);
894 if (!priv->eq_table.irq_names) {
895 err = -ENOMEM;
896 goto err_out_bitmap;
897 }
898
899 for (i = 0; i < dev->caps.num_comp_vectors; ++i) {
900 err = mlx4_create_eq(dev, dev->caps.num_cqs -
901 dev->caps.reserved_cqs +
902 MLX4_NUM_SPARE_EQE,
903 (dev->flags & MLX4_FLAG_MSI_X) ? i : 0,
904 &priv->eq_table.eq[i]);
905 if (err) {
906 --i;
907 goto err_out_unmap;
908 }
909 }
910
911 err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE,
912 (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0,
913 &priv->eq_table.eq[dev->caps.num_comp_vectors]);
914 if (err)
915 goto err_out_comp;
916
917 /*if additional completion vectors poolsize is 0 this loop will not run*/
918 for (i = dev->caps.num_comp_vectors + 1;
919 i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) {
920
921 err = mlx4_create_eq(dev, dev->caps.num_cqs -
922 dev->caps.reserved_cqs +
923 MLX4_NUM_SPARE_EQE,
924 (dev->flags & MLX4_FLAG_MSI_X) ? i : 0,
925 &priv->eq_table.eq[i]);
926 if (err) {
927 --i;
928 goto err_out_unmap;
929 }
930 }
931
932
933 if (dev->flags & MLX4_FLAG_MSI_X) {
934 const char *eq_name;
935
936 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) {
937 if (i < dev->caps.num_comp_vectors) {
938 snprintf(priv->eq_table.irq_names +
939 i * MLX4_IRQNAME_SIZE,
940 MLX4_IRQNAME_SIZE,
941 "mlx4-comp-%d@pci:%s", i,
942 pci_name(dev->pdev));
943 } else {
944 snprintf(priv->eq_table.irq_names +
945 i * MLX4_IRQNAME_SIZE,
946 MLX4_IRQNAME_SIZE,
947 "mlx4-async@pci:%s",
948 pci_name(dev->pdev));
949 }
950
951 eq_name = priv->eq_table.irq_names +
952 i * MLX4_IRQNAME_SIZE;
953 err = request_irq(priv->eq_table.eq[i].irq,
954 mlx4_msi_x_interrupt, 0, eq_name,
955 priv->eq_table.eq + i);
956 if (err)
957 goto err_out_async;
958
959 priv->eq_table.eq[i].have_irq = 1;
960 }
961 } else {
962 snprintf(priv->eq_table.irq_names,
963 MLX4_IRQNAME_SIZE,
964 DRV_NAME "@pci:%s",
965 pci_name(dev->pdev));
966 err = request_irq(dev->pdev->irq, mlx4_interrupt,
967 IRQF_SHARED, priv->eq_table.irq_names, dev);
968 if (err)
969 goto err_out_async;
970
971 priv->eq_table.have_irq = 1;
972 }
973
974 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
975 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn);
976 if (err)
977 mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
978 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn, err);
979
980 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
981 eq_set_ci(&priv->eq_table.eq[i], 1);
982
983 return 0;
984
985 err_out_async:
986 mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]);
987
988 err_out_comp:
989 i = dev->caps.num_comp_vectors - 1;
990
991 err_out_unmap:
992 while (i >= 0) {
993 mlx4_free_eq(dev, &priv->eq_table.eq[i]);
994 --i;
995 }
996 if (!mlx4_is_slave(dev))
997 mlx4_unmap_clr_int(dev);
998 mlx4_free_irqs(dev);
999
1000 err_out_bitmap:
1001 mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
1002
1003 err_out_free:
1004 kfree(priv->eq_table.uar_map);
1005
1006 return err;
1007 }
1008
1009 void mlx4_cleanup_eq_table(struct mlx4_dev *dev)
1010 {
1011 struct mlx4_priv *priv = mlx4_priv(dev);
1012 int i;
1013
1014 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1,
1015 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn);
1016
1017 mlx4_free_irqs(dev);
1018
1019 for (i = 0; i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i)
1020 mlx4_free_eq(dev, &priv->eq_table.eq[i]);
1021
1022 if (!mlx4_is_slave(dev))
1023 mlx4_unmap_clr_int(dev);
1024
1025 for (i = 0; i < mlx4_num_eq_uar(dev); ++i)
1026 if (priv->eq_table.uar_map[i])
1027 iounmap(priv->eq_table.uar_map[i]);
1028
1029 mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
1030
1031 kfree(priv->eq_table.uar_map);
1032 }
1033
1034 /* A test that verifies that we can accept interrupts on all
1035 * the irq vectors of the device.
1036 * Interrupts are checked using the NOP command.
1037 */
1038 int mlx4_test_interrupts(struct mlx4_dev *dev)
1039 {
1040 struct mlx4_priv *priv = mlx4_priv(dev);
1041 int i;
1042 int err;
1043
1044 err = mlx4_NOP(dev);
1045 /* When not in MSI_X, there is only one irq to check */
1046 if (!(dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(dev))
1047 return err;
1048
1049 /* A loop over all completion vectors, for each vector we will check
1050 * whether it works by mapping command completions to that vector
1051 * and performing a NOP command
1052 */
1053 for(i = 0; !err && (i < dev->caps.num_comp_vectors); ++i) {
1054 /* Temporary use polling for command completions */
1055 mlx4_cmd_use_polling(dev);
1056
1057 /* Map the new eq to handle all asyncronous events */
1058 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1059 priv->eq_table.eq[i].eqn);
1060 if (err) {
1061 mlx4_warn(dev, "Failed mapping eq for interrupt test\n");
1062 mlx4_cmd_use_events(dev);
1063 break;
1064 }
1065
1066 /* Go back to using events */
1067 mlx4_cmd_use_events(dev);
1068 err = mlx4_NOP(dev);
1069 }
1070
1071 /* Return to default */
1072 mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1073 priv->eq_table.eq[dev->caps.num_comp_vectors].eqn);
1074 return err;
1075 }
1076 EXPORT_SYMBOL(mlx4_test_interrupts);
1077
1078 int mlx4_assign_eq(struct mlx4_dev *dev, char *name, struct cpu_rmap *rmap,
1079 int *vector)
1080 {
1081
1082 struct mlx4_priv *priv = mlx4_priv(dev);
1083 int vec = 0, err = 0, i;
1084
1085 mutex_lock(&priv->msix_ctl.pool_lock);
1086 for (i = 0; !vec && i < dev->caps.comp_pool; i++) {
1087 if (~priv->msix_ctl.pool_bm & 1ULL << i) {
1088 priv->msix_ctl.pool_bm |= 1ULL << i;
1089 vec = dev->caps.num_comp_vectors + 1 + i;
1090 snprintf(priv->eq_table.irq_names +
1091 vec * MLX4_IRQNAME_SIZE,
1092 MLX4_IRQNAME_SIZE, "%s", name);
1093 #ifdef CONFIG_RFS_ACCEL
1094 if (rmap) {
1095 err = irq_cpu_rmap_add(rmap,
1096 priv->eq_table.eq[vec].irq);
1097 if (err)
1098 mlx4_warn(dev, "Failed adding irq rmap\n");
1099 }
1100 #endif
1101 err = request_irq(priv->eq_table.eq[vec].irq,
1102 mlx4_msi_x_interrupt, 0,
1103 &priv->eq_table.irq_names[vec<<5],
1104 priv->eq_table.eq + vec);
1105 if (err) {
1106 /*zero out bit by fliping it*/
1107 priv->msix_ctl.pool_bm ^= 1 << i;
1108 vec = 0;
1109 continue;
1110 /*we dont want to break here*/
1111 }
1112 eq_set_ci(&priv->eq_table.eq[vec], 1);
1113 }
1114 }
1115 mutex_unlock(&priv->msix_ctl.pool_lock);
1116
1117 if (vec) {
1118 *vector = vec;
1119 } else {
1120 *vector = 0;
1121 err = (i == dev->caps.comp_pool) ? -ENOSPC : err;
1122 }
1123 return err;
1124 }
1125 EXPORT_SYMBOL(mlx4_assign_eq);
1126
1127 void mlx4_release_eq(struct mlx4_dev *dev, int vec)
1128 {
1129 struct mlx4_priv *priv = mlx4_priv(dev);
1130 /*bm index*/
1131 int i = vec - dev->caps.num_comp_vectors - 1;
1132
1133 if (likely(i >= 0)) {
1134 /*sanity check , making sure were not trying to free irq's
1135 Belonging to a legacy EQ*/
1136 mutex_lock(&priv->msix_ctl.pool_lock);
1137 if (priv->msix_ctl.pool_bm & 1ULL << i) {
1138 free_irq(priv->eq_table.eq[vec].irq,
1139 &priv->eq_table.eq[vec]);
1140 priv->msix_ctl.pool_bm &= ~(1ULL << i);
1141 }
1142 mutex_unlock(&priv->msix_ctl.pool_lock);
1143 }
1144
1145 }
1146 EXPORT_SYMBOL(mlx4_release_eq);
1147
ubuntu-zesty-kernel mirror