2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/interrupt.h>
34 #include <linux/module.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/cmd.h>
37 #include "mlx5_core.h"
38 #ifdef CONFIG_MLX5_CORE_EN
43 MLX5_EQE_SIZE
= sizeof(struct mlx5_eqe
),
44 MLX5_EQE_OWNER_INIT_VAL
= 0x1,
48 MLX5_EQ_STATE_ARMED
= 0x9,
49 MLX5_EQ_STATE_FIRED
= 0xa,
50 MLX5_EQ_STATE_ALWAYS_ARMED
= 0xb,
54 MLX5_NUM_SPARE_EQE
= 0x80,
55 MLX5_NUM_ASYNC_EQE
= 0x100,
56 MLX5_NUM_CMD_EQE
= 32,
60 MLX5_EQ_DOORBEL_OFFSET
= 0x40,
63 #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG) | \
64 (1ull << MLX5_EVENT_TYPE_COMM_EST) | \
65 (1ull << MLX5_EVENT_TYPE_SQ_DRAINED) | \
66 (1ull << MLX5_EVENT_TYPE_CQ_ERROR) | \
67 (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR) | \
68 (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED) | \
69 (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
70 (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR) | \
71 (1ull << MLX5_EVENT_TYPE_PORT_CHANGE) | \
72 (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR) | \
73 (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE) | \
74 (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
87 static int mlx5_cmd_destroy_eq(struct mlx5_core_dev
*dev
, u8 eqn
)
89 u32 out
[MLX5_ST_SZ_DW(destroy_eq_out
)] = {0};
90 u32 in
[MLX5_ST_SZ_DW(destroy_eq_in
)] = {0};
93 MLX5_SET(destroy_eq_in
, in
, opcode
, MLX5_CMD_OP_DESTROY_EQ
);
94 MLX5_SET(destroy_eq_in
, in
, eq_number
, eqn
);
96 err
= mlx5_cmd_exec(dev
, in
, sizeof(in
), out
, sizeof(out
));
97 return err
? : mlx5_cmd_status_to_err_v2(out
);
101 static struct mlx5_eqe
*get_eqe(struct mlx5_eq
*eq
, u32 entry
)
103 return mlx5_buf_offset(&eq
->buf
, entry
* MLX5_EQE_SIZE
);
106 static struct mlx5_eqe
*next_eqe_sw(struct mlx5_eq
*eq
)
108 struct mlx5_eqe
*eqe
= get_eqe(eq
, eq
->cons_index
& (eq
->nent
- 1));
110 return ((eqe
->owner
& 1) ^ !!(eq
->cons_index
& eq
->nent
)) ? NULL
: eqe
;
113 static const char *eqe_type_str(u8 type
)
116 case MLX5_EVENT_TYPE_COMP
:
117 return "MLX5_EVENT_TYPE_COMP";
118 case MLX5_EVENT_TYPE_PATH_MIG
:
119 return "MLX5_EVENT_TYPE_PATH_MIG";
120 case MLX5_EVENT_TYPE_COMM_EST
:
121 return "MLX5_EVENT_TYPE_COMM_EST";
122 case MLX5_EVENT_TYPE_SQ_DRAINED
:
123 return "MLX5_EVENT_TYPE_SQ_DRAINED";
124 case MLX5_EVENT_TYPE_SRQ_LAST_WQE
:
125 return "MLX5_EVENT_TYPE_SRQ_LAST_WQE";
126 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT
:
127 return "MLX5_EVENT_TYPE_SRQ_RQ_LIMIT";
128 case MLX5_EVENT_TYPE_CQ_ERROR
:
129 return "MLX5_EVENT_TYPE_CQ_ERROR";
130 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR
:
131 return "MLX5_EVENT_TYPE_WQ_CATAS_ERROR";
132 case MLX5_EVENT_TYPE_PATH_MIG_FAILED
:
133 return "MLX5_EVENT_TYPE_PATH_MIG_FAILED";
134 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR
:
135 return "MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR";
136 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR
:
137 return "MLX5_EVENT_TYPE_WQ_ACCESS_ERROR";
138 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR
:
139 return "MLX5_EVENT_TYPE_SRQ_CATAS_ERROR";
140 case MLX5_EVENT_TYPE_INTERNAL_ERROR
:
141 return "MLX5_EVENT_TYPE_INTERNAL_ERROR";
142 case MLX5_EVENT_TYPE_PORT_CHANGE
:
143 return "MLX5_EVENT_TYPE_PORT_CHANGE";
144 case MLX5_EVENT_TYPE_GPIO_EVENT
:
145 return "MLX5_EVENT_TYPE_GPIO_EVENT";
146 case MLX5_EVENT_TYPE_REMOTE_CONFIG
:
147 return "MLX5_EVENT_TYPE_REMOTE_CONFIG";
148 case MLX5_EVENT_TYPE_DB_BF_CONGESTION
:
149 return "MLX5_EVENT_TYPE_DB_BF_CONGESTION";
150 case MLX5_EVENT_TYPE_STALL_EVENT
:
151 return "MLX5_EVENT_TYPE_STALL_EVENT";
152 case MLX5_EVENT_TYPE_CMD
:
153 return "MLX5_EVENT_TYPE_CMD";
154 case MLX5_EVENT_TYPE_PAGE_REQUEST
:
155 return "MLX5_EVENT_TYPE_PAGE_REQUEST";
156 case MLX5_EVENT_TYPE_PAGE_FAULT
:
157 return "MLX5_EVENT_TYPE_PAGE_FAULT";
159 return "Unrecognized event";
163 static enum mlx5_dev_event
port_subtype_event(u8 subtype
)
166 case MLX5_PORT_CHANGE_SUBTYPE_DOWN
:
167 return MLX5_DEV_EVENT_PORT_DOWN
;
168 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE
:
169 return MLX5_DEV_EVENT_PORT_UP
;
170 case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED
:
171 return MLX5_DEV_EVENT_PORT_INITIALIZED
;
172 case MLX5_PORT_CHANGE_SUBTYPE_LID
:
173 return MLX5_DEV_EVENT_LID_CHANGE
;
174 case MLX5_PORT_CHANGE_SUBTYPE_PKEY
:
175 return MLX5_DEV_EVENT_PKEY_CHANGE
;
176 case MLX5_PORT_CHANGE_SUBTYPE_GUID
:
177 return MLX5_DEV_EVENT_GUID_CHANGE
;
178 case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG
:
179 return MLX5_DEV_EVENT_CLIENT_REREG
;
184 static void eq_update_ci(struct mlx5_eq
*eq
, int arm
)
186 __be32 __iomem
*addr
= eq
->doorbell
+ (arm
? 0 : 2);
187 u32 val
= (eq
->cons_index
& 0xffffff) | (eq
->eqn
<< 24);
188 __raw_writel((__force u32
) cpu_to_be32(val
), addr
);
189 /* We still want ordering, just not swabbing, so add a barrier */
193 static int mlx5_eq_int(struct mlx5_core_dev
*dev
, struct mlx5_eq
*eq
)
195 struct mlx5_eqe
*eqe
;
202 while ((eqe
= next_eqe_sw(eq
))) {
204 * Make sure we read EQ entry contents after we've
205 * checked the ownership bit.
209 mlx5_core_dbg(eq
->dev
, "eqn %d, eqe type %s\n",
210 eq
->eqn
, eqe_type_str(eqe
->type
));
212 case MLX5_EVENT_TYPE_COMP
:
213 cqn
= be32_to_cpu(eqe
->data
.comp
.cqn
) & 0xffffff;
214 mlx5_cq_completion(dev
, cqn
);
217 case MLX5_EVENT_TYPE_PATH_MIG
:
218 case MLX5_EVENT_TYPE_COMM_EST
:
219 case MLX5_EVENT_TYPE_SQ_DRAINED
:
220 case MLX5_EVENT_TYPE_SRQ_LAST_WQE
:
221 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR
:
222 case MLX5_EVENT_TYPE_PATH_MIG_FAILED
:
223 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR
:
224 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR
:
225 rsn
= be32_to_cpu(eqe
->data
.qp_srq
.qp_srq_n
) & 0xffffff;
226 rsn
|= (eqe
->data
.qp_srq
.type
<< MLX5_USER_INDEX_LEN
);
227 mlx5_core_dbg(dev
, "event %s(%d) arrived on resource 0x%x\n",
228 eqe_type_str(eqe
->type
), eqe
->type
, rsn
);
229 mlx5_rsc_event(dev
, rsn
, eqe
->type
);
232 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT
:
233 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR
:
234 rsn
= be32_to_cpu(eqe
->data
.qp_srq
.qp_srq_n
) & 0xffffff;
235 mlx5_core_dbg(dev
, "SRQ event %s(%d): srqn 0x%x\n",
236 eqe_type_str(eqe
->type
), eqe
->type
, rsn
);
237 mlx5_srq_event(dev
, rsn
, eqe
->type
);
240 case MLX5_EVENT_TYPE_CMD
:
241 mlx5_cmd_comp_handler(dev
, be32_to_cpu(eqe
->data
.cmd
.vector
));
244 case MLX5_EVENT_TYPE_PORT_CHANGE
:
245 port
= (eqe
->data
.port
.port
>> 4) & 0xf;
246 switch (eqe
->sub_type
) {
247 case MLX5_PORT_CHANGE_SUBTYPE_DOWN
:
248 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE
:
249 case MLX5_PORT_CHANGE_SUBTYPE_LID
:
250 case MLX5_PORT_CHANGE_SUBTYPE_PKEY
:
251 case MLX5_PORT_CHANGE_SUBTYPE_GUID
:
252 case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG
:
253 case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED
:
255 dev
->event(dev
, port_subtype_event(eqe
->sub_type
),
256 (unsigned long)port
);
259 mlx5_core_warn(dev
, "Port event with unrecognized subtype: port %d, sub_type %d\n",
260 port
, eqe
->sub_type
);
263 case MLX5_EVENT_TYPE_CQ_ERROR
:
264 cqn
= be32_to_cpu(eqe
->data
.cq_err
.cqn
) & 0xffffff;
265 mlx5_core_warn(dev
, "CQ error on CQN 0x%x, syndrom 0x%x\n",
266 cqn
, eqe
->data
.cq_err
.syndrome
);
267 mlx5_cq_event(dev
, cqn
, eqe
->type
);
270 case MLX5_EVENT_TYPE_PAGE_REQUEST
:
272 u16 func_id
= be16_to_cpu(eqe
->data
.req_pages
.func_id
);
273 s32 npages
= be32_to_cpu(eqe
->data
.req_pages
.num_pages
);
275 mlx5_core_dbg(dev
, "page request for func 0x%x, npages %d\n",
277 mlx5_core_req_pages_handler(dev
, func_id
, npages
);
281 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
282 case MLX5_EVENT_TYPE_PAGE_FAULT
:
283 mlx5_eq_pagefault(dev
, eqe
);
287 #ifdef CONFIG_MLX5_CORE_EN
288 case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE
:
289 mlx5_eswitch_vport_event(dev
->priv
.eswitch
, eqe
);
293 mlx5_core_warn(dev
, "Unhandled event 0x%x on EQ 0x%x\n",
302 /* The HCA will think the queue has overflowed if we
303 * don't tell it we've been processing events. We
304 * create our EQs with MLX5_NUM_SPARE_EQE extra
305 * entries, so we must update our consumer index at
308 if (unlikely(set_ci
>= MLX5_NUM_SPARE_EQE
)) {
317 tasklet_schedule(&eq
->tasklet_ctx
.task
);
322 static irqreturn_t
mlx5_msix_handler(int irq
, void *eq_ptr
)
324 struct mlx5_eq
*eq
= eq_ptr
;
325 struct mlx5_core_dev
*dev
= eq
->dev
;
327 mlx5_eq_int(dev
, eq
);
329 /* MSI-X vectors always belong to us */
333 static void init_eq_buf(struct mlx5_eq
*eq
)
335 struct mlx5_eqe
*eqe
;
338 for (i
= 0; i
< eq
->nent
; i
++) {
339 eqe
= get_eqe(eq
, i
);
340 eqe
->owner
= MLX5_EQE_OWNER_INIT_VAL
;
344 int mlx5_create_map_eq(struct mlx5_core_dev
*dev
, struct mlx5_eq
*eq
, u8 vecidx
,
345 int nent
, u64 mask
, const char *name
, struct mlx5_uar
*uar
)
347 u32 out
[MLX5_ST_SZ_DW(create_eq_out
)] = {0};
348 struct mlx5_priv
*priv
= &dev
->priv
;
355 eq
->nent
= roundup_pow_of_two(nent
+ MLX5_NUM_SPARE_EQE
);
357 err
= mlx5_buf_alloc(dev
, eq
->nent
* MLX5_EQE_SIZE
, &eq
->buf
);
363 inlen
= MLX5_ST_SZ_BYTES(create_eq_in
) +
364 MLX5_FLD_SZ_BYTES(create_eq_in
, pas
[0]) * eq
->buf
.npages
;
366 in
= mlx5_vzalloc(inlen
);
372 pas
= (__be64
*)MLX5_ADDR_OF(create_eq_in
, in
, pas
);
373 mlx5_fill_page_array(&eq
->buf
, pas
);
375 MLX5_SET(create_eq_in
, in
, opcode
, MLX5_CMD_OP_CREATE_EQ
);
376 MLX5_SET64(create_eq_in
, in
, event_bitmask
, mask
);
378 eqc
= MLX5_ADDR_OF(create_eq_in
, in
, eq_context_entry
);
379 MLX5_SET(eqc
, eqc
, log_eq_size
, ilog2(eq
->nent
));
380 MLX5_SET(eqc
, eqc
, uar_page
, uar
->index
);
381 MLX5_SET(eqc
, eqc
, intr
, vecidx
);
382 MLX5_SET(eqc
, eqc
, log_page_size
,
383 eq
->buf
.page_shift
- MLX5_ADAPTER_PAGE_SHIFT
);
385 err
= mlx5_cmd_exec(dev
, in
, inlen
, out
, sizeof(out
));
386 err
= err
? : mlx5_cmd_status_to_err_v2(out
);
390 snprintf(priv
->irq_info
[vecidx
].name
, MLX5_MAX_IRQ_NAME
, "%s@pci:%s",
391 name
, pci_name(dev
->pdev
));
393 eq
->eqn
= MLX5_GET(create_eq_out
, out
, eq_number
);
394 eq
->irqn
= priv
->msix_arr
[vecidx
].vector
;
396 eq
->doorbell
= uar
->map
+ MLX5_EQ_DOORBEL_OFFSET
;
397 err
= request_irq(eq
->irqn
, mlx5_msix_handler
, 0,
398 priv
->irq_info
[vecidx
].name
, eq
);
402 err
= mlx5_debug_eq_add(dev
, eq
);
406 INIT_LIST_HEAD(&eq
->tasklet_ctx
.list
);
407 INIT_LIST_HEAD(&eq
->tasklet_ctx
.process_list
);
408 spin_lock_init(&eq
->tasklet_ctx
.lock
);
409 tasklet_init(&eq
->tasklet_ctx
.task
, mlx5_cq_tasklet_cb
,
410 (unsigned long)&eq
->tasklet_ctx
);
412 /* EQs are created in ARMED state
420 free_irq(priv
->msix_arr
[vecidx
].vector
, eq
);
423 mlx5_cmd_destroy_eq(dev
, eq
->eqn
);
429 mlx5_buf_free(dev
, &eq
->buf
);
432 EXPORT_SYMBOL_GPL(mlx5_create_map_eq
);
434 int mlx5_destroy_unmap_eq(struct mlx5_core_dev
*dev
, struct mlx5_eq
*eq
)
438 mlx5_debug_eq_remove(dev
, eq
);
439 free_irq(eq
->irqn
, eq
);
440 err
= mlx5_cmd_destroy_eq(dev
, eq
->eqn
);
442 mlx5_core_warn(dev
, "failed to destroy a previously created eq: eqn %d\n",
444 synchronize_irq(eq
->irqn
);
445 tasklet_disable(&eq
->tasklet_ctx
.task
);
446 mlx5_buf_free(dev
, &eq
->buf
);
450 EXPORT_SYMBOL_GPL(mlx5_destroy_unmap_eq
);
452 u32
mlx5_get_msix_vec(struct mlx5_core_dev
*dev
, int vecidx
)
454 return dev
->priv
.msix_arr
[MLX5_EQ_VEC_ASYNC
].vector
;
457 int mlx5_eq_init(struct mlx5_core_dev
*dev
)
461 spin_lock_init(&dev
->priv
.eq_table
.lock
);
463 err
= mlx5_eq_debugfs_init(dev
);
469 void mlx5_eq_cleanup(struct mlx5_core_dev
*dev
)
471 mlx5_eq_debugfs_cleanup(dev
);
474 int mlx5_start_eqs(struct mlx5_core_dev
*dev
)
476 struct mlx5_eq_table
*table
= &dev
->priv
.eq_table
;
477 u32 async_event_mask
= MLX5_ASYNC_EVENT_MASK
;
480 if (MLX5_CAP_GEN(dev
, pg
))
481 async_event_mask
|= (1ull << MLX5_EVENT_TYPE_PAGE_FAULT
);
483 if (MLX5_CAP_GEN(dev
, port_type
) == MLX5_CAP_PORT_TYPE_ETH
&&
484 MLX5_CAP_GEN(dev
, vport_group_manager
) &&
485 mlx5_core_is_pf(dev
))
486 async_event_mask
|= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE
);
488 err
= mlx5_create_map_eq(dev
, &table
->cmd_eq
, MLX5_EQ_VEC_CMD
,
489 MLX5_NUM_CMD_EQE
, 1ull << MLX5_EVENT_TYPE_CMD
,
490 "mlx5_cmd_eq", &dev
->priv
.uuari
.uars
[0]);
492 mlx5_core_warn(dev
, "failed to create cmd EQ %d\n", err
);
496 mlx5_cmd_use_events(dev
);
498 err
= mlx5_create_map_eq(dev
, &table
->async_eq
, MLX5_EQ_VEC_ASYNC
,
499 MLX5_NUM_ASYNC_EQE
, async_event_mask
,
500 "mlx5_async_eq", &dev
->priv
.uuari
.uars
[0]);
502 mlx5_core_warn(dev
, "failed to create async EQ %d\n", err
);
506 err
= mlx5_create_map_eq(dev
, &table
->pages_eq
,
508 /* TODO: sriov max_vf + */ 1,
509 1 << MLX5_EVENT_TYPE_PAGE_REQUEST
, "mlx5_pages_eq",
510 &dev
->priv
.uuari
.uars
[0]);
512 mlx5_core_warn(dev
, "failed to create pages EQ %d\n", err
);
519 mlx5_destroy_unmap_eq(dev
, &table
->async_eq
);
522 mlx5_cmd_use_polling(dev
);
523 mlx5_destroy_unmap_eq(dev
, &table
->cmd_eq
);
527 int mlx5_stop_eqs(struct mlx5_core_dev
*dev
)
529 struct mlx5_eq_table
*table
= &dev
->priv
.eq_table
;
532 err
= mlx5_destroy_unmap_eq(dev
, &table
->pages_eq
);
536 mlx5_destroy_unmap_eq(dev
, &table
->async_eq
);
537 mlx5_cmd_use_polling(dev
);
539 err
= mlx5_destroy_unmap_eq(dev
, &table
->cmd_eq
);
541 mlx5_cmd_use_events(dev
);
546 int mlx5_core_eq_query(struct mlx5_core_dev
*dev
, struct mlx5_eq
*eq
,
547 u32
*out
, int outlen
)
549 u32 in
[MLX5_ST_SZ_DW(query_eq_in
)] = {0};
552 MLX5_SET(query_eq_in
, in
, opcode
, MLX5_CMD_OP_QUERY_EQ
);
553 MLX5_SET(query_eq_in
, in
, eq_number
, eq
->eqn
);
555 err
= mlx5_cmd_exec(dev
, in
, sizeof(in
), out
, outlen
);
556 return err
? : mlx5_cmd_status_to_err_v2(out
);
558 EXPORT_SYMBOL_GPL(mlx5_core_eq_query
);