2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/string.h>
36 #include <linux/slab.h>
38 #include <rdma/ib_verbs.h>
39 #include <rdma/ib_mad.h>
40 #include <rdma/ib_smi.h>
42 #include "mthca_dev.h"
43 #include "mthca_cmd.h"
46 MTHCA_VENDOR_CLASS1
= 0x9,
47 MTHCA_VENDOR_CLASS2
= 0xa
50 static int mthca_update_rate(struct mthca_dev
*dev
, u8 port_num
)
52 struct ib_port_attr
*tprops
= NULL
;
55 tprops
= kmalloc(sizeof *tprops
, GFP_KERNEL
);
59 ret
= ib_query_port(&dev
->ib_dev
, port_num
, tprops
);
61 printk(KERN_WARNING
"ib_query_port failed (%d) for %s port %d\n",
62 ret
, dev
->ib_dev
.name
, port_num
);
66 dev
->rate
[port_num
- 1] = tprops
->active_speed
*
67 ib_width_enum_to_int(tprops
->active_width
);
74 static void update_sm_ah(struct mthca_dev
*dev
,
75 u8 port_num
, u16 lid
, u8 sl
)
78 struct rdma_ah_attr ah_attr
;
81 if (!dev
->send_agent
[port_num
- 1][0])
84 memset(&ah_attr
, 0, sizeof ah_attr
);
85 ah_attr
.type
= rdma_ah_find_type(&dev
->ib_dev
, port_num
);
86 rdma_ah_set_dlid(&ah_attr
, lid
);
87 rdma_ah_set_sl(&ah_attr
, sl
);
88 rdma_ah_set_port_num(&ah_attr
, port_num
);
90 new_ah
= rdma_create_ah(dev
->send_agent
[port_num
- 1][0]->qp
->pd
,
95 spin_lock_irqsave(&dev
->sm_lock
, flags
);
96 if (dev
->sm_ah
[port_num
- 1])
97 rdma_destroy_ah(dev
->sm_ah
[port_num
- 1]);
98 dev
->sm_ah
[port_num
- 1] = new_ah
;
99 spin_unlock_irqrestore(&dev
->sm_lock
, flags
);
103 * Snoop SM MADs for port info and P_Key table sets, so we can
104 * synthesize LID change and P_Key change events.
106 static void smp_snoop(struct ib_device
*ibdev
,
108 const struct ib_mad
*mad
,
111 struct ib_event event
;
113 if ((mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
114 mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) &&
115 mad
->mad_hdr
.method
== IB_MGMT_METHOD_SET
) {
116 if (mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_PORT_INFO
) {
117 struct ib_port_info
*pinfo
=
118 (struct ib_port_info
*) ((struct ib_smp
*) mad
)->data
;
119 u16 lid
= be16_to_cpu(pinfo
->lid
);
121 mthca_update_rate(to_mdev(ibdev
), port_num
);
122 update_sm_ah(to_mdev(ibdev
), port_num
,
123 be16_to_cpu(pinfo
->sm_lid
),
124 pinfo
->neighbormtu_mastersmsl
& 0xf);
126 event
.device
= ibdev
;
127 event
.element
.port_num
= port_num
;
129 if (pinfo
->clientrereg_resv_subnetto
& 0x80) {
130 event
.event
= IB_EVENT_CLIENT_REREGISTER
;
131 ib_dispatch_event(&event
);
134 if (prev_lid
!= lid
) {
135 event
.event
= IB_EVENT_LID_CHANGE
;
136 ib_dispatch_event(&event
);
140 if (mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_PKEY_TABLE
) {
141 event
.device
= ibdev
;
142 event
.event
= IB_EVENT_PKEY_CHANGE
;
143 event
.element
.port_num
= port_num
;
144 ib_dispatch_event(&event
);
149 static void node_desc_override(struct ib_device
*dev
,
152 if ((mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
153 mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) &&
154 mad
->mad_hdr
.method
== IB_MGMT_METHOD_GET_RESP
&&
155 mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_NODE_DESC
) {
156 mutex_lock(&to_mdev(dev
)->cap_mask_mutex
);
157 memcpy(((struct ib_smp
*) mad
)->data
, dev
->node_desc
,
158 IB_DEVICE_NODE_DESC_MAX
);
159 mutex_unlock(&to_mdev(dev
)->cap_mask_mutex
);
163 static void forward_trap(struct mthca_dev
*dev
,
165 const struct ib_mad
*mad
)
167 int qpn
= mad
->mad_hdr
.mgmt_class
!= IB_MGMT_CLASS_SUBN_LID_ROUTED
;
168 struct ib_mad_send_buf
*send_buf
;
169 struct ib_mad_agent
*agent
= dev
->send_agent
[port_num
- 1][qpn
];
174 send_buf
= ib_create_send_mad(agent
, qpn
, 0, 0, IB_MGMT_MAD_HDR
,
175 IB_MGMT_MAD_DATA
, GFP_ATOMIC
,
176 IB_MGMT_BASE_VERSION
);
177 if (IS_ERR(send_buf
))
180 * We rely here on the fact that MLX QPs don't use the
181 * address handle after the send is posted (this is
182 * wrong following the IB spec strictly, but we know
183 * it's OK for our devices).
185 spin_lock_irqsave(&dev
->sm_lock
, flags
);
186 memcpy(send_buf
->mad
, mad
, sizeof *mad
);
187 if ((send_buf
->ah
= dev
->sm_ah
[port_num
- 1]))
188 ret
= ib_post_send_mad(send_buf
, NULL
);
191 spin_unlock_irqrestore(&dev
->sm_lock
, flags
);
194 ib_free_send_mad(send_buf
);
198 int mthca_process_mad(struct ib_device
*ibdev
,
201 const struct ib_wc
*in_wc
,
202 const struct ib_grh
*in_grh
,
203 const struct ib_mad_hdr
*in
, size_t in_mad_size
,
204 struct ib_mad_hdr
*out
, size_t *out_mad_size
,
205 u16
*out_mad_pkey_index
)
208 u16 slid
= in_wc
? in_wc
->slid
: be16_to_cpu(IB_LID_PERMISSIVE
);
210 struct ib_port_attr pattr
;
211 const struct ib_mad
*in_mad
= (const struct ib_mad
*)in
;
212 struct ib_mad
*out_mad
= (struct ib_mad
*)out
;
214 if (WARN_ON_ONCE(in_mad_size
!= sizeof(*in_mad
) ||
215 *out_mad_size
!= sizeof(*out_mad
)))
216 return IB_MAD_RESULT_FAILURE
;
218 /* Forward locally generated traps to the SM */
219 if (in_mad
->mad_hdr
.method
== IB_MGMT_METHOD_TRAP
&&
221 forward_trap(to_mdev(ibdev
), port_num
, in_mad
);
222 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_CONSUMED
;
226 * Only handle SM gets, sets and trap represses for SM class
228 * Only handle PMA and Mellanox vendor-specific class gets and
229 * sets for other classes.
231 if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
232 in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) {
233 if (in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_GET
&&
234 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_SET
&&
235 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_TRAP_REPRESS
)
236 return IB_MAD_RESULT_SUCCESS
;
239 * Don't process SMInfo queries or vendor-specific
240 * MADs -- the SMA can't handle them.
242 if (in_mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_SM_INFO
||
243 ((in_mad
->mad_hdr
.attr_id
& IB_SMP_ATTR_VENDOR_MASK
) ==
244 IB_SMP_ATTR_VENDOR_MASK
))
245 return IB_MAD_RESULT_SUCCESS
;
246 } else if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_PERF_MGMT
||
247 in_mad
->mad_hdr
.mgmt_class
== MTHCA_VENDOR_CLASS1
||
248 in_mad
->mad_hdr
.mgmt_class
== MTHCA_VENDOR_CLASS2
) {
249 if (in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_GET
&&
250 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_SET
)
251 return IB_MAD_RESULT_SUCCESS
;
253 return IB_MAD_RESULT_SUCCESS
;
254 if ((in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
255 in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) &&
256 in_mad
->mad_hdr
.method
== IB_MGMT_METHOD_SET
&&
257 in_mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_PORT_INFO
&&
258 !ib_query_port(ibdev
, port_num
, &pattr
))
259 prev_lid
= pattr
.lid
;
261 err
= mthca_MAD_IFC(to_mdev(ibdev
),
262 mad_flags
& IB_MAD_IGNORE_MKEY
,
263 mad_flags
& IB_MAD_IGNORE_BKEY
,
264 port_num
, in_wc
, in_grh
, in_mad
, out_mad
);
266 return IB_MAD_RESULT_SUCCESS
;
268 mthca_err(to_mdev(ibdev
), "MAD_IFC returned %d\n", err
);
269 return IB_MAD_RESULT_FAILURE
;
272 if (!out_mad
->mad_hdr
.status
) {
273 smp_snoop(ibdev
, port_num
, in_mad
, prev_lid
);
274 node_desc_override(ibdev
, out_mad
);
277 /* set return bit in status of directed route responses */
278 if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
)
279 out_mad
->mad_hdr
.status
|= cpu_to_be16(1 << 15);
281 if (in_mad
->mad_hdr
.method
== IB_MGMT_METHOD_TRAP_REPRESS
)
282 /* no response for trap repress */
283 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_CONSUMED
;
285 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_REPLY
;
288 static void send_handler(struct ib_mad_agent
*agent
,
289 struct ib_mad_send_wc
*mad_send_wc
)
291 ib_free_send_mad(mad_send_wc
->send_buf
);
294 int mthca_create_agents(struct mthca_dev
*dev
)
296 struct ib_mad_agent
*agent
;
300 spin_lock_init(&dev
->sm_lock
);
302 for (p
= 0; p
< dev
->limits
.num_ports
; ++p
)
303 for (q
= 0; q
<= 1; ++q
) {
304 agent
= ib_register_mad_agent(&dev
->ib_dev
, p
+ 1,
305 q
? IB_QPT_GSI
: IB_QPT_SMI
,
306 NULL
, 0, send_handler
,
309 ret
= PTR_ERR(agent
);
312 dev
->send_agent
[p
][q
] = agent
;
316 for (p
= 1; p
<= dev
->limits
.num_ports
; ++p
) {
317 ret
= mthca_update_rate(dev
, p
);
319 mthca_err(dev
, "Failed to obtain port %d rate."
328 for (p
= 0; p
< dev
->limits
.num_ports
; ++p
)
329 for (q
= 0; q
<= 1; ++q
)
330 if (dev
->send_agent
[p
][q
])
331 ib_unregister_mad_agent(dev
->send_agent
[p
][q
]);
336 void mthca_free_agents(struct mthca_dev
*dev
)
338 struct ib_mad_agent
*agent
;
341 for (p
= 0; p
< dev
->limits
.num_ports
; ++p
) {
342 for (q
= 0; q
<= 1; ++q
) {
343 agent
= dev
->send_agent
[p
][q
];
344 dev
->send_agent
[p
][q
] = NULL
;
345 ib_unregister_mad_agent(agent
);
349 rdma_destroy_ah(dev
->sm_ah
[p
]);