2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * $Id: mthca_mad.c 1349 2004-12-16 21:09:43Z roland $
37 #include <rdma/ib_verbs.h>
38 #include <rdma/ib_mad.h>
39 #include <rdma/ib_smi.h>
41 #include "mthca_dev.h"
42 #include "mthca_cmd.h"
45 MTHCA_VENDOR_CLASS1
= 0x9,
46 MTHCA_VENDOR_CLASS2
= 0xa
49 struct mthca_trap_mad
{
51 DECLARE_PCI_UNMAP_ADDR(mapping
)
54 static void update_sm_ah(struct mthca_dev
*dev
,
55 u8 port_num
, u16 lid
, u8 sl
)
58 struct ib_ah_attr ah_attr
;
61 if (!dev
->send_agent
[port_num
- 1][0])
64 memset(&ah_attr
, 0, sizeof ah_attr
);
67 ah_attr
.port_num
= port_num
;
69 new_ah
= ib_create_ah(dev
->send_agent
[port_num
- 1][0]->qp
->pd
,
74 spin_lock_irqsave(&dev
->sm_lock
, flags
);
75 if (dev
->sm_ah
[port_num
- 1])
76 ib_destroy_ah(dev
->sm_ah
[port_num
- 1]);
77 dev
->sm_ah
[port_num
- 1] = new_ah
;
78 spin_unlock_irqrestore(&dev
->sm_lock
, flags
);
82 * Snoop SM MADs for port info and P_Key table sets, so we can
83 * synthesize LID change and P_Key change events.
85 static void smp_snoop(struct ib_device
*ibdev
,
89 struct ib_event event
;
91 if ((mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
92 mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) &&
93 mad
->mad_hdr
.method
== IB_MGMT_METHOD_SET
) {
94 if (mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_PORT_INFO
) {
95 update_sm_ah(to_mdev(ibdev
), port_num
,
96 be16_to_cpup((__be16
*) (mad
->data
+ 58)),
97 (*(u8
*) (mad
->data
+ 76)) & 0xf);
100 event
.event
= IB_EVENT_LID_CHANGE
;
101 event
.element
.port_num
= port_num
;
102 ib_dispatch_event(&event
);
105 if (mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_PKEY_TABLE
) {
106 event
.device
= ibdev
;
107 event
.event
= IB_EVENT_PKEY_CHANGE
;
108 event
.element
.port_num
= port_num
;
109 ib_dispatch_event(&event
);
114 static void forward_trap(struct mthca_dev
*dev
,
118 int qpn
= mad
->mad_hdr
.mgmt_class
!= IB_MGMT_CLASS_SUBN_LID_ROUTED
;
119 struct mthca_trap_mad
*tmad
;
120 struct ib_sge gather_list
;
121 struct ib_send_wr
*bad_wr
, wr
= {
122 .opcode
= IB_WR_SEND
,
123 .sg_list
= &gather_list
,
125 .send_flags
= IB_SEND_SIGNALED
,
129 .remote_qkey
= qpn
? IB_QP1_QKEY
: 0,
134 struct ib_mad_agent
*agent
= dev
->send_agent
[port_num
- 1][qpn
];
139 tmad
= kmalloc(sizeof *tmad
, GFP_KERNEL
);
143 tmad
->mad
= kmalloc(sizeof *tmad
->mad
, GFP_KERNEL
);
149 memcpy(tmad
->mad
, mad
, sizeof *mad
);
151 wr
.wr
.ud
.mad_hdr
= &tmad
->mad
->mad_hdr
;
152 wr
.wr_id
= (unsigned long) tmad
;
154 gather_list
.addr
= dma_map_single(agent
->device
->dma_device
,
158 gather_list
.length
= sizeof *tmad
->mad
;
159 gather_list
.lkey
= to_mpd(agent
->qp
->pd
)->ntmr
.ibmr
.lkey
;
160 pci_unmap_addr_set(tmad
, mapping
, gather_list
.addr
);
163 * We rely here on the fact that MLX QPs don't use the
164 * address handle after the send is posted (this is
165 * wrong following the IB spec strictly, but we know
166 * it's OK for our devices).
168 spin_lock_irqsave(&dev
->sm_lock
, flags
);
169 wr
.wr
.ud
.ah
= dev
->sm_ah
[port_num
- 1];
171 ret
= ib_post_send_mad(agent
, &wr
, &bad_wr
);
174 spin_unlock_irqrestore(&dev
->sm_lock
, flags
);
177 dma_unmap_single(agent
->device
->dma_device
,
178 pci_unmap_addr(tmad
, mapping
),
187 int mthca_process_mad(struct ib_device
*ibdev
,
191 struct ib_grh
*in_grh
,
192 struct ib_mad
*in_mad
,
193 struct ib_mad
*out_mad
)
197 u16 slid
= in_wc
? in_wc
->slid
: be16_to_cpu(IB_LID_PERMISSIVE
);
199 /* Forward locally generated traps to the SM */
200 if (in_mad
->mad_hdr
.method
== IB_MGMT_METHOD_TRAP
&&
202 forward_trap(to_mdev(ibdev
), port_num
, in_mad
);
203 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_CONSUMED
;
207 * Only handle SM gets, sets and trap represses for SM class
209 * Only handle PMA and Mellanox vendor-specific class gets and
210 * sets for other classes.
212 if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
213 in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) {
214 if (in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_GET
&&
215 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_SET
&&
216 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_TRAP_REPRESS
)
217 return IB_MAD_RESULT_SUCCESS
;
220 * Don't process SMInfo queries or vendor-specific
221 * MADs -- the SMA can't handle them.
223 if (in_mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_SM_INFO
||
224 ((in_mad
->mad_hdr
.attr_id
& IB_SMP_ATTR_VENDOR_MASK
) ==
225 IB_SMP_ATTR_VENDOR_MASK
))
226 return IB_MAD_RESULT_SUCCESS
;
227 } else if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_PERF_MGMT
||
228 in_mad
->mad_hdr
.mgmt_class
== MTHCA_VENDOR_CLASS1
||
229 in_mad
->mad_hdr
.mgmt_class
== MTHCA_VENDOR_CLASS2
) {
230 if (in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_GET
&&
231 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_SET
)
232 return IB_MAD_RESULT_SUCCESS
;
234 return IB_MAD_RESULT_SUCCESS
;
236 err
= mthca_MAD_IFC(to_mdev(ibdev
),
237 mad_flags
& IB_MAD_IGNORE_MKEY
,
238 mad_flags
& IB_MAD_IGNORE_BKEY
,
239 port_num
, in_wc
, in_grh
, in_mad
, out_mad
,
242 mthca_err(to_mdev(ibdev
), "MAD_IFC failed\n");
243 return IB_MAD_RESULT_FAILURE
;
245 if (status
== MTHCA_CMD_STAT_BAD_PKT
)
246 return IB_MAD_RESULT_SUCCESS
;
248 mthca_err(to_mdev(ibdev
), "MAD_IFC returned status %02x\n",
250 return IB_MAD_RESULT_FAILURE
;
253 if (!out_mad
->mad_hdr
.status
)
254 smp_snoop(ibdev
, port_num
, in_mad
);
256 /* set return bit in status of directed route responses */
257 if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
)
258 out_mad
->mad_hdr
.status
|= cpu_to_be16(1 << 15);
260 if (in_mad
->mad_hdr
.method
== IB_MGMT_METHOD_TRAP_REPRESS
)
261 /* no response for trap repress */
262 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_CONSUMED
;
264 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_REPLY
;
267 static void send_handler(struct ib_mad_agent
*agent
,
268 struct ib_mad_send_wc
*mad_send_wc
)
270 struct mthca_trap_mad
*tmad
=
271 (void *) (unsigned long) mad_send_wc
->wr_id
;
273 dma_unmap_single(agent
->device
->dma_device
,
274 pci_unmap_addr(tmad
, mapping
),
281 int mthca_create_agents(struct mthca_dev
*dev
)
283 struct ib_mad_agent
*agent
;
286 spin_lock_init(&dev
->sm_lock
);
288 for (p
= 0; p
< dev
->limits
.num_ports
; ++p
)
289 for (q
= 0; q
<= 1; ++q
) {
290 agent
= ib_register_mad_agent(&dev
->ib_dev
, p
+ 1,
291 q
? IB_QPT_GSI
: IB_QPT_SMI
,
292 NULL
, 0, send_handler
,
296 dev
->send_agent
[p
][q
] = agent
;
302 for (p
= 0; p
< dev
->limits
.num_ports
; ++p
)
303 for (q
= 0; q
<= 1; ++q
)
304 if (dev
->send_agent
[p
][q
])
305 ib_unregister_mad_agent(dev
->send_agent
[p
][q
]);
307 return PTR_ERR(agent
);
310 void mthca_free_agents(struct mthca_dev
*dev
)
312 struct ib_mad_agent
*agent
;
315 for (p
= 0; p
< dev
->limits
.num_ports
; ++p
) {
316 for (q
= 0; q
<= 1; ++q
) {
317 agent
= dev
->send_agent
[p
][q
];
318 dev
->send_agent
[p
][q
] = NULL
;
319 ib_unregister_mad_agent(agent
);
323 ib_destroy_ah(dev
->sm_ah
[p
]);