]>
Commit | Line | Data |
---|---|---|
77241056 | 1 | /* |
05d6ac1d | 2 | * Copyright(c) 2015, 2016 Intel Corporation. |
77241056 MM |
3 | * |
4 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
5 | * redistributing this file, you may do so under either license. | |
6 | * | |
7 | * GPL LICENSE SUMMARY | |
8 | * | |
77241056 MM |
9 | * This program is free software; you can redistribute it and/or modify |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but | |
14 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | * General Public License for more details. | |
17 | * | |
18 | * BSD LICENSE | |
19 | * | |
77241056 MM |
20 | * Redistribution and use in source and binary forms, with or without |
21 | * modification, are permitted provided that the following conditions | |
22 | * are met: | |
23 | * | |
24 | * - Redistributions of source code must retain the above copyright | |
25 | * notice, this list of conditions and the following disclaimer. | |
26 | * - Redistributions in binary form must reproduce the above copyright | |
27 | * notice, this list of conditions and the following disclaimer in | |
28 | * the documentation and/or other materials provided with the | |
29 | * distribution. | |
30 | * - Neither the name of Intel Corporation nor the names of its | |
31 | * contributors may be used to endorse or promote products derived | |
32 | * from this software without specific prior written permission. | |
33 | * | |
34 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
35 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
36 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
37 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
38 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
39 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
40 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
41 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
42 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
43 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
44 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
45 | * | |
46 | */ | |
47 | ||
48 | #include <linux/err.h> | |
49 | #include <linux/vmalloc.h> | |
50 | #include <linux/hash.h> | |
51 | #include <linux/module.h> | |
77241056 | 52 | #include <linux/seq_file.h> |
ec4274f1 DD |
53 | #include <rdma/rdma_vt.h> |
54 | #include <rdma/rdmavt_qp.h> | |
1ac57c50 | 55 | #include <rdma/ib_verbs.h> |
77241056 MM |
56 | |
57 | #include "hfi.h" | |
58 | #include "qp.h" | |
59 | #include "trace.h" | |
45842abb | 60 | #include "verbs_txreq.h" |
77241056 | 61 | |
a2c2d608 | 62 | unsigned int hfi1_qp_table_size = 256; |
77241056 MM |
63 | module_param_named(qp_table_size, hfi1_qp_table_size, uint, S_IRUGO); |
64 | MODULE_PARM_DESC(qp_table_size, "QP table size"); | |
65 | ||
895420dd | 66 | static void flush_tx_list(struct rvt_qp *qp); |
77241056 MM |
67 | static int iowait_sleep( |
68 | struct sdma_engine *sde, | |
69 | struct iowait *wait, | |
70 | struct sdma_txreq *stx, | |
bcad2913 KW |
71 | unsigned int seq, |
72 | bool pkts_sent); | |
77241056 | 73 | static void iowait_wakeup(struct iowait *wait, int reason); |
a545f530 | 74 | static void iowait_sdma_drained(struct iowait *wait); |
91702b4a | 75 | static void qp_pio_drain(struct rvt_qp *qp); |
77241056 | 76 | |
1ac57c50 MM |
77 | const struct rvt_operation_params hfi1_post_parms[RVT_OPERATION_MAX] = { |
78 | [IB_WR_RDMA_WRITE] = { | |
79 | .length = sizeof(struct ib_rdma_wr), | |
80 | .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC), | |
81 | }, | |
82 | ||
83 | [IB_WR_RDMA_READ] = { | |
84 | .length = sizeof(struct ib_rdma_wr), | |
85 | .qpt_support = BIT(IB_QPT_RC), | |
86 | .flags = RVT_OPERATION_ATOMIC, | |
87 | }, | |
88 | ||
89 | [IB_WR_ATOMIC_CMP_AND_SWP] = { | |
90 | .length = sizeof(struct ib_atomic_wr), | |
91 | .qpt_support = BIT(IB_QPT_RC), | |
92 | .flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE, | |
93 | }, | |
94 | ||
95 | [IB_WR_ATOMIC_FETCH_AND_ADD] = { | |
96 | .length = sizeof(struct ib_atomic_wr), | |
97 | .qpt_support = BIT(IB_QPT_RC), | |
98 | .flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE, | |
99 | }, | |
100 | ||
101 | [IB_WR_RDMA_WRITE_WITH_IMM] = { | |
102 | .length = sizeof(struct ib_rdma_wr), | |
103 | .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC), | |
104 | }, | |
105 | ||
106 | [IB_WR_SEND] = { | |
107 | .length = sizeof(struct ib_send_wr), | |
108 | .qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) | | |
109 | BIT(IB_QPT_UC) | BIT(IB_QPT_RC), | |
110 | }, | |
111 | ||
112 | [IB_WR_SEND_WITH_IMM] = { | |
113 | .length = sizeof(struct ib_send_wr), | |
114 | .qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) | | |
115 | BIT(IB_QPT_UC) | BIT(IB_QPT_RC), | |
116 | }, | |
117 | ||
c72cfe3e JX |
118 | [IB_WR_REG_MR] = { |
119 | .length = sizeof(struct ib_reg_wr), | |
120 | .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC), | |
121 | .flags = RVT_OPERATION_LOCAL, | |
122 | }, | |
123 | ||
124 | [IB_WR_LOCAL_INV] = { | |
125 | .length = sizeof(struct ib_send_wr), | |
126 | .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC), | |
127 | .flags = RVT_OPERATION_LOCAL, | |
128 | }, | |
129 | ||
130 | [IB_WR_SEND_WITH_INV] = { | |
131 | .length = sizeof(struct ib_send_wr), | |
132 | .qpt_support = BIT(IB_QPT_RC), | |
133 | }, | |
134 | ||
1ac57c50 MM |
135 | }; |
136 | ||
895420dd | 137 | static void flush_tx_list(struct rvt_qp *qp) |
77241056 | 138 | { |
4c6829c5 DD |
139 | struct hfi1_qp_priv *priv = qp->priv; |
140 | ||
141 | while (!list_empty(&priv->s_iowait.tx_head)) { | |
77241056 MM |
142 | struct sdma_txreq *tx; |
143 | ||
144 | tx = list_first_entry( | |
4c6829c5 | 145 | &priv->s_iowait.tx_head, |
77241056 MM |
146 | struct sdma_txreq, |
147 | list); | |
148 | list_del_init(&tx->list); | |
149 | hfi1_put_txreq( | |
150 | container_of(tx, struct verbs_txreq, txreq)); | |
151 | } | |
152 | } | |
153 | ||
895420dd | 154 | static void flush_iowait(struct rvt_qp *qp) |
77241056 | 155 | { |
4c6829c5 | 156 | struct hfi1_qp_priv *priv = qp->priv; |
77241056 | 157 | unsigned long flags; |
4e045572 | 158 | seqlock_t *lock = priv->s_iowait.lock; |
77241056 | 159 | |
4e045572 MM |
160 | if (!lock) |
161 | return; | |
162 | write_seqlock_irqsave(lock, flags); | |
4c6829c5 DD |
163 | if (!list_empty(&priv->s_iowait.list)) { |
164 | list_del_init(&priv->s_iowait.list); | |
4e045572 | 165 | priv->s_iowait.lock = NULL; |
4d6f85c3 | 166 | rvt_put_qp(qp); |
77241056 | 167 | } |
4e045572 | 168 | write_sequnlock_irqrestore(lock, flags); |
77241056 MM |
169 | } |
170 | ||
171 | static inline int opa_mtu_enum_to_int(int mtu) | |
172 | { | |
173 | switch (mtu) { | |
174 | case OPA_MTU_8192: return 8192; | |
175 | case OPA_MTU_10240: return 10240; | |
176 | default: return -1; | |
177 | } | |
178 | } | |
179 | ||
180 | /** | |
181 | * This function is what we would push to the core layer if we wanted to be a | |
182 | * "first class citizen". Instead we hide this here and rely on Verbs ULPs | |
183 | * to blindly pass the MTU enum value from the PathRecord to us. | |
77241056 MM |
184 | */ |
185 | static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu) | |
186 | { | |
ef699e84 | 187 | int val; |
77241056 | 188 | |
ef699e84 SS |
189 | /* Constraining 10KB packets to 8KB packets */ |
190 | if (mtu == (enum ib_mtu)OPA_MTU_10240) | |
191 | mtu = OPA_MTU_8192; | |
192 | val = opa_mtu_enum_to_int((int)mtu); | |
77241056 MM |
193 | if (val > 0) |
194 | return val; | |
195 | return ib_mtu_enum_to_int(mtu); | |
196 | } | |
197 | ||
ec4274f1 DD |
198 | int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
199 | int attr_mask, struct ib_udata *udata) | |
77241056 | 200 | { |
ec4274f1 | 201 | struct ib_qp *ibqp = &qp->ibqp; |
77241056 | 202 | struct hfi1_ibdev *dev = to_idev(ibqp->device); |
d7b8ba51 | 203 | struct hfi1_devdata *dd = dd_from_dev(dev); |
ec4274f1 | 204 | u8 sc; |
77241056 MM |
205 | |
206 | if (attr_mask & IB_QP_AV) { | |
d7b8ba51 | 207 | sc = ah_to_sc(ibqp->device, &attr->ah_attr); |
31e7af1c IW |
208 | if (sc == 0xf) |
209 | return -EINVAL; | |
210 | ||
d7b8ba51 MM |
211 | if (!qp_to_sdma_engine(qp, sc) && |
212 | dd->flags & HFI1_HAS_SEND_DMA) | |
ec4274f1 | 213 | return -EINVAL; |
721d0427 JJ |
214 | |
215 | if (!qp_to_send_context(qp, sc)) | |
216 | return -EINVAL; | |
77241056 MM |
217 | } |
218 | ||
219 | if (attr_mask & IB_QP_ALT_PATH) { | |
d7b8ba51 | 220 | sc = ah_to_sc(ibqp->device, &attr->alt_ah_attr); |
31e7af1c IW |
221 | if (sc == 0xf) |
222 | return -EINVAL; | |
223 | ||
d7b8ba51 MM |
224 | if (!qp_to_sdma_engine(qp, sc) && |
225 | dd->flags & HFI1_HAS_SEND_DMA) | |
ec4274f1 | 226 | return -EINVAL; |
721d0427 JJ |
227 | |
228 | if (!qp_to_send_context(qp, sc)) | |
229 | return -EINVAL; | |
77241056 MM |
230 | } |
231 | ||
ec4274f1 DD |
232 | return 0; |
233 | } | |
77241056 | 234 | |
d98bb7f7 DH |
235 | /* |
236 | * qp_set_16b - Set the hdr_type based on whether the slid or the | |
237 | * dlid in the connection is extended. Only applicable for RC and UC | |
238 | * QPs. UD QPs determine this on the fly from the ah in the wqe | |
239 | */ | |
240 | static inline void qp_set_16b(struct rvt_qp *qp) | |
241 | { | |
242 | struct hfi1_pportdata *ppd; | |
243 | struct hfi1_ibport *ibp; | |
244 | struct hfi1_qp_priv *priv = qp->priv; | |
245 | ||
246 | /* Update ah_attr to account for extended LIDs */ | |
247 | hfi1_update_ah_attr(qp->ibqp.device, &qp->remote_ah_attr); | |
248 | ||
249 | /* Create 32 bit LIDs */ | |
250 | hfi1_make_opa_lid(&qp->remote_ah_attr); | |
251 | ||
252 | if (!(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) | |
253 | return; | |
254 | ||
255 | ibp = to_iport(qp->ibqp.device, qp->port_num); | |
256 | ppd = ppd_from_ibp(ibp); | |
257 | priv->hdr_type = hfi1_get_hdr_type(ppd->lid, &qp->remote_ah_attr); | |
258 | } | |
259 | ||
ec4274f1 DD |
260 | void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
261 | int attr_mask, struct ib_udata *udata) | |
262 | { | |
263 | struct ib_qp *ibqp = &qp->ibqp; | |
264 | struct hfi1_qp_priv *priv = qp->priv; | |
77241056 MM |
265 | |
266 | if (attr_mask & IB_QP_AV) { | |
4c6829c5 DD |
267 | priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); |
268 | priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); | |
cef504c5 | 269 | priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); |
d98bb7f7 | 270 | qp_set_16b(qp); |
77241056 MM |
271 | } |
272 | ||
ec4274f1 DD |
273 | if (attr_mask & IB_QP_PATH_MIG_STATE && |
274 | attr->path_mig_state == IB_MIG_MIGRATED && | |
275 | qp->s_mig_state == IB_MIG_ARMED) { | |
276 | qp->s_flags |= RVT_S_AHG_CLEAR; | |
277 | priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr); | |
278 | priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); | |
cef504c5 | 279 | priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc); |
d98bb7f7 | 280 | qp_set_16b(qp); |
77241056 | 281 | } |
77241056 MM |
282 | } |
283 | ||
46a80d62 MM |
284 | /** |
285 | * hfi1_check_send_wqe - validate wqe | |
286 | * @qp - The qp | |
287 | * @wqe - The built wqe | |
288 | * | |
289 | * validate wqe. This is called | |
290 | * prior to inserting the wqe into | |
291 | * the ring but after the wqe has been | |
292 | * setup. | |
293 | * | |
294 | * Returns 0 on success, -EINVAL on failure | |
295 | * | |
296 | */ | |
297 | int hfi1_check_send_wqe(struct rvt_qp *qp, | |
298 | struct rvt_swqe *wqe) | |
31e7af1c IW |
299 | { |
300 | struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); | |
46a80d62 | 301 | struct rvt_ah *ah; |
31e7af1c | 302 | |
46a80d62 MM |
303 | switch (qp->ibqp.qp_type) { |
304 | case IB_QPT_RC: | |
305 | case IB_QPT_UC: | |
306 | if (wqe->length > 0x80000000U) | |
307 | return -EINVAL; | |
308 | break; | |
309 | case IB_QPT_SMI: | |
310 | ah = ibah_to_rvtah(wqe->ud_wr.ah); | |
311 | if (wqe->length > (1 << ah->log_pmtu)) | |
312 | return -EINVAL; | |
313 | break; | |
314 | case IB_QPT_GSI: | |
315 | case IB_QPT_UD: | |
316 | ah = ibah_to_rvtah(wqe->ud_wr.ah); | |
317 | if (wqe->length > (1 << ah->log_pmtu)) | |
318 | return -EINVAL; | |
d8966fcd | 319 | if (ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)] == 0xf) |
46a80d62 MM |
320 | return -EINVAL; |
321 | default: | |
322 | break; | |
31e7af1c | 323 | } |
91702b4a | 324 | return wqe->length <= piothreshold; |
31e7af1c IW |
325 | } |
326 | ||
46a80d62 MM |
327 | /** |
328 | * _hfi1_schedule_send - schedule progress | |
329 | * @qp: the QP | |
330 | * | |
331 | * This schedules qp progress w/o regard to the s_flags. | |
332 | * | |
333 | * It is only used in the post send, which doesn't hold | |
334 | * the s_lock. | |
335 | */ | |
336 | void _hfi1_schedule_send(struct rvt_qp *qp) | |
337 | { | |
338 | struct hfi1_qp_priv *priv = qp->priv; | |
339 | struct hfi1_ibport *ibp = | |
340 | to_iport(qp->ibqp.device, qp->port_num); | |
341 | struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); | |
342 | struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); | |
343 | ||
344 | iowait_schedule(&priv->s_iowait, ppd->hfi1_wq, | |
345 | priv->s_sde ? | |
346 | priv->s_sde->cpu : | |
347 | cpumask_first(cpumask_of_node(dd->node))); | |
348 | } | |
349 | ||
14553ca1 MM |
350 | static void qp_pio_drain(struct rvt_qp *qp) |
351 | { | |
352 | struct hfi1_ibdev *dev; | |
353 | struct hfi1_qp_priv *priv = qp->priv; | |
354 | ||
355 | if (!priv->s_sendcontext) | |
356 | return; | |
357 | dev = to_idev(qp->ibqp.device); | |
358 | while (iowait_pio_pending(&priv->s_iowait)) { | |
359 | write_seqlock_irq(&dev->iowait_lock); | |
360 | hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 1); | |
361 | write_sequnlock_irq(&dev->iowait_lock); | |
362 | iowait_pio_drain(&priv->s_iowait); | |
363 | write_seqlock_irq(&dev->iowait_lock); | |
364 | hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 0); | |
365 | write_sequnlock_irq(&dev->iowait_lock); | |
366 | } | |
367 | } | |
368 | ||
46a80d62 MM |
369 | /** |
370 | * hfi1_schedule_send - schedule progress | |
371 | * @qp: the QP | |
372 | * | |
373 | * This schedules qp progress and caller should hold | |
374 | * the s_lock. | |
375 | */ | |
376 | void hfi1_schedule_send(struct rvt_qp *qp) | |
377 | { | |
68e78b3d | 378 | lockdep_assert_held(&qp->s_lock); |
46a80d62 MM |
379 | if (hfi1_send_ok(qp)) |
380 | _hfi1_schedule_send(qp); | |
381 | } | |
382 | ||
895420dd | 383 | void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag) |
77241056 MM |
384 | { |
385 | unsigned long flags; | |
386 | ||
387 | spin_lock_irqsave(&qp->s_lock, flags); | |
388 | if (qp->s_flags & flag) { | |
389 | qp->s_flags &= ~flag; | |
390 | trace_hfi1_qpwakeup(qp, flag); | |
391 | hfi1_schedule_send(qp); | |
392 | } | |
393 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
394 | /* Notify hfi1_destroy_qp() if it is waiting. */ | |
4d6f85c3 | 395 | rvt_put_qp(qp); |
77241056 MM |
396 | } |
397 | ||
398 | static int iowait_sleep( | |
399 | struct sdma_engine *sde, | |
400 | struct iowait *wait, | |
401 | struct sdma_txreq *stx, | |
bcad2913 KW |
402 | uint seq, |
403 | bool pkts_sent) | |
77241056 MM |
404 | { |
405 | struct verbs_txreq *tx = container_of(stx, struct verbs_txreq, txreq); | |
895420dd | 406 | struct rvt_qp *qp; |
4c6829c5 | 407 | struct hfi1_qp_priv *priv; |
77241056 MM |
408 | unsigned long flags; |
409 | int ret = 0; | |
410 | struct hfi1_ibdev *dev; | |
411 | ||
412 | qp = tx->qp; | |
4c6829c5 | 413 | priv = qp->priv; |
77241056 MM |
414 | |
415 | spin_lock_irqsave(&qp->s_lock, flags); | |
83693bd1 | 416 | if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { |
77241056 MM |
417 | /* |
418 | * If we couldn't queue the DMA request, save the info | |
419 | * and try again later rather than destroying the | |
420 | * buffer and undoing the side effects of the copy. | |
421 | */ | |
422 | /* Make a common routine? */ | |
423 | dev = &sde->dd->verbs_dev; | |
424 | list_add_tail(&stx->list, &wait->tx_head); | |
425 | write_seqlock(&dev->iowait_lock); | |
426 | if (sdma_progress(sde, seq, stx)) | |
427 | goto eagain; | |
4c6829c5 | 428 | if (list_empty(&priv->s_iowait.list)) { |
77241056 MM |
429 | struct hfi1_ibport *ibp = |
430 | to_iport(qp->ibqp.device, qp->port_num); | |
431 | ||
4eb06882 | 432 | ibp->rvp.n_dmawait++; |
54d10c1e | 433 | qp->s_flags |= RVT_S_WAIT_DMA_DESC; |
bcad2913 KW |
434 | iowait_queue(pkts_sent, &priv->s_iowait, |
435 | &sde->dmawait); | |
4e045572 | 436 | priv->s_iowait.lock = &dev->iowait_lock; |
54d10c1e | 437 | trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC); |
4d6f85c3 | 438 | rvt_get_qp(qp); |
77241056 MM |
439 | } |
440 | write_sequnlock(&dev->iowait_lock); | |
54d10c1e | 441 | qp->s_flags &= ~RVT_S_BUSY; |
77241056 MM |
442 | spin_unlock_irqrestore(&qp->s_lock, flags); |
443 | ret = -EBUSY; | |
444 | } else { | |
445 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
446 | hfi1_put_txreq(tx); | |
447 | } | |
448 | return ret; | |
449 | eagain: | |
450 | write_sequnlock(&dev->iowait_lock); | |
451 | spin_unlock_irqrestore(&qp->s_lock, flags); | |
452 | list_del_init(&stx->list); | |
453 | return -EAGAIN; | |
454 | } | |
455 | ||
456 | static void iowait_wakeup(struct iowait *wait, int reason) | |
457 | { | |
895420dd | 458 | struct rvt_qp *qp = iowait_to_qp(wait); |
77241056 MM |
459 | |
460 | WARN_ON(reason != SDMA_AVAIL_REASON); | |
54d10c1e | 461 | hfi1_qp_wakeup(qp, RVT_S_WAIT_DMA_DESC); |
77241056 MM |
462 | } |
463 | ||
a545f530 MM |
464 | static void iowait_sdma_drained(struct iowait *wait) |
465 | { | |
466 | struct rvt_qp *qp = iowait_to_qp(wait); | |
7049de65 | 467 | unsigned long flags; |
a545f530 MM |
468 | |
469 | /* | |
470 | * This happens when the send engine notes | |
471 | * a QP in the error state and cannot | |
472 | * do the flush work until that QP's | |
473 | * sdma work has finished. | |
474 | */ | |
7049de65 | 475 | spin_lock_irqsave(&qp->s_lock, flags); |
a545f530 MM |
476 | if (qp->s_flags & RVT_S_WAIT_DMA) { |
477 | qp->s_flags &= ~RVT_S_WAIT_DMA; | |
478 | hfi1_schedule_send(qp); | |
479 | } | |
7049de65 | 480 | spin_unlock_irqrestore(&qp->s_lock, flags); |
a545f530 MM |
481 | } |
482 | ||
77241056 MM |
483 | /** |
484 | * | |
485 | * qp_to_sdma_engine - map a qp to a send engine | |
486 | * @qp: the QP | |
487 | * @sc5: the 5 bit sc | |
488 | * | |
489 | * Return: | |
490 | * A send engine for the qp or NULL for SMI type qp. | |
491 | */ | |
895420dd | 492 | struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5) |
77241056 MM |
493 | { |
494 | struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); | |
495 | struct sdma_engine *sde; | |
496 | ||
497 | if (!(dd->flags & HFI1_HAS_SEND_DMA)) | |
498 | return NULL; | |
499 | switch (qp->ibqp.qp_type) { | |
77241056 MM |
500 | case IB_QPT_SMI: |
501 | return NULL; | |
502 | default: | |
503 | break; | |
504 | } | |
505 | sde = sdma_select_engine_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, sc5); | |
506 | return sde; | |
507 | } | |
508 | ||
35f6befc JJ |
509 | /* |
510 | * qp_to_send_context - map a qp to a send context | |
511 | * @qp: the QP | |
512 | * @sc5: the 5 bit sc | |
513 | * | |
514 | * Return: | |
515 | * A send context for the qp | |
516 | */ | |
517 | struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5) | |
518 | { | |
519 | struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device); | |
520 | ||
521 | switch (qp->ibqp.qp_type) { | |
522 | case IB_QPT_SMI: | |
523 | /* SMA packets to VL15 */ | |
524 | return dd->vld[15].sc; | |
525 | default: | |
526 | break; | |
527 | } | |
528 | ||
529 | return pio_select_send_context_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, | |
530 | sc5); | |
531 | } | |
532 | ||
77241056 MM |
533 | struct qp_iter { |
534 | struct hfi1_ibdev *dev; | |
895420dd | 535 | struct rvt_qp *qp; |
77241056 MM |
536 | int specials; |
537 | int n; | |
538 | }; | |
539 | ||
540 | struct qp_iter *qp_iter_init(struct hfi1_ibdev *dev) | |
541 | { | |
542 | struct qp_iter *iter; | |
543 | ||
544 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); | |
545 | if (!iter) | |
546 | return NULL; | |
547 | ||
548 | iter->dev = dev; | |
ec3f2c12 | 549 | iter->specials = dev->rdi.ibdev.phys_port_cnt * 2; |
77241056 MM |
550 | |
551 | return iter; | |
552 | } | |
553 | ||
554 | int qp_iter_next(struct qp_iter *iter) | |
555 | { | |
556 | struct hfi1_ibdev *dev = iter->dev; | |
557 | int n = iter->n; | |
558 | int ret = 1; | |
895420dd DD |
559 | struct rvt_qp *pqp = iter->qp; |
560 | struct rvt_qp *qp; | |
77241056 MM |
561 | |
562 | /* | |
563 | * The approach is to consider the special qps | |
564 | * as an additional table entries before the | |
565 | * real hash table. Since the qp code sets | |
566 | * the qp->next hash link to NULL, this works just fine. | |
567 | * | |
568 | * iter->specials is 2 * # ports | |
569 | * | |
570 | * n = 0..iter->specials is the special qp indices | |
571 | * | |
1c4b7d97 | 572 | * n = iter->specials..dev->rdi.qp_dev->qp_table_size+iter->specials are |
77241056 MM |
573 | * the potential hash bucket entries |
574 | * | |
575 | */ | |
1c4b7d97 | 576 | for (; n < dev->rdi.qp_dev->qp_table_size + iter->specials; n++) { |
77241056 MM |
577 | if (pqp) { |
578 | qp = rcu_dereference(pqp->next); | |
579 | } else { | |
580 | if (n < iter->specials) { | |
581 | struct hfi1_pportdata *ppd; | |
582 | struct hfi1_ibport *ibp; | |
583 | int pidx; | |
584 | ||
ec3f2c12 | 585 | pidx = n % dev->rdi.ibdev.phys_port_cnt; |
77241056 MM |
586 | ppd = &dd_from_dev(dev)->pport[pidx]; |
587 | ibp = &ppd->ibport_data; | |
588 | ||
589 | if (!(n & 1)) | |
4eb06882 | 590 | qp = rcu_dereference(ibp->rvp.qp[0]); |
77241056 | 591 | else |
4eb06882 | 592 | qp = rcu_dereference(ibp->rvp.qp[1]); |
77241056 MM |
593 | } else { |
594 | qp = rcu_dereference( | |
1c4b7d97 | 595 | dev->rdi.qp_dev->qp_table[ |
77241056 MM |
596 | (n - iter->specials)]); |
597 | } | |
598 | } | |
599 | pqp = qp; | |
600 | if (qp) { | |
601 | iter->qp = qp; | |
602 | iter->n = n; | |
603 | return 0; | |
604 | } | |
605 | } | |
606 | return ret; | |
607 | } | |
608 | ||
609 | static const char * const qp_type_str[] = { | |
610 | "SMI", "GSI", "RC", "UC", "UD", | |
611 | }; | |
612 | ||
895420dd | 613 | static int qp_idle(struct rvt_qp *qp) |
77241056 MM |
614 | { |
615 | return | |
616 | qp->s_last == qp->s_acked && | |
617 | qp->s_acked == qp->s_cur && | |
618 | qp->s_cur == qp->s_tail && | |
619 | qp->s_tail == qp->s_head; | |
620 | } | |
621 | ||
622 | void qp_iter_print(struct seq_file *s, struct qp_iter *iter) | |
623 | { | |
895420dd DD |
624 | struct rvt_swqe *wqe; |
625 | struct rvt_qp *qp = iter->qp; | |
4c6829c5 | 626 | struct hfi1_qp_priv *priv = qp->priv; |
77241056 | 627 | struct sdma_engine *sde; |
721d0427 | 628 | struct send_context *send_context; |
642aaab5 | 629 | struct rvt_ack_entry *e = NULL; |
77241056 | 630 | |
4c6829c5 | 631 | sde = qp_to_sdma_engine(qp, priv->s_sc); |
83693bd1 | 632 | wqe = rvt_get_swqe_ptr(qp, qp->s_last); |
721d0427 | 633 | send_context = qp_to_send_context(qp, priv->s_sc); |
642aaab5 KW |
634 | if (qp->s_ack_queue) |
635 | e = &qp->s_ack_queue[qp->s_tail_ack_queue]; | |
77241056 | 636 | seq_printf(s, |
642aaab5 | 637 | "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d E %x %x %x\n", |
77241056 MM |
638 | iter->n, |
639 | qp_idle(qp) ? "I" : "B", | |
640 | qp->ibqp.qp_num, | |
641 | atomic_read(&qp->refcount), | |
642 | qp_type_str[qp->ibqp.qp_type], | |
643 | qp->state, | |
644 | wqe ? wqe->wr.opcode : 0, | |
645 | qp->s_hdrwords, | |
646 | qp->s_flags, | |
14553ca1 MM |
647 | iowait_sdma_pending(&priv->s_iowait), |
648 | iowait_pio_pending(&priv->s_iowait), | |
4c6829c5 | 649 | !list_empty(&priv->s_iowait.list), |
77241056 MM |
650 | qp->timeout, |
651 | wqe ? wqe->ssn : 0, | |
652 | qp->s_lsn, | |
653 | qp->s_last_psn, | |
654 | qp->s_psn, qp->s_next_psn, | |
655 | qp->s_sending_psn, qp->s_sending_hpsn, | |
d7c76e91 | 656 | qp->r_psn, |
77241056 MM |
657 | qp->s_last, qp->s_acked, qp->s_cur, |
658 | qp->s_tail, qp->s_head, qp->s_size, | |
3585254d | 659 | qp->s_avail, |
ff8d836e KW |
660 | /* ack_queue ring pointers, size */ |
661 | qp->s_tail_ack_queue, qp->r_head_ack_queue, | |
662 | HFI1_MAX_RDMA_ATOMIC, | |
663 | /* remote QP info */ | |
77241056 | 664 | qp->remote_qpn, |
d8966fcd DC |
665 | rdma_ah_get_dlid(&qp->remote_ah_attr), |
666 | rdma_ah_get_sl(&qp->remote_ah_attr), | |
77241056 | 667 | qp->pmtu, |
20658661 | 668 | qp->s_retry, |
77241056 | 669 | qp->s_retry_cnt, |
77241056 | 670 | qp->s_rnr_retry_cnt, |
d7c76e91 | 671 | qp->s_rnr_retry, |
77241056 | 672 | sde, |
721d0427 | 673 | sde ? sde->this_idx : 0, |
77e7639f | 674 | send_context, |
0358a440 VM |
675 | send_context ? send_context->sw_index : 0, |
676 | ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->head, | |
ef086c0d | 677 | ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->tail, |
642aaab5 KW |
678 | qp->pid, |
679 | /* ack queue information */ | |
680 | e ? e->opcode : 0, | |
681 | e ? e->psn : 0, | |
682 | e ? e->lpsn : 0); | |
77241056 MM |
683 | } |
684 | ||
0f4d027c | 685 | void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp) |
a2c2d608 DD |
686 | { |
687 | struct hfi1_qp_priv *priv; | |
688 | ||
0f4d027c | 689 | priv = kzalloc_node(sizeof(*priv), GFP_KERNEL, rdi->dparms.node); |
a2c2d608 DD |
690 | if (!priv) |
691 | return ERR_PTR(-ENOMEM); | |
692 | ||
693 | priv->owner = qp; | |
694 | ||
0f4d027c | 695 | priv->s_ahg = kzalloc_node(sizeof(*priv->s_ahg), GFP_KERNEL, |
a9b6b3bc DC |
696 | rdi->dparms.node); |
697 | if (!priv->s_ahg) { | |
a2c2d608 DD |
698 | kfree(priv); |
699 | return ERR_PTR(-ENOMEM); | |
700 | } | |
5a648dfa MM |
701 | iowait_init( |
702 | &priv->s_iowait, | |
703 | 1, | |
704 | _hfi1_do_send, | |
705 | iowait_sleep, | |
706 | iowait_wakeup, | |
707 | iowait_sdma_drained); | |
a2c2d608 DD |
708 | return priv; |
709 | } | |
710 | ||
711 | void qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp) | |
712 | { | |
713 | struct hfi1_qp_priv *priv = qp->priv; | |
714 | ||
a9b6b3bc | 715 | kfree(priv->s_ahg); |
a2c2d608 DD |
716 | kfree(priv); |
717 | } | |
718 | ||
719 | unsigned free_all_qps(struct rvt_dev_info *rdi) | |
720 | { | |
721 | struct hfi1_ibdev *verbs_dev = container_of(rdi, | |
722 | struct hfi1_ibdev, | |
723 | rdi); | |
724 | struct hfi1_devdata *dd = container_of(verbs_dev, | |
725 | struct hfi1_devdata, | |
726 | verbs_dev); | |
727 | int n; | |
728 | unsigned qp_inuse = 0; | |
729 | ||
730 | for (n = 0; n < dd->num_pports; n++) { | |
731 | struct hfi1_ibport *ibp = &dd->pport[n].ibport_data; | |
732 | ||
a2c2d608 DD |
733 | rcu_read_lock(); |
734 | if (rcu_dereference(ibp->rvp.qp[0])) | |
735 | qp_inuse++; | |
736 | if (rcu_dereference(ibp->rvp.qp[1])) | |
737 | qp_inuse++; | |
738 | rcu_read_unlock(); | |
739 | } | |
740 | ||
741 | return qp_inuse; | |
742 | } | |
743 | ||
ec4274f1 DD |
744 | void flush_qp_waiters(struct rvt_qp *qp) |
745 | { | |
68e78b3d | 746 | lockdep_assert_held(&qp->s_lock); |
ec4274f1 DD |
747 | flush_iowait(qp); |
748 | } | |
749 | ||
750 | void stop_send_queue(struct rvt_qp *qp) | |
751 | { | |
752 | struct hfi1_qp_priv *priv = qp->priv; | |
753 | ||
754 | cancel_work_sync(&priv->s_iowait.iowork); | |
755 | } | |
756 | ||
757 | void quiesce_qp(struct rvt_qp *qp) | |
758 | { | |
759 | struct hfi1_qp_priv *priv = qp->priv; | |
760 | ||
761 | iowait_sdma_drain(&priv->s_iowait); | |
14553ca1 | 762 | qp_pio_drain(qp); |
ec4274f1 DD |
763 | flush_tx_list(qp); |
764 | } | |
765 | ||
a2c2d608 DD |
766 | void notify_qp_reset(struct rvt_qp *qp) |
767 | { | |
688f21c0 | 768 | qp->r_adefered = 0; |
a2c2d608 DD |
769 | clear_ahg(qp); |
770 | } | |
771 | ||
c2f3ffb0 MM |
772 | /* |
773 | * Switch to alternate path. | |
774 | * The QP s_lock should be held and interrupts disabled. | |
775 | */ | |
895420dd | 776 | void hfi1_migrate_qp(struct rvt_qp *qp) |
c2f3ffb0 | 777 | { |
4c6829c5 | 778 | struct hfi1_qp_priv *priv = qp->priv; |
c2f3ffb0 MM |
779 | struct ib_event ev; |
780 | ||
781 | qp->s_mig_state = IB_MIG_MIGRATED; | |
782 | qp->remote_ah_attr = qp->alt_ah_attr; | |
d8966fcd | 783 | qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr); |
c2f3ffb0 | 784 | qp->s_pkey_index = qp->s_alt_pkey_index; |
54d10c1e | 785 | qp->s_flags |= RVT_S_AHG_CLEAR; |
4c6829c5 DD |
786 | priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr); |
787 | priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc); | |
d98bb7f7 | 788 | qp_set_16b(qp); |
c2f3ffb0 MM |
789 | |
790 | ev.device = qp->ibqp.device; | |
791 | ev.element.qp = &qp->ibqp; | |
792 | ev.event = IB_EVENT_PATH_MIG; | |
793 | qp->ibqp.event_handler(&ev, qp->ibqp.qp_context); | |
794 | } | |
ec4274f1 DD |
795 | |
796 | int mtu_to_path_mtu(u32 mtu) | |
797 | { | |
798 | return mtu_to_enum(mtu, OPA_MTU_8192); | |
799 | } | |
800 | ||
801 | u32 mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu) | |
802 | { | |
803 | u32 mtu; | |
804 | struct hfi1_ibdev *verbs_dev = container_of(rdi, | |
805 | struct hfi1_ibdev, | |
806 | rdi); | |
807 | struct hfi1_devdata *dd = container_of(verbs_dev, | |
808 | struct hfi1_devdata, | |
809 | verbs_dev); | |
810 | struct hfi1_ibport *ibp; | |
811 | u8 sc, vl; | |
812 | ||
813 | ibp = &dd->pport[qp->port_num - 1].ibport_data; | |
d8966fcd | 814 | sc = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; |
ec4274f1 DD |
815 | vl = sc_to_vlt(dd, sc); |
816 | ||
817 | mtu = verbs_mtu_enum_to_int(qp->ibqp.device, pmtu); | |
818 | if (vl < PER_VL_SEND_CONTEXTS) | |
819 | mtu = min_t(u32, mtu, dd->vld[vl].mtu); | |
820 | return mtu; | |
821 | } | |
822 | ||
823 | int get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp, | |
824 | struct ib_qp_attr *attr) | |
825 | { | |
826 | int mtu, pidx = qp->port_num - 1; | |
827 | struct hfi1_ibdev *verbs_dev = container_of(rdi, | |
828 | struct hfi1_ibdev, | |
829 | rdi); | |
830 | struct hfi1_devdata *dd = container_of(verbs_dev, | |
831 | struct hfi1_devdata, | |
832 | verbs_dev); | |
833 | mtu = verbs_mtu_enum_to_int(qp->ibqp.device, attr->path_mtu); | |
834 | if (mtu == -1) | |
835 | return -1; /* values less than 0 are error */ | |
836 | ||
837 | if (mtu > dd->pport[pidx].ibmtu) | |
838 | return mtu_to_enum(dd->pport[pidx].ibmtu, IB_MTU_2048); | |
839 | else | |
840 | return attr->path_mtu; | |
841 | } | |
842 | ||
843 | void notify_error_qp(struct rvt_qp *qp) | |
844 | { | |
ec4274f1 | 845 | struct hfi1_qp_priv *priv = qp->priv; |
a8715b97 | 846 | seqlock_t *lock = priv->s_iowait.lock; |
ec4274f1 | 847 | |
a8715b97 MM |
848 | if (lock) { |
849 | write_seqlock(lock); | |
850 | if (!list_empty(&priv->s_iowait.list) && | |
851 | !(qp->s_flags & RVT_S_BUSY)) { | |
852 | qp->s_flags &= ~RVT_S_ANY_WAIT_IO; | |
853 | list_del_init(&priv->s_iowait.list); | |
854 | priv->s_iowait.lock = NULL; | |
855 | rvt_put_qp(qp); | |
856 | } | |
857 | write_sequnlock(lock); | |
ec4274f1 | 858 | } |
ec4274f1 DD |
859 | |
860 | if (!(qp->s_flags & RVT_S_BUSY)) { | |
861 | qp->s_hdrwords = 0; | |
862 | if (qp->s_rdma_mr) { | |
863 | rvt_put_mr(qp->s_rdma_mr); | |
864 | qp->s_rdma_mr = NULL; | |
865 | } | |
866 | flush_tx_list(qp); | |
867 | } | |
868 | } | |
869 | ||
0ec79e87 KW |
870 | /** |
871 | * hfi1_error_port_qps - put a port's RC/UC qps into error state | |
872 | * @ibp: the ibport. | |
873 | * @sl: the service level. | |
874 | * | |
875 | * This function places all RC/UC qps with a given service level into error | |
876 | * state. It is generally called to force upper lay apps to abandon stale qps | |
877 | * after an sl->sc mapping change. | |
878 | */ | |
879 | void hfi1_error_port_qps(struct hfi1_ibport *ibp, u8 sl) | |
880 | { | |
881 | struct rvt_qp *qp = NULL; | |
882 | struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); | |
883 | struct hfi1_ibdev *dev = &ppd->dd->verbs_dev; | |
884 | int n; | |
885 | int lastwqe; | |
886 | struct ib_event ev; | |
887 | ||
888 | rcu_read_lock(); | |
889 | ||
890 | /* Deal only with RC/UC qps that use the given SL. */ | |
891 | for (n = 0; n < dev->rdi.qp_dev->qp_table_size; n++) { | |
892 | for (qp = rcu_dereference(dev->rdi.qp_dev->qp_table[n]); qp; | |
893 | qp = rcu_dereference(qp->next)) { | |
894 | if (qp->port_num == ppd->port && | |
895 | (qp->ibqp.qp_type == IB_QPT_UC || | |
896 | qp->ibqp.qp_type == IB_QPT_RC) && | |
d8966fcd | 897 | rdma_ah_get_sl(&qp->remote_ah_attr) == sl && |
0ec79e87 KW |
898 | (ib_rvt_state_ops[qp->state] & |
899 | RVT_POST_SEND_OK)) { | |
900 | spin_lock_irq(&qp->r_lock); | |
901 | spin_lock(&qp->s_hlock); | |
902 | spin_lock(&qp->s_lock); | |
903 | lastwqe = rvt_error_qp(qp, | |
904 | IB_WC_WR_FLUSH_ERR); | |
905 | spin_unlock(&qp->s_lock); | |
906 | spin_unlock(&qp->s_hlock); | |
907 | spin_unlock_irq(&qp->r_lock); | |
908 | if (lastwqe) { | |
909 | ev.device = qp->ibqp.device; | |
910 | ev.element.qp = &qp->ibqp; | |
911 | ev.event = | |
912 | IB_EVENT_QP_LAST_WQE_REACHED; | |
913 | qp->ibqp.event_handler(&ev, | |
914 | qp->ibqp.qp_context); | |
915 | } | |
916 | } | |
917 | } | |
918 | } | |
919 | ||
920 | rcu_read_unlock(); | |
921 | } |