]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/staging/rdma/hfi1/qp.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
staging/rdma/hfi1: Remove ibport and use rdmavt version
[mirror_ubuntu-artful-kernel.git] / drivers / staging / rdma / hfi1 / qp.c
1 /*
2 *
3 * This file is provided under a dual BSD/GPLv2 license. When using or
4 * redistributing this file, you may do so under either license.
5 *
6 * GPL LICENSE SUMMARY
7 *
8 * Copyright(c) 2015 Intel Corporation.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it will be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * General Public License for more details.
18 *
19 * BSD LICENSE
20 *
21 * Copyright(c) 2015 Intel Corporation.
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 *
27 * - Redistributions of source code must retain the above copyright
28 * notice, this list of conditions and the following disclaimer.
29 * - Redistributions in binary form must reproduce the above copyright
30 * notice, this list of conditions and the following disclaimer in
31 * the documentation and/or other materials provided with the
32 * distribution.
33 * - Neither the name of Intel Corporation nor the names of its
34 * contributors may be used to endorse or promote products derived
35 * from this software without specific prior written permission.
36 *
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 *
49 */
50
51 #include <linux/err.h>
52 #include <linux/vmalloc.h>
53 #include <linux/hash.h>
54 #include <linux/module.h>
55 #include <linux/random.h>
56 #include <linux/seq_file.h>
57
58 #include "hfi.h"
59 #include "qp.h"
60 #include "trace.h"
61 #include "sdma.h"
62
63 #define BITS_PER_PAGE (PAGE_SIZE*BITS_PER_BYTE)
64 #define BITS_PER_PAGE_MASK (BITS_PER_PAGE-1)
65
66 static unsigned int hfi1_qp_table_size = 256;
67 module_param_named(qp_table_size, hfi1_qp_table_size, uint, S_IRUGO);
68 MODULE_PARM_DESC(qp_table_size, "QP table size");
69
70 static void flush_tx_list(struct rvt_qp *qp);
71 static int iowait_sleep(
72 struct sdma_engine *sde,
73 struct iowait *wait,
74 struct sdma_txreq *stx,
75 unsigned seq);
76 static void iowait_wakeup(struct iowait *wait, int reason);
77
78 static inline unsigned mk_qpn(struct hfi1_qpn_table *qpt,
79 struct qpn_map *map, unsigned off)
80 {
81 return (map - qpt->map) * BITS_PER_PAGE + off;
82 }
83
84 /*
85 * Convert the AETH credit code into the number of credits.
86 */
87 static const u16 credit_table[31] = {
88 0, /* 0 */
89 1, /* 1 */
90 2, /* 2 */
91 3, /* 3 */
92 4, /* 4 */
93 6, /* 5 */
94 8, /* 6 */
95 12, /* 7 */
96 16, /* 8 */
97 24, /* 9 */
98 32, /* A */
99 48, /* B */
100 64, /* C */
101 96, /* D */
102 128, /* E */
103 192, /* F */
104 256, /* 10 */
105 384, /* 11 */
106 512, /* 12 */
107 768, /* 13 */
108 1024, /* 14 */
109 1536, /* 15 */
110 2048, /* 16 */
111 3072, /* 17 */
112 4096, /* 18 */
113 6144, /* 19 */
114 8192, /* 1A */
115 12288, /* 1B */
116 16384, /* 1C */
117 24576, /* 1D */
118 32768 /* 1E */
119 };
120
121 static void get_map_page(struct hfi1_qpn_table *qpt, struct qpn_map *map)
122 {
123 unsigned long page = get_zeroed_page(GFP_KERNEL);
124
125 /*
126 * Free the page if someone raced with us installing it.
127 */
128
129 spin_lock(&qpt->lock);
130 if (map->page)
131 free_page(page);
132 else
133 map->page = (void *)page;
134 spin_unlock(&qpt->lock);
135 }
136
137 /*
138 * Allocate the next available QPN or
139 * zero/one for QP type IB_QPT_SMI/IB_QPT_GSI.
140 */
141 static int alloc_qpn(struct hfi1_devdata *dd, struct hfi1_qpn_table *qpt,
142 enum ib_qp_type type, u8 port)
143 {
144 u32 i, offset, max_scan, qpn;
145 struct qpn_map *map;
146 u32 ret;
147
148 if (type == IB_QPT_SMI || type == IB_QPT_GSI) {
149 unsigned n;
150
151 ret = type == IB_QPT_GSI;
152 n = 1 << (ret + 2 * (port - 1));
153 spin_lock(&qpt->lock);
154 if (qpt->flags & n)
155 ret = -EINVAL;
156 else
157 qpt->flags |= n;
158 spin_unlock(&qpt->lock);
159 goto bail;
160 }
161
162 qpn = qpt->last + qpt->incr;
163 if (qpn >= QPN_MAX)
164 qpn = qpt->incr | ((qpt->last & 1) ^ 1);
165 /* offset carries bit 0 */
166 offset = qpn & BITS_PER_PAGE_MASK;
167 map = &qpt->map[qpn / BITS_PER_PAGE];
168 max_scan = qpt->nmaps - !offset;
169 for (i = 0;;) {
170 if (unlikely(!map->page)) {
171 get_map_page(qpt, map);
172 if (unlikely(!map->page))
173 break;
174 }
175 do {
176 if (!test_and_set_bit(offset, map->page)) {
177 qpt->last = qpn;
178 ret = qpn;
179 goto bail;
180 }
181 offset += qpt->incr;
182 /*
183 * This qpn might be bogus if offset >= BITS_PER_PAGE.
184 * That is OK. It gets re-assigned below
185 */
186 qpn = mk_qpn(qpt, map, offset);
187 } while (offset < BITS_PER_PAGE && qpn < QPN_MAX);
188 /*
189 * In order to keep the number of pages allocated to a
190 * minimum, we scan the all existing pages before increasing
191 * the size of the bitmap table.
192 */
193 if (++i > max_scan) {
194 if (qpt->nmaps == QPNMAP_ENTRIES)
195 break;
196 map = &qpt->map[qpt->nmaps++];
197 /* start at incr with current bit 0 */
198 offset = qpt->incr | (offset & 1);
199 } else if (map < &qpt->map[qpt->nmaps]) {
200 ++map;
201 /* start at incr with current bit 0 */
202 offset = qpt->incr | (offset & 1);
203 } else {
204 map = &qpt->map[0];
205 /* wrap to first map page, invert bit 0 */
206 offset = qpt->incr | ((offset & 1) ^ 1);
207 }
208 /* there can be no bits at shift and below */
209 WARN_ON(offset & (dd->qos_shift - 1));
210 qpn = mk_qpn(qpt, map, offset);
211 }
212
213 ret = -ENOMEM;
214
215 bail:
216 return ret;
217 }
218
219 static void free_qpn(struct hfi1_qpn_table *qpt, u32 qpn)
220 {
221 struct qpn_map *map;
222
223 map = qpt->map + qpn / BITS_PER_PAGE;
224 if (map->page)
225 clear_bit(qpn & BITS_PER_PAGE_MASK, map->page);
226 }
227
228 /*
229 * Put the QP into the hash table.
230 * The hash table holds a reference to the QP.
231 */
232 static void insert_qp(struct hfi1_ibdev *dev, struct rvt_qp *qp)
233 {
234 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
235 unsigned long flags;
236
237 atomic_inc(&qp->refcount);
238 spin_lock_irqsave(&dev->qp_dev->qpt_lock, flags);
239
240 if (qp->ibqp.qp_num <= 1) {
241 rcu_assign_pointer(ibp->rvp.qp[qp->ibqp.qp_num], qp);
242 } else {
243 u32 n = qpn_hash(dev->qp_dev, qp->ibqp.qp_num);
244
245 qp->next = dev->qp_dev->qp_table[n];
246 rcu_assign_pointer(dev->qp_dev->qp_table[n], qp);
247 trace_hfi1_qpinsert(qp, n);
248 }
249
250 spin_unlock_irqrestore(&dev->qp_dev->qpt_lock, flags);
251 }
252
253 /*
254 * Remove the QP from the table so it can't be found asynchronously by
255 * the receive interrupt routine.
256 */
257 static void remove_qp(struct hfi1_ibdev *dev, struct rvt_qp *qp)
258 {
259 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
260 u32 n = qpn_hash(dev->qp_dev, qp->ibqp.qp_num);
261 unsigned long flags;
262 int removed = 1;
263
264 spin_lock_irqsave(&dev->qp_dev->qpt_lock, flags);
265
266 if (rcu_dereference_protected(ibp->rvp.qp[0],
267 lockdep_is_held(
268 &dev->qp_dev->qpt_lock)) == qp) {
269 RCU_INIT_POINTER(ibp->rvp.qp[0], NULL);
270 } else if (rcu_dereference_protected(ibp->rvp.qp[1],
271 lockdep_is_held(&dev->qp_dev->qpt_lock)) == qp) {
272 RCU_INIT_POINTER(ibp->rvp.qp[1], NULL);
273 } else {
274 struct rvt_qp *q;
275 struct rvt_qp __rcu **qpp;
276
277 removed = 0;
278 qpp = &dev->qp_dev->qp_table[n];
279 for (; (q = rcu_dereference_protected(*qpp,
280 lockdep_is_held(&dev->qp_dev->qpt_lock)))
281 != NULL;
282 qpp = &q->next)
283 if (q == qp) {
284 RCU_INIT_POINTER(*qpp,
285 rcu_dereference_protected(qp->next,
286 lockdep_is_held(&dev->qp_dev->qpt_lock)));
287 removed = 1;
288 trace_hfi1_qpremove(qp, n);
289 break;
290 }
291 }
292
293 spin_unlock_irqrestore(&dev->qp_dev->qpt_lock, flags);
294 if (removed) {
295 synchronize_rcu();
296 if (atomic_dec_and_test(&qp->refcount))
297 wake_up(&qp->wait);
298 }
299 }
300
301 /**
302 * free_all_qps - check for QPs still in use
303 * @qpt: the QP table to empty
304 *
305 * There should not be any QPs still in use.
306 * Free memory for table.
307 */
308 static unsigned free_all_qps(struct hfi1_devdata *dd)
309 {
310 struct hfi1_ibdev *dev = &dd->verbs_dev;
311 unsigned long flags;
312 struct rvt_qp *qp;
313 unsigned n, qp_inuse = 0;
314
315 for (n = 0; n < dd->num_pports; n++) {
316 struct hfi1_ibport *ibp = &dd->pport[n].ibport_data;
317
318 if (!hfi1_mcast_tree_empty(ibp))
319 qp_inuse++;
320 rcu_read_lock();
321 if (rcu_dereference(ibp->rvp.qp[0]))
322 qp_inuse++;
323 if (rcu_dereference(ibp->rvp.qp[1]))
324 qp_inuse++;
325 rcu_read_unlock();
326 }
327
328 if (!dev->qp_dev)
329 goto bail;
330 spin_lock_irqsave(&dev->qp_dev->qpt_lock, flags);
331 for (n = 0; n < dev->qp_dev->qp_table_size; n++) {
332 qp = rcu_dereference_protected(dev->qp_dev->qp_table[n],
333 lockdep_is_held(&dev->qp_dev->qpt_lock));
334 RCU_INIT_POINTER(dev->qp_dev->qp_table[n], NULL);
335
336 for (; qp; qp = rcu_dereference_protected(qp->next,
337 lockdep_is_held(&dev->qp_dev->qpt_lock)))
338 qp_inuse++;
339 }
340 spin_unlock_irqrestore(&dev->qp_dev->qpt_lock, flags);
341 synchronize_rcu();
342 bail:
343 return qp_inuse;
344 }
345
346 /**
347 * reset_qp - initialize the QP state to the reset state
348 * @qp: the QP to reset
349 * @type: the QP type
350 */
351 static void reset_qp(struct rvt_qp *qp, enum ib_qp_type type)
352 {
353 struct hfi1_qp_priv *priv = qp->priv;
354 qp->remote_qpn = 0;
355 qp->qkey = 0;
356 qp->qp_access_flags = 0;
357 iowait_init(
358 &priv->s_iowait,
359 1,
360 hfi1_do_send,
361 iowait_sleep,
362 iowait_wakeup);
363 qp->s_flags &= HFI1_S_SIGNAL_REQ_WR;
364 qp->s_hdrwords = 0;
365 qp->s_wqe = NULL;
366 qp->s_draining = 0;
367 qp->s_next_psn = 0;
368 qp->s_last_psn = 0;
369 qp->s_sending_psn = 0;
370 qp->s_sending_hpsn = 0;
371 qp->s_psn = 0;
372 qp->r_psn = 0;
373 qp->r_msn = 0;
374 if (type == IB_QPT_RC) {
375 qp->s_state = IB_OPCODE_RC_SEND_LAST;
376 qp->r_state = IB_OPCODE_RC_SEND_LAST;
377 } else {
378 qp->s_state = IB_OPCODE_UC_SEND_LAST;
379 qp->r_state = IB_OPCODE_UC_SEND_LAST;
380 }
381 qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
382 qp->r_nak_state = 0;
383 priv->r_adefered = 0;
384 qp->r_aflags = 0;
385 qp->r_flags = 0;
386 qp->s_head = 0;
387 qp->s_tail = 0;
388 qp->s_cur = 0;
389 qp->s_acked = 0;
390 qp->s_last = 0;
391 qp->s_ssn = 1;
392 qp->s_lsn = 0;
393 clear_ahg(qp);
394 qp->s_mig_state = IB_MIG_MIGRATED;
395 memset(qp->s_ack_queue, 0, sizeof(qp->s_ack_queue));
396 qp->r_head_ack_queue = 0;
397 qp->s_tail_ack_queue = 0;
398 qp->s_num_rd_atomic = 0;
399 if (qp->r_rq.wq) {
400 qp->r_rq.wq->head = 0;
401 qp->r_rq.wq->tail = 0;
402 }
403 qp->r_sge.num_sge = 0;
404 }
405
406 static void clear_mr_refs(struct rvt_qp *qp, int clr_sends)
407 {
408 unsigned n;
409
410 if (test_and_clear_bit(HFI1_R_REWIND_SGE, &qp->r_aflags))
411 hfi1_put_ss(&qp->s_rdma_read_sge);
412
413 hfi1_put_ss(&qp->r_sge);
414
415 if (clr_sends) {
416 while (qp->s_last != qp->s_head) {
417 struct rvt_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
418 unsigned i;
419
420 for (i = 0; i < wqe->wr.num_sge; i++) {
421 struct rvt_sge *sge = &wqe->sg_list[i];
422
423 rvt_put_mr(sge->mr);
424 }
425 if (qp->ibqp.qp_type == IB_QPT_UD ||
426 qp->ibqp.qp_type == IB_QPT_SMI ||
427 qp->ibqp.qp_type == IB_QPT_GSI)
428 atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
429 if (++qp->s_last >= qp->s_size)
430 qp->s_last = 0;
431 }
432 if (qp->s_rdma_mr) {
433 rvt_put_mr(qp->s_rdma_mr);
434 qp->s_rdma_mr = NULL;
435 }
436 }
437
438 if (qp->ibqp.qp_type != IB_QPT_RC)
439 return;
440
441 for (n = 0; n < ARRAY_SIZE(qp->s_ack_queue); n++) {
442 struct rvt_ack_entry *e = &qp->s_ack_queue[n];
443
444 if (e->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST &&
445 e->rdma_sge.mr) {
446 rvt_put_mr(e->rdma_sge.mr);
447 e->rdma_sge.mr = NULL;
448 }
449 }
450 }
451
452 /**
453 * hfi1_error_qp - put a QP into the error state
454 * @qp: the QP to put into the error state
455 * @err: the receive completion error to signal if a RWQE is active
456 *
457 * Flushes both send and receive work queues.
458 * Returns true if last WQE event should be generated.
459 * The QP r_lock and s_lock should be held and interrupts disabled.
460 * If we are already in error state, just return.
461 */
462 int hfi1_error_qp(struct rvt_qp *qp, enum ib_wc_status err)
463 {
464 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
465 struct hfi1_qp_priv *priv = qp->priv;
466 struct ib_wc wc;
467 int ret = 0;
468
469 if (qp->state == IB_QPS_ERR || qp->state == IB_QPS_RESET)
470 goto bail;
471
472 qp->state = IB_QPS_ERR;
473
474 if (qp->s_flags & (HFI1_S_TIMER | HFI1_S_WAIT_RNR)) {
475 qp->s_flags &= ~(HFI1_S_TIMER | HFI1_S_WAIT_RNR);
476 del_timer(&qp->s_timer);
477 }
478
479 if (qp->s_flags & HFI1_S_ANY_WAIT_SEND)
480 qp->s_flags &= ~HFI1_S_ANY_WAIT_SEND;
481
482 write_seqlock(&dev->iowait_lock);
483 if (!list_empty(&priv->s_iowait.list) && !(qp->s_flags & HFI1_S_BUSY)) {
484 qp->s_flags &= ~HFI1_S_ANY_WAIT_IO;
485 list_del_init(&priv->s_iowait.list);
486 if (atomic_dec_and_test(&qp->refcount))
487 wake_up(&qp->wait);
488 }
489 write_sequnlock(&dev->iowait_lock);
490
491 if (!(qp->s_flags & HFI1_S_BUSY)) {
492 qp->s_hdrwords = 0;
493 if (qp->s_rdma_mr) {
494 rvt_put_mr(qp->s_rdma_mr);
495 qp->s_rdma_mr = NULL;
496 }
497 flush_tx_list(qp);
498 }
499
500 /* Schedule the sending tasklet to drain the send work queue. */
501 if (qp->s_last != qp->s_head)
502 hfi1_schedule_send(qp);
503
504 clear_mr_refs(qp, 0);
505
506 memset(&wc, 0, sizeof(wc));
507 wc.qp = &qp->ibqp;
508 wc.opcode = IB_WC_RECV;
509
510 if (test_and_clear_bit(HFI1_R_WRID_VALID, &qp->r_aflags)) {
511 wc.wr_id = qp->r_wr_id;
512 wc.status = err;
513 hfi1_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
514 }
515 wc.status = IB_WC_WR_FLUSH_ERR;
516
517 if (qp->r_rq.wq) {
518 struct rvt_rwq *wq;
519 u32 head;
520 u32 tail;
521
522 spin_lock(&qp->r_rq.lock);
523
524 /* sanity check pointers before trusting them */
525 wq = qp->r_rq.wq;
526 head = wq->head;
527 if (head >= qp->r_rq.size)
528 head = 0;
529 tail = wq->tail;
530 if (tail >= qp->r_rq.size)
531 tail = 0;
532 while (tail != head) {
533 wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;
534 if (++tail >= qp->r_rq.size)
535 tail = 0;
536 hfi1_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
537 }
538 wq->tail = tail;
539
540 spin_unlock(&qp->r_rq.lock);
541 } else if (qp->ibqp.event_handler)
542 ret = 1;
543
544 bail:
545 return ret;
546 }
547
548 static void flush_tx_list(struct rvt_qp *qp)
549 {
550 struct hfi1_qp_priv *priv = qp->priv;
551
552 while (!list_empty(&priv->s_iowait.tx_head)) {
553 struct sdma_txreq *tx;
554
555 tx = list_first_entry(
556 &priv->s_iowait.tx_head,
557 struct sdma_txreq,
558 list);
559 list_del_init(&tx->list);
560 hfi1_put_txreq(
561 container_of(tx, struct verbs_txreq, txreq));
562 }
563 }
564
565 static void flush_iowait(struct rvt_qp *qp)
566 {
567 struct hfi1_qp_priv *priv = qp->priv;
568 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
569 unsigned long flags;
570
571 write_seqlock_irqsave(&dev->iowait_lock, flags);
572 if (!list_empty(&priv->s_iowait.list)) {
573 list_del_init(&priv->s_iowait.list);
574 if (atomic_dec_and_test(&qp->refcount))
575 wake_up(&qp->wait);
576 }
577 write_sequnlock_irqrestore(&dev->iowait_lock, flags);
578 }
579
580 static inline int opa_mtu_enum_to_int(int mtu)
581 {
582 switch (mtu) {
583 case OPA_MTU_8192: return 8192;
584 case OPA_MTU_10240: return 10240;
585 default: return -1;
586 }
587 }
588
589 /**
590 * This function is what we would push to the core layer if we wanted to be a
591 * "first class citizen". Instead we hide this here and rely on Verbs ULPs
592 * to blindly pass the MTU enum value from the PathRecord to us.
593 *
594 * The actual flag used to determine "8k MTU" will change and is currently
595 * unknown.
596 */
597 static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu)
598 {
599 int val = opa_mtu_enum_to_int((int)mtu);
600
601 if (val > 0)
602 return val;
603 return ib_mtu_enum_to_int(mtu);
604 }
605
606
607 /**
608 * hfi1_modify_qp - modify the attributes of a queue pair
609 * @ibqp: the queue pair who's attributes we're modifying
610 * @attr: the new attributes
611 * @attr_mask: the mask of attributes to modify
612 * @udata: user data for libibverbs.so
613 *
614 * Returns 0 on success, otherwise returns an errno.
615 */
616 int hfi1_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
617 int attr_mask, struct ib_udata *udata)
618 {
619 struct hfi1_ibdev *dev = to_idev(ibqp->device);
620 struct rvt_qp *qp = to_iqp(ibqp);
621 struct hfi1_qp_priv *priv = qp->priv;
622 enum ib_qp_state cur_state, new_state;
623 struct ib_event ev;
624 int lastwqe = 0;
625 int mig = 0;
626 int ret;
627 u32 pmtu = 0; /* for gcc warning only */
628 struct hfi1_devdata *dd = dd_from_dev(dev);
629
630 spin_lock_irq(&qp->r_lock);
631 spin_lock(&qp->s_lock);
632
633 cur_state = attr_mask & IB_QP_CUR_STATE ?
634 attr->cur_qp_state : qp->state;
635 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
636
637 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
638 attr_mask, IB_LINK_LAYER_UNSPECIFIED))
639 goto inval;
640
641 if (attr_mask & IB_QP_AV) {
642 u8 sc;
643
644 if (attr->ah_attr.dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE))
645 goto inval;
646 if (rvt_check_ah(qp->ibqp.device, &attr->ah_attr))
647 goto inval;
648 sc = ah_to_sc(ibqp->device, &attr->ah_attr);
649 if (!qp_to_sdma_engine(qp, sc) &&
650 dd->flags & HFI1_HAS_SEND_DMA)
651 goto inval;
652 }
653
654 if (attr_mask & IB_QP_ALT_PATH) {
655 u8 sc;
656
657 if (attr->alt_ah_attr.dlid >=
658 be16_to_cpu(IB_MULTICAST_LID_BASE))
659 goto inval;
660 if (rvt_check_ah(qp->ibqp.device, &attr->alt_ah_attr))
661 goto inval;
662 if (attr->alt_pkey_index >= hfi1_get_npkeys(dd))
663 goto inval;
664 sc = ah_to_sc(ibqp->device, &attr->alt_ah_attr);
665 if (!qp_to_sdma_engine(qp, sc) &&
666 dd->flags & HFI1_HAS_SEND_DMA)
667 goto inval;
668 }
669
670 if (attr_mask & IB_QP_PKEY_INDEX)
671 if (attr->pkey_index >= hfi1_get_npkeys(dd))
672 goto inval;
673
674 if (attr_mask & IB_QP_MIN_RNR_TIMER)
675 if (attr->min_rnr_timer > 31)
676 goto inval;
677
678 if (attr_mask & IB_QP_PORT)
679 if (qp->ibqp.qp_type == IB_QPT_SMI ||
680 qp->ibqp.qp_type == IB_QPT_GSI ||
681 attr->port_num == 0 ||
682 attr->port_num > ibqp->device->phys_port_cnt)
683 goto inval;
684
685 if (attr_mask & IB_QP_DEST_QPN)
686 if (attr->dest_qp_num > HFI1_QPN_MASK)
687 goto inval;
688
689 if (attr_mask & IB_QP_RETRY_CNT)
690 if (attr->retry_cnt > 7)
691 goto inval;
692
693 if (attr_mask & IB_QP_RNR_RETRY)
694 if (attr->rnr_retry > 7)
695 goto inval;
696
697 /*
698 * Don't allow invalid path_mtu values. OK to set greater
699 * than the active mtu (or even the max_cap, if we have tuned
700 * that to a small mtu. We'll set qp->path_mtu
701 * to the lesser of requested attribute mtu and active,
702 * for packetizing messages.
703 * Note that the QP port has to be set in INIT and MTU in RTR.
704 */
705 if (attr_mask & IB_QP_PATH_MTU) {
706 int mtu, pidx = qp->port_num - 1;
707
708 dd = dd_from_dev(dev);
709 mtu = verbs_mtu_enum_to_int(ibqp->device, attr->path_mtu);
710 if (mtu == -1)
711 goto inval;
712
713 if (mtu > dd->pport[pidx].ibmtu)
714 pmtu = mtu_to_enum(dd->pport[pidx].ibmtu, IB_MTU_2048);
715 else
716 pmtu = attr->path_mtu;
717 }
718
719 if (attr_mask & IB_QP_PATH_MIG_STATE) {
720 if (attr->path_mig_state == IB_MIG_REARM) {
721 if (qp->s_mig_state == IB_MIG_ARMED)
722 goto inval;
723 if (new_state != IB_QPS_RTS)
724 goto inval;
725 } else if (attr->path_mig_state == IB_MIG_MIGRATED) {
726 if (qp->s_mig_state == IB_MIG_REARM)
727 goto inval;
728 if (new_state != IB_QPS_RTS && new_state != IB_QPS_SQD)
729 goto inval;
730 if (qp->s_mig_state == IB_MIG_ARMED)
731 mig = 1;
732 } else
733 goto inval;
734 }
735
736 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
737 if (attr->max_dest_rd_atomic > HFI1_MAX_RDMA_ATOMIC)
738 goto inval;
739
740 switch (new_state) {
741 case IB_QPS_RESET:
742 if (qp->state != IB_QPS_RESET) {
743 qp->state = IB_QPS_RESET;
744 flush_iowait(qp);
745 qp->s_flags &= ~(HFI1_S_TIMER | HFI1_S_ANY_WAIT);
746 spin_unlock(&qp->s_lock);
747 spin_unlock_irq(&qp->r_lock);
748 /* Stop the sending work queue and retry timer */
749 cancel_work_sync(&priv->s_iowait.iowork);
750 del_timer_sync(&qp->s_timer);
751 iowait_sdma_drain(&priv->s_iowait);
752 flush_tx_list(qp);
753 remove_qp(dev, qp);
754 wait_event(qp->wait, !atomic_read(&qp->refcount));
755 spin_lock_irq(&qp->r_lock);
756 spin_lock(&qp->s_lock);
757 clear_mr_refs(qp, 1);
758 clear_ahg(qp);
759 reset_qp(qp, ibqp->qp_type);
760 }
761 break;
762
763 case IB_QPS_RTR:
764 /* Allow event to re-trigger if QP set to RTR more than once */
765 qp->r_flags &= ~HFI1_R_COMM_EST;
766 qp->state = new_state;
767 break;
768
769 case IB_QPS_SQD:
770 qp->s_draining = qp->s_last != qp->s_cur;
771 qp->state = new_state;
772 break;
773
774 case IB_QPS_SQE:
775 if (qp->ibqp.qp_type == IB_QPT_RC)
776 goto inval;
777 qp->state = new_state;
778 break;
779
780 case IB_QPS_ERR:
781 lastwqe = hfi1_error_qp(qp, IB_WC_WR_FLUSH_ERR);
782 break;
783
784 default:
785 qp->state = new_state;
786 break;
787 }
788
789 if (attr_mask & IB_QP_PKEY_INDEX)
790 qp->s_pkey_index = attr->pkey_index;
791
792 if (attr_mask & IB_QP_PORT)
793 qp->port_num = attr->port_num;
794
795 if (attr_mask & IB_QP_DEST_QPN)
796 qp->remote_qpn = attr->dest_qp_num;
797
798 if (attr_mask & IB_QP_SQ_PSN) {
799 qp->s_next_psn = attr->sq_psn & PSN_MODIFY_MASK;
800 qp->s_psn = qp->s_next_psn;
801 qp->s_sending_psn = qp->s_next_psn;
802 qp->s_last_psn = qp->s_next_psn - 1;
803 qp->s_sending_hpsn = qp->s_last_psn;
804 }
805
806 if (attr_mask & IB_QP_RQ_PSN)
807 qp->r_psn = attr->rq_psn & PSN_MODIFY_MASK;
808
809 if (attr_mask & IB_QP_ACCESS_FLAGS)
810 qp->qp_access_flags = attr->qp_access_flags;
811
812 if (attr_mask & IB_QP_AV) {
813 qp->remote_ah_attr = attr->ah_attr;
814 qp->s_srate = attr->ah_attr.static_rate;
815 qp->srate_mbps = ib_rate_to_mbps(qp->s_srate);
816 priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr);
817 priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
818 }
819
820 if (attr_mask & IB_QP_ALT_PATH) {
821 qp->alt_ah_attr = attr->alt_ah_attr;
822 qp->s_alt_pkey_index = attr->alt_pkey_index;
823 }
824
825 if (attr_mask & IB_QP_PATH_MIG_STATE) {
826 qp->s_mig_state = attr->path_mig_state;
827 if (mig) {
828 qp->remote_ah_attr = qp->alt_ah_attr;
829 qp->port_num = qp->alt_ah_attr.port_num;
830 qp->s_pkey_index = qp->s_alt_pkey_index;
831 qp->s_flags |= HFI1_S_AHG_CLEAR;
832 priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr);
833 priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
834 }
835 }
836
837 if (attr_mask & IB_QP_PATH_MTU) {
838 struct hfi1_ibport *ibp;
839 u8 sc, vl;
840 u32 mtu;
841
842 dd = dd_from_dev(dev);
843 ibp = &dd->pport[qp->port_num - 1].ibport_data;
844
845 sc = ibp->sl_to_sc[qp->remote_ah_attr.sl];
846 vl = sc_to_vlt(dd, sc);
847
848 mtu = verbs_mtu_enum_to_int(ibqp->device, pmtu);
849 if (vl < PER_VL_SEND_CONTEXTS)
850 mtu = min_t(u32, mtu, dd->vld[vl].mtu);
851 pmtu = mtu_to_enum(mtu, OPA_MTU_8192);
852
853 qp->path_mtu = pmtu;
854 qp->pmtu = mtu;
855 }
856
857 if (attr_mask & IB_QP_RETRY_CNT) {
858 qp->s_retry_cnt = attr->retry_cnt;
859 qp->s_retry = attr->retry_cnt;
860 }
861
862 if (attr_mask & IB_QP_RNR_RETRY) {
863 qp->s_rnr_retry_cnt = attr->rnr_retry;
864 qp->s_rnr_retry = attr->rnr_retry;
865 }
866
867 if (attr_mask & IB_QP_MIN_RNR_TIMER)
868 qp->r_min_rnr_timer = attr->min_rnr_timer;
869
870 if (attr_mask & IB_QP_TIMEOUT) {
871 qp->timeout = attr->timeout;
872 qp->timeout_jiffies =
873 usecs_to_jiffies((4096UL * (1UL << qp->timeout)) /
874 1000UL);
875 }
876
877 if (attr_mask & IB_QP_QKEY)
878 qp->qkey = attr->qkey;
879
880 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
881 qp->r_max_rd_atomic = attr->max_dest_rd_atomic;
882
883 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
884 qp->s_max_rd_atomic = attr->max_rd_atomic;
885
886 spin_unlock(&qp->s_lock);
887 spin_unlock_irq(&qp->r_lock);
888
889 if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
890 insert_qp(dev, qp);
891
892 if (lastwqe) {
893 ev.device = qp->ibqp.device;
894 ev.element.qp = &qp->ibqp;
895 ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
896 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
897 }
898 if (mig) {
899 ev.device = qp->ibqp.device;
900 ev.element.qp = &qp->ibqp;
901 ev.event = IB_EVENT_PATH_MIG;
902 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
903 }
904 ret = 0;
905 goto bail;
906
907 inval:
908 spin_unlock(&qp->s_lock);
909 spin_unlock_irq(&qp->r_lock);
910 ret = -EINVAL;
911
912 bail:
913 return ret;
914 }
915
916 int hfi1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
917 int attr_mask, struct ib_qp_init_attr *init_attr)
918 {
919 struct rvt_qp *qp = to_iqp(ibqp);
920
921 attr->qp_state = qp->state;
922 attr->cur_qp_state = attr->qp_state;
923 attr->path_mtu = qp->path_mtu;
924 attr->path_mig_state = qp->s_mig_state;
925 attr->qkey = qp->qkey;
926 attr->rq_psn = mask_psn(qp->r_psn);
927 attr->sq_psn = mask_psn(qp->s_next_psn);
928 attr->dest_qp_num = qp->remote_qpn;
929 attr->qp_access_flags = qp->qp_access_flags;
930 attr->cap.max_send_wr = qp->s_size - 1;
931 attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1;
932 attr->cap.max_send_sge = qp->s_max_sge;
933 attr->cap.max_recv_sge = qp->r_rq.max_sge;
934 attr->cap.max_inline_data = 0;
935 attr->ah_attr = qp->remote_ah_attr;
936 attr->alt_ah_attr = qp->alt_ah_attr;
937 attr->pkey_index = qp->s_pkey_index;
938 attr->alt_pkey_index = qp->s_alt_pkey_index;
939 attr->en_sqd_async_notify = 0;
940 attr->sq_draining = qp->s_draining;
941 attr->max_rd_atomic = qp->s_max_rd_atomic;
942 attr->max_dest_rd_atomic = qp->r_max_rd_atomic;
943 attr->min_rnr_timer = qp->r_min_rnr_timer;
944 attr->port_num = qp->port_num;
945 attr->timeout = qp->timeout;
946 attr->retry_cnt = qp->s_retry_cnt;
947 attr->rnr_retry = qp->s_rnr_retry_cnt;
948 attr->alt_port_num = qp->alt_ah_attr.port_num;
949 attr->alt_timeout = qp->alt_timeout;
950
951 init_attr->event_handler = qp->ibqp.event_handler;
952 init_attr->qp_context = qp->ibqp.qp_context;
953 init_attr->send_cq = qp->ibqp.send_cq;
954 init_attr->recv_cq = qp->ibqp.recv_cq;
955 init_attr->srq = qp->ibqp.srq;
956 init_attr->cap = attr->cap;
957 if (qp->s_flags & HFI1_S_SIGNAL_REQ_WR)
958 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
959 else
960 init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
961 init_attr->qp_type = qp->ibqp.qp_type;
962 init_attr->port_num = qp->port_num;
963 return 0;
964 }
965
966 /**
967 * hfi1_compute_aeth - compute the AETH (syndrome + MSN)
968 * @qp: the queue pair to compute the AETH for
969 *
970 * Returns the AETH.
971 */
972 __be32 hfi1_compute_aeth(struct rvt_qp *qp)
973 {
974 u32 aeth = qp->r_msn & HFI1_MSN_MASK;
975
976 if (qp->ibqp.srq) {
977 /*
978 * Shared receive queues don't generate credits.
979 * Set the credit field to the invalid value.
980 */
981 aeth |= HFI1_AETH_CREDIT_INVAL << HFI1_AETH_CREDIT_SHIFT;
982 } else {
983 u32 min, max, x;
984 u32 credits;
985 struct rvt_rwq *wq = qp->r_rq.wq;
986 u32 head;
987 u32 tail;
988
989 /* sanity check pointers before trusting them */
990 head = wq->head;
991 if (head >= qp->r_rq.size)
992 head = 0;
993 tail = wq->tail;
994 if (tail >= qp->r_rq.size)
995 tail = 0;
996 /*
997 * Compute the number of credits available (RWQEs).
998 * There is a small chance that the pair of reads are
999 * not atomic, which is OK, since the fuzziness is
1000 * resolved as further ACKs go out.
1001 */
1002 credits = head - tail;
1003 if ((int)credits < 0)
1004 credits += qp->r_rq.size;
1005 /*
1006 * Binary search the credit table to find the code to
1007 * use.
1008 */
1009 min = 0;
1010 max = 31;
1011 for (;;) {
1012 x = (min + max) / 2;
1013 if (credit_table[x] == credits)
1014 break;
1015 if (credit_table[x] > credits)
1016 max = x;
1017 else if (min == x)
1018 break;
1019 else
1020 min = x;
1021 }
1022 aeth |= x << HFI1_AETH_CREDIT_SHIFT;
1023 }
1024 return cpu_to_be32(aeth);
1025 }
1026
1027 /**
1028 * hfi1_create_qp - create a queue pair for a device
1029 * @ibpd: the protection domain who's device we create the queue pair for
1030 * @init_attr: the attributes of the queue pair
1031 * @udata: user data for libibverbs.so
1032 *
1033 * Returns the queue pair on success, otherwise returns an errno.
1034 *
1035 * Called by the ib_create_qp() core verbs function.
1036 */
1037 struct ib_qp *hfi1_create_qp(struct ib_pd *ibpd,
1038 struct ib_qp_init_attr *init_attr,
1039 struct ib_udata *udata)
1040 {
1041 struct rvt_qp *qp;
1042 struct hfi1_qp_priv *priv;
1043 int err;
1044 struct rvt_swqe *swq = NULL;
1045 struct hfi1_ibdev *dev;
1046 struct hfi1_devdata *dd;
1047 size_t sz;
1048 size_t sg_list_sz;
1049 struct ib_qp *ret;
1050
1051 if (init_attr->cap.max_send_sge > hfi1_max_sges ||
1052 init_attr->cap.max_send_wr > hfi1_max_qp_wrs ||
1053 init_attr->create_flags) {
1054 ret = ERR_PTR(-EINVAL);
1055 goto bail;
1056 }
1057
1058 /* Check receive queue parameters if no SRQ is specified. */
1059 if (!init_attr->srq) {
1060 if (init_attr->cap.max_recv_sge > hfi1_max_sges ||
1061 init_attr->cap.max_recv_wr > hfi1_max_qp_wrs) {
1062 ret = ERR_PTR(-EINVAL);
1063 goto bail;
1064 }
1065 if (init_attr->cap.max_send_sge +
1066 init_attr->cap.max_send_wr +
1067 init_attr->cap.max_recv_sge +
1068 init_attr->cap.max_recv_wr == 0) {
1069 ret = ERR_PTR(-EINVAL);
1070 goto bail;
1071 }
1072 }
1073
1074 switch (init_attr->qp_type) {
1075 case IB_QPT_SMI:
1076 case IB_QPT_GSI:
1077 if (init_attr->port_num == 0 ||
1078 init_attr->port_num > ibpd->device->phys_port_cnt) {
1079 ret = ERR_PTR(-EINVAL);
1080 goto bail;
1081 }
1082 case IB_QPT_UC:
1083 case IB_QPT_RC:
1084 case IB_QPT_UD:
1085 sz = sizeof(struct rvt_sge) *
1086 init_attr->cap.max_send_sge +
1087 sizeof(struct rvt_swqe);
1088 swq = vmalloc((init_attr->cap.max_send_wr + 1) * sz);
1089 if (swq == NULL) {
1090 ret = ERR_PTR(-ENOMEM);
1091 goto bail;
1092 }
1093 sz = sizeof(*qp);
1094 sg_list_sz = 0;
1095 if (init_attr->srq) {
1096 struct rvt_srq *srq = ibsrq_to_rvtsrq(init_attr->srq);
1097
1098 if (srq->rq.max_sge > 1)
1099 sg_list_sz = sizeof(*qp->r_sg_list) *
1100 (srq->rq.max_sge - 1);
1101 } else if (init_attr->cap.max_recv_sge > 1)
1102 sg_list_sz = sizeof(*qp->r_sg_list) *
1103 (init_attr->cap.max_recv_sge - 1);
1104 qp = kzalloc(sz + sg_list_sz, GFP_KERNEL);
1105 if (!qp) {
1106 ret = ERR_PTR(-ENOMEM);
1107 goto bail_swq;
1108 }
1109 RCU_INIT_POINTER(qp->next, NULL);
1110 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1111 if (!priv) {
1112 ret = ERR_PTR(-ENOMEM);
1113 goto bail_qp_priv;
1114 }
1115 priv->owner = qp;
1116 priv->s_hdr = kzalloc(sizeof(*priv->s_hdr), GFP_KERNEL);
1117 if (!priv->s_hdr) {
1118 ret = ERR_PTR(-ENOMEM);
1119 goto bail_qp;
1120 }
1121 qp->priv = priv;
1122 qp->timeout_jiffies =
1123 usecs_to_jiffies((4096UL * (1UL << qp->timeout)) /
1124 1000UL);
1125 if (init_attr->srq)
1126 sz = 0;
1127 else {
1128 qp->r_rq.size = init_attr->cap.max_recv_wr + 1;
1129 qp->r_rq.max_sge = init_attr->cap.max_recv_sge;
1130 sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) +
1131 sizeof(struct rvt_rwqe);
1132 qp->r_rq.wq = vmalloc_user(sizeof(struct rvt_rwq) +
1133 qp->r_rq.size * sz);
1134 if (!qp->r_rq.wq) {
1135 ret = ERR_PTR(-ENOMEM);
1136 goto bail_qp;
1137 }
1138 }
1139
1140 /*
1141 * ib_create_qp() will initialize qp->ibqp
1142 * except for qp->ibqp.qp_num.
1143 */
1144 spin_lock_init(&qp->r_lock);
1145 spin_lock_init(&qp->s_lock);
1146 spin_lock_init(&qp->r_rq.lock);
1147 atomic_set(&qp->refcount, 0);
1148 init_waitqueue_head(&qp->wait);
1149 init_timer(&qp->s_timer);
1150 qp->s_timer.data = (unsigned long)qp;
1151 INIT_LIST_HEAD(&qp->rspwait);
1152 qp->state = IB_QPS_RESET;
1153 qp->s_wq = swq;
1154 qp->s_size = init_attr->cap.max_send_wr + 1;
1155 qp->s_max_sge = init_attr->cap.max_send_sge;
1156 if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR)
1157 qp->s_flags = HFI1_S_SIGNAL_REQ_WR;
1158 dev = to_idev(ibpd->device);
1159 dd = dd_from_dev(dev);
1160 err = alloc_qpn(dd, &dev->qp_dev->qpn_table, init_attr->qp_type,
1161 init_attr->port_num);
1162 if (err < 0) {
1163 ret = ERR_PTR(err);
1164 vfree(qp->r_rq.wq);
1165 goto bail_qp;
1166 }
1167 qp->ibqp.qp_num = err;
1168 qp->port_num = init_attr->port_num;
1169 reset_qp(qp, init_attr->qp_type);
1170
1171 break;
1172
1173 default:
1174 /* Don't support raw QPs */
1175 ret = ERR_PTR(-ENOSYS);
1176 goto bail;
1177 }
1178
1179 init_attr->cap.max_inline_data = 0;
1180
1181 /*
1182 * Return the address of the RWQ as the offset to mmap.
1183 * See hfi1_mmap() for details.
1184 */
1185 if (udata && udata->outlen >= sizeof(__u64)) {
1186 if (!qp->r_rq.wq) {
1187 __u64 offset = 0;
1188
1189 err = ib_copy_to_udata(udata, &offset,
1190 sizeof(offset));
1191 if (err) {
1192 ret = ERR_PTR(err);
1193 goto bail_ip;
1194 }
1195 } else {
1196 u32 s = sizeof(struct rvt_rwq) + qp->r_rq.size * sz;
1197
1198 qp->ip = hfi1_create_mmap_info(dev, s,
1199 ibpd->uobject->context,
1200 qp->r_rq.wq);
1201 if (!qp->ip) {
1202 ret = ERR_PTR(-ENOMEM);
1203 goto bail_ip;
1204 }
1205
1206 err = ib_copy_to_udata(udata, &(qp->ip->offset),
1207 sizeof(qp->ip->offset));
1208 if (err) {
1209 ret = ERR_PTR(err);
1210 goto bail_ip;
1211 }
1212 }
1213 }
1214
1215 spin_lock(&dev->n_qps_lock);
1216 if (dev->n_qps_allocated == hfi1_max_qps) {
1217 spin_unlock(&dev->n_qps_lock);
1218 ret = ERR_PTR(-ENOMEM);
1219 goto bail_ip;
1220 }
1221
1222 dev->n_qps_allocated++;
1223 spin_unlock(&dev->n_qps_lock);
1224
1225 if (qp->ip) {
1226 spin_lock_irq(&dev->pending_lock);
1227 list_add(&qp->ip->pending_mmaps, &dev->pending_mmaps);
1228 spin_unlock_irq(&dev->pending_lock);
1229 }
1230
1231 ret = &qp->ibqp;
1232
1233 /*
1234 * We have our QP and its good, now keep track of what types of opcodes
1235 * can be processed on this QP. We do this by keeping track of what the
1236 * 3 high order bits of the opcode are.
1237 */
1238 switch (init_attr->qp_type) {
1239 case IB_QPT_SMI:
1240 case IB_QPT_GSI:
1241 case IB_QPT_UD:
1242 qp->allowed_ops = IB_OPCODE_UD_SEND_ONLY & OPCODE_QP_MASK;
1243 break;
1244 case IB_QPT_RC:
1245 qp->allowed_ops = IB_OPCODE_RC_SEND_ONLY & OPCODE_QP_MASK;
1246 break;
1247 case IB_QPT_UC:
1248 qp->allowed_ops = IB_OPCODE_UC_SEND_ONLY & OPCODE_QP_MASK;
1249 break;
1250 default:
1251 ret = ERR_PTR(-EINVAL);
1252 goto bail_ip;
1253 }
1254
1255 goto bail;
1256
1257 bail_ip:
1258 if (qp->ip)
1259 kref_put(&qp->ip->ref, hfi1_release_mmap_info);
1260 else
1261 vfree(qp->r_rq.wq);
1262 free_qpn(&dev->qp_dev->qpn_table, qp->ibqp.qp_num);
1263 bail_qp:
1264 kfree(priv->s_hdr);
1265 kfree(priv);
1266 bail_qp_priv:
1267 kfree(qp);
1268 bail_swq:
1269 vfree(swq);
1270 bail:
1271 return ret;
1272 }
1273
1274 /**
1275 * hfi1_destroy_qp - destroy a queue pair
1276 * @ibqp: the queue pair to destroy
1277 *
1278 * Returns 0 on success.
1279 *
1280 * Note that this can be called while the QP is actively sending or
1281 * receiving!
1282 */
1283 int hfi1_destroy_qp(struct ib_qp *ibqp)
1284 {
1285 struct rvt_qp *qp = to_iqp(ibqp);
1286 struct hfi1_ibdev *dev = to_idev(ibqp->device);
1287 struct hfi1_qp_priv *priv = qp->priv;
1288
1289 /* Make sure HW and driver activity is stopped. */
1290 spin_lock_irq(&qp->r_lock);
1291 spin_lock(&qp->s_lock);
1292 if (qp->state != IB_QPS_RESET) {
1293 qp->state = IB_QPS_RESET;
1294 flush_iowait(qp);
1295 qp->s_flags &= ~(HFI1_S_TIMER | HFI1_S_ANY_WAIT);
1296 spin_unlock(&qp->s_lock);
1297 spin_unlock_irq(&qp->r_lock);
1298 cancel_work_sync(&priv->s_iowait.iowork);
1299 del_timer_sync(&qp->s_timer);
1300 iowait_sdma_drain(&priv->s_iowait);
1301 flush_tx_list(qp);
1302 remove_qp(dev, qp);
1303 wait_event(qp->wait, !atomic_read(&qp->refcount));
1304 spin_lock_irq(&qp->r_lock);
1305 spin_lock(&qp->s_lock);
1306 clear_mr_refs(qp, 1);
1307 clear_ahg(qp);
1308 }
1309 spin_unlock(&qp->s_lock);
1310 spin_unlock_irq(&qp->r_lock);
1311
1312 /* all user's cleaned up, mark it available */
1313 free_qpn(&dev->qp_dev->qpn_table, qp->ibqp.qp_num);
1314 spin_lock(&dev->n_qps_lock);
1315 dev->n_qps_allocated--;
1316 spin_unlock(&dev->n_qps_lock);
1317
1318 if (qp->ip)
1319 kref_put(&qp->ip->ref, hfi1_release_mmap_info);
1320 else
1321 vfree(qp->r_rq.wq);
1322 vfree(qp->s_wq);
1323 kfree(priv->s_hdr);
1324 kfree(priv);
1325 kfree(qp);
1326 return 0;
1327 }
1328
1329 /**
1330 * init_qpn_table - initialize the QP number table for a device
1331 * @qpt: the QPN table
1332 */
1333 static int init_qpn_table(struct hfi1_devdata *dd, struct hfi1_qpn_table *qpt)
1334 {
1335 u32 offset, qpn, i;
1336 struct qpn_map *map;
1337 int ret = 0;
1338
1339 spin_lock_init(&qpt->lock);
1340
1341 qpt->last = 0;
1342 qpt->incr = 1 << dd->qos_shift;
1343
1344 /* insure we don't assign QPs from KDETH 64K window */
1345 qpn = kdeth_qp << 16;
1346 qpt->nmaps = qpn / BITS_PER_PAGE;
1347 /* This should always be zero */
1348 offset = qpn & BITS_PER_PAGE_MASK;
1349 map = &qpt->map[qpt->nmaps];
1350 dd_dev_info(dd, "Reserving QPNs for KDETH window from 0x%x to 0x%x\n",
1351 qpn, qpn + 65535);
1352 for (i = 0; i < 65536; i++) {
1353 if (!map->page) {
1354 get_map_page(qpt, map);
1355 if (!map->page) {
1356 ret = -ENOMEM;
1357 break;
1358 }
1359 }
1360 set_bit(offset, map->page);
1361 offset++;
1362 if (offset == BITS_PER_PAGE) {
1363 /* next page */
1364 qpt->nmaps++;
1365 map++;
1366 offset = 0;
1367 }
1368 }
1369 return ret;
1370 }
1371
1372 /**
1373 * free_qpn_table - free the QP number table for a device
1374 * @qpt: the QPN table
1375 */
1376 static void free_qpn_table(struct hfi1_qpn_table *qpt)
1377 {
1378 int i;
1379
1380 for (i = 0; i < ARRAY_SIZE(qpt->map); i++)
1381 free_page((unsigned long) qpt->map[i].page);
1382 }
1383
1384 /**
1385 * hfi1_get_credit - flush the send work queue of a QP
1386 * @qp: the qp who's send work queue to flush
1387 * @aeth: the Acknowledge Extended Transport Header
1388 *
1389 * The QP s_lock should be held.
1390 */
1391 void hfi1_get_credit(struct rvt_qp *qp, u32 aeth)
1392 {
1393 u32 credit = (aeth >> HFI1_AETH_CREDIT_SHIFT) & HFI1_AETH_CREDIT_MASK;
1394
1395 /*
1396 * If the credit is invalid, we can send
1397 * as many packets as we like. Otherwise, we have to
1398 * honor the credit field.
1399 */
1400 if (credit == HFI1_AETH_CREDIT_INVAL) {
1401 if (!(qp->s_flags & HFI1_S_UNLIMITED_CREDIT)) {
1402 qp->s_flags |= HFI1_S_UNLIMITED_CREDIT;
1403 if (qp->s_flags & HFI1_S_WAIT_SSN_CREDIT) {
1404 qp->s_flags &= ~HFI1_S_WAIT_SSN_CREDIT;
1405 hfi1_schedule_send(qp);
1406 }
1407 }
1408 } else if (!(qp->s_flags & HFI1_S_UNLIMITED_CREDIT)) {
1409 /* Compute new LSN (i.e., MSN + credit) */
1410 credit = (aeth + credit_table[credit]) & HFI1_MSN_MASK;
1411 if (cmp_msn(credit, qp->s_lsn) > 0) {
1412 qp->s_lsn = credit;
1413 if (qp->s_flags & HFI1_S_WAIT_SSN_CREDIT) {
1414 qp->s_flags &= ~HFI1_S_WAIT_SSN_CREDIT;
1415 hfi1_schedule_send(qp);
1416 }
1417 }
1418 }
1419 }
1420
1421 void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag)
1422 {
1423 unsigned long flags;
1424
1425 spin_lock_irqsave(&qp->s_lock, flags);
1426 if (qp->s_flags & flag) {
1427 qp->s_flags &= ~flag;
1428 trace_hfi1_qpwakeup(qp, flag);
1429 hfi1_schedule_send(qp);
1430 }
1431 spin_unlock_irqrestore(&qp->s_lock, flags);
1432 /* Notify hfi1_destroy_qp() if it is waiting. */
1433 if (atomic_dec_and_test(&qp->refcount))
1434 wake_up(&qp->wait);
1435 }
1436
1437 static int iowait_sleep(
1438 struct sdma_engine *sde,
1439 struct iowait *wait,
1440 struct sdma_txreq *stx,
1441 unsigned seq)
1442 {
1443 struct verbs_txreq *tx = container_of(stx, struct verbs_txreq, txreq);
1444 struct rvt_qp *qp;
1445 struct hfi1_qp_priv *priv;
1446 unsigned long flags;
1447 int ret = 0;
1448 struct hfi1_ibdev *dev;
1449
1450 qp = tx->qp;
1451 priv = qp->priv;
1452
1453 spin_lock_irqsave(&qp->s_lock, flags);
1454 if (ib_hfi1_state_ops[qp->state] & HFI1_PROCESS_RECV_OK) {
1455
1456 /*
1457 * If we couldn't queue the DMA request, save the info
1458 * and try again later rather than destroying the
1459 * buffer and undoing the side effects of the copy.
1460 */
1461 /* Make a common routine? */
1462 dev = &sde->dd->verbs_dev;
1463 list_add_tail(&stx->list, &wait->tx_head);
1464 write_seqlock(&dev->iowait_lock);
1465 if (sdma_progress(sde, seq, stx))
1466 goto eagain;
1467 if (list_empty(&priv->s_iowait.list)) {
1468 struct hfi1_ibport *ibp =
1469 to_iport(qp->ibqp.device, qp->port_num);
1470
1471 ibp->rvp.n_dmawait++;
1472 qp->s_flags |= HFI1_S_WAIT_DMA_DESC;
1473 list_add_tail(&priv->s_iowait.list, &sde->dmawait);
1474 trace_hfi1_qpsleep(qp, HFI1_S_WAIT_DMA_DESC);
1475 atomic_inc(&qp->refcount);
1476 }
1477 write_sequnlock(&dev->iowait_lock);
1478 qp->s_flags &= ~HFI1_S_BUSY;
1479 spin_unlock_irqrestore(&qp->s_lock, flags);
1480 ret = -EBUSY;
1481 } else {
1482 spin_unlock_irqrestore(&qp->s_lock, flags);
1483 hfi1_put_txreq(tx);
1484 }
1485 return ret;
1486 eagain:
1487 write_sequnlock(&dev->iowait_lock);
1488 spin_unlock_irqrestore(&qp->s_lock, flags);
1489 list_del_init(&stx->list);
1490 return -EAGAIN;
1491 }
1492
1493 static void iowait_wakeup(struct iowait *wait, int reason)
1494 {
1495 struct rvt_qp *qp = iowait_to_qp(wait);
1496
1497 WARN_ON(reason != SDMA_AVAIL_REASON);
1498 hfi1_qp_wakeup(qp, HFI1_S_WAIT_DMA_DESC);
1499 }
1500
1501 int hfi1_qp_init(struct hfi1_ibdev *dev)
1502 {
1503 struct hfi1_devdata *dd = dd_from_dev(dev);
1504 int i;
1505 int ret = -ENOMEM;
1506
1507 /* allocate parent object */
1508 dev->qp_dev = kzalloc(sizeof(*dev->qp_dev), GFP_KERNEL);
1509 if (!dev->qp_dev)
1510 goto nomem;
1511 /* allocate hash table */
1512 dev->qp_dev->qp_table_size = hfi1_qp_table_size;
1513 dev->qp_dev->qp_table_bits = ilog2(hfi1_qp_table_size);
1514 dev->qp_dev->qp_table =
1515 kmalloc(dev->qp_dev->qp_table_size *
1516 sizeof(*dev->qp_dev->qp_table),
1517 GFP_KERNEL);
1518 if (!dev->qp_dev->qp_table)
1519 goto nomem;
1520 for (i = 0; i < dev->qp_dev->qp_table_size; i++)
1521 RCU_INIT_POINTER(dev->qp_dev->qp_table[i], NULL);
1522 spin_lock_init(&dev->qp_dev->qpt_lock);
1523 /* initialize qpn map */
1524 ret = init_qpn_table(dd, &dev->qp_dev->qpn_table);
1525 if (ret)
1526 goto nomem;
1527 return ret;
1528 nomem:
1529 if (dev->qp_dev) {
1530 kfree(dev->qp_dev->qp_table);
1531 free_qpn_table(&dev->qp_dev->qpn_table);
1532 kfree(dev->qp_dev);
1533 }
1534 return ret;
1535 }
1536
1537 void hfi1_qp_exit(struct hfi1_ibdev *dev)
1538 {
1539 struct hfi1_devdata *dd = dd_from_dev(dev);
1540 u32 qps_inuse;
1541
1542 qps_inuse = free_all_qps(dd);
1543 if (qps_inuse)
1544 dd_dev_err(dd, "QP memory leak! %u still in use\n",
1545 qps_inuse);
1546 if (dev->qp_dev) {
1547 kfree(dev->qp_dev->qp_table);
1548 free_qpn_table(&dev->qp_dev->qpn_table);
1549 kfree(dev->qp_dev);
1550 }
1551 }
1552
1553 /**
1554 *
1555 * qp_to_sdma_engine - map a qp to a send engine
1556 * @qp: the QP
1557 * @sc5: the 5 bit sc
1558 *
1559 * Return:
1560 * A send engine for the qp or NULL for SMI type qp.
1561 */
1562 struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5)
1563 {
1564 struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
1565 struct sdma_engine *sde;
1566
1567 if (!(dd->flags & HFI1_HAS_SEND_DMA))
1568 return NULL;
1569 switch (qp->ibqp.qp_type) {
1570 case IB_QPT_SMI:
1571 return NULL;
1572 default:
1573 break;
1574 }
1575 sde = sdma_select_engine_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, sc5);
1576 return sde;
1577 }
1578
1579 struct qp_iter {
1580 struct hfi1_ibdev *dev;
1581 struct rvt_qp *qp;
1582 int specials;
1583 int n;
1584 };
1585
1586 struct qp_iter *qp_iter_init(struct hfi1_ibdev *dev)
1587 {
1588 struct qp_iter *iter;
1589
1590 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1591 if (!iter)
1592 return NULL;
1593
1594 iter->dev = dev;
1595 iter->specials = dev->rdi.ibdev.phys_port_cnt * 2;
1596 if (qp_iter_next(iter)) {
1597 kfree(iter);
1598 return NULL;
1599 }
1600
1601 return iter;
1602 }
1603
1604 int qp_iter_next(struct qp_iter *iter)
1605 {
1606 struct hfi1_ibdev *dev = iter->dev;
1607 int n = iter->n;
1608 int ret = 1;
1609 struct rvt_qp *pqp = iter->qp;
1610 struct rvt_qp *qp;
1611
1612 /*
1613 * The approach is to consider the special qps
1614 * as an additional table entries before the
1615 * real hash table. Since the qp code sets
1616 * the qp->next hash link to NULL, this works just fine.
1617 *
1618 * iter->specials is 2 * # ports
1619 *
1620 * n = 0..iter->specials is the special qp indices
1621 *
1622 * n = iter->specials..dev->qp_dev->qp_table_size+iter->specials are
1623 * the potential hash bucket entries
1624 *
1625 */
1626 for (; n < dev->qp_dev->qp_table_size + iter->specials; n++) {
1627 if (pqp) {
1628 qp = rcu_dereference(pqp->next);
1629 } else {
1630 if (n < iter->specials) {
1631 struct hfi1_pportdata *ppd;
1632 struct hfi1_ibport *ibp;
1633 int pidx;
1634
1635 pidx = n % dev->rdi.ibdev.phys_port_cnt;
1636 ppd = &dd_from_dev(dev)->pport[pidx];
1637 ibp = &ppd->ibport_data;
1638
1639 if (!(n & 1))
1640 qp = rcu_dereference(ibp->rvp.qp[0]);
1641 else
1642 qp = rcu_dereference(ibp->rvp.qp[1]);
1643 } else {
1644 qp = rcu_dereference(
1645 dev->qp_dev->qp_table[
1646 (n - iter->specials)]);
1647 }
1648 }
1649 pqp = qp;
1650 if (qp) {
1651 iter->qp = qp;
1652 iter->n = n;
1653 return 0;
1654 }
1655 }
1656 return ret;
1657 }
1658
1659 static const char * const qp_type_str[] = {
1660 "SMI", "GSI", "RC", "UC", "UD",
1661 };
1662
1663 static int qp_idle(struct rvt_qp *qp)
1664 {
1665 return
1666 qp->s_last == qp->s_acked &&
1667 qp->s_acked == qp->s_cur &&
1668 qp->s_cur == qp->s_tail &&
1669 qp->s_tail == qp->s_head;
1670 }
1671
1672 void qp_iter_print(struct seq_file *s, struct qp_iter *iter)
1673 {
1674 struct rvt_swqe *wqe;
1675 struct rvt_qp *qp = iter->qp;
1676 struct hfi1_qp_priv *priv = qp->priv;
1677 struct sdma_engine *sde;
1678
1679 sde = qp_to_sdma_engine(qp, priv->s_sc);
1680 wqe = get_swqe_ptr(qp, qp->s_last);
1681 seq_printf(s,
1682 "N %d %s QP%u R %u %s %u %u %u f=%x %u %u %u %u %u PSN %x %x %x %x %x (%u %u %u %u %u %u) QP%u LID %x SL %u MTU %d %u %u %u SDE %p,%u\n",
1683 iter->n,
1684 qp_idle(qp) ? "I" : "B",
1685 qp->ibqp.qp_num,
1686 atomic_read(&qp->refcount),
1687 qp_type_str[qp->ibqp.qp_type],
1688 qp->state,
1689 wqe ? wqe->wr.opcode : 0,
1690 qp->s_hdrwords,
1691 qp->s_flags,
1692 atomic_read(&priv->s_iowait.sdma_busy),
1693 !list_empty(&priv->s_iowait.list),
1694 qp->timeout,
1695 wqe ? wqe->ssn : 0,
1696 qp->s_lsn,
1697 qp->s_last_psn,
1698 qp->s_psn, qp->s_next_psn,
1699 qp->s_sending_psn, qp->s_sending_hpsn,
1700 qp->s_last, qp->s_acked, qp->s_cur,
1701 qp->s_tail, qp->s_head, qp->s_size,
1702 qp->remote_qpn,
1703 qp->remote_ah_attr.dlid,
1704 qp->remote_ah_attr.sl,
1705 qp->pmtu,
1706 qp->s_retry_cnt,
1707 qp->timeout,
1708 qp->s_rnr_retry_cnt,
1709 sde,
1710 sde ? sde->this_idx : 0);
1711 }
1712
1713 void qp_comm_est(struct rvt_qp *qp)
1714 {
1715 qp->r_flags |= HFI1_R_COMM_EST;
1716 if (qp->ibqp.event_handler) {
1717 struct ib_event ev;
1718
1719 ev.device = qp->ibqp.device;
1720 ev.element.qp = &qp->ibqp;
1721 ev.event = IB_EVENT_COMM_EST;
1722 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
1723 }
1724 }
1725
1726 /*
1727 * Switch to alternate path.
1728 * The QP s_lock should be held and interrupts disabled.
1729 */
1730 void hfi1_migrate_qp(struct rvt_qp *qp)
1731 {
1732 struct hfi1_qp_priv *priv = qp->priv;
1733 struct ib_event ev;
1734
1735 qp->s_mig_state = IB_MIG_MIGRATED;
1736 qp->remote_ah_attr = qp->alt_ah_attr;
1737 qp->port_num = qp->alt_ah_attr.port_num;
1738 qp->s_pkey_index = qp->s_alt_pkey_index;
1739 qp->s_flags |= HFI1_S_AHG_CLEAR;
1740 priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr);
1741 priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
1742
1743 ev.device = qp->ibqp.device;
1744 ev.element.qp = &qp->ibqp;
1745 ev.event = IB_EVENT_PATH_MIG;
1746 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
1747 }
Ubuntu Artful kernel mirror