2 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * $Id: mthca_srq.c 3047 2005-08-10 03:59:35Z roland $
35 #include <linux/slab.h>
36 #include <linux/string.h>
38 #include "mthca_dev.h"
39 #include "mthca_cmd.h"
40 #include "mthca_memfree.h"
41 #include "mthca_wqe.h"
44 MTHCA_MAX_DIRECT_SRQ_SIZE
= 4 * PAGE_SIZE
47 struct mthca_tavor_srq_context
{
48 __be64 wqe_base_ds
; /* low 6 bits is descriptor size */
56 struct mthca_arbel_srq_context
{
57 __be32 state_logsize_srqn
;
60 __be32 logstride_usrpage
;
63 __be16 limit_watermark
;
70 static void *get_wqe(struct mthca_srq
*srq
, int n
)
73 return srq
->queue
.direct
.buf
+ (n
<< srq
->wqe_shift
);
75 return srq
->queue
.page_list
[(n
<< srq
->wqe_shift
) >> PAGE_SHIFT
].buf
+
76 ((n
<< srq
->wqe_shift
) & (PAGE_SIZE
- 1));
80 * Return a pointer to the location within a WQE that we're using as a
81 * link when the WQE is in the free list. We use the imm field
82 * because in the Tavor case, posting a WQE may overwrite the next
83 * segment of the previous WQE, but a receive WQE will never touch the
84 * imm field. This avoids corrupting our free list if the previous
85 * WQE has already completed and been put on the free list when we
88 static inline int *wqe_to_link(void *wqe
)
90 return (int *) (wqe
+ offsetof(struct mthca_next_seg
, imm
));
93 static void mthca_tavor_init_srq_context(struct mthca_dev
*dev
,
95 struct mthca_srq
*srq
,
96 struct mthca_tavor_srq_context
*context
)
98 memset(context
, 0, sizeof *context
);
100 context
->wqe_base_ds
= cpu_to_be64(1 << (srq
->wqe_shift
- 4));
101 context
->state_pd
= cpu_to_be32(pd
->pd_num
);
102 context
->lkey
= cpu_to_be32(srq
->mr
.ibmr
.lkey
);
104 if (pd
->ibpd
.uobject
)
106 cpu_to_be32(to_mucontext(pd
->ibpd
.uobject
->context
)->uar
.index
);
108 context
->uar
= cpu_to_be32(dev
->driver_uar
.index
);
111 static void mthca_arbel_init_srq_context(struct mthca_dev
*dev
,
113 struct mthca_srq
*srq
,
114 struct mthca_arbel_srq_context
*context
)
118 memset(context
, 0, sizeof *context
);
120 logsize
= long_log2(srq
->max
) + srq
->wqe_shift
;
121 context
->state_logsize_srqn
= cpu_to_be32(logsize
<< 24 | srq
->srqn
);
122 context
->lkey
= cpu_to_be32(srq
->mr
.ibmr
.lkey
);
123 context
->db_index
= cpu_to_be32(srq
->db_index
);
124 context
->logstride_usrpage
= cpu_to_be32((srq
->wqe_shift
- 4) << 29);
125 if (pd
->ibpd
.uobject
)
126 context
->logstride_usrpage
|=
127 cpu_to_be32(to_mucontext(pd
->ibpd
.uobject
->context
)->uar
.index
);
129 context
->logstride_usrpage
|= cpu_to_be32(dev
->driver_uar
.index
);
130 context
->eq_pd
= cpu_to_be32(MTHCA_EQ_ASYNC
<< 24 | pd
->pd_num
);
133 static void mthca_free_srq_buf(struct mthca_dev
*dev
, struct mthca_srq
*srq
)
135 mthca_buf_free(dev
, srq
->max
<< srq
->wqe_shift
, &srq
->queue
,
136 srq
->is_direct
, &srq
->mr
);
140 static int mthca_alloc_srq_buf(struct mthca_dev
*dev
, struct mthca_pd
*pd
,
141 struct mthca_srq
*srq
)
143 struct mthca_data_seg
*scatter
;
148 if (pd
->ibpd
.uobject
)
151 srq
->wrid
= kmalloc(srq
->max
* sizeof (u64
), GFP_KERNEL
);
155 err
= mthca_buf_alloc(dev
, srq
->max
<< srq
->wqe_shift
,
156 MTHCA_MAX_DIRECT_SRQ_SIZE
,
157 &srq
->queue
, &srq
->is_direct
, pd
, 1, &srq
->mr
);
164 * Now initialize the SRQ buffer so that all of the WQEs are
165 * linked into the list of free WQEs. In addition, set the
166 * scatter list L_Keys to the sentry value of 0x100.
168 for (i
= 0; i
< srq
->max
; ++i
) {
169 wqe
= get_wqe(srq
, i
);
171 *wqe_to_link(wqe
) = i
< srq
->max
- 1 ? i
+ 1 : -1;
173 for (scatter
= wqe
+ sizeof (struct mthca_next_seg
);
174 (void *) scatter
< wqe
+ (1 << srq
->wqe_shift
);
176 scatter
->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
179 srq
->last
= get_wqe(srq
, srq
->max
- 1);
184 int mthca_alloc_srq(struct mthca_dev
*dev
, struct mthca_pd
*pd
,
185 struct ib_srq_attr
*attr
, struct mthca_srq
*srq
)
187 struct mthca_mailbox
*mailbox
;
192 /* Sanity check SRQ size before proceeding */
193 if (attr
->max_wr
> dev
->limits
.max_srq_wqes
||
194 attr
->max_sge
> dev
->limits
.max_sg
)
197 srq
->max
= attr
->max_wr
;
198 srq
->max_gs
= attr
->max_sge
;
201 if (mthca_is_memfree(dev
))
202 srq
->max
= roundup_pow_of_two(srq
->max
+ 1);
205 roundup_pow_of_two(sizeof (struct mthca_next_seg
) +
206 srq
->max_gs
* sizeof (struct mthca_data_seg
)));
207 srq
->wqe_shift
= long_log2(ds
);
209 srq
->srqn
= mthca_alloc(&dev
->srq_table
.alloc
);
213 if (mthca_is_memfree(dev
)) {
214 err
= mthca_table_get(dev
, dev
->srq_table
.table
, srq
->srqn
);
218 if (!pd
->ibpd
.uobject
) {
219 srq
->db_index
= mthca_alloc_db(dev
, MTHCA_DB_TYPE_SRQ
,
220 srq
->srqn
, &srq
->db
);
221 if (srq
->db_index
< 0) {
228 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
229 if (IS_ERR(mailbox
)) {
230 err
= PTR_ERR(mailbox
);
234 err
= mthca_alloc_srq_buf(dev
, pd
, srq
);
236 goto err_out_mailbox
;
238 spin_lock_init(&srq
->lock
);
239 atomic_set(&srq
->refcount
, 1);
240 init_waitqueue_head(&srq
->wait
);
242 if (mthca_is_memfree(dev
))
243 mthca_arbel_init_srq_context(dev
, pd
, srq
, mailbox
->buf
);
245 mthca_tavor_init_srq_context(dev
, pd
, srq
, mailbox
->buf
);
247 err
= mthca_SW2HW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
250 mthca_warn(dev
, "SW2HW_SRQ failed (%d)\n", err
);
251 goto err_out_free_buf
;
254 mthca_warn(dev
, "SW2HW_SRQ returned status 0x%02x\n",
257 goto err_out_free_buf
;
260 spin_lock_irq(&dev
->srq_table
.lock
);
261 if (mthca_array_set(&dev
->srq_table
.srq
,
262 srq
->srqn
& (dev
->limits
.num_srqs
- 1),
264 spin_unlock_irq(&dev
->srq_table
.lock
);
265 goto err_out_free_srq
;
267 spin_unlock_irq(&dev
->srq_table
.lock
);
269 mthca_free_mailbox(dev
, mailbox
);
272 srq
->last_free
= srq
->max
- 1;
277 err
= mthca_HW2SW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
279 mthca_warn(dev
, "HW2SW_SRQ failed (%d)\n", err
);
281 mthca_warn(dev
, "HW2SW_SRQ returned status 0x%02x\n", status
);
284 if (!pd
->ibpd
.uobject
)
285 mthca_free_srq_buf(dev
, srq
);
288 mthca_free_mailbox(dev
, mailbox
);
291 if (!pd
->ibpd
.uobject
&& mthca_is_memfree(dev
))
292 mthca_free_db(dev
, MTHCA_DB_TYPE_SRQ
, srq
->db_index
);
295 mthca_table_put(dev
, dev
->srq_table
.table
, srq
->srqn
);
298 mthca_free(&dev
->srq_table
.alloc
, srq
->srqn
);
303 void mthca_free_srq(struct mthca_dev
*dev
, struct mthca_srq
*srq
)
305 struct mthca_mailbox
*mailbox
;
309 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
310 if (IS_ERR(mailbox
)) {
311 mthca_warn(dev
, "No memory for mailbox to free SRQ.\n");
315 err
= mthca_HW2SW_SRQ(dev
, mailbox
, srq
->srqn
, &status
);
317 mthca_warn(dev
, "HW2SW_SRQ failed (%d)\n", err
);
319 mthca_warn(dev
, "HW2SW_SRQ returned status 0x%02x\n", status
);
321 spin_lock_irq(&dev
->srq_table
.lock
);
322 mthca_array_clear(&dev
->srq_table
.srq
,
323 srq
->srqn
& (dev
->limits
.num_srqs
- 1));
324 spin_unlock_irq(&dev
->srq_table
.lock
);
326 atomic_dec(&srq
->refcount
);
327 wait_event(srq
->wait
, !atomic_read(&srq
->refcount
));
329 if (!srq
->ibsrq
.uobject
) {
330 mthca_free_srq_buf(dev
, srq
);
331 if (mthca_is_memfree(dev
))
332 mthca_free_db(dev
, MTHCA_DB_TYPE_SRQ
, srq
->db_index
);
335 mthca_table_put(dev
, dev
->srq_table
.table
, srq
->srqn
);
336 mthca_free(&dev
->srq_table
.alloc
, srq
->srqn
);
337 mthca_free_mailbox(dev
, mailbox
);
340 int mthca_modify_srq(struct ib_srq
*ibsrq
, struct ib_srq_attr
*attr
,
341 enum ib_srq_attr_mask attr_mask
)
343 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
344 struct mthca_srq
*srq
= to_msrq(ibsrq
);
348 /* We don't support resizing SRQs (yet?) */
349 if (attr_mask
& IB_SRQ_MAX_WR
)
352 if (attr_mask
& IB_SRQ_LIMIT
) {
353 ret
= mthca_ARM_SRQ(dev
, srq
->srqn
, attr
->srq_limit
, &status
);
363 void mthca_srq_event(struct mthca_dev
*dev
, u32 srqn
,
364 enum ib_event_type event_type
)
366 struct mthca_srq
*srq
;
367 struct ib_event event
;
369 spin_lock(&dev
->srq_table
.lock
);
370 srq
= mthca_array_get(&dev
->srq_table
.srq
, srqn
& (dev
->limits
.num_srqs
- 1));
372 atomic_inc(&srq
->refcount
);
373 spin_unlock(&dev
->srq_table
.lock
);
376 mthca_warn(dev
, "Async event for bogus SRQ %08x\n", srqn
);
380 if (!srq
->ibsrq
.event_handler
)
383 event
.device
= &dev
->ib_dev
;
384 event
.event
= event_type
;
385 event
.element
.srq
= &srq
->ibsrq
;
386 srq
->ibsrq
.event_handler(&event
, srq
->ibsrq
.srq_context
);
389 if (atomic_dec_and_test(&srq
->refcount
))
394 * This function must be called with IRQs disabled.
396 void mthca_free_srq_wqe(struct mthca_srq
*srq
, u32 wqe_addr
)
400 ind
= wqe_addr
>> srq
->wqe_shift
;
402 spin_lock(&srq
->lock
);
404 if (likely(srq
->first_free
>= 0))
405 *wqe_to_link(get_wqe(srq
, srq
->last_free
)) = ind
;
407 srq
->first_free
= ind
;
409 *wqe_to_link(get_wqe(srq
, ind
)) = -1;
410 srq
->last_free
= ind
;
412 spin_unlock(&srq
->lock
);
415 int mthca_tavor_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
416 struct ib_recv_wr
**bad_wr
)
418 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
419 struct mthca_srq
*srq
= to_msrq(ibsrq
);
430 spin_lock_irqsave(&srq
->lock
, flags
);
432 first_ind
= srq
->first_free
;
434 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
435 ind
= srq
->first_free
;
438 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
444 wqe
= get_wqe(srq
, ind
);
445 next_ind
= *wqe_to_link(wqe
);
448 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
454 prev_wqe
= srq
->last
;
457 ((struct mthca_next_seg
*) wqe
)->nda_op
= 0;
458 ((struct mthca_next_seg
*) wqe
)->ee_nds
= 0;
459 /* flags field will always remain 0 */
461 wqe
+= sizeof (struct mthca_next_seg
);
463 if (unlikely(wr
->num_sge
> srq
->max_gs
)) {
466 srq
->last
= prev_wqe
;
470 for (i
= 0; i
< wr
->num_sge
; ++i
) {
471 ((struct mthca_data_seg
*) wqe
)->byte_count
=
472 cpu_to_be32(wr
->sg_list
[i
].length
);
473 ((struct mthca_data_seg
*) wqe
)->lkey
=
474 cpu_to_be32(wr
->sg_list
[i
].lkey
);
475 ((struct mthca_data_seg
*) wqe
)->addr
=
476 cpu_to_be64(wr
->sg_list
[i
].addr
);
477 wqe
+= sizeof (struct mthca_data_seg
);
480 if (i
< srq
->max_gs
) {
481 ((struct mthca_data_seg
*) wqe
)->byte_count
= 0;
482 ((struct mthca_data_seg
*) wqe
)->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
483 ((struct mthca_data_seg
*) wqe
)->addr
= 0;
486 ((struct mthca_next_seg
*) prev_wqe
)->nda_op
=
487 cpu_to_be32((ind
<< srq
->wqe_shift
) | 1);
489 ((struct mthca_next_seg
*) prev_wqe
)->ee_nds
=
490 cpu_to_be32(MTHCA_NEXT_DBD
);
492 srq
->wrid
[ind
] = wr
->wr_id
;
493 srq
->first_free
= next_ind
;
499 doorbell
[0] = cpu_to_be32(first_ind
<< srq
->wqe_shift
);
500 doorbell
[1] = cpu_to_be32((srq
->srqn
<< 8) | nreq
);
503 * Make sure that descriptors are written before
508 mthca_write64(doorbell
,
509 dev
->kar
+ MTHCA_RECEIVE_DOORBELL
,
510 MTHCA_GET_DOORBELL_LOCK(&dev
->doorbell_lock
));
513 spin_unlock_irqrestore(&srq
->lock
, flags
);
517 int mthca_arbel_post_srq_recv(struct ib_srq
*ibsrq
, struct ib_recv_wr
*wr
,
518 struct ib_recv_wr
**bad_wr
)
520 struct mthca_dev
*dev
= to_mdev(ibsrq
->device
);
521 struct mthca_srq
*srq
= to_msrq(ibsrq
);
530 spin_lock_irqsave(&srq
->lock
, flags
);
532 for (nreq
= 0; wr
; ++nreq
, wr
= wr
->next
) {
533 ind
= srq
->first_free
;
536 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
542 wqe
= get_wqe(srq
, ind
);
543 next_ind
= *wqe_to_link(wqe
);
546 mthca_err(dev
, "SRQ %06x full\n", srq
->srqn
);
552 ((struct mthca_next_seg
*) wqe
)->nda_op
=
553 cpu_to_be32((next_ind
<< srq
->wqe_shift
) | 1);
554 ((struct mthca_next_seg
*) wqe
)->ee_nds
= 0;
555 /* flags field will always remain 0 */
557 wqe
+= sizeof (struct mthca_next_seg
);
559 if (unlikely(wr
->num_sge
> srq
->max_gs
)) {
565 for (i
= 0; i
< wr
->num_sge
; ++i
) {
566 ((struct mthca_data_seg
*) wqe
)->byte_count
=
567 cpu_to_be32(wr
->sg_list
[i
].length
);
568 ((struct mthca_data_seg
*) wqe
)->lkey
=
569 cpu_to_be32(wr
->sg_list
[i
].lkey
);
570 ((struct mthca_data_seg
*) wqe
)->addr
=
571 cpu_to_be64(wr
->sg_list
[i
].addr
);
572 wqe
+= sizeof (struct mthca_data_seg
);
575 if (i
< srq
->max_gs
) {
576 ((struct mthca_data_seg
*) wqe
)->byte_count
= 0;
577 ((struct mthca_data_seg
*) wqe
)->lkey
= cpu_to_be32(MTHCA_INVAL_LKEY
);
578 ((struct mthca_data_seg
*) wqe
)->addr
= 0;
581 srq
->wrid
[ind
] = wr
->wr_id
;
582 srq
->first_free
= next_ind
;
586 srq
->counter
+= nreq
;
589 * Make sure that descriptors are written before
590 * we write doorbell record.
593 *srq
->db
= cpu_to_be32(srq
->counter
);
596 spin_unlock_irqrestore(&srq
->lock
, flags
);
600 int __devinit
mthca_init_srq_table(struct mthca_dev
*dev
)
604 if (!(dev
->mthca_flags
& MTHCA_FLAG_SRQ
))
607 spin_lock_init(&dev
->srq_table
.lock
);
609 err
= mthca_alloc_init(&dev
->srq_table
.alloc
,
610 dev
->limits
.num_srqs
,
611 dev
->limits
.num_srqs
- 1,
612 dev
->limits
.reserved_srqs
);
616 err
= mthca_array_init(&dev
->srq_table
.srq
,
617 dev
->limits
.num_srqs
);
619 mthca_alloc_cleanup(&dev
->srq_table
.alloc
);
624 void __devexit
mthca_cleanup_srq_table(struct mthca_dev
*dev
)
626 if (!(dev
->mthca_flags
& MTHCA_FLAG_SRQ
))
629 mthca_array_cleanup(&dev
->srq_table
.srq
, dev
->limits
.num_srqs
);
630 mthca_alloc_cleanup(&dev
->srq_table
.alloc
);