2 * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/random.h>
37 #include <linux/highmem.h>
38 #include <linux/slab.h>
39 #include <asm/byteorder.h>
41 #include <rdma/ib_verbs.h>
42 #include <rdma/iw_cm.h>
43 #include <rdma/ib_user_verbs.h>
47 #include <rdma/ib_umem.h>
49 atomic_t mod_qp_timouts
;
51 atomic_t sw_qps_destroyed
;
53 static void nes_unregister_ofa_device(struct nes_ib_device
*nesibdev
);
54 static int nes_dereg_mr(struct ib_mr
*ib_mr
);
59 static struct ib_mw
*nes_alloc_mw(struct ib_pd
*ibpd
, enum ib_mw_type type
,
60 struct ib_udata
*udata
)
62 struct nes_pd
*nespd
= to_nespd(ibpd
);
63 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
64 struct nes_device
*nesdev
= nesvnic
->nesdev
;
65 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
66 struct nes_cqp_request
*cqp_request
;
69 struct nes_hw_cqp_wqe
*cqp_wqe
;
73 u32 next_stag_index
= 0;
77 if (type
!= IB_MW_TYPE_1
)
78 return ERR_PTR(-EINVAL
);
80 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
81 stag_key
= (u8
)next_stag_index
;
85 next_stag_index
>>= 8;
86 next_stag_index
%= nesadapter
->max_mr
;
88 ret
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
89 nesadapter
->max_mr
, &stag_index
, &next_stag_index
, NES_RESOURCE_MW
);
94 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
96 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
97 return ERR_PTR(-ENOMEM
);
100 stag
= stag_index
<< 8;
102 stag
+= (u32
)stag_key
;
104 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, index = 0x%08X\n",
107 /* Register the region with the adapter */
108 cqp_request
= nes_get_cqp_request(nesdev
);
109 if (cqp_request
== NULL
) {
111 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
112 return ERR_PTR(-ENOMEM
);
115 cqp_request
->waiting
= 1;
116 cqp_wqe
= &cqp_request
->cqp_wqe
;
118 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] =
119 cpu_to_le32( NES_CQP_ALLOCATE_STAG
| NES_CQP_STAG_RIGHTS_REMOTE_READ
|
120 NES_CQP_STAG_RIGHTS_REMOTE_WRITE
| NES_CQP_STAG_VA_TO
|
121 NES_CQP_STAG_REM_ACC_EN
);
123 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
124 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
, (nespd
->pd_id
& 0x00007fff));
125 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
127 atomic_set(&cqp_request
->refcount
, 2);
128 nes_post_cqp_request(nesdev
, cqp_request
);
131 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
133 nes_debug(NES_DBG_MR
, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
134 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
135 stag
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
136 if ((!ret
) || (cqp_request
->major_code
)) {
137 nes_put_cqp_request(nesdev
, cqp_request
);
139 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
141 return ERR_PTR(-ETIME
);
143 return ERR_PTR(-ENOMEM
);
146 nes_put_cqp_request(nesdev
, cqp_request
);
148 nesmr
->ibmw
.rkey
= stag
;
149 nesmr
->mode
= IWNES_MEMREG_TYPE_MW
;
152 nesmr
->pbls_used
= 0;
161 static int nes_dealloc_mw(struct ib_mw
*ibmw
)
163 struct nes_mr
*nesmr
= to_nesmw(ibmw
);
164 struct nes_vnic
*nesvnic
= to_nesvnic(ibmw
->device
);
165 struct nes_device
*nesdev
= nesvnic
->nesdev
;
166 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
167 struct nes_hw_cqp_wqe
*cqp_wqe
;
168 struct nes_cqp_request
*cqp_request
;
172 /* Deallocate the window with the adapter */
173 cqp_request
= nes_get_cqp_request(nesdev
);
174 if (cqp_request
== NULL
) {
175 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
178 cqp_request
->waiting
= 1;
179 cqp_wqe
= &cqp_request
->cqp_wqe
;
180 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
181 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, NES_CQP_DEALLOCATE_STAG
);
182 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, ibmw
->rkey
);
184 atomic_set(&cqp_request
->refcount
, 2);
185 nes_post_cqp_request(nesdev
, cqp_request
);
188 nes_debug(NES_DBG_MR
, "Waiting for deallocate STag 0x%08X to complete.\n",
190 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
192 nes_debug(NES_DBG_MR
, "Deallocate STag completed, wait_event_timeout ret = %u,"
193 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
194 ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
197 else if (cqp_request
->major_code
)
200 nes_put_cqp_request(nesdev
, cqp_request
);
202 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
203 (ibmw
->rkey
& 0x0fffff00) >> 8);
213 static int alloc_fast_reg_mr(struct nes_device
*nesdev
, struct nes_pd
*nespd
,
214 u32 stag
, u32 page_count
)
216 struct nes_hw_cqp_wqe
*cqp_wqe
;
217 struct nes_cqp_request
*cqp_request
;
220 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
223 u64 region_length
= page_count
* PAGE_SIZE
;
226 cqp_request
= nes_get_cqp_request(nesdev
);
227 if (cqp_request
== NULL
) {
228 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
231 nes_debug(NES_DBG_MR
, "alloc_fast_reg_mr: page_count = %d, "
232 "region_length = %llu\n",
233 page_count
, region_length
);
234 cqp_request
->waiting
= 1;
235 cqp_wqe
= &cqp_request
->cqp_wqe
;
237 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
238 if (nesadapter
->free_4kpbl
> 0) {
239 nesadapter
->free_4kpbl
--;
240 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
242 /* No 4kpbl's available: */
243 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
244 nes_debug(NES_DBG_MR
, "Out of Pbls\n");
245 nes_free_cqp_request(nesdev
, cqp_request
);
249 opcode
= NES_CQP_ALLOCATE_STAG
| NES_CQP_STAG_MR
|
250 NES_CQP_STAG_PBL_BLK_SIZE
| NES_CQP_STAG_VA_TO
|
251 NES_CQP_STAG_REM_ACC_EN
;
253 * The current OFED API does not support the zero based TO option.
254 * If added then need to changed the NES_CQP_STAG_VA* option. Also,
255 * the API does not support that ability to have the MR set for local
256 * access only when created and not allow the SQ op to override. Given
257 * this the remote enable must be set here.
260 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
261 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
262 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX
, 1);
264 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] =
265 cpu_to_le32((u32
)(region_length
>> 8) & 0xff000000);
266 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] |=
267 cpu_to_le32(nespd
->pd_id
& 0x00007fff);
269 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
270 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_VA_LOW_IDX
, 0);
271 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_LOW_IDX
, 0);
272 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, 0);
273 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_LEN_IDX
, (page_count
* 8));
274 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE
);
277 atomic_set(&cqp_request
->refcount
, 2);
278 nes_post_cqp_request(nesdev
, cqp_request
);
281 ret
= wait_event_timeout(cqp_request
->waitq
,
282 (0 != cqp_request
->request_done
),
285 nes_debug(NES_DBG_MR
, "Allocate STag 0x%08X completed, "
286 "wait_event_timeout ret = %u, CQP Major:Minor codes = "
287 "0x%04X:0x%04X.\n", stag
, ret
, cqp_request
->major_code
,
288 cqp_request
->minor_code
);
289 major_code
= cqp_request
->major_code
;
290 nes_put_cqp_request(nesdev
, cqp_request
);
292 if (!ret
|| major_code
) {
293 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
294 nesadapter
->free_4kpbl
++;
295 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
308 static struct ib_mr
*nes_alloc_mr(struct ib_pd
*ibpd
,
309 enum ib_mr_type mr_type
,
312 struct nes_pd
*nespd
= to_nespd(ibpd
);
313 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
314 struct nes_device
*nesdev
= nesvnic
->nesdev
;
315 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
322 struct nes_mr
*nesmr
;
327 if (mr_type
!= IB_MR_TYPE_MEM_REG
)
328 return ERR_PTR(-EINVAL
);
330 if (max_num_sg
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
)))
331 return ERR_PTR(-E2BIG
);
334 * Note: Set to always use a fixed length single page entry PBL. This is to allow
335 * for the fast_reg_mr operation to always know the size of the PBL.
337 if (max_num_sg
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
)))
338 return ERR_PTR(-E2BIG
);
340 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
341 stag_key
= (u8
)next_stag_index
;
342 next_stag_index
>>= 8;
343 next_stag_index
%= nesadapter
->max_mr
;
345 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
346 nesadapter
->max_mr
, &stag_index
,
347 &next_stag_index
, NES_RESOURCE_FAST_MR
);
351 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
353 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
354 return ERR_PTR(-ENOMEM
);
357 stag
= stag_index
<< 8;
359 stag
+= (u32
)stag_key
;
361 nes_debug(NES_DBG_MR
, "Allocating STag 0x%08X index = 0x%08X\n",
364 ret
= alloc_fast_reg_mr(nesdev
, nespd
, stag
, max_num_sg
);
367 nesmr
->ibmr
.rkey
= stag
;
368 nesmr
->ibmr
.lkey
= stag
;
369 nesmr
->mode
= IWNES_MEMREG_TYPE_FMEM
;
373 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
374 return ERR_PTR(-ENOMEM
);
377 nesmr
->pages
= pci_alloc_consistent(nesdev
->pcidev
,
378 max_num_sg
* sizeof(u64
),
383 nesmr
->max_pages
= max_num_sg
;
390 return ERR_PTR(-ENOMEM
);
393 static int nes_set_page(struct ib_mr
*ibmr
, u64 addr
)
395 struct nes_mr
*nesmr
= to_nesmr(ibmr
);
397 if (unlikely(nesmr
->npages
== nesmr
->max_pages
))
400 nesmr
->pages
[nesmr
->npages
++] = cpu_to_le64(addr
);
405 static int nes_map_mr_sg(struct ib_mr
*ibmr
, struct scatterlist
*sg
,
406 int sg_nents
, unsigned int *sg_offset
)
408 struct nes_mr
*nesmr
= to_nesmr(ibmr
);
412 return ib_sg_to_pages(ibmr
, sg
, sg_nents
, sg_offset
, nes_set_page
);
418 static int nes_query_device(struct ib_device
*ibdev
, struct ib_device_attr
*props
,
419 struct ib_udata
*uhw
)
421 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
422 struct nes_device
*nesdev
= nesvnic
->nesdev
;
423 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
425 if (uhw
->inlen
|| uhw
->outlen
)
428 memset(props
, 0, sizeof(*props
));
429 memcpy(&props
->sys_image_guid
, nesvnic
->netdev
->dev_addr
, 6);
431 props
->fw_ver
= nesdev
->nesadapter
->firmware_version
;
432 props
->device_cap_flags
= nesdev
->nesadapter
->device_cap_flags
;
433 props
->vendor_id
= nesdev
->nesadapter
->vendor_id
;
434 props
->vendor_part_id
= nesdev
->nesadapter
->vendor_part_id
;
435 props
->hw_ver
= nesdev
->nesadapter
->hw_rev
;
436 props
->max_mr_size
= 0x80000000;
437 props
->max_qp
= nesibdev
->max_qp
;
438 props
->max_qp_wr
= nesdev
->nesadapter
->max_qp_wr
- 2;
439 props
->max_sge
= nesdev
->nesadapter
->max_sge
;
440 props
->max_cq
= nesibdev
->max_cq
;
441 props
->max_cqe
= nesdev
->nesadapter
->max_cqe
;
442 props
->max_mr
= nesibdev
->max_mr
;
443 props
->max_mw
= nesibdev
->max_mr
;
444 props
->max_pd
= nesibdev
->max_pd
;
445 props
->max_sge_rd
= 1;
446 switch (nesdev
->nesadapter
->max_irrq_wr
) {
448 props
->max_qp_rd_atom
= 2;
451 props
->max_qp_rd_atom
= 8;
454 props
->max_qp_rd_atom
= 32;
457 props
->max_qp_rd_atom
= 64;
460 props
->max_qp_rd_atom
= 0;
462 props
->max_qp_init_rd_atom
= props
->max_qp_rd_atom
;
463 props
->atomic_cap
= IB_ATOMIC_NONE
;
464 props
->max_map_per_fmr
= 1;
473 static int nes_query_port(struct ib_device
*ibdev
, u8 port
, struct ib_port_attr
*props
)
475 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
476 struct net_device
*netdev
= nesvnic
->netdev
;
478 /* props being zeroed by the caller, avoid zeroing it here */
480 props
->max_mtu
= IB_MTU_4096
;
481 props
->active_mtu
= ib_mtu_int_to_enum(netdev
->mtu
);
487 if (netif_queue_stopped(netdev
))
488 props
->state
= IB_PORT_DOWN
;
489 else if (nesvnic
->linkup
)
490 props
->state
= IB_PORT_ACTIVE
;
492 props
->state
= IB_PORT_DOWN
;
493 props
->phys_state
= 0;
494 props
->port_cap_flags
= IB_PORT_CM_SUP
| IB_PORT_REINIT_SUP
|
495 IB_PORT_VENDOR_CLASS_SUP
| IB_PORT_BOOT_MGMT_SUP
;
496 props
->gid_tbl_len
= 1;
497 props
->pkey_tbl_len
= 1;
498 props
->qkey_viol_cntr
= 0;
499 props
->active_width
= IB_WIDTH_4X
;
500 props
->active_speed
= IB_SPEED_SDR
;
501 props
->max_msg_sz
= 0x80000000;
509 static int nes_query_pkey(struct ib_device
*ibdev
, u8 port
, u16 index
, u16
*pkey
)
519 static int nes_query_gid(struct ib_device
*ibdev
, u8 port
,
520 int index
, union ib_gid
*gid
)
522 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
524 memset(&(gid
->raw
[0]), 0, sizeof(gid
->raw
));
525 memcpy(&(gid
->raw
[0]), nesvnic
->netdev
->dev_addr
, 6);
532 * nes_alloc_ucontext - Allocate the user context data structure. This keeps track
533 * of all objects associated with a particular user-mode client.
535 static struct ib_ucontext
*nes_alloc_ucontext(struct ib_device
*ibdev
,
536 struct ib_udata
*udata
)
538 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
539 struct nes_device
*nesdev
= nesvnic
->nesdev
;
540 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
541 struct nes_alloc_ucontext_req req
;
542 struct nes_alloc_ucontext_resp uresp
;
543 struct nes_ucontext
*nes_ucontext
;
544 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
547 if (ib_copy_from_udata(&req
, udata
, sizeof(struct nes_alloc_ucontext_req
))) {
548 printk(KERN_ERR PFX
"Invalid structure size on allocate user context.\n");
549 return ERR_PTR(-EINVAL
);
552 if (req
.userspace_ver
!= NES_ABI_USERSPACE_VER
) {
553 printk(KERN_ERR PFX
"Invalid userspace driver version detected. Detected version %d, should be %d\n",
554 req
.userspace_ver
, NES_ABI_USERSPACE_VER
);
555 return ERR_PTR(-EINVAL
);
559 memset(&uresp
, 0, sizeof uresp
);
561 uresp
.max_qps
= nesibdev
->max_qp
;
562 uresp
.max_pds
= nesibdev
->max_pd
;
563 uresp
.wq_size
= nesdev
->nesadapter
->max_qp_wr
* 2;
564 uresp
.virtwq
= nesadapter
->virtwq
;
565 uresp
.kernel_ver
= NES_ABI_KERNEL_VER
;
567 nes_ucontext
= kzalloc(sizeof *nes_ucontext
, GFP_KERNEL
);
569 return ERR_PTR(-ENOMEM
);
571 nes_ucontext
->nesdev
= nesdev
;
572 nes_ucontext
->mmap_wq_offset
= uresp
.max_pds
;
573 nes_ucontext
->mmap_cq_offset
= nes_ucontext
->mmap_wq_offset
+
574 ((sizeof(struct nes_hw_qp_wqe
) * uresp
.max_qps
* 2) + PAGE_SIZE
-1) /
578 if (ib_copy_to_udata(udata
, &uresp
, sizeof uresp
)) {
580 return ERR_PTR(-EFAULT
);
583 INIT_LIST_HEAD(&nes_ucontext
->cq_reg_mem_list
);
584 INIT_LIST_HEAD(&nes_ucontext
->qp_reg_mem_list
);
585 atomic_set(&nes_ucontext
->usecnt
, 1);
586 return &nes_ucontext
->ibucontext
;
591 * nes_dealloc_ucontext
593 static int nes_dealloc_ucontext(struct ib_ucontext
*context
)
595 /* struct nes_vnic *nesvnic = to_nesvnic(context->device); */
596 /* struct nes_device *nesdev = nesvnic->nesdev; */
597 struct nes_ucontext
*nes_ucontext
= to_nesucontext(context
);
599 if (!atomic_dec_and_test(&nes_ucontext
->usecnt
))
609 static int nes_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
612 struct nes_vnic
*nesvnic
= to_nesvnic(context
->device
);
613 struct nes_device
*nesdev
= nesvnic
->nesdev
;
614 /* struct nes_adapter *nesadapter = nesdev->nesadapter; */
615 struct nes_ucontext
*nes_ucontext
;
616 struct nes_qp
*nesqp
;
618 nes_ucontext
= to_nesucontext(context
);
621 if (vma
->vm_pgoff
>= nes_ucontext
->mmap_wq_offset
) {
622 index
= (vma
->vm_pgoff
- nes_ucontext
->mmap_wq_offset
) * PAGE_SIZE
;
623 index
/= ((sizeof(struct nes_hw_qp_wqe
) * nesdev
->nesadapter
->max_qp_wr
* 2) +
624 PAGE_SIZE
-1) & (~(PAGE_SIZE
-1));
625 if (!test_bit(index
, nes_ucontext
->allocated_wqs
)) {
626 nes_debug(NES_DBG_MMAP
, "wq %lu not allocated\n", index
);
629 nesqp
= nes_ucontext
->mmap_nesqp
[index
];
631 nes_debug(NES_DBG_MMAP
, "wq %lu has a NULL QP base.\n", index
);
634 if (remap_pfn_range(vma
, vma
->vm_start
,
635 virt_to_phys(nesqp
->hwqp
.sq_vbase
) >> PAGE_SHIFT
,
636 vma
->vm_end
- vma
->vm_start
,
637 vma
->vm_page_prot
)) {
638 nes_debug(NES_DBG_MMAP
, "remap_pfn_range failed.\n");
641 vma
->vm_private_data
= nesqp
;
644 index
= vma
->vm_pgoff
;
645 if (!test_bit(index
, nes_ucontext
->allocated_doorbells
))
648 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
649 if (io_remap_pfn_range(vma
, vma
->vm_start
,
650 (nesdev
->doorbell_start
+
651 ((nes_ucontext
->mmap_db_index
[index
] - nesdev
->base_doorbell_index
) * 4096))
652 >> PAGE_SHIFT
, PAGE_SIZE
, vma
->vm_page_prot
))
654 vma
->vm_private_data
= nes_ucontext
;
665 static struct ib_pd
*nes_alloc_pd(struct ib_device
*ibdev
,
666 struct ib_ucontext
*context
, struct ib_udata
*udata
)
668 struct nes_pd
*nespd
;
669 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
670 struct nes_device
*nesdev
= nesvnic
->nesdev
;
671 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
672 struct nes_ucontext
*nesucontext
;
673 struct nes_alloc_pd_resp uresp
;
677 nes_debug(NES_DBG_PD
, "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n",
678 nesvnic
, nesdev
->netdev
[0], nesdev
->netdev
[0]->name
, ibdev
, context
,
679 netdev_refcnt_read(nesvnic
->netdev
));
681 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_pds
,
682 nesadapter
->max_pd
, &pd_num
, &nesadapter
->next_pd
, NES_RESOURCE_PD
);
687 nespd
= kzalloc(sizeof (struct nes_pd
), GFP_KERNEL
);
689 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
690 return ERR_PTR(-ENOMEM
);
693 nes_debug(NES_DBG_PD
, "Allocating PD (%p) for ib device %s\n",
694 nespd
, nesvnic
->nesibdev
->ibdev
.name
);
696 nespd
->pd_id
= (pd_num
<< (PAGE_SHIFT
-12)) + nesadapter
->base_pd
;
699 nesucontext
= to_nesucontext(context
);
700 nespd
->mmap_db_index
= find_next_zero_bit(nesucontext
->allocated_doorbells
,
701 NES_MAX_USER_DB_REGIONS
, nesucontext
->first_free_db
);
702 nes_debug(NES_DBG_PD
, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n",
703 nespd
->mmap_db_index
, nespd
->pd_id
);
704 if (nespd
->mmap_db_index
>= NES_MAX_USER_DB_REGIONS
) {
705 nes_debug(NES_DBG_PD
, "mmap_db_index > MAX\n");
706 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
708 return ERR_PTR(-ENOMEM
);
711 uresp
.pd_id
= nespd
->pd_id
;
712 uresp
.mmap_db_index
= nespd
->mmap_db_index
;
713 if (ib_copy_to_udata(udata
, &uresp
, sizeof (struct nes_alloc_pd_resp
))) {
714 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
, pd_num
);
716 return ERR_PTR(-EFAULT
);
719 set_bit(nespd
->mmap_db_index
, nesucontext
->allocated_doorbells
);
720 nesucontext
->mmap_db_index
[nespd
->mmap_db_index
] = nespd
->pd_id
;
721 nesucontext
->first_free_db
= nespd
->mmap_db_index
+ 1;
724 nes_debug(NES_DBG_PD
, "PD%u structure located @%p.\n", nespd
->pd_id
, nespd
);
732 static int nes_dealloc_pd(struct ib_pd
*ibpd
)
734 struct nes_ucontext
*nesucontext
;
735 struct nes_pd
*nespd
= to_nespd(ibpd
);
736 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
737 struct nes_device
*nesdev
= nesvnic
->nesdev
;
738 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
740 if ((ibpd
->uobject
) && (ibpd
->uobject
->context
)) {
741 nesucontext
= to_nesucontext(ibpd
->uobject
->context
);
742 nes_debug(NES_DBG_PD
, "Clearing bit %u from allocated doorbells\n",
743 nespd
->mmap_db_index
);
744 clear_bit(nespd
->mmap_db_index
, nesucontext
->allocated_doorbells
);
745 nesucontext
->mmap_db_index
[nespd
->mmap_db_index
] = 0;
746 if (nesucontext
->first_free_db
> nespd
->mmap_db_index
) {
747 nesucontext
->first_free_db
= nespd
->mmap_db_index
;
751 nes_debug(NES_DBG_PD
, "Deallocating PD%u structure located @%p.\n",
752 nespd
->pd_id
, nespd
);
753 nes_free_resource(nesadapter
, nesadapter
->allocated_pds
,
754 (nespd
->pd_id
-nesadapter
->base_pd
)>>(PAGE_SHIFT
-12));
764 static struct ib_ah
*nes_create_ah(struct ib_pd
*pd
, struct ib_ah_attr
*ah_attr
,
765 struct ib_udata
*udata
)
767 return ERR_PTR(-ENOSYS
);
774 static int nes_destroy_ah(struct ib_ah
*ah
)
781 * nes_get_encoded_size
783 static inline u8
nes_get_encoded_size(int *size
)
789 } else if (*size
<= 128) {
792 } else if (*size
<= 512) {
796 return (encoded_size
);
804 static int nes_setup_virt_qp(struct nes_qp
*nesqp
, struct nes_pbl
*nespbl
,
805 struct nes_vnic
*nesvnic
, int sq_size
, int rq_size
)
812 struct nes_device
*nesdev
= nesvnic
->nesdev
;
813 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
818 pbl_entries
= nespbl
->pbl_size
>> 3;
819 nes_debug(NES_DBG_QP
, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n",
820 nespbl
->pbl_size
, pbl_entries
,
821 (void *)nespbl
->pbl_vbase
,
822 (unsigned long) nespbl
->pbl_pbase
);
823 pbl
= (__le64
*) nespbl
->pbl_vbase
; /* points to first pbl entry */
824 /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */
825 /* the first pbl to be fro the rq_vbase... */
826 rq_pbl_entries
= (rq_size
* sizeof(struct nes_hw_qp_wqe
)) >> 12;
827 sq_pbl_entries
= (sq_size
* sizeof(struct nes_hw_qp_wqe
)) >> 12;
828 nesqp
->hwqp
.sq_pbase
= (le32_to_cpu(((__le32
*)pbl
)[0])) | ((u64
)((le32_to_cpu(((__le32
*)pbl
)[1]))) << 32);
830 nes_debug(NES_DBG_QP
, "QP nespbl->page is NULL \n");
835 nesqp
->hwqp
.sq_vbase
= kmap(nespbl
->page
);
836 nesqp
->page
= nespbl
->page
;
837 if (!nesqp
->hwqp
.sq_vbase
) {
838 nes_debug(NES_DBG_QP
, "QP sq_vbase kmap failed\n");
843 /* Now to get to sq.. we need to calculate how many */
844 /* PBL entries were used by the rq.. */
845 pbl
+= sq_pbl_entries
;
846 nesqp
->hwqp
.rq_pbase
= (le32_to_cpu(((__le32
*)pbl
)[0])) | ((u64
)((le32_to_cpu(((__le32
*)pbl
)[1]))) << 32);
847 /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */
848 /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */
850 nes_debug(NES_DBG_QP
, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n",
851 nesqp
->hwqp
.sq_vbase
, (unsigned long) nesqp
->hwqp
.sq_pbase
,
852 nesqp
->hwqp
.rq_vbase
, (unsigned long) nesqp
->hwqp
.rq_pbase
);
853 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
854 if (!nesadapter
->free_256pbl
) {
855 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
857 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
862 nesadapter
->free_256pbl
--;
863 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
865 nesqp
->pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 256, &nesqp
->pbl_pbase
);
866 pblbuffer
= nesqp
->pbl_vbase
;
867 if (!nesqp
->pbl_vbase
) {
868 /* memory allocated during nes_reg_user_mr() */
869 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
872 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
873 nesadapter
->free_256pbl
++;
874 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
878 memset(nesqp
->pbl_vbase
, 0, 256);
879 /* fill in the page address in the pbl buffer.. */
880 tpbl
= pblbuffer
+ 16;
881 pbl
= (__le64
*)nespbl
->pbl_vbase
;
882 while (sq_pbl_entries
--)
885 while (rq_pbl_entries
--)
888 /* done with memory allocated during nes_reg_user_mr() */
889 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
894 max((u32
)sizeof(struct nes_qp_context
), ((u32
)256)) + 256; /* this is Q2 */
895 /* Round up to a multiple of a page */
896 nesqp
->qp_mem_size
+= PAGE_SIZE
- 1;
897 nesqp
->qp_mem_size
&= ~(PAGE_SIZE
- 1);
899 mem
= pci_alloc_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
900 &nesqp
->hwqp
.q2_pbase
);
903 pci_free_consistent(nesdev
->pcidev
, 256, nesqp
->pbl_vbase
, nesqp
->pbl_pbase
);
904 nesqp
->pbl_vbase
= NULL
;
905 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
906 nesadapter
->free_256pbl
++;
907 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
911 nesqp
->sq_kmapped
= 1;
912 nesqp
->hwqp
.q2_vbase
= mem
;
914 memset(nesqp
->hwqp
.q2_vbase
, 0, 256);
915 nesqp
->nesqp_context
= mem
;
916 memset(nesqp
->nesqp_context
, 0, sizeof(*nesqp
->nesqp_context
));
917 nesqp
->nesqp_context_pbase
= nesqp
->hwqp
.q2_pbase
+ 256;
926 static int nes_setup_mmap_qp(struct nes_qp
*nesqp
, struct nes_vnic
*nesvnic
,
927 int sq_size
, int rq_size
)
930 struct nes_device
*nesdev
= nesvnic
->nesdev
;
932 nesqp
->qp_mem_size
= (sizeof(struct nes_hw_qp_wqe
) * sq_size
) +
933 (sizeof(struct nes_hw_qp_wqe
) * rq_size
) +
934 max((u32
)sizeof(struct nes_qp_context
), ((u32
)256)) +
935 256; /* this is Q2 */
936 /* Round up to a multiple of a page */
937 nesqp
->qp_mem_size
+= PAGE_SIZE
- 1;
938 nesqp
->qp_mem_size
&= ~(PAGE_SIZE
- 1);
940 mem
= pci_alloc_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
941 &nesqp
->hwqp
.sq_pbase
);
944 nes_debug(NES_DBG_QP
, "PCI consistent memory for "
945 "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n",
946 mem
, (unsigned long)nesqp
->hwqp
.sq_pbase
, nesqp
->qp_mem_size
);
948 memset(mem
, 0, nesqp
->qp_mem_size
);
950 nesqp
->hwqp
.sq_vbase
= mem
;
951 mem
+= sizeof(struct nes_hw_qp_wqe
) * sq_size
;
953 nesqp
->hwqp
.rq_vbase
= mem
;
954 nesqp
->hwqp
.rq_pbase
= nesqp
->hwqp
.sq_pbase
+
955 sizeof(struct nes_hw_qp_wqe
) * sq_size
;
956 mem
+= sizeof(struct nes_hw_qp_wqe
) * rq_size
;
958 nesqp
->hwqp
.q2_vbase
= mem
;
959 nesqp
->hwqp
.q2_pbase
= nesqp
->hwqp
.rq_pbase
+
960 sizeof(struct nes_hw_qp_wqe
) * rq_size
;
962 memset(nesqp
->hwqp
.q2_vbase
, 0, 256);
964 nesqp
->nesqp_context
= mem
;
965 nesqp
->nesqp_context_pbase
= nesqp
->hwqp
.q2_pbase
+ 256;
966 memset(nesqp
->nesqp_context
, 0, sizeof(*nesqp
->nesqp_context
));
972 * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory.
974 static void nes_free_qp_mem(struct nes_device
*nesdev
,
975 struct nes_qp
*nesqp
, int virt_wqs
)
978 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
980 pci_free_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
,
981 nesqp
->hwqp
.sq_vbase
, nesqp
->hwqp
.sq_pbase
);
983 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
984 nesadapter
->free_256pbl
++;
985 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
986 pci_free_consistent(nesdev
->pcidev
, nesqp
->qp_mem_size
, nesqp
->hwqp
.q2_vbase
, nesqp
->hwqp
.q2_pbase
);
987 pci_free_consistent(nesdev
->pcidev
, 256, nesqp
->pbl_vbase
, nesqp
->pbl_pbase
);
988 nesqp
->pbl_vbase
= NULL
;
989 if (nesqp
->sq_kmapped
) {
990 nesqp
->sq_kmapped
= 0;
1000 static struct ib_qp
*nes_create_qp(struct ib_pd
*ibpd
,
1001 struct ib_qp_init_attr
*init_attr
, struct ib_udata
*udata
)
1005 struct nes_pd
*nespd
= to_nespd(ibpd
);
1006 struct nes_vnic
*nesvnic
= to_nesvnic(ibpd
->device
);
1007 struct nes_device
*nesdev
= nesvnic
->nesdev
;
1008 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1009 struct nes_qp
*nesqp
;
1010 struct nes_cq
*nescq
;
1011 struct nes_ucontext
*nes_ucontext
;
1012 struct nes_hw_cqp_wqe
*cqp_wqe
;
1013 struct nes_cqp_request
*cqp_request
;
1014 struct nes_create_qp_req req
;
1015 struct nes_create_qp_resp uresp
;
1016 struct nes_pbl
*nespbl
= NULL
;
1019 /* u32 counter = 0; */
1021 unsigned long flags
;
1031 if (init_attr
->create_flags
)
1032 return ERR_PTR(-EINVAL
);
1034 atomic_inc(&qps_created
);
1035 switch (init_attr
->qp_type
) {
1037 if (nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) {
1038 init_attr
->cap
.max_inline_data
= 0;
1040 init_attr
->cap
.max_inline_data
= 64;
1042 sq_size
= init_attr
->cap
.max_send_wr
;
1043 rq_size
= init_attr
->cap
.max_recv_wr
;
1045 /* check if the encoded sizes are OK or not... */
1046 sq_encoded_size
= nes_get_encoded_size(&sq_size
);
1047 rq_encoded_size
= nes_get_encoded_size(&rq_size
);
1049 if ((!sq_encoded_size
) || (!rq_encoded_size
)) {
1050 nes_debug(NES_DBG_QP
, "ERROR bad rq (%u) or sq (%u) size\n",
1052 return ERR_PTR(-EINVAL
);
1055 init_attr
->cap
.max_send_wr
= sq_size
-2;
1056 init_attr
->cap
.max_recv_wr
= rq_size
-1;
1057 nes_debug(NES_DBG_QP
, "RQ size=%u, SQ Size=%u\n", rq_size
, sq_size
);
1059 ret
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_qps
,
1060 nesadapter
->max_qp
, &qp_num
, &nesadapter
->next_qp
, NES_RESOURCE_QP
);
1062 return ERR_PTR(ret
);
1065 /* Need 512 (actually now 1024) byte alignment on this structure */
1066 mem
= kzalloc(sizeof(*nesqp
)+NES_SW_CONTEXT_ALIGN
-1, GFP_KERNEL
);
1068 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1069 return ERR_PTR(-ENOMEM
);
1071 u64nesqp
= (unsigned long)mem
;
1072 u64nesqp
+= ((u64
)NES_SW_CONTEXT_ALIGN
) - 1;
1073 u64temp
= ((u64
)NES_SW_CONTEXT_ALIGN
) - 1;
1074 u64nesqp
&= ~u64temp
;
1075 nesqp
= (struct nes_qp
*)(unsigned long)u64nesqp
;
1076 /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n",
1077 nesqp, mem, NES_SW_CONTEXT_ALIGN); */
1078 nesqp
->allocated_buffer
= mem
;
1081 if (ib_copy_from_udata(&req
, udata
, sizeof(struct nes_create_qp_req
))) {
1082 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1083 kfree(nesqp
->allocated_buffer
);
1084 nes_debug(NES_DBG_QP
, "ib_copy_from_udata() Failed \n");
1085 return ERR_PTR(-EFAULT
);
1087 if (req
.user_wqe_buffers
) {
1090 if (req
.user_qp_buffer
)
1091 nesqp
->nesuqp_addr
= req
.user_qp_buffer
;
1092 if ((ibpd
->uobject
) && (ibpd
->uobject
->context
)) {
1093 nesqp
->user_mode
= 1;
1094 nes_ucontext
= to_nesucontext(ibpd
->uobject
->context
);
1097 list_for_each_entry(nespbl
, &nes_ucontext
->qp_reg_mem_list
, list
) {
1098 if (nespbl
->user_base
== (unsigned long )req
.user_wqe_buffers
) {
1099 list_del(&nespbl
->list
);
1101 nes_debug(NES_DBG_QP
, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n",
1102 nespbl
, nespbl
->user_base
);
1107 nes_debug(NES_DBG_QP
, "Didn't Find PBL for virtual QP. address = %llx.\n",
1108 (long long unsigned int)req
.user_wqe_buffers
);
1109 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1110 kfree(nesqp
->allocated_buffer
);
1111 return ERR_PTR(-EFAULT
);
1115 nes_ucontext
= to_nesucontext(ibpd
->uobject
->context
);
1116 nesqp
->mmap_sq_db_index
=
1117 find_next_zero_bit(nes_ucontext
->allocated_wqs
,
1118 NES_MAX_USER_WQ_REGIONS
, nes_ucontext
->first_free_wq
);
1119 /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n",
1120 nespd->mmap_db_index); */
1121 if (nesqp
->mmap_sq_db_index
>= NES_MAX_USER_WQ_REGIONS
) {
1122 nes_debug(NES_DBG_QP
,
1123 "db index > max user regions, failing create QP\n");
1124 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1126 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1130 kfree(nesqp
->allocated_buffer
);
1131 return ERR_PTR(-ENOMEM
);
1133 set_bit(nesqp
->mmap_sq_db_index
, nes_ucontext
->allocated_wqs
);
1134 nes_ucontext
->mmap_nesqp
[nesqp
->mmap_sq_db_index
] = nesqp
;
1135 nes_ucontext
->first_free_wq
= nesqp
->mmap_sq_db_index
+ 1;
1137 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1138 kfree(nesqp
->allocated_buffer
);
1139 return ERR_PTR(-EFAULT
);
1142 err
= (!virt_wqs
) ? nes_setup_mmap_qp(nesqp
, nesvnic
, sq_size
, rq_size
) :
1143 nes_setup_virt_qp(nesqp
, nespbl
, nesvnic
, sq_size
, rq_size
);
1145 nes_debug(NES_DBG_QP
,
1146 "error geting qp mem code = %d\n", err
);
1147 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1148 kfree(nesqp
->allocated_buffer
);
1149 return ERR_PTR(-ENOMEM
);
1152 nesqp
->hwqp
.sq_size
= sq_size
;
1153 nesqp
->hwqp
.sq_encoded_size
= sq_encoded_size
;
1154 nesqp
->hwqp
.sq_head
= 1;
1155 nesqp
->hwqp
.rq_size
= rq_size
;
1156 nesqp
->hwqp
.rq_encoded_size
= rq_encoded_size
;
1157 /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n",
1158 (void *)nesqp->nesqp_context_pbase);
1160 nesqp
->hwqp
.qp_id
= qp_num
;
1161 nesqp
->ibqp
.qp_num
= nesqp
->hwqp
.qp_id
;
1162 nesqp
->nespd
= nespd
;
1164 nescq
= to_nescq(init_attr
->send_cq
);
1165 nesqp
->nesscq
= nescq
;
1166 nescq
= to_nescq(init_attr
->recv_cq
);
1167 nesqp
->nesrcq
= nescq
;
1169 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)PCI_FUNC(nesdev
->pcidev
->devfn
) <<
1170 NES_QPCONTEXT_MISC_PCI_FCN_SHIFT
);
1171 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)nesqp
->hwqp
.rq_encoded_size
<<
1172 NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT
);
1173 nesqp
->nesqp_context
->misc
|= cpu_to_le32((u32
)nesqp
->hwqp
.sq_encoded_size
<<
1174 NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT
);
1176 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN
);
1177 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN
);
1179 nesqp
->nesqp_context
->cqs
= cpu_to_le32(nesqp
->nesscq
->hw_cq
.cq_number
+
1180 ((u32
)nesqp
->nesrcq
->hw_cq
.cq_number
<< 16));
1181 u64temp
= (u64
)nesqp
->hwqp
.sq_pbase
;
1182 nesqp
->nesqp_context
->sq_addr_low
= cpu_to_le32((u32
)u64temp
);
1183 nesqp
->nesqp_context
->sq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1187 u64temp
= (u64
)nesqp
->hwqp
.sq_pbase
;
1188 nesqp
->nesqp_context
->sq_addr_low
= cpu_to_le32((u32
)u64temp
);
1189 nesqp
->nesqp_context
->sq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1190 u64temp
= (u64
)nesqp
->hwqp
.rq_pbase
;
1191 nesqp
->nesqp_context
->rq_addr_low
= cpu_to_le32((u32
)u64temp
);
1192 nesqp
->nesqp_context
->rq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1194 u64temp
= (u64
)nesqp
->pbl_pbase
;
1195 nesqp
->nesqp_context
->rq_addr_low
= cpu_to_le32((u32
)u64temp
);
1196 nesqp
->nesqp_context
->rq_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1199 /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n",
1200 nesvnic->next_qp_nic_index,
1201 nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */
1202 spin_lock_irqsave(&nesdev
->cqp
.lock
, flags
);
1203 nesqp
->nesqp_context
->misc2
|= cpu_to_le32(
1204 (u32
)nesvnic
->qp_nic_index
[nesvnic
->next_qp_nic_index
] <<
1205 NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT
);
1206 nesvnic
->next_qp_nic_index
++;
1207 if ((nesvnic
->next_qp_nic_index
> 3) ||
1208 (nesvnic
->qp_nic_index
[nesvnic
->next_qp_nic_index
] == 0xf)) {
1209 nesvnic
->next_qp_nic_index
= 0;
1211 spin_unlock_irqrestore(&nesdev
->cqp
.lock
, flags
);
1213 nesqp
->nesqp_context
->pd_index_wscale
|= cpu_to_le32((u32
)nesqp
->nespd
->pd_id
<< 16);
1214 u64temp
= (u64
)nesqp
->hwqp
.q2_pbase
;
1215 nesqp
->nesqp_context
->q2_addr_low
= cpu_to_le32((u32
)u64temp
);
1216 nesqp
->nesqp_context
->q2_addr_high
= cpu_to_le32((u32
)(u64temp
>> 32));
1217 nesqp
->nesqp_context
->aeq_token_low
= cpu_to_le32((u32
)((unsigned long)(nesqp
)));
1218 nesqp
->nesqp_context
->aeq_token_high
= cpu_to_le32((u32
)(upper_32_bits((unsigned long)(nesqp
))));
1219 nesqp
->nesqp_context
->ird_ord_sizes
= cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM
|
1220 NES_QPCONTEXT_ORDIRD_AAH
|
1221 ((((u32
)nesadapter
->max_irrq_wr
) <<
1222 NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT
) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK
));
1223 if (disable_mpa_crc
) {
1224 nes_debug(NES_DBG_QP
, "Disabling MPA crc checking due to module option.\n");
1225 nesqp
->nesqp_context
->ird_ord_sizes
|= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC
);
1230 cqp_request
= nes_get_cqp_request(nesdev
);
1231 if (cqp_request
== NULL
) {
1232 nes_debug(NES_DBG_QP
, "Failed to get a cqp_request\n");
1233 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1234 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1235 kfree(nesqp
->allocated_buffer
);
1236 return ERR_PTR(-ENOMEM
);
1238 cqp_request
->waiting
= 1;
1239 cqp_wqe
= &cqp_request
->cqp_wqe
;
1242 opcode
= NES_CQP_CREATE_QP
| NES_CQP_QP_TYPE_IWARP
|
1243 NES_CQP_QP_IWARP_STATE_IDLE
;
1245 opcode
= NES_CQP_CREATE_QP
| NES_CQP_QP_TYPE_IWARP
| NES_CQP_QP_VIRT_WQS
|
1246 NES_CQP_QP_IWARP_STATE_IDLE
;
1248 opcode
|= NES_CQP_QP_CQS_VALID
;
1249 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1250 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1251 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
, nesqp
->hwqp
.qp_id
);
1253 u64temp
= (u64
)nesqp
->nesqp_context_pbase
;
1254 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_CONTEXT_LOW_IDX
, u64temp
);
1256 atomic_set(&cqp_request
->refcount
, 2);
1257 nes_post_cqp_request(nesdev
, cqp_request
);
1260 nes_debug(NES_DBG_QP
, "Waiting for create iWARP QP%u to complete.\n",
1262 ret
= wait_event_timeout(cqp_request
->waitq
,
1263 (cqp_request
->request_done
!= 0), NES_EVENT_TIMEOUT
);
1264 nes_debug(NES_DBG_QP
, "Create iwarp QP%u completed, wait_event_timeout ret=%u,"
1265 " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u,"
1266 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1267 nesqp
->hwqp
.qp_id
, ret
, nesdev
->cqp
.sq_head
, nesdev
->cqp
.sq_tail
,
1268 cqp_request
->major_code
, cqp_request
->minor_code
);
1269 if ((!ret
) || (cqp_request
->major_code
)) {
1270 nes_put_cqp_request(nesdev
, cqp_request
);
1271 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1272 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1273 kfree(nesqp
->allocated_buffer
);
1275 return ERR_PTR(-ETIME
);
1277 return ERR_PTR(-EIO
);
1281 nes_put_cqp_request(nesdev
, cqp_request
);
1283 if (ibpd
->uobject
) {
1284 uresp
.mmap_sq_db_index
= nesqp
->mmap_sq_db_index
;
1285 uresp
.mmap_rq_db_index
= 0;
1286 uresp
.actual_sq_size
= sq_size
;
1287 uresp
.actual_rq_size
= rq_size
;
1288 uresp
.qp_id
= nesqp
->hwqp
.qp_id
;
1289 uresp
.nes_drv_opt
= nes_drv_opt
;
1290 if (ib_copy_to_udata(udata
, &uresp
, sizeof uresp
)) {
1291 nes_free_resource(nesadapter
, nesadapter
->allocated_qps
, qp_num
);
1292 nes_free_qp_mem(nesdev
, nesqp
,virt_wqs
);
1293 kfree(nesqp
->allocated_buffer
);
1294 return ERR_PTR(-EFAULT
);
1298 nes_debug(NES_DBG_QP
, "QP%u structure located @%p.Size = %u.\n",
1299 nesqp
->hwqp
.qp_id
, nesqp
, (u32
)sizeof(*nesqp
));
1300 spin_lock_init(&nesqp
->lock
);
1301 nes_add_ref(&nesqp
->ibqp
);
1304 nes_debug(NES_DBG_QP
, "Invalid QP type: %d\n", init_attr
->qp_type
);
1305 return ERR_PTR(-EINVAL
);
1307 init_completion(&nesqp
->sq_drained
);
1308 init_completion(&nesqp
->rq_drained
);
1310 nesqp
->sig_all
= (init_attr
->sq_sig_type
== IB_SIGNAL_ALL_WR
);
1311 init_timer(&nesqp
->terminate_timer
);
1312 nesqp
->terminate_timer
.function
= nes_terminate_timeout
;
1313 nesqp
->terminate_timer
.data
= (unsigned long)nesqp
;
1315 /* update the QP table */
1316 nesdev
->nesadapter
->qp_table
[nesqp
->hwqp
.qp_id
-NES_FIRST_QPN
] = nesqp
;
1317 nes_debug(NES_DBG_QP
, "netdev refcnt=%u\n",
1318 netdev_refcnt_read(nesvnic
->netdev
));
1320 return &nesqp
->ibqp
;
1326 static void nes_clean_cq(struct nes_qp
*nesqp
, struct nes_cq
*nescq
)
1332 unsigned long flags
= 0;
1334 spin_lock_irqsave(&nescq
->lock
, flags
);
1336 cq_head
= nescq
->hw_cq
.cq_head
;
1337 while (le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_OPCODE_IDX
]) & NES_CQE_VALID
) {
1339 lo
= le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
1340 hi
= le32_to_cpu(nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
]);
1341 u64temp
= (((u64
)hi
) << 32) | ((u64
)lo
);
1342 u64temp
&= ~(NES_SW_CONTEXT_ALIGN
-1);
1343 if (u64temp
== (u64
)(unsigned long)nesqp
) {
1344 /* Zero the context value so cqe will be ignored */
1345 nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
] = 0;
1346 nescq
->hw_cq
.cq_vbase
[cq_head
].cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
] = 0;
1349 if (++cq_head
>= nescq
->hw_cq
.cq_size
)
1353 spin_unlock_irqrestore(&nescq
->lock
, flags
);
1360 static int nes_destroy_qp(struct ib_qp
*ibqp
)
1362 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
1363 struct nes_ucontext
*nes_ucontext
;
1364 struct ib_qp_attr attr
;
1365 struct iw_cm_id
*cm_id
;
1366 struct iw_cm_event cm_event
;
1369 atomic_inc(&sw_qps_destroyed
);
1370 nesqp
->destroyed
= 1;
1372 /* Blow away the connection if it exists. */
1373 if (nesqp
->ibqp_state
>= IB_QPS_INIT
&& nesqp
->ibqp_state
<= IB_QPS_RTS
) {
1374 /* if (nesqp->ibqp_state == IB_QPS_RTS) { */
1375 attr
.qp_state
= IB_QPS_ERR
;
1376 nes_modify_qp(&nesqp
->ibqp
, &attr
, IB_QP_STATE
, NULL
);
1379 if (((nesqp
->ibqp_state
== IB_QPS_INIT
) ||
1380 (nesqp
->ibqp_state
== IB_QPS_RTR
)) && (nesqp
->cm_id
)) {
1381 cm_id
= nesqp
->cm_id
;
1382 cm_event
.event
= IW_CM_EVENT_CONNECT_REPLY
;
1383 cm_event
.status
= -ETIMEDOUT
;
1384 cm_event
.local_addr
= cm_id
->local_addr
;
1385 cm_event
.remote_addr
= cm_id
->remote_addr
;
1386 cm_event
.private_data
= NULL
;
1387 cm_event
.private_data_len
= 0;
1389 nes_debug(NES_DBG_QP
, "Generating a CM Timeout Event for "
1390 "QP%u. cm_id = %p, refcount = %u. \n",
1391 nesqp
->hwqp
.qp_id
, cm_id
, atomic_read(&nesqp
->refcount
));
1393 cm_id
->rem_ref(cm_id
);
1394 ret
= cm_id
->event_handler(cm_id
, &cm_event
);
1396 nes_debug(NES_DBG_QP
, "OFA CM event_handler returned, ret=%d\n", ret
);
1399 if (nesqp
->user_mode
) {
1400 if ((ibqp
->uobject
)&&(ibqp
->uobject
->context
)) {
1401 nes_ucontext
= to_nesucontext(ibqp
->uobject
->context
);
1402 clear_bit(nesqp
->mmap_sq_db_index
, nes_ucontext
->allocated_wqs
);
1403 nes_ucontext
->mmap_nesqp
[nesqp
->mmap_sq_db_index
] = NULL
;
1404 if (nes_ucontext
->first_free_wq
> nesqp
->mmap_sq_db_index
) {
1405 nes_ucontext
->first_free_wq
= nesqp
->mmap_sq_db_index
;
1408 if (nesqp
->pbl_pbase
&& nesqp
->sq_kmapped
) {
1409 nesqp
->sq_kmapped
= 0;
1410 kunmap(nesqp
->page
);
1413 /* Clean any pending completions from the cq(s) */
1415 nes_clean_cq(nesqp
, nesqp
->nesscq
);
1417 if ((nesqp
->nesrcq
) && (nesqp
->nesrcq
!= nesqp
->nesscq
))
1418 nes_clean_cq(nesqp
, nesqp
->nesrcq
);
1420 nes_rem_ref(&nesqp
->ibqp
);
1428 static struct ib_cq
*nes_create_cq(struct ib_device
*ibdev
,
1429 const struct ib_cq_init_attr
*attr
,
1430 struct ib_ucontext
*context
,
1431 struct ib_udata
*udata
)
1433 int entries
= attr
->cqe
;
1435 struct nes_vnic
*nesvnic
= to_nesvnic(ibdev
);
1436 struct nes_device
*nesdev
= nesvnic
->nesdev
;
1437 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1438 struct nes_cq
*nescq
;
1439 struct nes_ucontext
*nes_ucontext
= NULL
;
1440 struct nes_cqp_request
*cqp_request
;
1442 struct nes_hw_cqp_wqe
*cqp_wqe
;
1443 struct nes_pbl
*nespbl
= NULL
;
1444 struct nes_create_cq_req req
;
1445 struct nes_create_cq_resp resp
;
1448 u32 pbl_entries
= 1;
1450 unsigned long flags
;
1454 return ERR_PTR(-EINVAL
);
1456 if (entries
> nesadapter
->max_cqe
)
1457 return ERR_PTR(-EINVAL
);
1459 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_cqs
,
1460 nesadapter
->max_cq
, &cq_num
, &nesadapter
->next_cq
, NES_RESOURCE_CQ
);
1462 return ERR_PTR(err
);
1465 nescq
= kzalloc(sizeof(struct nes_cq
), GFP_KERNEL
);
1467 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1468 return ERR_PTR(-ENOMEM
);
1471 nescq
->hw_cq
.cq_size
= max(entries
+ 1, 5);
1472 nescq
->hw_cq
.cq_number
= cq_num
;
1473 nescq
->ibcq
.cqe
= nescq
->hw_cq
.cq_size
- 1;
1477 nes_ucontext
= to_nesucontext(context
);
1478 if (ib_copy_from_udata(&req
, udata
, sizeof (struct nes_create_cq_req
))) {
1479 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1481 return ERR_PTR(-EFAULT
);
1483 nesvnic
->mcrq_ucontext
= nes_ucontext
;
1484 nes_ucontext
->mcrqf
= req
.mcrqf
;
1485 if (nes_ucontext
->mcrqf
) {
1486 if (nes_ucontext
->mcrqf
& 0x80000000)
1487 nescq
->hw_cq
.cq_number
= nesvnic
->nic
.qp_id
+ 28 + 2 * ((nes_ucontext
->mcrqf
& 0xf) - 1);
1488 else if (nes_ucontext
->mcrqf
& 0x40000000)
1489 nescq
->hw_cq
.cq_number
= nes_ucontext
->mcrqf
& 0xffff;
1491 nescq
->hw_cq
.cq_number
= nesvnic
->mcrq_qp_id
+ nes_ucontext
->mcrqf
-1;
1492 nescq
->mcrqf
= nes_ucontext
->mcrqf
;
1493 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1495 nes_debug(NES_DBG_CQ
, "CQ Virtual Address = %08lX, size = %u.\n",
1496 (unsigned long)req
.user_cq_buffer
, entries
);
1498 list_for_each_entry(nespbl
, &nes_ucontext
->cq_reg_mem_list
, list
) {
1499 if (nespbl
->user_base
== (unsigned long )req
.user_cq_buffer
) {
1500 list_del(&nespbl
->list
);
1502 nes_debug(NES_DBG_CQ
, "Found PBL for virtual CQ. nespbl=%p.\n",
1508 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1510 return ERR_PTR(-EFAULT
);
1513 pbl_entries
= nespbl
->pbl_size
>> 3;
1514 nescq
->cq_mem_size
= 0;
1516 nescq
->cq_mem_size
= nescq
->hw_cq
.cq_size
* sizeof(struct nes_hw_cqe
);
1517 nes_debug(NES_DBG_CQ
, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n",
1518 entries
, nescq
->cq_mem_size
, nescq
->hw_cq
.cq_number
);
1520 /* allocate the physical buffer space */
1521 mem
= pci_zalloc_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
,
1522 &nescq
->hw_cq
.cq_pbase
);
1524 printk(KERN_ERR PFX
"Unable to allocate pci memory for cq\n");
1525 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1527 return ERR_PTR(-ENOMEM
);
1530 nescq
->hw_cq
.cq_vbase
= mem
;
1531 nescq
->hw_cq
.cq_head
= 0;
1532 nes_debug(NES_DBG_CQ
, "CQ%u virtual address @ %p, phys = 0x%08X\n",
1533 nescq
->hw_cq
.cq_number
, nescq
->hw_cq
.cq_vbase
,
1534 (u32
)nescq
->hw_cq
.cq_pbase
);
1537 nescq
->hw_cq
.ce_handler
= nes_iwarp_ce_handler
;
1538 spin_lock_init(&nescq
->lock
);
1540 /* send CreateCQ request to CQP */
1541 cqp_request
= nes_get_cqp_request(nesdev
);
1542 if (cqp_request
== NULL
) {
1543 nes_debug(NES_DBG_CQ
, "Failed to get a cqp_request.\n");
1545 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1546 nescq
->hw_cq
.cq_pbase
);
1548 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1549 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1553 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1555 return ERR_PTR(-ENOMEM
);
1557 cqp_request
->waiting
= 1;
1558 cqp_wqe
= &cqp_request
->cqp_wqe
;
1560 opcode
= NES_CQP_CREATE_CQ
| NES_CQP_CQ_CEQ_VALID
|
1561 NES_CQP_CQ_CHK_OVERFLOW
|
1562 NES_CQP_CQ_CEQE_MASK
| ((u32
)nescq
->hw_cq
.cq_size
<< 16);
1564 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1566 if (pbl_entries
!= 1) {
1567 if (pbl_entries
> 32) {
1569 nes_debug(NES_DBG_CQ
, "pbl_entries=%u, use a 4k PBL\n", pbl_entries
);
1570 if (nesadapter
->free_4kpbl
== 0) {
1571 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1572 nes_free_cqp_request(nesdev
, cqp_request
);
1574 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1575 nescq
->hw_cq
.cq_pbase
);
1577 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1578 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1581 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1583 return ERR_PTR(-ENOMEM
);
1585 opcode
|= (NES_CQP_CQ_VIRT
| NES_CQP_CQ_4KB_CHUNK
);
1586 nescq
->virtual_cq
= 2;
1587 nesadapter
->free_4kpbl
--;
1590 /* use 256 byte pbl */
1591 nes_debug(NES_DBG_CQ
, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries
);
1592 if (nesadapter
->free_256pbl
== 0) {
1593 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1594 nes_free_cqp_request(nesdev
, cqp_request
);
1596 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1597 nescq
->hw_cq
.cq_pbase
);
1599 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1600 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1603 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1605 return ERR_PTR(-ENOMEM
);
1607 opcode
|= NES_CQP_CQ_VIRT
;
1608 nescq
->virtual_cq
= 1;
1609 nesadapter
->free_256pbl
--;
1614 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1616 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1617 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1618 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
,
1619 (nescq
->hw_cq
.cq_number
| ((u32
)nesdev
->ceq_index
<< 16)));
1622 if (pbl_entries
!= 1)
1623 u64temp
= (u64
)nespbl
->pbl_pbase
;
1625 u64temp
= le64_to_cpu(nespbl
->pbl_vbase
[0]);
1626 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX
,
1627 nes_ucontext
->mmap_db_index
[0]);
1629 u64temp
= (u64
)nescq
->hw_cq
.cq_pbase
;
1630 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX
] = 0;
1632 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_CQ_WQE_PBL_LOW_IDX
, u64temp
);
1633 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX
] = 0;
1634 u64temp
= (u64
)(unsigned long)&nescq
->hw_cq
;
1635 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX
] =
1636 cpu_to_le32((u32
)(u64temp
>> 1));
1637 cqp_wqe
->wqe_words
[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX
] =
1638 cpu_to_le32(((u32
)((u64temp
) >> 33)) & 0x7FFFFFFF);
1640 atomic_set(&cqp_request
->refcount
, 2);
1641 nes_post_cqp_request(nesdev
, cqp_request
);
1644 nes_debug(NES_DBG_CQ
, "Waiting for create iWARP CQ%u to complete.\n",
1645 nescq
->hw_cq
.cq_number
);
1646 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
1647 NES_EVENT_TIMEOUT
* 2);
1648 nes_debug(NES_DBG_CQ
, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n",
1649 nescq
->hw_cq
.cq_number
, ret
);
1650 if ((!ret
) || (cqp_request
->major_code
)) {
1651 nes_put_cqp_request(nesdev
, cqp_request
);
1653 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
, mem
,
1654 nescq
->hw_cq
.cq_pbase
);
1656 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
1657 nespbl
->pbl_vbase
, nespbl
->pbl_pbase
);
1660 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1662 return ERR_PTR(-EIO
);
1664 nes_put_cqp_request(nesdev
, cqp_request
);
1667 /* free the nespbl */
1668 pci_free_consistent(nesdev
->pcidev
, nespbl
->pbl_size
, nespbl
->pbl_vbase
,
1671 resp
.cq_id
= nescq
->hw_cq
.cq_number
;
1672 resp
.cq_size
= nescq
->hw_cq
.cq_size
;
1673 resp
.mmap_db_index
= 0;
1674 if (ib_copy_to_udata(udata
, &resp
, sizeof resp
- sizeof resp
.reserved
)) {
1675 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, cq_num
);
1677 return ERR_PTR(-EFAULT
);
1681 return &nescq
->ibcq
;
1688 static int nes_destroy_cq(struct ib_cq
*ib_cq
)
1690 struct nes_cq
*nescq
;
1691 struct nes_device
*nesdev
;
1692 struct nes_vnic
*nesvnic
;
1693 struct nes_adapter
*nesadapter
;
1694 struct nes_hw_cqp_wqe
*cqp_wqe
;
1695 struct nes_cqp_request
*cqp_request
;
1696 unsigned long flags
;
1703 nescq
= to_nescq(ib_cq
);
1704 nesvnic
= to_nesvnic(ib_cq
->device
);
1705 nesdev
= nesvnic
->nesdev
;
1706 nesadapter
= nesdev
->nesadapter
;
1708 nes_debug(NES_DBG_CQ
, "Destroy CQ%u\n", nescq
->hw_cq
.cq_number
);
1710 /* Send DestroyCQ request to CQP */
1711 cqp_request
= nes_get_cqp_request(nesdev
);
1712 if (cqp_request
== NULL
) {
1713 nes_debug(NES_DBG_CQ
, "Failed to get a cqp_request.\n");
1716 cqp_request
->waiting
= 1;
1717 cqp_wqe
= &cqp_request
->cqp_wqe
;
1718 opcode
= NES_CQP_DESTROY_CQ
| (nescq
->hw_cq
.cq_size
<< 16);
1719 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1720 if (nescq
->virtual_cq
== 1) {
1721 nesadapter
->free_256pbl
++;
1722 if (nesadapter
->free_256pbl
> nesadapter
->max_256pbl
) {
1723 printk(KERN_ERR PFX
"%s: free 256B PBLs(%u) has exceeded the max(%u)\n",
1724 __func__
, nesadapter
->free_256pbl
, nesadapter
->max_256pbl
);
1726 } else if (nescq
->virtual_cq
== 2) {
1727 nesadapter
->free_4kpbl
++;
1728 if (nesadapter
->free_4kpbl
> nesadapter
->max_4kpbl
) {
1729 printk(KERN_ERR PFX
"%s: free 4K PBLs(%u) has exceeded the max(%u)\n",
1730 __func__
, nesadapter
->free_4kpbl
, nesadapter
->max_4kpbl
);
1732 opcode
|= NES_CQP_CQ_4KB_CHUNK
;
1735 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1737 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1738 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1739 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
,
1740 (nescq
->hw_cq
.cq_number
| ((u32
)PCI_FUNC(nesdev
->pcidev
->devfn
) << 16)));
1742 nes_free_resource(nesadapter
, nesadapter
->allocated_cqs
, nescq
->hw_cq
.cq_number
);
1744 atomic_set(&cqp_request
->refcount
, 2);
1745 nes_post_cqp_request(nesdev
, cqp_request
);
1748 nes_debug(NES_DBG_CQ
, "Waiting for destroy iWARP CQ%u to complete.\n",
1749 nescq
->hw_cq
.cq_number
);
1750 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
1752 nes_debug(NES_DBG_CQ
, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u,"
1753 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1754 nescq
->hw_cq
.cq_number
, ret
, cqp_request
->major_code
,
1755 cqp_request
->minor_code
);
1757 nes_debug(NES_DBG_CQ
, "iWARP CQ%u destroy timeout expired\n",
1758 nescq
->hw_cq
.cq_number
);
1760 } else if (cqp_request
->major_code
) {
1761 nes_debug(NES_DBG_CQ
, "iWARP CQ%u destroy failed\n",
1762 nescq
->hw_cq
.cq_number
);
1767 nes_put_cqp_request(nesdev
, cqp_request
);
1769 if (nescq
->cq_mem_size
)
1770 pci_free_consistent(nesdev
->pcidev
, nescq
->cq_mem_size
,
1771 nescq
->hw_cq
.cq_vbase
, nescq
->hw_cq
.cq_pbase
);
1780 static u32
root_256(struct nes_device
*nesdev
,
1781 struct nes_root_vpbl
*root_vpbl
,
1782 struct nes_root_vpbl
*new_root
,
1788 if (pbl_count_4k
== 1) {
1789 new_root
->pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
,
1790 512, &new_root
->pbl_pbase
);
1792 if (new_root
->pbl_vbase
== NULL
)
1795 leaf_pbl
= (u64
)root_vpbl
->pbl_pbase
;
1796 for (i
= 0; i
< 16; i
++) {
1797 new_root
->pbl_vbase
[i
].pa_low
=
1798 cpu_to_le32((u32
)leaf_pbl
);
1799 new_root
->pbl_vbase
[i
].pa_high
=
1800 cpu_to_le32((u32
)((((u64
)leaf_pbl
) >> 32)));
1804 for (i
= 3; i
>= 0; i
--) {
1806 root_vpbl
->pbl_vbase
[j
] = root_vpbl
->pbl_vbase
[i
];
1807 leaf_pbl
= le32_to_cpu(root_vpbl
->pbl_vbase
[j
].pa_low
) +
1808 (((u64
)le32_to_cpu(root_vpbl
->pbl_vbase
[j
].pa_high
))
1810 for (k
= 1; k
< 16; k
++) {
1812 root_vpbl
->pbl_vbase
[j
+ k
].pa_low
=
1813 cpu_to_le32((u32
)leaf_pbl
);
1814 root_vpbl
->pbl_vbase
[j
+ k
].pa_high
=
1815 cpu_to_le32((u32
)((((u64
)leaf_pbl
) >> 32)));
1827 static int nes_reg_mr(struct nes_device
*nesdev
, struct nes_pd
*nespd
,
1828 u32 stag
, u64 region_length
, struct nes_root_vpbl
*root_vpbl
,
1829 dma_addr_t single_buffer
, u16 pbl_count_4k
,
1830 u16 residual_page_count_4k
, int acc
, u64
*iova_start
,
1831 u16
*actual_pbl_cnt
, u8
*used_4k_pbls
)
1833 struct nes_hw_cqp_wqe
*cqp_wqe
;
1834 struct nes_cqp_request
*cqp_request
;
1835 unsigned long flags
;
1837 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
1839 u16 pbl_count_256
= 0;
1841 u8 use_256_pbls
= 0;
1843 u16 use_two_level
= (pbl_count_4k
> 1) ? 1 : 0;
1844 struct nes_root_vpbl new_root
= { 0, NULL
, NULL
};
1848 /* Register the region with the adapter */
1849 cqp_request
= nes_get_cqp_request(nesdev
);
1850 if (cqp_request
== NULL
) {
1851 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
1854 cqp_request
->waiting
= 1;
1855 cqp_wqe
= &cqp_request
->cqp_wqe
;
1858 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1860 pg_cnt
= ((pbl_count_4k
- 1) * 512) + residual_page_count_4k
;
1861 pbl_count_256
= (pg_cnt
+ 31) / 32;
1863 if (pbl_count_256
<= nesadapter
->free_256pbl
)
1865 else if (pbl_count_4k
<= nesadapter
->free_4kpbl
)
1867 } else if (pg_cnt
<= 2048) {
1868 if (((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) &&
1869 (nesadapter
->free_4kpbl
> (nesadapter
->max_4kpbl
>> 1))) {
1871 } else if ((pbl_count_256
+ 1) <= nesadapter
->free_256pbl
) {
1874 } else if ((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) {
1878 if ((pbl_count_4k
+ 1) <= nesadapter
->free_4kpbl
)
1883 pbl_count
= pbl_count_256
;
1884 nesadapter
->free_256pbl
-= pbl_count
+ use_two_level
;
1885 } else if (use_4k_pbls
) {
1886 pbl_count
= pbl_count_4k
;
1887 nesadapter
->free_4kpbl
-= pbl_count
+ use_two_level
;
1889 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1890 nes_debug(NES_DBG_MR
, "Out of Pbls\n");
1891 nes_free_cqp_request(nesdev
, cqp_request
);
1895 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1898 if (use_256_pbls
&& use_two_level
) {
1899 if (root_256(nesdev
, root_vpbl
, &new_root
, pbl_count_4k
) == 1) {
1900 if (new_root
.pbl_pbase
!= 0)
1901 root_vpbl
= &new_root
;
1903 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1904 nesadapter
->free_256pbl
+= pbl_count_256
+ use_two_level
;
1907 if (pbl_count_4k
== 1)
1909 pbl_count
= pbl_count_4k
;
1911 if ((pbl_count_4k
+ use_two_level
) <= nesadapter
->free_4kpbl
) {
1912 nesadapter
->free_4kpbl
-= pbl_count
+ use_two_level
;
1915 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1917 if (use_4k_pbls
== 0)
1922 opcode
= NES_CQP_REGISTER_STAG
| NES_CQP_STAG_RIGHTS_LOCAL_READ
|
1923 NES_CQP_STAG_VA_TO
| NES_CQP_STAG_MR
;
1924 if (acc
& IB_ACCESS_LOCAL_WRITE
)
1925 opcode
|= NES_CQP_STAG_RIGHTS_LOCAL_WRITE
;
1926 if (acc
& IB_ACCESS_REMOTE_WRITE
)
1927 opcode
|= NES_CQP_STAG_RIGHTS_REMOTE_WRITE
| NES_CQP_STAG_REM_ACC_EN
;
1928 if (acc
& IB_ACCESS_REMOTE_READ
)
1929 opcode
|= NES_CQP_STAG_RIGHTS_REMOTE_READ
| NES_CQP_STAG_REM_ACC_EN
;
1930 if (acc
& IB_ACCESS_MW_BIND
)
1931 opcode
|= NES_CQP_STAG_RIGHTS_WINDOW_BIND
| NES_CQP_STAG_REM_ACC_EN
;
1933 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
1934 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
, opcode
);
1935 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_VA_LOW_IDX
, *iova_start
);
1936 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_LEN_LOW_IDX
, region_length
);
1938 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] =
1939 cpu_to_le32((u32
)(region_length
>> 8) & 0xff000000);
1940 cqp_wqe
->wqe_words
[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX
] |=
1941 cpu_to_le32(nespd
->pd_id
& 0x00007fff);
1942 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, stag
);
1944 if (pbl_count
== 0) {
1945 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, single_buffer
);
1947 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PA_LOW_IDX
, root_vpbl
->pbl_pbase
);
1948 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX
, pbl_count
);
1949 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_PBL_LEN_IDX
, (pg_cnt
* 8));
1952 cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE
);
1956 atomic_set(&cqp_request
->refcount
, 2);
1957 nes_post_cqp_request(nesdev
, cqp_request
);
1960 ret
= wait_event_timeout(cqp_request
->waitq
, (0 != cqp_request
->request_done
),
1962 nes_debug(NES_DBG_MR
, "Register STag 0x%08X completed, wait_event_timeout ret = %u,"
1963 " CQP Major:Minor codes = 0x%04X:0x%04X.\n",
1964 stag
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
1965 major_code
= cqp_request
->major_code
;
1966 nes_put_cqp_request(nesdev
, cqp_request
);
1968 if ((!ret
|| major_code
) && pbl_count
!= 0) {
1969 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
1971 nesadapter
->free_256pbl
+= pbl_count
+ use_two_level
;
1972 else if (use_4k_pbls
)
1973 nesadapter
->free_4kpbl
+= pbl_count
+ use_two_level
;
1974 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
1976 if (new_root
.pbl_pbase
)
1977 pci_free_consistent(nesdev
->pcidev
, 512, new_root
.pbl_vbase
,
1978 new_root
.pbl_pbase
);
1982 else if (major_code
)
1985 *actual_pbl_cnt
= pbl_count
+ use_two_level
;
1986 *used_4k_pbls
= use_4k_pbls
;
1994 struct ib_mr
*nes_reg_phys_mr(struct ib_pd
*ib_pd
, u64 addr
, u64 size
,
1995 int acc
, u64
*iova_start
)
1998 struct nes_pd
*nespd
= to_nespd(ib_pd
);
1999 struct nes_vnic
*nesvnic
= to_nesvnic(ib_pd
->device
);
2000 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2001 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2002 struct nes_mr
*nesmr
;
2004 struct nes_vpbl vpbl
;
2005 struct nes_root_vpbl root_vpbl
;
2009 u32 next_stag_index
= 0;
2018 vpbl
.pbl_vbase
= NULL
;
2019 root_vpbl
.pbl_vbase
= NULL
;
2020 root_vpbl
.pbl_pbase
= 0;
2022 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
2023 stag_key
= (u8
)next_stag_index
;
2027 next_stag_index
>>= 8;
2028 next_stag_index
%= nesadapter
->max_mr
;
2030 if ((addr
^ *iova_start
) & ~PAGE_MASK
)
2031 return ERR_PTR(-EINVAL
);
2033 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
, nesadapter
->max_mr
,
2034 &stag_index
, &next_stag_index
, NES_RESOURCE_PHYS_MR
);
2036 return ERR_PTR(err
);
2039 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2041 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2042 return ERR_PTR(-ENOMEM
);
2045 /* Allocate a 4K buffer for the PBL */
2046 vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 4096,
2048 nes_debug(NES_DBG_MR
, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n",
2049 vpbl
.pbl_vbase
, (unsigned long)vpbl
.pbl_pbase
);
2050 if (!vpbl
.pbl_vbase
) {
2051 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2052 ibmr
= ERR_PTR(-ENOMEM
);
2060 if (mask
& ~PAGE_MASK
) {
2061 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2062 nes_debug(NES_DBG_MR
, "Invalid buffer addr or size\n");
2063 ibmr
= ERR_PTR(-EINVAL
);
2068 region_length
+= size
;
2069 vpbl
.pbl_vbase
[0].pa_low
= cpu_to_le32((u32
)addr
& PAGE_MASK
);
2070 vpbl
.pbl_vbase
[0].pa_high
= cpu_to_le32((u32
)((((u64
)addr
) >> 32)));
2072 stag
= stag_index
<< 8;
2074 stag
+= (u32
)stag_key
;
2076 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, VA = 0x%016lX,"
2077 " length = 0x%016lX, index = 0x%08X\n",
2078 stag
, (unsigned long)*iova_start
, (unsigned long)region_length
, stag_index
);
2080 /* Make the leaf PBL the root if only one PBL */
2081 root_vpbl
.pbl_pbase
= vpbl
.pbl_pbase
;
2088 ret
= nes_reg_mr(nesdev
, nespd
, stag
, region_length
, &root_vpbl
,
2089 addr
, pbl_count
, 1, acc
, iova_start
,
2090 &nesmr
->pbls_used
, &nesmr
->pbl_4k
);
2093 nesmr
->ibmr
.rkey
= stag
;
2094 nesmr
->ibmr
.lkey
= stag
;
2095 nesmr
->mode
= IWNES_MEMREG_TYPE_MEM
;
2096 ibmr
= &nesmr
->ibmr
;
2099 ibmr
= ERR_PTR(-ENOMEM
);
2103 /* single PBL case */
2104 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
, vpbl
.pbl_pbase
);
2112 static struct ib_mr
*nes_get_dma_mr(struct ib_pd
*pd
, int acc
)
2116 nes_debug(NES_DBG_MR
, "\n");
2118 return nes_reg_phys_mr(pd
, 0, 0xffffffffffULL
, acc
, &kva
);
2124 static struct ib_mr
*nes_reg_user_mr(struct ib_pd
*pd
, u64 start
, u64 length
,
2125 u64 virt
, int acc
, struct ib_udata
*udata
)
2130 dma_addr_t last_dma_addr
= 0;
2131 dma_addr_t first_dma_addr
= 0;
2132 struct nes_pd
*nespd
= to_nespd(pd
);
2133 struct nes_vnic
*nesvnic
= to_nesvnic(pd
->device
);
2134 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2135 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2136 struct ib_mr
*ibmr
= ERR_PTR(-EINVAL
);
2137 struct scatterlist
*sg
;
2138 struct nes_ucontext
*nes_ucontext
;
2139 struct nes_pbl
*nespbl
;
2140 struct nes_mr
*nesmr
;
2141 struct ib_umem
*region
;
2142 struct nes_mem_reg_req req
;
2143 struct nes_vpbl vpbl
;
2144 struct nes_root_vpbl root_vpbl
;
2145 int entry
, page_index
;
2147 int err
, pbl_depth
= 0;
2152 u32 next_stag_index
;
2154 u32 root_pbl_index
= 0;
2155 u32 cur_pbl_index
= 0;
2162 region
= ib_umem_get(pd
->uobject
->context
, start
, length
, acc
, 0);
2163 if (IS_ERR(region
)) {
2164 return (struct ib_mr
*)region
;
2167 nes_debug(NES_DBG_MR
, "User base = 0x%lX, Virt base = 0x%lX, length = %u,"
2168 " offset = %u, page size = %u.\n",
2169 (unsigned long int)start
, (unsigned long int)virt
, (u32
)length
,
2170 ib_umem_offset(region
), region
->page_size
);
2172 skip_pages
= ((u32
)ib_umem_offset(region
)) >> 12;
2174 if (ib_copy_from_udata(&req
, udata
, sizeof(req
))) {
2175 ib_umem_release(region
);
2176 return ERR_PTR(-EFAULT
);
2178 nes_debug(NES_DBG_MR
, "Memory Registration type = %08X.\n", req
.reg_type
);
2180 switch (req
.reg_type
) {
2181 case IWNES_MEMREG_TYPE_MEM
:
2184 vpbl
.pbl_vbase
= NULL
;
2185 root_vpbl
.pbl_vbase
= NULL
;
2186 root_vpbl
.pbl_pbase
= 0;
2188 get_random_bytes(&next_stag_index
, sizeof(next_stag_index
));
2189 stag_key
= (u8
)next_stag_index
;
2191 driver_key
= next_stag_index
& 0x70000000;
2193 next_stag_index
>>= 8;
2194 next_stag_index
%= nesadapter
->max_mr
;
2196 err
= nes_alloc_resource(nesadapter
, nesadapter
->allocated_mrs
,
2197 nesadapter
->max_mr
, &stag_index
, &next_stag_index
, NES_RESOURCE_USER_MR
);
2199 ib_umem_release(region
);
2200 return ERR_PTR(err
);
2203 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2205 ib_umem_release(region
);
2206 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2207 return ERR_PTR(-ENOMEM
);
2209 nesmr
->region
= region
;
2211 for_each_sg(region
->sg_head
.sgl
, sg
, region
->nmap
, entry
) {
2212 if (sg_dma_address(sg
) & ~PAGE_MASK
) {
2213 ib_umem_release(region
);
2214 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2215 nes_debug(NES_DBG_MR
, "Unaligned Memory Buffer: 0x%x\n",
2216 (unsigned int) sg_dma_address(sg
));
2217 ibmr
= ERR_PTR(-EINVAL
);
2219 goto reg_user_mr_err
;
2222 if (!sg_dma_len(sg
)) {
2223 ib_umem_release(region
);
2224 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2226 nes_debug(NES_DBG_MR
, "Invalid Buffer Size\n");
2227 ibmr
= ERR_PTR(-EINVAL
);
2229 goto reg_user_mr_err
;
2232 region_length
+= sg_dma_len(sg
);
2233 chunk_pages
= sg_dma_len(sg
) >> 12;
2234 region_length
-= skip_pages
<< 12;
2235 for (page_index
= skip_pages
; page_index
< chunk_pages
; page_index
++) {
2237 if ((page_count
!= 0) && (page_count
<< 12) - (ib_umem_offset(region
) & (4096 - 1)) >= region
->length
)
2239 if ((page_count
&0x01FF) == 0) {
2240 if (page_count
>= 1024 * 512) {
2241 ib_umem_release(region
);
2242 nes_free_resource(nesadapter
,
2243 nesadapter
->allocated_mrs
, stag_index
);
2245 ibmr
= ERR_PTR(-E2BIG
);
2246 goto reg_user_mr_err
;
2248 if (root_pbl_index
== 1) {
2249 root_vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
,
2250 8192, &root_vpbl
.pbl_pbase
);
2251 nes_debug(NES_DBG_MR
, "Allocating root PBL, va = %p, pa = 0x%08X\n",
2252 root_vpbl
.pbl_vbase
, (unsigned int)root_vpbl
.pbl_pbase
);
2253 if (!root_vpbl
.pbl_vbase
) {
2254 ib_umem_release(region
);
2255 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2257 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2260 ibmr
= ERR_PTR(-ENOMEM
);
2261 goto reg_user_mr_err
;
2263 root_vpbl
.leaf_vpbl
= kzalloc(sizeof(*root_vpbl
.leaf_vpbl
)*1024,
2265 if (!root_vpbl
.leaf_vpbl
) {
2266 ib_umem_release(region
);
2267 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2268 root_vpbl
.pbl_pbase
);
2269 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2271 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2274 ibmr
= ERR_PTR(-ENOMEM
);
2275 goto reg_user_mr_err
;
2277 root_vpbl
.pbl_vbase
[0].pa_low
=
2278 cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2279 root_vpbl
.pbl_vbase
[0].pa_high
=
2280 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
) >> 32)));
2281 root_vpbl
.leaf_vpbl
[0] = vpbl
;
2283 vpbl
.pbl_vbase
= pci_alloc_consistent(nesdev
->pcidev
, 4096,
2285 nes_debug(NES_DBG_MR
, "Allocating leaf PBL, va = %p, pa = 0x%08X\n",
2286 vpbl
.pbl_vbase
, (unsigned int)vpbl
.pbl_pbase
);
2287 if (!vpbl
.pbl_vbase
) {
2288 ib_umem_release(region
);
2289 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
, stag_index
);
2290 ibmr
= ERR_PTR(-ENOMEM
);
2292 goto reg_user_mr_err
;
2294 if (1 <= root_pbl_index
) {
2295 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_low
=
2296 cpu_to_le32((u32
)vpbl
.pbl_pbase
);
2297 root_vpbl
.pbl_vbase
[root_pbl_index
].pa_high
=
2298 cpu_to_le32((u32
)((((u64
)vpbl
.pbl_pbase
)>>32)));
2299 root_vpbl
.leaf_vpbl
[root_pbl_index
] = vpbl
;
2305 if (page_count
!= 0) {
2306 if ((last_dma_addr
+4096) !=
2307 (sg_dma_address(sg
)+
2310 last_dma_addr
= sg_dma_address(sg
)+
2313 first_dma_addr
= sg_dma_address(sg
)+
2315 last_dma_addr
= first_dma_addr
;
2319 vpbl
.pbl_vbase
[cur_pbl_index
].pa_low
=
2320 cpu_to_le32((u32
)(sg_dma_address(sg
)+
2321 (page_index
*4096)));
2322 vpbl
.pbl_vbase
[cur_pbl_index
].pa_high
=
2323 cpu_to_le32((u32
)((((u64
)(sg_dma_address(sg
)+
2324 (page_index
*4096))) >> 32)));
2331 nes_debug(NES_DBG_MR
, "calculating stag, stag_index=0x%08x, driver_key=0x%08x,"
2332 " stag_key=0x%08x\n",
2333 stag_index
, driver_key
, stag_key
);
2334 stag
= stag_index
<< 8;
2336 stag
+= (u32
)stag_key
;
2339 /* Make the leaf PBL the root if only one PBL */
2340 if (root_pbl_index
== 1) {
2341 root_vpbl
.pbl_pbase
= vpbl
.pbl_pbase
;
2347 pbl_count
= root_pbl_index
;
2350 nes_debug(NES_DBG_MR
, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X,"
2351 " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n",
2352 stag
, (unsigned int)iova_start
,
2353 (unsigned int)region_length
, stag_index
,
2354 (unsigned long long)region
->length
, pbl_count
);
2355 ret
= nes_reg_mr(nesdev
, nespd
, stag
, region
->length
, &root_vpbl
,
2356 first_dma_addr
, pbl_count
, (u16
)cur_pbl_index
, acc
,
2357 &iova_start
, &nesmr
->pbls_used
, &nesmr
->pbl_4k
);
2359 nes_debug(NES_DBG_MR
, "ret=%d\n", ret
);
2362 nesmr
->ibmr
.rkey
= stag
;
2363 nesmr
->ibmr
.lkey
= stag
;
2364 nesmr
->mode
= IWNES_MEMREG_TYPE_MEM
;
2365 ibmr
= &nesmr
->ibmr
;
2367 ib_umem_release(region
);
2369 ibmr
= ERR_PTR(-ENOMEM
);
2373 /* free the resources */
2374 if (root_pbl_index
== 1) {
2375 pci_free_consistent(nesdev
->pcidev
, 4096, vpbl
.pbl_vbase
,
2378 for (page_index
=0; page_index
<root_pbl_index
; page_index
++) {
2379 pci_free_consistent(nesdev
->pcidev
, 4096,
2380 root_vpbl
.leaf_vpbl
[page_index
].pbl_vbase
,
2381 root_vpbl
.leaf_vpbl
[page_index
].pbl_pbase
);
2383 kfree(root_vpbl
.leaf_vpbl
);
2384 pci_free_consistent(nesdev
->pcidev
, 8192, root_vpbl
.pbl_vbase
,
2385 root_vpbl
.pbl_pbase
);
2388 nes_debug(NES_DBG_MR
, "Leaving, ibmr=%p", ibmr
);
2391 case IWNES_MEMREG_TYPE_QP
:
2392 case IWNES_MEMREG_TYPE_CQ
:
2393 if (!region
->length
) {
2394 nes_debug(NES_DBG_MR
, "Unable to register zero length region for CQ\n");
2395 ib_umem_release(region
);
2396 return ERR_PTR(-EINVAL
);
2398 nespbl
= kzalloc(sizeof(*nespbl
), GFP_KERNEL
);
2400 ib_umem_release(region
);
2401 return ERR_PTR(-ENOMEM
);
2403 nesmr
= kzalloc(sizeof(*nesmr
), GFP_KERNEL
);
2405 ib_umem_release(region
);
2407 return ERR_PTR(-ENOMEM
);
2409 nesmr
->region
= region
;
2410 nes_ucontext
= to_nesucontext(pd
->uobject
->context
);
2411 pbl_depth
= region
->length
>> 12;
2412 pbl_depth
+= (region
->length
& (4096-1)) ? 1 : 0;
2413 nespbl
->pbl_size
= pbl_depth
*sizeof(u64
);
2414 if (req
.reg_type
== IWNES_MEMREG_TYPE_QP
) {
2415 nes_debug(NES_DBG_MR
, "Attempting to allocate QP PBL memory");
2417 nes_debug(NES_DBG_MR
, "Attempting to allocate CP PBL memory");
2420 nes_debug(NES_DBG_MR
, " %u bytes, %u entries.\n",
2421 nespbl
->pbl_size
, pbl_depth
);
2422 pbl
= pci_alloc_consistent(nesdev
->pcidev
, nespbl
->pbl_size
,
2423 &nespbl
->pbl_pbase
);
2425 ib_umem_release(region
);
2428 nes_debug(NES_DBG_MR
, "Unable to allocate PBL memory\n");
2429 return ERR_PTR(-ENOMEM
);
2432 nespbl
->pbl_vbase
= (u64
*)pbl
;
2433 nespbl
->user_base
= start
;
2434 nes_debug(NES_DBG_MR
, "Allocated PBL memory, %u bytes, pbl_pbase=%lx,"
2435 " pbl_vbase=%p user_base=0x%lx\n",
2436 nespbl
->pbl_size
, (unsigned long) nespbl
->pbl_pbase
,
2437 (void *) nespbl
->pbl_vbase
, nespbl
->user_base
);
2439 for_each_sg(region
->sg_head
.sgl
, sg
, region
->nmap
, entry
) {
2440 chunk_pages
= sg_dma_len(sg
) >> 12;
2441 chunk_pages
+= (sg_dma_len(sg
) & (4096-1)) ? 1 : 0;
2443 nespbl
->page
= sg_page(sg
);
2447 for (page_index
= 0; page_index
< chunk_pages
; page_index
++) {
2448 ((__le32
*)pbl
)[0] = cpu_to_le32((u32
)
2449 (sg_dma_address(sg
)+
2450 (page_index
*4096)));
2451 ((__le32
*)pbl
)[1] = cpu_to_le32(((u64
)
2452 (sg_dma_address(sg
)+
2453 (page_index
*4096)))>>32);
2454 nes_debug(NES_DBG_MR
, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl
,
2455 (unsigned long long)*pbl
,
2456 le32_to_cpu(((__le32
*)pbl
)[1]), le32_to_cpu(((__le32
*)pbl
)[0]));
2461 if (req
.reg_type
== IWNES_MEMREG_TYPE_QP
) {
2462 list_add_tail(&nespbl
->list
, &nes_ucontext
->qp_reg_mem_list
);
2464 list_add_tail(&nespbl
->list
, &nes_ucontext
->cq_reg_mem_list
);
2466 nesmr
->ibmr
.rkey
= -1;
2467 nesmr
->ibmr
.lkey
= -1;
2468 nesmr
->mode
= req
.reg_type
;
2469 return &nesmr
->ibmr
;
2472 ib_umem_release(region
);
2473 return ERR_PTR(-ENOSYS
);
2480 static int nes_dereg_mr(struct ib_mr
*ib_mr
)
2482 struct nes_mr
*nesmr
= to_nesmr(ib_mr
);
2483 struct nes_vnic
*nesvnic
= to_nesvnic(ib_mr
->device
);
2484 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2485 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
2486 struct nes_hw_cqp_wqe
*cqp_wqe
;
2487 struct nes_cqp_request
*cqp_request
;
2488 unsigned long flags
;
2495 pci_free_consistent(nesdev
->pcidev
,
2496 nesmr
->max_pages
* sizeof(u64
),
2500 if (nesmr
->region
) {
2501 ib_umem_release(nesmr
->region
);
2503 if (nesmr
->mode
!= IWNES_MEMREG_TYPE_MEM
) {
2508 /* Deallocate the region with the adapter */
2510 cqp_request
= nes_get_cqp_request(nesdev
);
2511 if (cqp_request
== NULL
) {
2512 nes_debug(NES_DBG_MR
, "Failed to get a cqp_request.\n");
2515 cqp_request
->waiting
= 1;
2516 cqp_wqe
= &cqp_request
->cqp_wqe
;
2518 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
2519 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
,
2520 NES_CQP_DEALLOCATE_STAG
| NES_CQP_STAG_VA_TO
|
2521 NES_CQP_STAG_DEALLOC_PBLS
| NES_CQP_STAG_MR
);
2522 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_STAG_WQE_STAG_IDX
, ib_mr
->rkey
);
2524 atomic_set(&cqp_request
->refcount
, 2);
2525 nes_post_cqp_request(nesdev
, cqp_request
);
2528 nes_debug(NES_DBG_MR
, "Waiting for deallocate STag 0x%08X completed\n", ib_mr
->rkey
);
2529 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
2531 nes_debug(NES_DBG_MR
, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u,"
2532 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
2533 ib_mr
->rkey
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
2535 major_code
= cqp_request
->major_code
;
2536 minor_code
= cqp_request
->minor_code
;
2538 nes_put_cqp_request(nesdev
, cqp_request
);
2541 nes_debug(NES_DBG_MR
, "Timeout waiting to destroy STag,"
2542 " ib_mr=%p, rkey = 0x%08X\n",
2543 ib_mr
, ib_mr
->rkey
);
2545 } else if (major_code
) {
2546 nes_debug(NES_DBG_MR
, "Error (0x%04X:0x%04X) while attempting"
2547 " to destroy STag, ib_mr=%p, rkey = 0x%08X\n",
2548 major_code
, minor_code
, ib_mr
, ib_mr
->rkey
);
2552 if (nesmr
->pbls_used
!= 0) {
2553 spin_lock_irqsave(&nesadapter
->pbl_lock
, flags
);
2554 if (nesmr
->pbl_4k
) {
2555 nesadapter
->free_4kpbl
+= nesmr
->pbls_used
;
2556 if (nesadapter
->free_4kpbl
> nesadapter
->max_4kpbl
)
2557 printk(KERN_ERR PFX
"free 4KB PBLs(%u) has "
2558 "exceeded the max(%u)\n",
2559 nesadapter
->free_4kpbl
,
2560 nesadapter
->max_4kpbl
);
2562 nesadapter
->free_256pbl
+= nesmr
->pbls_used
;
2563 if (nesadapter
->free_256pbl
> nesadapter
->max_256pbl
)
2564 printk(KERN_ERR PFX
"free 256B PBLs(%u) has "
2565 "exceeded the max(%u)\n",
2566 nesadapter
->free_256pbl
,
2567 nesadapter
->max_256pbl
);
2569 spin_unlock_irqrestore(&nesadapter
->pbl_lock
, flags
);
2571 nes_free_resource(nesadapter
, nesadapter
->allocated_mrs
,
2572 (ib_mr
->rkey
& 0x0fffff00) >> 8);
2583 static ssize_t
show_rev(struct device
*dev
, struct device_attribute
*attr
,
2586 struct nes_ib_device
*nesibdev
=
2587 container_of(dev
, struct nes_ib_device
, ibdev
.dev
);
2588 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
2590 nes_debug(NES_DBG_INIT
, "\n");
2591 return sprintf(buf
, "%x\n", nesvnic
->nesdev
->nesadapter
->hw_rev
);
2598 static ssize_t
show_hca(struct device
*dev
, struct device_attribute
*attr
,
2601 nes_debug(NES_DBG_INIT
, "\n");
2602 return sprintf(buf
, "NES020\n");
2609 static ssize_t
show_board(struct device
*dev
, struct device_attribute
*attr
,
2612 nes_debug(NES_DBG_INIT
, "\n");
2613 return sprintf(buf
, "%.*s\n", 32, "NES020 Board ID");
2617 static DEVICE_ATTR(hw_rev
, S_IRUGO
, show_rev
, NULL
);
2618 static DEVICE_ATTR(hca_type
, S_IRUGO
, show_hca
, NULL
);
2619 static DEVICE_ATTR(board_id
, S_IRUGO
, show_board
, NULL
);
2621 static struct device_attribute
*nes_dev_attributes
[] = {
2631 static int nes_query_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2632 int attr_mask
, struct ib_qp_init_attr
*init_attr
)
2634 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
2636 nes_debug(NES_DBG_QP
, "\n");
2638 attr
->qp_access_flags
= 0;
2639 attr
->cap
.max_send_wr
= nesqp
->hwqp
.sq_size
;
2640 attr
->cap
.max_recv_wr
= nesqp
->hwqp
.rq_size
;
2641 attr
->cap
.max_recv_sge
= 1;
2642 if (nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
)
2643 attr
->cap
.max_inline_data
= 0;
2645 attr
->cap
.max_inline_data
= 64;
2647 init_attr
->event_handler
= nesqp
->ibqp
.event_handler
;
2648 init_attr
->qp_context
= nesqp
->ibqp
.qp_context
;
2649 init_attr
->send_cq
= nesqp
->ibqp
.send_cq
;
2650 init_attr
->recv_cq
= nesqp
->ibqp
.recv_cq
;
2651 init_attr
->srq
= nesqp
->ibqp
.srq
;
2652 init_attr
->cap
= attr
->cap
;
2661 int nes_hw_modify_qp(struct nes_device
*nesdev
, struct nes_qp
*nesqp
,
2662 u32 next_iwarp_state
, u32 termlen
, u32 wait_completion
)
2664 struct nes_hw_cqp_wqe
*cqp_wqe
;
2665 /* struct iw_cm_id *cm_id = nesqp->cm_id; */
2666 /* struct iw_cm_event cm_event; */
2667 struct nes_cqp_request
*cqp_request
;
2671 nes_debug(NES_DBG_MOD_QP
, "QP%u, refcount=%d\n",
2672 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
2674 cqp_request
= nes_get_cqp_request(nesdev
);
2675 if (cqp_request
== NULL
) {
2676 nes_debug(NES_DBG_MOD_QP
, "Failed to get a cqp_request.\n");
2679 if (wait_completion
) {
2680 cqp_request
->waiting
= 1;
2682 cqp_request
->waiting
= 0;
2684 cqp_wqe
= &cqp_request
->cqp_wqe
;
2686 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_OPCODE_IDX
,
2687 NES_CQP_MODIFY_QP
| NES_CQP_QP_TYPE_IWARP
| next_iwarp_state
);
2688 nes_debug(NES_DBG_MOD_QP
, "using next_iwarp_state=%08x, wqe_words=%08x\n",
2689 next_iwarp_state
, le32_to_cpu(cqp_wqe
->wqe_words
[NES_CQP_WQE_OPCODE_IDX
]));
2690 nes_fill_init_cqp_wqe(cqp_wqe
, nesdev
);
2691 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_WQE_ID_IDX
, nesqp
->hwqp
.qp_id
);
2692 set_wqe_64bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_CONTEXT_LOW_IDX
, (u64
)nesqp
->nesqp_context_pbase
);
2694 /* If sending a terminate message, fill in the length (in words) */
2695 if (((next_iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
) == NES_CQP_QP_IWARP_STATE_TERMINATE
) &&
2696 !(next_iwarp_state
& NES_CQP_QP_TERM_DONT_SEND_TERM_MSG
)) {
2697 termlen
= ((termlen
+ 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT
;
2698 set_wqe_32bit_value(cqp_wqe
->wqe_words
, NES_CQP_QP_WQE_NEW_MSS_IDX
, termlen
);
2701 atomic_set(&cqp_request
->refcount
, 2);
2702 nes_post_cqp_request(nesdev
, cqp_request
);
2705 if (wait_completion
) {
2706 /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n",
2707 nesqp->hwqp.qp_id); */
2708 ret
= wait_event_timeout(cqp_request
->waitq
, (cqp_request
->request_done
!= 0),
2710 nes_debug(NES_DBG_MOD_QP
, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, "
2711 "CQP Major:Minor codes = 0x%04X:0x%04X.\n",
2712 nesqp
->hwqp
.qp_id
, ret
, cqp_request
->major_code
, cqp_request
->minor_code
);
2713 major_code
= cqp_request
->major_code
;
2715 nes_debug(NES_DBG_MOD_QP
, "Modify iwarp QP%u failed"
2716 "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n",
2717 nesqp
->hwqp
.qp_id
, cqp_request
->major_code
,
2718 cqp_request
->minor_code
, next_iwarp_state
);
2721 nes_put_cqp_request(nesdev
, cqp_request
);
2725 else if (major_code
)
2738 int nes_modify_qp(struct ib_qp
*ibqp
, struct ib_qp_attr
*attr
,
2739 int attr_mask
, struct ib_udata
*udata
)
2741 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
2742 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
2743 struct nes_device
*nesdev
= nesvnic
->nesdev
;
2746 u32 next_iwarp_state
= 0;
2748 unsigned long qplockflags
;
2750 u16 original_last_aeq
;
2751 u8 issue_modify_qp
= 0;
2754 nes_debug(NES_DBG_MOD_QP
, "QP%u: QP State=%u, cur QP State=%u,"
2755 " iwarp_state=0x%X, refcount=%d\n",
2756 nesqp
->hwqp
.qp_id
, attr
->qp_state
, nesqp
->ibqp_state
,
2757 nesqp
->iwarp_state
, atomic_read(&nesqp
->refcount
));
2759 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
2761 nes_debug(NES_DBG_MOD_QP
, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X,"
2762 " QP Access Flags=0x%X, attr_mask = 0x%0x\n",
2763 nesqp
->hwqp
.qp_id
, nesqp
->hw_iwarp_state
,
2764 nesqp
->hw_tcp_state
, attr
->qp_access_flags
, attr_mask
);
2766 if (attr_mask
& IB_QP_STATE
) {
2767 switch (attr
->qp_state
) {
2769 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = init\n",
2771 if (nesqp
->iwarp_state
> (u32
)NES_CQP_QP_IWARP_STATE_IDLE
) {
2772 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2775 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_IDLE
;
2776 issue_modify_qp
= 1;
2779 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = rtr\n",
2781 if (nesqp
->iwarp_state
>(u32
)NES_CQP_QP_IWARP_STATE_IDLE
) {
2782 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2785 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_IDLE
;
2786 issue_modify_qp
= 1;
2789 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = rts\n",
2791 if (nesqp
->iwarp_state
>(u32
)NES_CQP_QP_IWARP_STATE_RTS
) {
2792 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2795 if (nesqp
->cm_id
== NULL
) {
2796 nes_debug(NES_DBG_MOD_QP
, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n",
2797 nesqp
->hwqp
.qp_id
);
2798 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2801 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_RTS
;
2802 if (nesqp
->iwarp_state
!= NES_CQP_QP_IWARP_STATE_RTS
)
2803 next_iwarp_state
|= NES_CQP_QP_CONTEXT_VALID
|
2804 NES_CQP_QP_ARP_VALID
| NES_CQP_QP_ORD_VALID
;
2805 issue_modify_qp
= 1;
2806 nesqp
->hw_tcp_state
= NES_AEQE_TCP_STATE_ESTABLISHED
;
2807 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_RTS
;
2808 nesqp
->hte_added
= 1;
2811 issue_modify_qp
= 1;
2812 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n",
2813 nesqp
->hwqp
.qp_id
, nesqp
->hwqp
.sq_head
, nesqp
->hwqp
.sq_tail
);
2814 if (nesqp
->iwarp_state
== (u32
)NES_CQP_QP_IWARP_STATE_CLOSING
) {
2815 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2818 if (nesqp
->iwarp_state
> (u32
)NES_CQP_QP_IWARP_STATE_CLOSING
) {
2819 nes_debug(NES_DBG_MOD_QP
, "QP%u: State change to closing"
2820 " ignored due to current iWARP state\n",
2822 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2825 if (nesqp
->hw_iwarp_state
!= NES_AEQE_IWARP_STATE_RTS
) {
2826 nes_debug(NES_DBG_MOD_QP
, "QP%u: State change to closing"
2827 " already done based on hw state.\n",
2829 issue_modify_qp
= 0;
2831 switch (nesqp
->hw_iwarp_state
) {
2832 case NES_AEQE_IWARP_STATE_CLOSING
:
2833 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_CLOSING
;
2835 case NES_AEQE_IWARP_STATE_TERMINATE
:
2836 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_TERMINATE
;
2838 case NES_AEQE_IWARP_STATE_ERROR
:
2839 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_ERROR
;
2842 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_CLOSING
;
2843 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_CLOSING
;
2849 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = terminate\n",
2851 if (nesqp
->iwarp_state
>=(u32
)NES_CQP_QP_IWARP_STATE_TERMINATE
) {
2852 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2855 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
2856 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_TERMINATE
;
2857 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_TERMINATE
;
2858 issue_modify_qp
= 1;
2862 if (nesqp
->iwarp_state
== (u32
)NES_CQP_QP_IWARP_STATE_ERROR
) {
2863 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2866 nes_debug(NES_DBG_MOD_QP
, "QP%u: new state = error\n",
2868 if (nesqp
->term_flags
)
2869 del_timer(&nesqp
->terminate_timer
);
2871 next_iwarp_state
= NES_CQP_QP_IWARP_STATE_ERROR
;
2872 /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */
2873 if (nesqp
->hte_added
) {
2874 nes_debug(NES_DBG_MOD_QP
, "set CQP_QP_DEL_HTE\n");
2875 next_iwarp_state
|= NES_CQP_QP_DEL_HTE
;
2876 nesqp
->hte_added
= 0;
2878 if ((nesqp
->hw_tcp_state
> NES_AEQE_TCP_STATE_CLOSED
) &&
2879 (nesdev
->iw_status
) &&
2880 (nesqp
->hw_tcp_state
!= NES_AEQE_TCP_STATE_TIME_WAIT
)) {
2881 next_iwarp_state
|= NES_CQP_QP_RESET
;
2883 nes_debug(NES_DBG_MOD_QP
, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n",
2884 nesqp
->hwqp
.qp_id
, nesqp
->hw_tcp_state
);
2887 issue_modify_qp
= 1;
2888 nesqp
->hw_iwarp_state
= NES_AEQE_IWARP_STATE_ERROR
;
2891 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2896 nesqp
->ibqp_state
= attr
->qp_state
;
2897 nesqp
->iwarp_state
= next_iwarp_state
& NES_CQP_QP_IWARP_STATE_MASK
;
2898 nes_debug(NES_DBG_MOD_QP
, "Change nesqp->iwarp_state=%08x\n",
2899 nesqp
->iwarp_state
);
2902 if (attr_mask
& IB_QP_ACCESS_FLAGS
) {
2903 if (attr
->qp_access_flags
& IB_ACCESS_LOCAL_WRITE
) {
2904 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
|
2905 NES_QPCONTEXT_MISC_RDMA_READ_EN
);
2906 issue_modify_qp
= 1;
2908 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_WRITE
) {
2909 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
);
2910 issue_modify_qp
= 1;
2912 if (attr
->qp_access_flags
& IB_ACCESS_REMOTE_READ
) {
2913 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN
);
2914 issue_modify_qp
= 1;
2916 if (attr
->qp_access_flags
& IB_ACCESS_MW_BIND
) {
2917 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN
);
2918 issue_modify_qp
= 1;
2921 if (nesqp
->user_mode
) {
2922 nesqp
->nesqp_context
->misc
|= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN
|
2923 NES_QPCONTEXT_MISC_RDMA_READ_EN
);
2924 issue_modify_qp
= 1;
2928 original_last_aeq
= nesqp
->last_aeq
;
2929 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2931 nes_debug(NES_DBG_MOD_QP
, "issue_modify_qp=%u\n", issue_modify_qp
);
2936 if (issue_modify_qp
) {
2937 nes_debug(NES_DBG_MOD_QP
, "call nes_hw_modify_qp\n");
2938 ret
= nes_hw_modify_qp(nesdev
, nesqp
, next_iwarp_state
, 0, 1);
2940 nes_debug(NES_DBG_MOD_QP
, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)"
2941 " failed for QP%u.\n",
2942 next_iwarp_state
, nesqp
->hwqp
.qp_id
);
2946 if ((issue_modify_qp
) && (nesqp
->ibqp_state
> IB_QPS_RTS
)) {
2947 nes_debug(NES_DBG_MOD_QP
, "QP%u Issued ModifyQP refcount (%d),"
2948 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
2949 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
2950 original_last_aeq
, nesqp
->last_aeq
);
2951 if (!ret
|| original_last_aeq
!= NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE
) {
2953 if (nesqp
->cm_id
&& nesqp
->hw_tcp_state
!= 0) {
2954 nes_debug(NES_DBG_MOD_QP
, "QP%u Queuing fake disconnect for QP refcount (%d),"
2955 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
2956 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
2957 original_last_aeq
, nesqp
->last_aeq
);
2958 /* this one is for the cm_disconnect thread */
2959 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
2960 nesqp
->hw_tcp_state
= NES_AEQE_TCP_STATE_CLOSED
;
2961 nesqp
->last_aeq
= NES_AEQE_AEID_RESET_SENT
;
2962 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2963 nes_cm_disconn(nesqp
);
2965 nes_debug(NES_DBG_MOD_QP
, "QP%u No fake disconnect, QP refcount=%d\n",
2966 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
2969 spin_lock_irqsave(&nesqp
->lock
, qplockflags
);
2971 /* These two are for the timer thread */
2972 if (atomic_inc_return(&nesqp
->close_timer_started
) == 1) {
2973 nesqp
->cm_id
->add_ref(nesqp
->cm_id
);
2974 nes_debug(NES_DBG_MOD_QP
, "QP%u Not decrementing QP refcount (%d),"
2975 " need ae to finish up, original_last_aeq = 0x%04X."
2976 " last_aeq = 0x%04X, scheduling timer.\n",
2977 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
2978 original_last_aeq
, nesqp
->last_aeq
);
2979 schedule_nes_timer(nesqp
->cm_node
, (struct sk_buff
*) nesqp
, NES_TIMER_TYPE_CLOSE
, 1, 0);
2981 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2983 spin_unlock_irqrestore(&nesqp
->lock
, qplockflags
);
2984 nes_debug(NES_DBG_MOD_QP
, "QP%u Not decrementing QP refcount (%d),"
2985 " need ae to finish up, original_last_aeq = 0x%04X."
2986 " last_aeq = 0x%04X.\n",
2987 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
2988 original_last_aeq
, nesqp
->last_aeq
);
2992 nes_debug(NES_DBG_MOD_QP
, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
2993 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
2994 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
2995 original_last_aeq
, nesqp
->last_aeq
);
2998 nes_debug(NES_DBG_MOD_QP
, "QP%u Decrementing QP refcount (%d), No ae to finish up,"
2999 " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n",
3000 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
),
3001 original_last_aeq
, nesqp
->last_aeq
);
3006 nes_debug(NES_DBG_MOD_QP
, "QP%u Leaving, refcount=%d\n",
3007 nesqp
->hwqp
.qp_id
, atomic_read(&nesqp
->refcount
));
3014 * nes_muticast_attach
3016 static int nes_multicast_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
3018 nes_debug(NES_DBG_INIT
, "\n");
3024 * nes_multicast_detach
3026 static int nes_multicast_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
3028 nes_debug(NES_DBG_INIT
, "\n");
3036 static int nes_process_mad(struct ib_device
*ibdev
, int mad_flags
,
3037 u8 port_num
, const struct ib_wc
*in_wc
, const struct ib_grh
*in_grh
,
3038 const struct ib_mad_hdr
*in
, size_t in_mad_size
,
3039 struct ib_mad_hdr
*out
, size_t *out_mad_size
,
3040 u16
*out_mad_pkey_index
)
3042 nes_debug(NES_DBG_INIT
, "\n");
3047 fill_wqe_sg_send(struct nes_hw_qp_wqe
*wqe
, struct ib_send_wr
*ib_wr
, u32 uselkey
)
3050 int total_payload_length
= 0;
3051 for (sge_index
= 0; sge_index
< ib_wr
->num_sge
; sge_index
++) {
3052 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX
+(sge_index
*4),
3053 ib_wr
->sg_list
[sge_index
].addr
);
3054 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_LENGTH0_IDX
+ (sge_index
*4),
3055 ib_wr
->sg_list
[sge_index
].length
);
3057 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
+ (sge_index
*4),
3058 (ib_wr
->sg_list
[sge_index
].lkey
));
3060 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
+ (sge_index
*4), 0);
3062 total_payload_length
+= ib_wr
->sg_list
[sge_index
].length
;
3064 nes_debug(NES_DBG_IW_TX
, "UC UC UC, sending total_payload_length=%u \n",
3065 total_payload_length
);
3066 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3067 total_payload_length
);
3073 static int nes_post_send(struct ib_qp
*ibqp
, struct ib_send_wr
*ib_wr
,
3074 struct ib_send_wr
**bad_wr
)
3077 unsigned long flags
= 0;
3078 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
3079 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3080 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
3081 struct nes_hw_qp_wqe
*wqe
;
3083 u32 qsize
= nesqp
->hwqp
.sq_size
;
3089 if (nesqp
->ibqp_state
> IB_QPS_RTS
) {
3094 spin_lock_irqsave(&nesqp
->lock
, flags
);
3096 head
= nesqp
->hwqp
.sq_head
;
3099 /* Check for QP error */
3100 if (nesqp
->term_flags
) {
3105 /* Check for SQ overflow */
3106 if (((head
+ (2 * qsize
) - nesqp
->hwqp
.sq_tail
) % qsize
) == (qsize
- 1)) {
3111 wqe
= &nesqp
->hwqp
.sq_vbase
[head
];
3112 /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n",
3113 nesqp->hwqp.qp_id, wqe, head); */
3114 nes_fill_init_qp_wqe(wqe
, nesqp
, head
);
3115 u64temp
= (u64
)(ib_wr
->wr_id
);
3116 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
,
3118 switch (ib_wr
->opcode
) {
3120 case IB_WR_SEND_WITH_INV
:
3121 if (IB_WR_SEND
== ib_wr
->opcode
) {
3122 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3123 wqe_misc
= NES_IWARP_SQ_OP_SENDSE
;
3125 wqe_misc
= NES_IWARP_SQ_OP_SEND
;
3127 if (ib_wr
->send_flags
& IB_SEND_SOLICITED
)
3128 wqe_misc
= NES_IWARP_SQ_OP_SENDSEINV
;
3130 wqe_misc
= NES_IWARP_SQ_OP_SENDINV
;
3132 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX
,
3133 ib_wr
->ex
.invalidate_rkey
);
3136 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3141 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
3142 wqe_misc
|= NES_IWARP_SQ_WQE_LOCAL_FENCE
;
3144 if ((ib_wr
->send_flags
& IB_SEND_INLINE
) &&
3145 ((nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) == 0) &&
3146 (ib_wr
->sg_list
[0].length
<= 64)) {
3147 memcpy(&wqe
->wqe_words
[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX
],
3148 (void *)(unsigned long)ib_wr
->sg_list
[0].addr
, ib_wr
->sg_list
[0].length
);
3149 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3150 ib_wr
->sg_list
[0].length
);
3151 wqe_misc
|= NES_IWARP_SQ_WQE_IMM_DATA
;
3153 fill_wqe_sg_send(wqe
, ib_wr
, 1);
3157 case IB_WR_RDMA_WRITE
:
3158 wqe_misc
= NES_IWARP_SQ_OP_RDMAW
;
3159 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3160 nes_debug(NES_DBG_IW_TX
, "Exceeded max sge, ib_wr=%u, max=%u\n",
3161 ib_wr
->num_sge
, nesdev
->nesadapter
->max_sge
);
3166 if (ib_wr
->send_flags
& IB_SEND_FENCE
)
3167 wqe_misc
|= NES_IWARP_SQ_WQE_LOCAL_FENCE
;
3169 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_STAG_IDX
,
3170 rdma_wr(ib_wr
)->rkey
);
3171 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX
,
3172 rdma_wr(ib_wr
)->remote_addr
);
3174 if ((ib_wr
->send_flags
& IB_SEND_INLINE
) &&
3175 ((nes_drv_opt
& NES_DRV_OPT_NO_INLINE_DATA
) == 0) &&
3176 (ib_wr
->sg_list
[0].length
<= 64)) {
3177 memcpy(&wqe
->wqe_words
[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX
],
3178 (void *)(unsigned long)ib_wr
->sg_list
[0].addr
, ib_wr
->sg_list
[0].length
);
3179 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
,
3180 ib_wr
->sg_list
[0].length
);
3181 wqe_misc
|= NES_IWARP_SQ_WQE_IMM_DATA
;
3183 fill_wqe_sg_send(wqe
, ib_wr
, 1);
3186 wqe
->wqe_words
[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
] =
3187 wqe
->wqe_words
[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
];
3189 case IB_WR_RDMA_READ
:
3190 case IB_WR_RDMA_READ_WITH_INV
:
3191 /* iWARP only supports 1 sge for RDMA reads */
3192 if (ib_wr
->num_sge
> 1) {
3193 nes_debug(NES_DBG_IW_TX
, "Exceeded max sge, ib_wr=%u, max=1\n",
3198 if (ib_wr
->opcode
== IB_WR_RDMA_READ
) {
3199 wqe_misc
= NES_IWARP_SQ_OP_RDMAR
;
3201 wqe_misc
= NES_IWARP_SQ_OP_RDMAR_LOCINV
;
3202 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX
,
3203 ib_wr
->ex
.invalidate_rkey
);
3206 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX
,
3207 rdma_wr(ib_wr
)->remote_addr
);
3208 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_STAG_IDX
,
3209 rdma_wr(ib_wr
)->rkey
);
3210 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
,
3211 ib_wr
->sg_list
->length
);
3212 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX
,
3213 ib_wr
->sg_list
->addr
);
3214 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_STAG0_IDX
,
3215 ib_wr
->sg_list
->lkey
);
3217 case IB_WR_LOCAL_INV
:
3218 wqe_misc
= NES_IWARP_SQ_OP_LOCINV
;
3219 set_wqe_32bit_value(wqe
->wqe_words
,
3220 NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX
,
3221 ib_wr
->ex
.invalidate_rkey
);
3225 struct nes_mr
*mr
= to_nesmr(reg_wr(ib_wr
)->mr
);
3226 int page_shift
= ilog2(reg_wr(ib_wr
)->mr
->page_size
);
3227 int flags
= reg_wr(ib_wr
)->access
;
3229 if (mr
->npages
> (NES_4K_PBL_CHUNK_SIZE
/ sizeof(u64
))) {
3230 nes_debug(NES_DBG_IW_TX
, "SQ_FMR: bad page_list_len\n");
3234 wqe_misc
= NES_IWARP_SQ_OP_FAST_REG
;
3235 set_wqe_64bit_value(wqe
->wqe_words
,
3236 NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX
,
3238 set_wqe_32bit_value(wqe
->wqe_words
,
3239 NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX
,
3241 set_wqe_32bit_value(wqe
->wqe_words
,
3242 NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX
, 0);
3243 set_wqe_32bit_value(wqe
->wqe_words
,
3244 NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX
,
3245 reg_wr(ib_wr
)->key
);
3247 if (page_shift
== 12) {
3248 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K
;
3249 } else if (page_shift
== 21) {
3250 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M
;
3252 nes_debug(NES_DBG_IW_TX
, "Invalid page shift,"
3253 " ib_wr=%u, max=1\n", ib_wr
->num_sge
);
3258 /* Set access_flags */
3259 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ
;
3260 if (flags
& IB_ACCESS_LOCAL_WRITE
)
3261 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE
;
3263 if (flags
& IB_ACCESS_REMOTE_WRITE
)
3264 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE
;
3266 if (flags
& IB_ACCESS_REMOTE_READ
)
3267 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ
;
3269 if (flags
& IB_ACCESS_MW_BIND
)
3270 wqe_misc
|= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND
;
3272 /* Fill in PBL info: */
3273 set_wqe_64bit_value(wqe
->wqe_words
,
3274 NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX
,
3277 set_wqe_32bit_value(wqe
->wqe_words
,
3278 NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX
,
3281 nes_debug(NES_DBG_IW_TX
, "SQ_REG_MR: iova_start: %llx, "
3282 "length: %d, rkey: %0x, pgl_paddr: %llx, "
3283 "page_list_len: %u, wqe_misc: %x\n",
3284 (unsigned long long) mr
->ibmr
.iova
,
3287 (unsigned long long) mr
->paddr
,
3301 if ((ib_wr
->send_flags
& IB_SEND_SIGNALED
) || nesqp
->sig_all
)
3302 wqe_misc
|= NES_IWARP_SQ_WQE_SIGNALED_COMPL
;
3304 wqe
->wqe_words
[NES_IWARP_SQ_WQE_MISC_IDX
] = cpu_to_le32(wqe_misc
);
3306 ib_wr
= ib_wr
->next
;
3314 nesqp
->hwqp
.sq_head
= head
;
3317 counter
= min(wqe_count
, ((u32
)255));
3318 wqe_count
-= counter
;
3319 nes_write32(nesdev
->regs
+ NES_WQE_ALLOC
,
3320 (counter
<< 24) | 0x00800000 | nesqp
->hwqp
.qp_id
);
3323 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
3335 static int nes_post_recv(struct ib_qp
*ibqp
, struct ib_recv_wr
*ib_wr
,
3336 struct ib_recv_wr
**bad_wr
)
3339 unsigned long flags
= 0;
3340 struct nes_vnic
*nesvnic
= to_nesvnic(ibqp
->device
);
3341 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3342 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
3343 struct nes_hw_qp_wqe
*wqe
;
3346 u32 qsize
= nesqp
->hwqp
.rq_size
;
3350 u32 total_payload_length
;
3352 if (nesqp
->ibqp_state
> IB_QPS_RTS
) {
3357 spin_lock_irqsave(&nesqp
->lock
, flags
);
3359 head
= nesqp
->hwqp
.rq_head
;
3362 /* Check for QP error */
3363 if (nesqp
->term_flags
) {
3368 if (ib_wr
->num_sge
> nesdev
->nesadapter
->max_sge
) {
3372 /* Check for RQ overflow */
3373 if (((head
+ (2 * qsize
) - nesqp
->hwqp
.rq_tail
) % qsize
) == (qsize
- 1)) {
3378 nes_debug(NES_DBG_IW_RX
, "ibwr sge count = %u.\n", ib_wr
->num_sge
);
3379 wqe
= &nesqp
->hwqp
.rq_vbase
[head
];
3381 /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n",
3382 nesqp->hwqp.qp_id, wqe, head); */
3383 nes_fill_init_qp_wqe(wqe
, nesqp
, head
);
3384 u64temp
= (u64
)(ib_wr
->wr_id
);
3385 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
,
3387 total_payload_length
= 0;
3388 for (sge_index
=0; sge_index
< ib_wr
->num_sge
; sge_index
++) {
3389 set_wqe_64bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX
+(sge_index
*4),
3390 ib_wr
->sg_list
[sge_index
].addr
);
3391 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_LENGTH0_IDX
+(sge_index
*4),
3392 ib_wr
->sg_list
[sge_index
].length
);
3393 set_wqe_32bit_value(wqe
->wqe_words
,NES_IWARP_RQ_WQE_STAG0_IDX
+(sge_index
*4),
3394 ib_wr
->sg_list
[sge_index
].lkey
);
3396 total_payload_length
+= ib_wr
->sg_list
[sge_index
].length
;
3398 set_wqe_32bit_value(wqe
->wqe_words
, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX
,
3399 total_payload_length
);
3401 ib_wr
= ib_wr
->next
;
3408 nesqp
->hwqp
.rq_head
= head
;
3411 counter
= min(wqe_count
, ((u32
)255));
3412 wqe_count
-= counter
;
3413 nes_write32(nesdev
->regs
+NES_WQE_ALLOC
, (counter
<<24) | nesqp
->hwqp
.qp_id
);
3416 spin_unlock_irqrestore(&nesqp
->lock
, flags
);
3425 * nes_drain_sq - drain sq
3426 * @ibqp: pointer to ibqp
3428 static void nes_drain_sq(struct ib_qp
*ibqp
)
3430 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
3432 if (nesqp
->hwqp
.sq_tail
!= nesqp
->hwqp
.sq_head
)
3433 wait_for_completion(&nesqp
->sq_drained
);
3437 * nes_drain_rq - drain rq
3438 * @ibqp: pointer to ibqp
3440 static void nes_drain_rq(struct ib_qp
*ibqp
)
3442 struct nes_qp
*nesqp
= to_nesqp(ibqp
);
3444 if (nesqp
->hwqp
.rq_tail
!= nesqp
->hwqp
.rq_head
)
3445 wait_for_completion(&nesqp
->rq_drained
);
3451 static int nes_poll_cq(struct ib_cq
*ibcq
, int num_entries
, struct ib_wc
*entry
)
3455 unsigned long flags
= 0;
3456 struct nes_vnic
*nesvnic
= to_nesvnic(ibcq
->device
);
3457 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3458 struct nes_cq
*nescq
= to_nescq(ibcq
);
3459 struct nes_qp
*nesqp
;
3460 struct nes_hw_cqe cqe
;
3467 u32 move_cq_head
= 1;
3470 nes_debug(NES_DBG_CQ
, "\n");
3472 spin_lock_irqsave(&nescq
->lock
, flags
);
3474 head
= nescq
->hw_cq
.cq_head
;
3475 cq_size
= nescq
->hw_cq
.cq_size
;
3477 while (cqe_count
< num_entries
) {
3478 if ((le32_to_cpu(nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_OPCODE_IDX
]) &
3479 NES_CQE_VALID
) == 0)
3483 * Make sure we read CQ entry contents *after*
3484 * we've checked the valid bit.
3488 cqe
= nescq
->hw_cq
.cq_vbase
[head
];
3489 u32temp
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
3490 wqe_index
= u32temp
& (nesdev
->nesadapter
->max_qp_wr
- 1);
3491 u32temp
&= ~(NES_SW_CONTEXT_ALIGN
-1);
3492 /* parse CQE, get completion context from WQE (either rq or sq) */
3493 u64temp
= (((u64
)(le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_HIGH_IDX
])))<<32) |
3497 nesqp
= (struct nes_qp
*)(unsigned long)u64temp
;
3498 memset(entry
, 0, sizeof *entry
);
3499 if (cqe
.cqe_words
[NES_CQE_ERROR_CODE_IDX
] == 0) {
3500 entry
->status
= IB_WC_SUCCESS
;
3502 err_code
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_ERROR_CODE_IDX
]);
3503 if (NES_IWARP_CQE_MAJOR_DRV
== (err_code
>> 16)) {
3504 entry
->status
= err_code
& 0x0000ffff;
3506 /* The rest of the cqe's will be marked as flushed */
3507 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_ERROR_CODE_IDX
] =
3508 cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH
<< 16) |
3509 NES_IWARP_CQE_MINOR_FLUSH
);
3511 entry
->status
= IB_WC_WR_FLUSH_ERR
;
3514 entry
->qp
= &nesqp
->ibqp
;
3515 entry
->src_qp
= nesqp
->hwqp
.qp_id
;
3517 if (le32_to_cpu(cqe
.cqe_words
[NES_CQE_OPCODE_IDX
]) & NES_CQE_SQ
) {
3518 if (nesqp
->skip_lsmm
) {
3519 nesqp
->skip_lsmm
= 0;
3520 nesqp
->hwqp
.sq_tail
++;
3523 /* Working on a SQ Completion*/
3524 wrid
= (((u64
)(cpu_to_le32((u32
)nesqp
->hwqp
.sq_vbase
[wqe_index
].
3525 wqe_words
[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX
]))) << 32) |
3526 ((u64
)(cpu_to_le32((u32
)nesqp
->hwqp
.sq_vbase
[wqe_index
].
3527 wqe_words
[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX
])));
3528 entry
->byte_len
= le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3529 wqe_words
[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX
]);
3531 switch (le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3532 wqe_words
[NES_IWARP_SQ_WQE_MISC_IDX
]) & 0x3f) {
3533 case NES_IWARP_SQ_OP_RDMAW
:
3534 nes_debug(NES_DBG_CQ
, "Operation = RDMA WRITE.\n");
3535 entry
->opcode
= IB_WC_RDMA_WRITE
;
3537 case NES_IWARP_SQ_OP_RDMAR
:
3538 nes_debug(NES_DBG_CQ
, "Operation = RDMA READ.\n");
3539 entry
->opcode
= IB_WC_RDMA_READ
;
3540 entry
->byte_len
= le32_to_cpu(nesqp
->hwqp
.sq_vbase
[wqe_index
].
3541 wqe_words
[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX
]);
3543 case NES_IWARP_SQ_OP_SENDINV
:
3544 case NES_IWARP_SQ_OP_SENDSEINV
:
3545 case NES_IWARP_SQ_OP_SEND
:
3546 case NES_IWARP_SQ_OP_SENDSE
:
3547 nes_debug(NES_DBG_CQ
, "Operation = Send.\n");
3548 entry
->opcode
= IB_WC_SEND
;
3550 case NES_IWARP_SQ_OP_LOCINV
:
3551 entry
->opcode
= IB_WC_LOCAL_INV
;
3553 case NES_IWARP_SQ_OP_FAST_REG
:
3554 entry
->opcode
= IB_WC_REG_MR
;
3558 nesqp
->hwqp
.sq_tail
= (wqe_index
+1)&(nesqp
->hwqp
.sq_size
- 1);
3559 if ((entry
->status
!= IB_WC_SUCCESS
) && (nesqp
->hwqp
.sq_tail
!= nesqp
->hwqp
.sq_head
)) {
3561 wq_tail
= nesqp
->hwqp
.sq_tail
;
3564 /* Working on a RQ Completion*/
3565 entry
->byte_len
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_PAYLOAD_LENGTH_IDX
]);
3566 wrid
= ((u64
)(le32_to_cpu(nesqp
->hwqp
.rq_vbase
[wqe_index
].wqe_words
[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX
]))) |
3567 ((u64
)(le32_to_cpu(nesqp
->hwqp
.rq_vbase
[wqe_index
].wqe_words
[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX
]))<<32);
3568 entry
->opcode
= IB_WC_RECV
;
3570 nesqp
->hwqp
.rq_tail
= (wqe_index
+1)&(nesqp
->hwqp
.rq_size
- 1);
3571 if ((entry
->status
!= IB_WC_SUCCESS
) && (nesqp
->hwqp
.rq_tail
!= nesqp
->hwqp
.rq_head
)) {
3573 wq_tail
= nesqp
->hwqp
.rq_tail
;
3577 if (nesqp
->iwarp_state
> NES_CQP_QP_IWARP_STATE_RTS
) {
3578 if (nesqp
->hwqp
.sq_tail
== nesqp
->hwqp
.sq_head
)
3579 complete(&nesqp
->sq_drained
);
3580 if (nesqp
->hwqp
.rq_tail
== nesqp
->hwqp
.rq_head
)
3581 complete(&nesqp
->rq_drained
);
3584 entry
->wr_id
= wrid
;
3590 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_OPCODE_IDX
] = 0;
3591 if (++head
>= cq_size
)
3593 nescq
->polled_completions
++;
3595 if ((nescq
->polled_completions
> (cq_size
/ 2)) ||
3596 (nescq
->polled_completions
== 255)) {
3597 nes_debug(NES_DBG_CQ
, "CQ%u Issuing CQE Allocate since more than half of cqes"
3598 " are pending %u of %u.\n",
3599 nescq
->hw_cq
.cq_number
, nescq
->polled_completions
, cq_size
);
3600 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
,
3601 nescq
->hw_cq
.cq_number
| (nescq
->polled_completions
<< 16));
3602 nescq
->polled_completions
= 0;
3605 /* Update the wqe index and set status to flush */
3606 wqe_index
= le32_to_cpu(cqe
.cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
]);
3607 wqe_index
= (wqe_index
& (~(nesdev
->nesadapter
->max_qp_wr
- 1))) | wq_tail
;
3608 nescq
->hw_cq
.cq_vbase
[head
].cqe_words
[NES_CQE_COMP_COMP_CTX_LOW_IDX
] =
3609 cpu_to_le32(wqe_index
);
3610 move_cq_head
= 1; /* ready for next pass */
3614 if (nescq
->polled_completions
) {
3615 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
,
3616 nescq
->hw_cq
.cq_number
| (nescq
->polled_completions
<< 16));
3617 nescq
->polled_completions
= 0;
3620 nescq
->hw_cq
.cq_head
= head
;
3621 nes_debug(NES_DBG_CQ
, "Reporting %u completions for CQ%u.\n",
3622 cqe_count
, nescq
->hw_cq
.cq_number
);
3624 spin_unlock_irqrestore(&nescq
->lock
, flags
);
3633 static int nes_req_notify_cq(struct ib_cq
*ibcq
, enum ib_cq_notify_flags notify_flags
)
3635 struct nes_vnic
*nesvnic
= to_nesvnic(ibcq
->device
);
3636 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3637 struct nes_cq
*nescq
= to_nescq(ibcq
);
3640 nes_debug(NES_DBG_CQ
, "Requesting notification for CQ%u.\n",
3641 nescq
->hw_cq
.cq_number
);
3643 cq_arm
= nescq
->hw_cq
.cq_number
;
3644 if ((notify_flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_NEXT_COMP
)
3645 cq_arm
|= NES_CQE_ALLOC_NOTIFY_NEXT
;
3646 else if ((notify_flags
& IB_CQ_SOLICITED_MASK
) == IB_CQ_SOLICITED
)
3647 cq_arm
|= NES_CQE_ALLOC_NOTIFY_SE
;
3651 nes_write32(nesdev
->regs
+NES_CQE_ALLOC
, cq_arm
);
3652 nes_read32(nesdev
->regs
+NES_CQE_ALLOC
);
3657 static int nes_port_immutable(struct ib_device
*ibdev
, u8 port_num
,
3658 struct ib_port_immutable
*immutable
)
3660 struct ib_port_attr attr
;
3663 immutable
->core_cap_flags
= RDMA_CORE_PORT_IWARP
;
3665 err
= nes_query_port(ibdev
, port_num
, &attr
);
3669 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
3670 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
3675 static void get_dev_fw_str(struct ib_device
*dev
, char *str
,
3678 struct nes_ib_device
*nesibdev
=
3679 container_of(dev
, struct nes_ib_device
, ibdev
);
3680 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
3682 nes_debug(NES_DBG_INIT
, "\n");
3683 snprintf(str
, str_len
, "%u.%u",
3684 (nesvnic
->nesdev
->nesadapter
->firmware_version
>> 16),
3685 (nesvnic
->nesdev
->nesadapter
->firmware_version
& 0x000000ff));
3689 * nes_init_ofa_device
3691 struct nes_ib_device
*nes_init_ofa_device(struct net_device
*netdev
)
3693 struct nes_ib_device
*nesibdev
;
3694 struct nes_vnic
*nesvnic
= netdev_priv(netdev
);
3695 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3697 nesibdev
= (struct nes_ib_device
*)ib_alloc_device(sizeof(struct nes_ib_device
));
3698 if (nesibdev
== NULL
) {
3701 strlcpy(nesibdev
->ibdev
.name
, "nes%d", IB_DEVICE_NAME_MAX
);
3702 nesibdev
->ibdev
.owner
= THIS_MODULE
;
3704 nesibdev
->ibdev
.node_type
= RDMA_NODE_RNIC
;
3705 memset(&nesibdev
->ibdev
.node_guid
, 0, sizeof(nesibdev
->ibdev
.node_guid
));
3706 memcpy(&nesibdev
->ibdev
.node_guid
, netdev
->dev_addr
, 6);
3708 nesibdev
->ibdev
.uverbs_cmd_mask
=
3709 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT
) |
3710 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE
) |
3711 (1ull << IB_USER_VERBS_CMD_QUERY_PORT
) |
3712 (1ull << IB_USER_VERBS_CMD_ALLOC_PD
) |
3713 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD
) |
3714 (1ull << IB_USER_VERBS_CMD_REG_MR
) |
3715 (1ull << IB_USER_VERBS_CMD_DEREG_MR
) |
3716 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL
) |
3717 (1ull << IB_USER_VERBS_CMD_CREATE_CQ
) |
3718 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ
) |
3719 (1ull << IB_USER_VERBS_CMD_CREATE_AH
) |
3720 (1ull << IB_USER_VERBS_CMD_DESTROY_AH
) |
3721 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ
) |
3722 (1ull << IB_USER_VERBS_CMD_CREATE_QP
) |
3723 (1ull << IB_USER_VERBS_CMD_MODIFY_QP
) |
3724 (1ull << IB_USER_VERBS_CMD_POLL_CQ
) |
3725 (1ull << IB_USER_VERBS_CMD_DESTROY_QP
) |
3726 (1ull << IB_USER_VERBS_CMD_ALLOC_MW
) |
3727 (1ull << IB_USER_VERBS_CMD_BIND_MW
) |
3728 (1ull << IB_USER_VERBS_CMD_DEALLOC_MW
) |
3729 (1ull << IB_USER_VERBS_CMD_POST_RECV
) |
3730 (1ull << IB_USER_VERBS_CMD_POST_SEND
);
3732 nesibdev
->ibdev
.phys_port_cnt
= 1;
3733 nesibdev
->ibdev
.num_comp_vectors
= 1;
3734 nesibdev
->ibdev
.dev
.parent
= &nesdev
->pcidev
->dev
;
3735 nesibdev
->ibdev
.query_device
= nes_query_device
;
3736 nesibdev
->ibdev
.query_port
= nes_query_port
;
3737 nesibdev
->ibdev
.query_pkey
= nes_query_pkey
;
3738 nesibdev
->ibdev
.query_gid
= nes_query_gid
;
3739 nesibdev
->ibdev
.alloc_ucontext
= nes_alloc_ucontext
;
3740 nesibdev
->ibdev
.dealloc_ucontext
= nes_dealloc_ucontext
;
3741 nesibdev
->ibdev
.mmap
= nes_mmap
;
3742 nesibdev
->ibdev
.alloc_pd
= nes_alloc_pd
;
3743 nesibdev
->ibdev
.dealloc_pd
= nes_dealloc_pd
;
3744 nesibdev
->ibdev
.create_ah
= nes_create_ah
;
3745 nesibdev
->ibdev
.destroy_ah
= nes_destroy_ah
;
3746 nesibdev
->ibdev
.create_qp
= nes_create_qp
;
3747 nesibdev
->ibdev
.modify_qp
= nes_modify_qp
;
3748 nesibdev
->ibdev
.query_qp
= nes_query_qp
;
3749 nesibdev
->ibdev
.destroy_qp
= nes_destroy_qp
;
3750 nesibdev
->ibdev
.create_cq
= nes_create_cq
;
3751 nesibdev
->ibdev
.destroy_cq
= nes_destroy_cq
;
3752 nesibdev
->ibdev
.poll_cq
= nes_poll_cq
;
3753 nesibdev
->ibdev
.get_dma_mr
= nes_get_dma_mr
;
3754 nesibdev
->ibdev
.reg_user_mr
= nes_reg_user_mr
;
3755 nesibdev
->ibdev
.dereg_mr
= nes_dereg_mr
;
3756 nesibdev
->ibdev
.alloc_mw
= nes_alloc_mw
;
3757 nesibdev
->ibdev
.dealloc_mw
= nes_dealloc_mw
;
3759 nesibdev
->ibdev
.alloc_mr
= nes_alloc_mr
;
3760 nesibdev
->ibdev
.map_mr_sg
= nes_map_mr_sg
;
3762 nesibdev
->ibdev
.attach_mcast
= nes_multicast_attach
;
3763 nesibdev
->ibdev
.detach_mcast
= nes_multicast_detach
;
3764 nesibdev
->ibdev
.process_mad
= nes_process_mad
;
3766 nesibdev
->ibdev
.req_notify_cq
= nes_req_notify_cq
;
3767 nesibdev
->ibdev
.post_send
= nes_post_send
;
3768 nesibdev
->ibdev
.post_recv
= nes_post_recv
;
3769 nesibdev
->ibdev
.drain_sq
= nes_drain_sq
;
3770 nesibdev
->ibdev
.drain_rq
= nes_drain_rq
;
3772 nesibdev
->ibdev
.iwcm
= kzalloc(sizeof(*nesibdev
->ibdev
.iwcm
), GFP_KERNEL
);
3773 if (nesibdev
->ibdev
.iwcm
== NULL
) {
3774 ib_dealloc_device(&nesibdev
->ibdev
);
3777 nesibdev
->ibdev
.iwcm
->add_ref
= nes_add_ref
;
3778 nesibdev
->ibdev
.iwcm
->rem_ref
= nes_rem_ref
;
3779 nesibdev
->ibdev
.iwcm
->get_qp
= nes_get_qp
;
3780 nesibdev
->ibdev
.iwcm
->connect
= nes_connect
;
3781 nesibdev
->ibdev
.iwcm
->accept
= nes_accept
;
3782 nesibdev
->ibdev
.iwcm
->reject
= nes_reject
;
3783 nesibdev
->ibdev
.iwcm
->create_listen
= nes_create_listen
;
3784 nesibdev
->ibdev
.iwcm
->destroy_listen
= nes_destroy_listen
;
3785 nesibdev
->ibdev
.get_port_immutable
= nes_port_immutable
;
3786 nesibdev
->ibdev
.get_dev_fw_str
= get_dev_fw_str
;
3787 memcpy(nesibdev
->ibdev
.iwcm
->ifname
, netdev
->name
,
3788 sizeof(nesibdev
->ibdev
.iwcm
->ifname
));
3795 * nes_handle_delayed_event
3797 static void nes_handle_delayed_event(unsigned long data
)
3799 struct nes_vnic
*nesvnic
= (void *) data
;
3801 if (nesvnic
->delayed_event
!= nesvnic
->last_dispatched_event
) {
3802 struct ib_event event
;
3804 event
.device
= &nesvnic
->nesibdev
->ibdev
;
3807 event
.event
= nesvnic
->delayed_event
;
3808 event
.element
.port_num
= nesvnic
->logical_port
+ 1;
3809 ib_dispatch_event(&event
);
3813 nesvnic
->event_timer
.function
= NULL
;
3817 void nes_port_ibevent(struct nes_vnic
*nesvnic
)
3819 struct nes_ib_device
*nesibdev
= nesvnic
->nesibdev
;
3820 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3821 struct ib_event event
;
3822 event
.device
= &nesibdev
->ibdev
;
3823 event
.element
.port_num
= nesvnic
->logical_port
+ 1;
3824 event
.event
= nesdev
->iw_status
? IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
3826 if (!nesvnic
->event_timer
.function
) {
3827 ib_dispatch_event(&event
);
3828 nesvnic
->last_dispatched_event
= event
.event
;
3829 nesvnic
->event_timer
.function
= nes_handle_delayed_event
;
3830 nesvnic
->event_timer
.data
= (unsigned long) nesvnic
;
3831 nesvnic
->event_timer
.expires
= jiffies
+ NES_EVENT_DELAY
;
3832 add_timer(&nesvnic
->event_timer
);
3834 mod_timer(&nesvnic
->event_timer
, jiffies
+ NES_EVENT_DELAY
);
3836 nesvnic
->delayed_event
= event
.event
;
3841 * nes_destroy_ofa_device
3843 void nes_destroy_ofa_device(struct nes_ib_device
*nesibdev
)
3845 if (nesibdev
== NULL
)
3848 nes_unregister_ofa_device(nesibdev
);
3850 kfree(nesibdev
->ibdev
.iwcm
);
3851 ib_dealloc_device(&nesibdev
->ibdev
);
3856 * nes_register_ofa_device
3858 int nes_register_ofa_device(struct nes_ib_device
*nesibdev
)
3860 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
3861 struct nes_device
*nesdev
= nesvnic
->nesdev
;
3862 struct nes_adapter
*nesadapter
= nesdev
->nesadapter
;
3865 ret
= ib_register_device(&nesvnic
->nesibdev
->ibdev
, NULL
);
3870 /* Get the resources allocated to this device */
3871 nesibdev
->max_cq
= (nesadapter
->max_cq
-NES_FIRST_QPN
) / nesadapter
->port_count
;
3872 nesibdev
->max_mr
= nesadapter
->max_mr
/ nesadapter
->port_count
;
3873 nesibdev
->max_qp
= (nesadapter
->max_qp
-NES_FIRST_QPN
) / nesadapter
->port_count
;
3874 nesibdev
->max_pd
= nesadapter
->max_pd
/ nesadapter
->port_count
;
3876 for (i
= 0; i
< ARRAY_SIZE(nes_dev_attributes
); ++i
) {
3877 ret
= device_create_file(&nesibdev
->ibdev
.dev
, nes_dev_attributes
[i
]);
3881 device_remove_file(&nesibdev
->ibdev
.dev
,
3882 nes_dev_attributes
[i
]);
3884 ib_unregister_device(&nesibdev
->ibdev
);
3889 nesvnic
->of_device_registered
= 1;
3896 * nes_unregister_ofa_device
3898 static void nes_unregister_ofa_device(struct nes_ib_device
*nesibdev
)
3900 struct nes_vnic
*nesvnic
= nesibdev
->nesvnic
;
3903 for (i
= 0; i
< ARRAY_SIZE(nes_dev_attributes
); ++i
) {
3904 device_remove_file(&nesibdev
->ibdev
.dev
, nes_dev_attributes
[i
]);
3907 if (nesvnic
->of_device_registered
) {
3908 ib_unregister_device(&nesibdev
->ibdev
);
3911 nesvnic
->of_device_registered
= 0;