1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include "i40iw_osdep.h"
36 #include "i40iw_register.h"
37 #include "i40iw_status.h"
38 #include "i40iw_hmc.h"
41 #include "i40iw_type.h"
44 #include "i40iw_virtchnl.h"
47 * i40iw_insert_wqe_hdr - write wqe header
48 * @wqe: cqp wqe for header
49 * @header: header for the cqp wqe
51 static inline void i40iw_insert_wqe_hdr(u64
*wqe
, u64 header
)
53 wmb(); /* make sure WQE is populated before polarity is set */
54 set_64bit_val(wqe
, 24, header
);
58 * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
59 * @cqp: struct for cqp hw
60 * @val: cqp tail register value
61 * @tail:wqtail register value
62 * @error: cqp processing err
64 static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp
*cqp
,
69 if (cqp
->dev
->is_pf
) {
70 *val
= i40iw_rd32(cqp
->dev
->hw
, I40E_PFPE_CQPTAIL
);
71 *tail
= RS_32(*val
, I40E_PFPE_CQPTAIL_WQTAIL
);
72 *error
= RS_32(*val
, I40E_PFPE_CQPTAIL_CQP_OP_ERR
);
74 *val
= i40iw_rd32(cqp
->dev
->hw
, I40E_VFPE_CQPTAIL1
);
75 *tail
= RS_32(*val
, I40E_VFPE_CQPTAIL_WQTAIL
);
76 *error
= RS_32(*val
, I40E_VFPE_CQPTAIL_CQP_OP_ERR
);
81 * i40iw_cqp_poll_registers - poll cqp registers
82 * @cqp: struct for cqp hw
83 * @tail:wqtail register value
84 * @count: how many times to try for completion
86 static enum i40iw_status_code
i40iw_cqp_poll_registers(
87 struct i40iw_sc_cqp
*cqp
,
92 u32 newtail
, error
, val
;
96 i40iw_get_cqp_reg_info(cqp
, &val
, &newtail
, &error
);
98 error
= (cqp
->dev
->is_pf
) ?
99 i40iw_rd32(cqp
->dev
->hw
, I40E_PFPE_CQPERRCODES
) :
100 i40iw_rd32(cqp
->dev
->hw
, I40E_VFPE_CQPERRCODES1
);
101 return I40IW_ERR_CQP_COMPL_ERROR
;
103 if (newtail
!= tail
) {
105 I40IW_RING_MOVE_TAIL(cqp
->sq_ring
);
106 cqp
->dev
->cqp_cmd_stats
[OP_COMPLETED_COMMANDS
]++;
109 udelay(I40IW_SLEEP_COUNT
);
111 return I40IW_ERR_TIMEOUT
;
115 * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer
116 * @buf: ptr to fpm commit buffer
117 * @info: ptr to i40iw_hmc_obj_info struct
118 * @sd: number of SDs for HMC objects
120 * parses fpm commit info and copy base value
121 * of hmc objects in hmc_info
123 static enum i40iw_status_code
i40iw_sc_parse_fpm_commit_buf(
125 struct i40iw_hmc_obj_info
*info
,
134 /* copy base values in obj_info */
135 for (i
= I40IW_HMC_IW_QP
, j
= 0; i
<= I40IW_HMC_IW_PBLE
; i
++, j
+= 8) {
136 if ((i
== I40IW_HMC_IW_SRQ
) ||
137 (i
== I40IW_HMC_IW_FSIMC
) ||
138 (i
== I40IW_HMC_IW_FSIAV
)) {
143 get_64bit_val(buf
, j
, &temp
);
144 info
[i
].base
= RS_64_1(temp
, 32) * 512;
145 if (info
[i
].base
> base
) {
149 if (i
== I40IW_HMC_IW_APBVT_ENTRY
) {
153 if (i
== I40IW_HMC_IW_QP
)
154 info
[i
].cnt
= (u32
)RS_64(temp
, I40IW_QUERY_FPM_MAX_QPS
);
155 else if (i
== I40IW_HMC_IW_CQ
)
156 info
[i
].cnt
= (u32
)RS_64(temp
, I40IW_QUERY_FPM_MAX_CQS
);
158 info
[i
].cnt
= (u32
)(temp
);
160 size
= info
[k
].cnt
* info
[k
].size
+ info
[k
].base
;
162 *sd
= (u32
)((size
>> 21) + 1); /* add 1 for remainder */
164 *sd
= (u32
)(size
>> 21);
170 * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
171 * @buf: ptr to fpm query buffer
172 * @buf_idx: index into buf
173 * @info: ptr to i40iw_hmc_obj_info struct
174 * @rsrc_idx: resource index into info
176 * Decode a 64 bit value from fpm query buffer into max count and size
178 static u64
i40iw_sc_decode_fpm_query(u64
*buf
,
180 struct i40iw_hmc_obj_info
*obj_info
,
186 get_64bit_val(buf
, buf_idx
, &temp
);
187 obj_info
[rsrc_idx
].max_cnt
= (u32
)temp
;
188 size
= (u32
)RS_64_1(temp
, 32);
189 obj_info
[rsrc_idx
].size
= LS_64_1(1, size
);
195 * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
196 * @buf: ptr to fpm query buffer
197 * @info: ptr to i40iw_hmc_obj_info struct
198 * @hmc_fpm_misc: ptr to fpm data
200 * parses fpm query buffer and copy max_cnt and
201 * size value of hmc objects in hmc_info
203 static enum i40iw_status_code
i40iw_sc_parse_fpm_query_buf(
205 struct i40iw_hmc_info
*hmc_info
,
206 struct i40iw_hmc_fpm_misc
*hmc_fpm_misc
)
208 struct i40iw_hmc_obj_info
*obj_info
;
213 obj_info
= hmc_info
->hmc_obj
;
215 get_64bit_val(buf
, 0, &temp
);
216 hmc_info
->first_sd_index
= (u16
)RS_64(temp
, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX
);
217 max_pe_sds
= (u16
)RS_64(temp
, I40IW_QUERY_FPM_MAX_PE_SDS
);
219 /* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */
220 if (hmc_info
->hmc_fn_id
>= I40IW_FIRST_VF_FPM_ID
)
222 hmc_fpm_misc
->max_sds
= max_pe_sds
;
223 hmc_info
->sd_table
.sd_cnt
= max_pe_sds
+ hmc_info
->first_sd_index
;
225 get_64bit_val(buf
, 8, &temp
);
226 obj_info
[I40IW_HMC_IW_QP
].max_cnt
= (u32
)RS_64(temp
, I40IW_QUERY_FPM_MAX_QPS
);
227 size
= (u32
)RS_64_1(temp
, 32);
228 obj_info
[I40IW_HMC_IW_QP
].size
= LS_64_1(1, size
);
230 get_64bit_val(buf
, 16, &temp
);
231 obj_info
[I40IW_HMC_IW_CQ
].max_cnt
= (u32
)RS_64(temp
, I40IW_QUERY_FPM_MAX_CQS
);
232 size
= (u32
)RS_64_1(temp
, 32);
233 obj_info
[I40IW_HMC_IW_CQ
].size
= LS_64_1(1, size
);
235 i40iw_sc_decode_fpm_query(buf
, 32, obj_info
, I40IW_HMC_IW_HTE
);
236 i40iw_sc_decode_fpm_query(buf
, 40, obj_info
, I40IW_HMC_IW_ARP
);
238 obj_info
[I40IW_HMC_IW_APBVT_ENTRY
].size
= 8192;
239 obj_info
[I40IW_HMC_IW_APBVT_ENTRY
].max_cnt
= 1;
241 i40iw_sc_decode_fpm_query(buf
, 48, obj_info
, I40IW_HMC_IW_MR
);
242 i40iw_sc_decode_fpm_query(buf
, 56, obj_info
, I40IW_HMC_IW_XF
);
244 get_64bit_val(buf
, 64, &temp
);
245 obj_info
[I40IW_HMC_IW_XFFL
].max_cnt
= (u32
)temp
;
246 obj_info
[I40IW_HMC_IW_XFFL
].size
= 4;
247 hmc_fpm_misc
->xf_block_size
= RS_64(temp
, I40IW_QUERY_FPM_XFBLOCKSIZE
);
248 if (!hmc_fpm_misc
->xf_block_size
)
249 return I40IW_ERR_INVALID_SIZE
;
251 i40iw_sc_decode_fpm_query(buf
, 72, obj_info
, I40IW_HMC_IW_Q1
);
253 get_64bit_val(buf
, 80, &temp
);
254 obj_info
[I40IW_HMC_IW_Q1FL
].max_cnt
= (u32
)temp
;
255 obj_info
[I40IW_HMC_IW_Q1FL
].size
= 4;
256 hmc_fpm_misc
->q1_block_size
= RS_64(temp
, I40IW_QUERY_FPM_Q1BLOCKSIZE
);
257 if (!hmc_fpm_misc
->q1_block_size
)
258 return I40IW_ERR_INVALID_SIZE
;
260 i40iw_sc_decode_fpm_query(buf
, 88, obj_info
, I40IW_HMC_IW_TIMER
);
262 get_64bit_val(buf
, 112, &temp
);
263 obj_info
[I40IW_HMC_IW_PBLE
].max_cnt
= (u32
)temp
;
264 obj_info
[I40IW_HMC_IW_PBLE
].size
= 8;
266 get_64bit_val(buf
, 120, &temp
);
267 hmc_fpm_misc
->max_ceqs
= (u8
)RS_64(temp
, I40IW_QUERY_FPM_MAX_CEQS
);
268 hmc_fpm_misc
->ht_multiplier
= RS_64(temp
, I40IW_QUERY_FPM_HTMULTIPLIER
);
269 hmc_fpm_misc
->timer_bucket
= RS_64(temp
, I40IW_QUERY_FPM_TIMERBUCKET
);
275 * i40iw_fill_qos_list - Change all unknown qs handles to available ones
276 * @qs_list: list of qs_handles to be fixed with valid qs_handles
278 static void i40iw_fill_qos_list(u16
*qs_list
)
280 u16 qshandle
= qs_list
[0];
283 for (i
= 0; i
< I40IW_MAX_USER_PRIORITY
; i
++) {
284 if (qs_list
[i
] == QS_HANDLE_UNKNOWN
)
285 qs_list
[i
] = qshandle
;
287 qshandle
= qs_list
[i
];
292 * i40iw_qp_from_entry - Given entry, get to the qp structure
293 * @entry: Points to list of qp structure
295 static struct i40iw_sc_qp
*i40iw_qp_from_entry(struct list_head
*entry
)
300 return (struct i40iw_sc_qp
*)((char *)entry
- offsetof(struct i40iw_sc_qp
, list
));
304 * i40iw_get_qp - get the next qp from the list given current qp
305 * @head: Listhead of qp's
308 static struct i40iw_sc_qp
*i40iw_get_qp(struct list_head
*head
, struct i40iw_sc_qp
*qp
)
310 struct list_head
*entry
= NULL
;
311 struct list_head
*lastentry
;
313 if (list_empty(head
))
319 lastentry
= &qp
->list
;
320 entry
= (lastentry
!= head
) ? lastentry
->next
: NULL
;
323 return i40iw_qp_from_entry(entry
);
327 * i40iw_change_l2params - given the new l2 parameters, change all qp
328 * @vsi: pointer to the vsi structure
329 * @l2params: New paramaters from l2
331 void i40iw_change_l2params(struct i40iw_sc_vsi
*vsi
, struct i40iw_l2params
*l2params
)
333 struct i40iw_sc_dev
*dev
= vsi
->dev
;
334 struct i40iw_sc_qp
*qp
= NULL
;
335 bool qs_handle_change
= false;
340 vsi
->mss
= l2params
->mss
;
342 i40iw_fill_qos_list(l2params
->qs_handle_list
);
343 for (i
= 0; i
< I40IW_MAX_USER_PRIORITY
; i
++) {
344 qs_handle
= l2params
->qs_handle_list
[i
];
345 if (vsi
->qos
[i
].qs_handle
!= qs_handle
)
346 qs_handle_change
= true;
347 spin_lock_irqsave(&vsi
->qos
[i
].lock
, flags
);
348 qp
= i40iw_get_qp(&vsi
->qos
[i
].qplist
, qp
);
350 if (qs_handle_change
) {
351 qp
->qs_handle
= qs_handle
;
352 /* issue cqp suspend command */
353 i40iw_qp_suspend_resume(dev
, qp
, true);
355 qp
= i40iw_get_qp(&vsi
->qos
[i
].qplist
, qp
);
357 spin_unlock_irqrestore(&vsi
->qos
[i
].lock
, flags
);
358 vsi
->qos
[i
].qs_handle
= qs_handle
;
363 * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp
364 * @qp: qp to be removed from qos
366 static void i40iw_qp_rem_qos(struct i40iw_sc_qp
*qp
)
368 struct i40iw_sc_vsi
*vsi
= qp
->vsi
;
373 spin_lock_irqsave(&vsi
->qos
[qp
->user_pri
].lock
, flags
);
375 spin_unlock_irqrestore(&vsi
->qos
[qp
->user_pri
].lock
, flags
);
379 * i40iw_qp_add_qos - called during setctx fot qp to be added to qos
380 * @qp: qp to be added to qos
382 void i40iw_qp_add_qos(struct i40iw_sc_qp
*qp
)
384 struct i40iw_sc_vsi
*vsi
= qp
->vsi
;
389 spin_lock_irqsave(&vsi
->qos
[qp
->user_pri
].lock
, flags
);
390 qp
->qs_handle
= vsi
->qos
[qp
->user_pri
].qs_handle
;
391 list_add(&qp
->list
, &vsi
->qos
[qp
->user_pri
].qplist
);
392 qp
->on_qoslist
= true;
393 spin_unlock_irqrestore(&vsi
->qos
[qp
->user_pri
].lock
, flags
);
397 * i40iw_sc_pd_init - initialize sc pd struct
398 * @dev: sc device struct
400 * @pd_id: pd_id for allocated pd
401 * @abi_ver: ABI version from user context, -1 if not valid
403 static void i40iw_sc_pd_init(struct i40iw_sc_dev
*dev
,
404 struct i40iw_sc_pd
*pd
,
408 pd
->size
= sizeof(*pd
);
410 pd
->abi_ver
= abi_ver
;
415 * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size
416 * @wqsize: size of the wq (sq, rq, srq) to encoded_size
417 * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
419 u8
i40iw_get_encoded_wqe_size(u32 wqsize
, bool cqpsq
)
423 /* cqp sq's hw coded value starts from 1 for size of 4
424 * while it starts from 0 for qp' wq's.
435 * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair
436 * @cqp: IWARP control queue pair pointer
437 * @info: IWARP control queue pair init info pointer
439 * Initializes the object and context buffers for a control Queue Pair.
441 static enum i40iw_status_code
i40iw_sc_cqp_init(struct i40iw_sc_cqp
*cqp
,
442 struct i40iw_cqp_init_info
*info
)
446 if ((info
->sq_size
> I40IW_CQP_SW_SQSIZE_2048
) ||
447 (info
->sq_size
< I40IW_CQP_SW_SQSIZE_4
) ||
448 ((info
->sq_size
& (info
->sq_size
- 1))))
449 return I40IW_ERR_INVALID_SIZE
;
451 hw_sq_size
= i40iw_get_encoded_wqe_size(info
->sq_size
, true);
452 cqp
->size
= sizeof(*cqp
);
453 cqp
->sq_size
= info
->sq_size
;
454 cqp
->hw_sq_size
= hw_sq_size
;
455 cqp
->sq_base
= info
->sq
;
456 cqp
->host_ctx
= info
->host_ctx
;
457 cqp
->sq_pa
= info
->sq_pa
;
458 cqp
->host_ctx_pa
= info
->host_ctx_pa
;
459 cqp
->dev
= info
->dev
;
460 cqp
->struct_ver
= info
->struct_ver
;
461 cqp
->scratch_array
= info
->scratch_array
;
463 cqp
->en_datacenter_tcp
= info
->en_datacenter_tcp
;
464 cqp
->enabled_vf_count
= info
->enabled_vf_count
;
465 cqp
->hmc_profile
= info
->hmc_profile
;
466 info
->dev
->cqp
= cqp
;
468 I40IW_RING_INIT(cqp
->sq_ring
, cqp
->sq_size
);
469 cqp
->dev
->cqp_cmd_stats
[OP_REQUESTED_COMMANDS
] = 0;
470 cqp
->dev
->cqp_cmd_stats
[OP_COMPLETED_COMMANDS
] = 0;
472 i40iw_debug(cqp
->dev
, I40IW_DEBUG_WQE
,
473 "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",
474 __func__
, cqp
->sq_size
, cqp
->hw_sq_size
,
475 cqp
->sq_base
, cqp
->sq_pa
, cqp
, cqp
->polarity
);
480 * i40iw_sc_cqp_create - create cqp during bringup
481 * @cqp: struct for cqp hw
482 * @maj_err: If error, major err number
483 * @min_err: If error, minor err number
485 static enum i40iw_status_code
i40iw_sc_cqp_create(struct i40iw_sc_cqp
*cqp
,
490 u32 cnt
= 0, p1
, p2
, val
= 0, err_code
;
491 enum i40iw_status_code ret_code
;
496 ret_code
= i40iw_allocate_dma_mem(cqp
->dev
->hw
,
499 I40IW_SD_BUF_ALIGNMENT
);
504 temp
= LS_64(cqp
->hw_sq_size
, I40IW_CQPHC_SQSIZE
) |
505 LS_64(cqp
->struct_ver
, I40IW_CQPHC_SVER
);
507 set_64bit_val(cqp
->host_ctx
, 0, temp
);
508 set_64bit_val(cqp
->host_ctx
, 8, cqp
->sq_pa
);
509 temp
= LS_64(cqp
->enabled_vf_count
, I40IW_CQPHC_ENABLED_VFS
) |
510 LS_64(cqp
->hmc_profile
, I40IW_CQPHC_HMC_PROFILE
);
511 set_64bit_val(cqp
->host_ctx
, 16, temp
);
512 set_64bit_val(cqp
->host_ctx
, 24, (uintptr_t)cqp
);
513 set_64bit_val(cqp
->host_ctx
, 32, 0);
514 set_64bit_val(cqp
->host_ctx
, 40, 0);
515 set_64bit_val(cqp
->host_ctx
, 48, 0);
516 set_64bit_val(cqp
->host_ctx
, 56, 0);
518 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CQP_HOST_CTX",
519 cqp
->host_ctx
, I40IW_CQP_CTX_SIZE
* 8);
521 p1
= RS_32_1(cqp
->host_ctx_pa
, 32);
522 p2
= (u32
)cqp
->host_ctx_pa
;
524 if (cqp
->dev
->is_pf
) {
525 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CCQPHIGH
, p1
);
526 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CCQPLOW
, p2
);
528 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CCQPHIGH1
, p1
);
529 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CCQPLOW1
, p2
);
532 if (cnt
++ > I40IW_DONE_COUNT
) {
533 i40iw_free_dma_mem(cqp
->dev
->hw
, &cqp
->sdbuf
);
534 ret_code
= I40IW_ERR_TIMEOUT
;
536 * read PFPE_CQPERRORCODES register to get the minor
537 * and major error code
540 err_code
= i40iw_rd32(cqp
->dev
->hw
, I40E_PFPE_CQPERRCODES
);
542 err_code
= i40iw_rd32(cqp
->dev
->hw
, I40E_VFPE_CQPERRCODES1
);
543 *min_err
= RS_32(err_code
, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE
);
544 *maj_err
= RS_32(err_code
, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE
);
547 udelay(I40IW_SLEEP_COUNT
);
549 val
= i40iw_rd32(cqp
->dev
->hw
, I40E_PFPE_CCQPSTATUS
);
551 val
= i40iw_rd32(cqp
->dev
->hw
, I40E_VFPE_CCQPSTATUS1
);
556 cqp
->process_cqp_sds
= i40iw_update_sds_noccq
;
561 * i40iw_sc_cqp_post_sq - post of cqp's sq
562 * @cqp: struct for cqp hw
564 void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp
*cqp
)
567 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CQPDB
, I40IW_RING_GETCURRENT_HEAD(cqp
->sq_ring
));
569 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CQPDB1
, I40IW_RING_GETCURRENT_HEAD(cqp
->sq_ring
));
571 i40iw_debug(cqp
->dev
,
573 "%s: HEAD_TAIL[%04d,%04d,%04d]\n",
581 * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
582 * @cqp: struct for cqp hw
583 * @wqe_idx: we index of cqp ring
585 u64
*i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp
*cqp
, u64 scratch
)
589 enum i40iw_status_code ret_code
;
591 if (I40IW_RING_FULL_ERR(cqp
->sq_ring
)) {
592 i40iw_debug(cqp
->dev
,
594 "%s: ring is full head %x tail %x size %x\n",
601 I40IW_ATOMIC_RING_MOVE_HEAD(cqp
->sq_ring
, wqe_idx
, ret_code
);
602 cqp
->dev
->cqp_cmd_stats
[OP_REQUESTED_COMMANDS
]++;
606 cqp
->polarity
= !cqp
->polarity
;
608 wqe
= cqp
->sq_base
[wqe_idx
].elem
;
609 cqp
->scratch_array
[wqe_idx
] = scratch
;
610 I40IW_CQP_INIT_WQE(wqe
);
616 * i40iw_sc_cqp_destroy - destroy cqp during close
617 * @cqp: struct for cqp hw
619 static enum i40iw_status_code
i40iw_sc_cqp_destroy(struct i40iw_sc_cqp
*cqp
)
621 u32 cnt
= 0, val
= 1;
622 enum i40iw_status_code ret_code
= 0;
625 if (cqp
->dev
->is_pf
) {
626 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CCQPHIGH
, 0);
627 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CCQPLOW
, 0);
628 cqpstat_addr
= I40E_PFPE_CCQPSTATUS
;
630 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CCQPHIGH1
, 0);
631 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CCQPLOW1
, 0);
632 cqpstat_addr
= I40E_VFPE_CCQPSTATUS1
;
635 if (cnt
++ > I40IW_DONE_COUNT
) {
636 ret_code
= I40IW_ERR_TIMEOUT
;
639 udelay(I40IW_SLEEP_COUNT
);
640 val
= i40iw_rd32(cqp
->dev
->hw
, cqpstat_addr
);
643 i40iw_free_dma_mem(cqp
->dev
->hw
, &cqp
->sdbuf
);
648 * i40iw_sc_ccq_arm - enable intr for control cq
649 * @ccq: ccq sc struct
651 static void i40iw_sc_ccq_arm(struct i40iw_sc_cq
*ccq
)
658 /* write to cq doorbell shadow area */
659 /* arm next se should always be zero */
660 get_64bit_val(ccq
->cq_uk
.shadow_area
, 32, &temp_val
);
662 sw_cq_sel
= (u16
)RS_64(temp_val
, I40IW_CQ_DBSA_SW_CQ_SELECT
);
663 arm_next_se
= (u8
)RS_64(temp_val
, I40IW_CQ_DBSA_ARM_NEXT_SE
);
665 arm_seq_num
= (u8
)RS_64(temp_val
, I40IW_CQ_DBSA_ARM_SEQ_NUM
);
668 temp_val
= LS_64(arm_seq_num
, I40IW_CQ_DBSA_ARM_SEQ_NUM
) |
669 LS_64(sw_cq_sel
, I40IW_CQ_DBSA_SW_CQ_SELECT
) |
670 LS_64(arm_next_se
, I40IW_CQ_DBSA_ARM_NEXT_SE
) |
671 LS_64(1, I40IW_CQ_DBSA_ARM_NEXT
);
673 set_64bit_val(ccq
->cq_uk
.shadow_area
, 32, temp_val
);
675 wmb(); /* make sure shadow area is updated before arming */
678 i40iw_wr32(ccq
->dev
->hw
, I40E_PFPE_CQARM
, ccq
->cq_uk
.cq_id
);
680 i40iw_wr32(ccq
->dev
->hw
, I40E_VFPE_CQARM1
, ccq
->cq_uk
.cq_id
);
684 * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry
685 * @ccq: ccq sc struct
686 * @info: completion q entry to return
688 static enum i40iw_status_code
i40iw_sc_ccq_get_cqe_info(
689 struct i40iw_sc_cq
*ccq
,
690 struct i40iw_ccq_cqe_info
*info
)
692 u64 qp_ctx
, temp
, temp1
;
694 struct i40iw_sc_cqp
*cqp
;
697 enum i40iw_status_code ret_code
= 0;
699 if (ccq
->cq_uk
.avoid_mem_cflct
)
700 cqe
= (u64
*)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq
->cq_uk
);
702 cqe
= (u64
*)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq
->cq_uk
);
704 get_64bit_val(cqe
, 24, &temp
);
705 polarity
= (u8
)RS_64(temp
, I40IW_CQ_VALID
);
706 if (polarity
!= ccq
->cq_uk
.polarity
)
707 return I40IW_ERR_QUEUE_EMPTY
;
709 get_64bit_val(cqe
, 8, &qp_ctx
);
710 cqp
= (struct i40iw_sc_cqp
*)(unsigned long)qp_ctx
;
711 info
->error
= (bool)RS_64(temp
, I40IW_CQ_ERROR
);
712 info
->min_err_code
= (u16
)RS_64(temp
, I40IW_CQ_MINERR
);
714 info
->maj_err_code
= (u16
)RS_64(temp
, I40IW_CQ_MAJERR
);
715 info
->min_err_code
= (u16
)RS_64(temp
, I40IW_CQ_MINERR
);
717 wqe_idx
= (u32
)RS_64(temp
, I40IW_CQ_WQEIDX
);
718 info
->scratch
= cqp
->scratch_array
[wqe_idx
];
720 get_64bit_val(cqe
, 16, &temp1
);
721 info
->op_ret_val
= (u32
)RS_64(temp1
, I40IW_CCQ_OPRETVAL
);
722 get_64bit_val(cqp
->sq_base
[wqe_idx
].elem
, 24, &temp1
);
723 info
->op_code
= (u8
)RS_64(temp1
, I40IW_CQPSQ_OPCODE
);
726 /* move the head for cq */
727 I40IW_RING_MOVE_HEAD(ccq
->cq_uk
.cq_ring
, ret_code
);
728 if (I40IW_RING_GETCURRENT_HEAD(ccq
->cq_uk
.cq_ring
) == 0)
729 ccq
->cq_uk
.polarity
^= 1;
731 /* update cq tail in cq shadow memory also */
732 I40IW_RING_MOVE_TAIL(ccq
->cq_uk
.cq_ring
);
733 set_64bit_val(ccq
->cq_uk
.shadow_area
,
735 I40IW_RING_GETCURRENT_HEAD(ccq
->cq_uk
.cq_ring
));
736 wmb(); /* write shadow area before tail */
737 I40IW_RING_MOVE_TAIL(cqp
->sq_ring
);
738 ccq
->dev
->cqp_cmd_stats
[OP_COMPLETED_COMMANDS
]++;
744 * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
745 * @cqp: struct for cqp hw
746 * @op_code: cqp opcode for completion
747 * @info: completion q entry to return
749 static enum i40iw_status_code
i40iw_sc_poll_for_cqp_op_done(
750 struct i40iw_sc_cqp
*cqp
,
752 struct i40iw_ccq_cqe_info
*compl_info
)
754 struct i40iw_ccq_cqe_info info
;
755 struct i40iw_sc_cq
*ccq
;
756 enum i40iw_status_code ret_code
= 0;
759 memset(&info
, 0, sizeof(info
));
762 if (cnt
++ > I40IW_DONE_COUNT
)
763 return I40IW_ERR_TIMEOUT
;
765 if (i40iw_sc_ccq_get_cqe_info(ccq
, &info
)) {
766 udelay(I40IW_SLEEP_COUNT
);
771 ret_code
= I40IW_ERR_CQP_COMPL_ERROR
;
774 /* check if opcode is cq create */
775 if (op_code
!= info
.op_code
) {
776 i40iw_debug(cqp
->dev
, I40IW_DEBUG_WQE
,
777 "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",
778 __func__
, op_code
, info
.op_code
);
780 /* success, exit out of the loop */
781 if (op_code
== info
.op_code
)
786 memcpy(compl_info
, &info
, sizeof(*compl_info
));
792 * i40iw_sc_manage_push_page - Handle push page
793 * @cqp: struct for cqp hw
794 * @info: push page info
795 * @scratch: u64 saved to be used during cqp completion
796 * @post_sq: flag for cqp db to ring
798 static enum i40iw_status_code
i40iw_sc_manage_push_page(
799 struct i40iw_sc_cqp
*cqp
,
800 struct i40iw_cqp_manage_push_page_info
*info
,
807 if (info
->push_idx
>= I40IW_MAX_PUSH_PAGE_COUNT
)
808 return I40IW_ERR_INVALID_PUSH_PAGE_INDEX
;
810 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
812 return I40IW_ERR_RING_FULL
;
814 set_64bit_val(wqe
, 16, info
->qs_handle
);
816 header
= LS_64(info
->push_idx
, I40IW_CQPSQ_MPP_PPIDX
) |
817 LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES
, I40IW_CQPSQ_OPCODE
) |
818 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
) |
819 LS_64(info
->free_page
, I40IW_CQPSQ_MPP_FREE_PAGE
);
821 i40iw_insert_wqe_hdr(wqe
, header
);
823 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_PUSH_PAGES WQE",
824 wqe
, I40IW_CQP_WQE_SIZE
* 8);
827 i40iw_sc_cqp_post_sq(cqp
);
832 * i40iw_sc_manage_hmc_pm_func_table - manage of function table
833 * @cqp: struct for cqp hw
834 * @scratch: u64 saved to be used during cqp completion
835 * @vf_index: vf index for cqp
836 * @free_pm_fcn: function number
837 * @post_sq: flag for cqp db to ring
839 static enum i40iw_status_code
i40iw_sc_manage_hmc_pm_func_table(
840 struct i40iw_sc_cqp
*cqp
,
849 if (vf_index
>= I40IW_MAX_VF_PER_PF
)
850 return I40IW_ERR_INVALID_VF_ID
;
851 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
853 return I40IW_ERR_RING_FULL
;
855 header
= LS_64(vf_index
, I40IW_CQPSQ_MHMC_VFIDX
) |
856 LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE
, I40IW_CQPSQ_OPCODE
) |
857 LS_64(free_pm_fcn
, I40IW_CQPSQ_MHMC_FREEPMFN
) |
858 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
860 i40iw_insert_wqe_hdr(wqe
, header
);
861 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_HMC_PM_FUNC_TABLE WQE",
862 wqe
, I40IW_CQP_WQE_SIZE
* 8);
864 i40iw_sc_cqp_post_sq(cqp
);
869 * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile
870 * @cqp: struct for cqp hw
871 * @scratch: u64 saved to be used during cqp completion
872 * @hmc_profile_type: type of profile to set
873 * @vf_num: vf number for profile
874 * @post_sq: flag for cqp db to ring
875 * @poll_registers: flag to poll register for cqp completion
877 static enum i40iw_status_code
i40iw_sc_set_hmc_resource_profile(
878 struct i40iw_sc_cqp
*cqp
,
881 u8 vf_num
, bool post_sq
,
886 u32 val
, tail
, error
;
887 enum i40iw_status_code ret_code
= 0;
889 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
891 return I40IW_ERR_RING_FULL
;
893 set_64bit_val(wqe
, 16,
894 (LS_64(hmc_profile_type
, I40IW_CQPSQ_SHMCRP_HMC_PROFILE
) |
895 LS_64(vf_num
, I40IW_CQPSQ_SHMCRP_VFNUM
)));
897 header
= LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE
, I40IW_CQPSQ_OPCODE
) |
898 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
900 i40iw_insert_wqe_hdr(wqe
, header
);
902 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_HMC_PM_FUNC_TABLE WQE",
903 wqe
, I40IW_CQP_WQE_SIZE
* 8);
905 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
907 return I40IW_ERR_CQP_COMPL_ERROR
;
910 i40iw_sc_cqp_post_sq(cqp
);
912 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, 1000000);
914 ret_code
= i40iw_sc_poll_for_cqp_op_done(cqp
,
915 I40IW_CQP_OP_SHMC_PAGES_ALLOCATED
,
923 * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
924 * @cqp: struct for cqp hw
926 static enum i40iw_status_code
i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp
*cqp
)
928 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE
, NULL
);
932 * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
933 * @cqp: struct for cqp hw
935 static enum i40iw_status_code
i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp
*cqp
)
937 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_COMMIT_FPM_VALUES
, NULL
);
941 * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values
942 * @cqp: struct for cqp hw
943 * @scratch: u64 saved to be used during cqp completion
944 * @hmc_fn_id: hmc function id
945 * @commit_fpm_mem; Memory for fpm values
946 * @post_sq: flag for cqp db to ring
947 * @wait_type: poll ccq or cqp registers for cqp completion
949 static enum i40iw_status_code
i40iw_sc_commit_fpm_values(
950 struct i40iw_sc_cqp
*cqp
,
953 struct i40iw_dma_mem
*commit_fpm_mem
,
959 u32 tail
, val
, error
;
960 enum i40iw_status_code ret_code
= 0;
962 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
964 return I40IW_ERR_RING_FULL
;
966 set_64bit_val(wqe
, 16, hmc_fn_id
);
967 set_64bit_val(wqe
, 32, commit_fpm_mem
->pa
);
969 header
= LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES
, I40IW_CQPSQ_OPCODE
) |
970 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
972 i40iw_insert_wqe_hdr(wqe
, header
);
974 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "COMMIT_FPM_VALUES WQE",
975 wqe
, I40IW_CQP_WQE_SIZE
* 8);
977 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
979 return I40IW_ERR_CQP_COMPL_ERROR
;
982 i40iw_sc_cqp_post_sq(cqp
);
984 if (wait_type
== I40IW_CQP_WAIT_POLL_REGS
)
985 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, I40IW_DONE_COUNT
);
986 else if (wait_type
== I40IW_CQP_WAIT_POLL_CQ
)
987 ret_code
= i40iw_sc_commit_fpm_values_done(cqp
);
994 * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
995 * @cqp: struct for cqp hw
997 static enum i40iw_status_code
i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp
*cqp
)
999 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_QUERY_FPM_VALUES
, NULL
);
1003 * i40iw_sc_query_fpm_values - cqp wqe query fpm values
1004 * @cqp: struct for cqp hw
1005 * @scratch: u64 saved to be used during cqp completion
1006 * @hmc_fn_id: hmc function id
1007 * @query_fpm_mem: memory for return fpm values
1008 * @post_sq: flag for cqp db to ring
1009 * @wait_type: poll ccq or cqp registers for cqp completion
1011 static enum i40iw_status_code
i40iw_sc_query_fpm_values(
1012 struct i40iw_sc_cqp
*cqp
,
1015 struct i40iw_dma_mem
*query_fpm_mem
,
1021 u32 tail
, val
, error
;
1022 enum i40iw_status_code ret_code
= 0;
1024 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1026 return I40IW_ERR_RING_FULL
;
1028 set_64bit_val(wqe
, 16, hmc_fn_id
);
1029 set_64bit_val(wqe
, 32, query_fpm_mem
->pa
);
1031 header
= LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES
, I40IW_CQPSQ_OPCODE
) |
1032 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1034 i40iw_insert_wqe_hdr(wqe
, header
);
1036 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QUERY_FPM WQE",
1037 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1039 /* read the tail from CQP_TAIL register */
1040 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
1043 return I40IW_ERR_CQP_COMPL_ERROR
;
1046 i40iw_sc_cqp_post_sq(cqp
);
1047 if (wait_type
== I40IW_CQP_WAIT_POLL_REGS
)
1048 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, I40IW_DONE_COUNT
);
1049 else if (wait_type
== I40IW_CQP_WAIT_POLL_CQ
)
1050 ret_code
= i40iw_sc_query_fpm_values_done(cqp
);
1057 * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry
1058 * @cqp: struct for cqp hw
1059 * @info: arp entry information
1060 * @scratch: u64 saved to be used during cqp completion
1061 * @post_sq: flag for cqp db to ring
1063 static enum i40iw_status_code
i40iw_sc_add_arp_cache_entry(
1064 struct i40iw_sc_cqp
*cqp
,
1065 struct i40iw_add_arp_cache_entry_info
*info
,
1072 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1074 return I40IW_ERR_RING_FULL
;
1075 set_64bit_val(wqe
, 8, info
->reach_max
);
1077 temp
= info
->mac_addr
[5] |
1078 LS_64_1(info
->mac_addr
[4], 8) |
1079 LS_64_1(info
->mac_addr
[3], 16) |
1080 LS_64_1(info
->mac_addr
[2], 24) |
1081 LS_64_1(info
->mac_addr
[1], 32) |
1082 LS_64_1(info
->mac_addr
[0], 40);
1084 set_64bit_val(wqe
, 16, temp
);
1086 header
= info
->arp_index
|
1087 LS_64(I40IW_CQP_OP_MANAGE_ARP
, I40IW_CQPSQ_OPCODE
) |
1088 LS_64((info
->permanent
? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT
) |
1089 LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID
) |
1090 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1092 i40iw_insert_wqe_hdr(wqe
, header
);
1094 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "ARP_CACHE_ENTRY WQE",
1095 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1098 i40iw_sc_cqp_post_sq(cqp
);
1103 * i40iw_sc_del_arp_cache_entry - dele arp cache entry
1104 * @cqp: struct for cqp hw
1105 * @scratch: u64 saved to be used during cqp completion
1106 * @arp_index: arp index to delete arp entry
1107 * @post_sq: flag for cqp db to ring
1109 static enum i40iw_status_code
i40iw_sc_del_arp_cache_entry(
1110 struct i40iw_sc_cqp
*cqp
,
1118 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1120 return I40IW_ERR_RING_FULL
;
1122 header
= arp_index
|
1123 LS_64(I40IW_CQP_OP_MANAGE_ARP
, I40IW_CQPSQ_OPCODE
) |
1124 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1125 i40iw_insert_wqe_hdr(wqe
, header
);
1127 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "ARP_CACHE_DEL_ENTRY WQE",
1128 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1131 i40iw_sc_cqp_post_sq(cqp
);
1136 * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
1137 * @cqp: struct for cqp hw
1138 * @scratch: u64 saved to be used during cqp completion
1139 * @arp_index: arp index to delete arp entry
1140 * @post_sq: flag for cqp db to ring
1142 static enum i40iw_status_code
i40iw_sc_query_arp_cache_entry(
1143 struct i40iw_sc_cqp
*cqp
,
1151 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1153 return I40IW_ERR_RING_FULL
;
1155 header
= arp_index
|
1156 LS_64(I40IW_CQP_OP_MANAGE_ARP
, I40IW_CQPSQ_OPCODE
) |
1157 LS_64(1, I40IW_CQPSQ_MAT_QUERY
) |
1158 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1160 i40iw_insert_wqe_hdr(wqe
, header
);
1162 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QUERY_ARP_CACHE_ENTRY WQE",
1163 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1166 i40iw_sc_cqp_post_sq(cqp
);
1171 * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries
1172 * @cqp: struct for cqp hw
1173 * @info: info for apbvt entry to add or delete
1174 * @scratch: u64 saved to be used during cqp completion
1175 * @post_sq: flag for cqp db to ring
1177 static enum i40iw_status_code
i40iw_sc_manage_apbvt_entry(
1178 struct i40iw_sc_cqp
*cqp
,
1179 struct i40iw_apbvt_info
*info
,
1186 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1188 return I40IW_ERR_RING_FULL
;
1190 set_64bit_val(wqe
, 16, info
->port
);
1192 header
= LS_64(I40IW_CQP_OP_MANAGE_APBVT
, I40IW_CQPSQ_OPCODE
) |
1193 LS_64(info
->add
, I40IW_CQPSQ_MAPT_ADDPORT
) |
1194 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1196 i40iw_insert_wqe_hdr(wqe
, header
);
1198 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_APBVT WQE",
1199 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1202 i40iw_sc_cqp_post_sq(cqp
);
1207 * i40iw_sc_manage_qhash_table_entry - manage quad hash entries
1208 * @cqp: struct for cqp hw
1209 * @info: info for quad hash to manage
1210 * @scratch: u64 saved to be used during cqp completion
1211 * @post_sq: flag for cqp db to ring
1213 * This is called before connection establishment is started. For passive connections, when
1214 * listener is created, it will call with entry type of I40IW_QHASH_TYPE_TCP_SYN with local
1215 * ip address and tcp port. When SYN is received (passive connections) or
1216 * sent (active connections), this routine is called with entry type of
1217 * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.
1219 * When iwarp connection is done and its state moves to RTS, the quad hash entry in
1220 * the hardware will point to iwarp's qp number and requires no calls from the driver.
1222 static enum i40iw_status_code
i40iw_sc_manage_qhash_table_entry(
1223 struct i40iw_sc_cqp
*cqp
,
1224 struct i40iw_qhash_table_info
*info
,
1232 struct i40iw_sc_vsi
*vsi
= info
->vsi
;
1234 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1236 return I40IW_ERR_RING_FULL
;
1238 temp
= info
->mac_addr
[5] |
1239 LS_64_1(info
->mac_addr
[4], 8) |
1240 LS_64_1(info
->mac_addr
[3], 16) |
1241 LS_64_1(info
->mac_addr
[2], 24) |
1242 LS_64_1(info
->mac_addr
[1], 32) |
1243 LS_64_1(info
->mac_addr
[0], 40);
1245 set_64bit_val(wqe
, 0, temp
);
1247 qw1
= LS_64(info
->qp_num
, I40IW_CQPSQ_QHASH_QPN
) |
1248 LS_64(info
->dest_port
, I40IW_CQPSQ_QHASH_DEST_PORT
);
1249 if (info
->ipv4_valid
) {
1252 LS_64(info
->dest_ip
[0], I40IW_CQPSQ_QHASH_ADDR3
));
1256 LS_64(info
->dest_ip
[0], I40IW_CQPSQ_QHASH_ADDR0
) |
1257 LS_64(info
->dest_ip
[1], I40IW_CQPSQ_QHASH_ADDR1
));
1261 LS_64(info
->dest_ip
[2], I40IW_CQPSQ_QHASH_ADDR2
) |
1262 LS_64(info
->dest_ip
[3], I40IW_CQPSQ_QHASH_ADDR3
));
1264 qw2
= LS_64(vsi
->qos
[info
->user_pri
].qs_handle
, I40IW_CQPSQ_QHASH_QS_HANDLE
);
1265 if (info
->vlan_valid
)
1266 qw2
|= LS_64(info
->vlan_id
, I40IW_CQPSQ_QHASH_VLANID
);
1267 set_64bit_val(wqe
, 16, qw2
);
1268 if (info
->entry_type
== I40IW_QHASH_TYPE_TCP_ESTABLISHED
) {
1269 qw1
|= LS_64(info
->src_port
, I40IW_CQPSQ_QHASH_SRC_PORT
);
1270 if (!info
->ipv4_valid
) {
1273 LS_64(info
->src_ip
[0], I40IW_CQPSQ_QHASH_ADDR0
) |
1274 LS_64(info
->src_ip
[1], I40IW_CQPSQ_QHASH_ADDR1
));
1277 LS_64(info
->src_ip
[2], I40IW_CQPSQ_QHASH_ADDR2
) |
1278 LS_64(info
->src_ip
[3], I40IW_CQPSQ_QHASH_ADDR3
));
1282 LS_64(info
->src_ip
[0], I40IW_CQPSQ_QHASH_ADDR3
));
1286 set_64bit_val(wqe
, 8, qw1
);
1287 temp
= LS_64(cqp
->polarity
, I40IW_CQPSQ_QHASH_WQEVALID
) |
1288 LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY
, I40IW_CQPSQ_QHASH_OPCODE
) |
1289 LS_64(info
->manage
, I40IW_CQPSQ_QHASH_MANAGE
) |
1290 LS_64(info
->ipv4_valid
, I40IW_CQPSQ_QHASH_IPV4VALID
) |
1291 LS_64(info
->vlan_valid
, I40IW_CQPSQ_QHASH_VLANVALID
) |
1292 LS_64(info
->entry_type
, I40IW_CQPSQ_QHASH_ENTRYTYPE
);
1294 i40iw_insert_wqe_hdr(wqe
, temp
);
1296 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_QHASH WQE",
1297 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1300 i40iw_sc_cqp_post_sq(cqp
);
1305 * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry
1306 * @cqp: struct for cqp hw
1307 * @scratch: u64 saved to be used during cqp completion
1308 * @post_sq: flag for cqp db to ring
1310 static enum i40iw_status_code
i40iw_sc_alloc_local_mac_ipaddr_entry(
1311 struct i40iw_sc_cqp
*cqp
,
1318 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1320 return I40IW_ERR_RING_FULL
;
1321 header
= LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY
, I40IW_CQPSQ_OPCODE
) |
1322 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1324 i40iw_insert_wqe_hdr(wqe
, header
);
1325 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "ALLOCATE_LOCAL_MAC_IPADDR WQE",
1326 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1328 i40iw_sc_cqp_post_sq(cqp
);
1333 * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry
1334 * @cqp: struct for cqp hw
1335 * @info:mac addr info
1336 * @scratch: u64 saved to be used during cqp completion
1337 * @post_sq: flag for cqp db to ring
1339 static enum i40iw_status_code
i40iw_sc_add_local_mac_ipaddr_entry(
1340 struct i40iw_sc_cqp
*cqp
,
1341 struct i40iw_local_mac_ipaddr_entry_info
*info
,
1348 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1350 return I40IW_ERR_RING_FULL
;
1351 temp
= info
->mac_addr
[5] |
1352 LS_64_1(info
->mac_addr
[4], 8) |
1353 LS_64_1(info
->mac_addr
[3], 16) |
1354 LS_64_1(info
->mac_addr
[2], 24) |
1355 LS_64_1(info
->mac_addr
[1], 32) |
1356 LS_64_1(info
->mac_addr
[0], 40);
1358 set_64bit_val(wqe
, 32, temp
);
1360 header
= LS_64(info
->entry_idx
, I40IW_CQPSQ_MLIPA_IPTABLEIDX
) |
1361 LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE
, I40IW_CQPSQ_OPCODE
) |
1362 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1364 i40iw_insert_wqe_hdr(wqe
, header
);
1366 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "ADD_LOCAL_MAC_IPADDR WQE",
1367 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1370 i40iw_sc_cqp_post_sq(cqp
);
1375 * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac
1376 * @cqp: struct for cqp hw
1377 * @scratch: u64 saved to be used during cqp completion
1378 * @entry_idx: index of mac entry
1379 * @ ignore_ref_count: to force mac adde delete
1380 * @post_sq: flag for cqp db to ring
1382 static enum i40iw_status_code
i40iw_sc_del_local_mac_ipaddr_entry(
1383 struct i40iw_sc_cqp
*cqp
,
1386 u8 ignore_ref_count
,
1392 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1394 return I40IW_ERR_RING_FULL
;
1395 header
= LS_64(entry_idx
, I40IW_CQPSQ_MLIPA_IPTABLEIDX
) |
1396 LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE
, I40IW_CQPSQ_OPCODE
) |
1397 LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY
) |
1398 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
) |
1399 LS_64(ignore_ref_count
, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT
);
1401 i40iw_insert_wqe_hdr(wqe
, header
);
1403 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "DEL_LOCAL_MAC_IPADDR WQE",
1404 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1407 i40iw_sc_cqp_post_sq(cqp
);
1412 * i40iw_sc_cqp_nop - send a nop wqe
1413 * @cqp: struct for cqp hw
1414 * @scratch: u64 saved to be used during cqp completion
1415 * @post_sq: flag for cqp db to ring
1417 static enum i40iw_status_code
i40iw_sc_cqp_nop(struct i40iw_sc_cqp
*cqp
,
1424 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1426 return I40IW_ERR_RING_FULL
;
1427 header
= LS_64(I40IW_CQP_OP_NOP
, I40IW_CQPSQ_OPCODE
) |
1428 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1429 i40iw_insert_wqe_hdr(wqe
, header
);
1430 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "NOP WQE",
1431 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1434 i40iw_sc_cqp_post_sq(cqp
);
1439 * i40iw_sc_ceq_init - initialize ceq
1440 * @ceq: ceq sc structure
1441 * @info: ceq initialization info
1443 static enum i40iw_status_code
i40iw_sc_ceq_init(struct i40iw_sc_ceq
*ceq
,
1444 struct i40iw_ceq_init_info
*info
)
1448 if ((info
->elem_cnt
< I40IW_MIN_CEQ_ENTRIES
) ||
1449 (info
->elem_cnt
> I40IW_MAX_CEQ_ENTRIES
))
1450 return I40IW_ERR_INVALID_SIZE
;
1452 if (info
->ceq_id
>= I40IW_MAX_CEQID
)
1453 return I40IW_ERR_INVALID_CEQ_ID
;
1455 pble_obj_cnt
= info
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
1457 if (info
->virtual_map
&& (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
1458 return I40IW_ERR_INVALID_PBLE_INDEX
;
1460 ceq
->size
= sizeof(*ceq
);
1461 ceq
->ceqe_base
= (struct i40iw_ceqe
*)info
->ceqe_base
;
1462 ceq
->ceq_id
= info
->ceq_id
;
1463 ceq
->dev
= info
->dev
;
1464 ceq
->elem_cnt
= info
->elem_cnt
;
1465 ceq
->ceq_elem_pa
= info
->ceqe_pa
;
1466 ceq
->virtual_map
= info
->virtual_map
;
1468 ceq
->pbl_chunk_size
= (ceq
->virtual_map
? info
->pbl_chunk_size
: 0);
1469 ceq
->first_pm_pbl_idx
= (ceq
->virtual_map
? info
->first_pm_pbl_idx
: 0);
1470 ceq
->pbl_list
= (ceq
->virtual_map
? info
->pbl_list
: NULL
);
1472 ceq
->tph_en
= info
->tph_en
;
1473 ceq
->tph_val
= info
->tph_val
;
1475 I40IW_RING_INIT(ceq
->ceq_ring
, ceq
->elem_cnt
);
1476 ceq
->dev
->ceq
[info
->ceq_id
] = ceq
;
1482 * i40iw_sc_ceq_create - create ceq wqe
1483 * @ceq: ceq sc structure
1484 * @scratch: u64 saved to be used during cqp completion
1485 * @post_sq: flag for cqp db to ring
1487 static enum i40iw_status_code
i40iw_sc_ceq_create(struct i40iw_sc_ceq
*ceq
,
1491 struct i40iw_sc_cqp
*cqp
;
1495 cqp
= ceq
->dev
->cqp
;
1496 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1498 return I40IW_ERR_RING_FULL
;
1499 set_64bit_val(wqe
, 16, ceq
->elem_cnt
);
1500 set_64bit_val(wqe
, 32, (ceq
->virtual_map
? 0 : ceq
->ceq_elem_pa
));
1501 set_64bit_val(wqe
, 48, (ceq
->virtual_map
? ceq
->first_pm_pbl_idx
: 0));
1502 set_64bit_val(wqe
, 56, LS_64(ceq
->tph_val
, I40IW_CQPSQ_TPHVAL
));
1504 header
= ceq
->ceq_id
|
1505 LS_64(I40IW_CQP_OP_CREATE_CEQ
, I40IW_CQPSQ_OPCODE
) |
1506 LS_64(ceq
->pbl_chunk_size
, I40IW_CQPSQ_CEQ_LPBLSIZE
) |
1507 LS_64(ceq
->virtual_map
, I40IW_CQPSQ_CEQ_VMAP
) |
1508 LS_64(ceq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
1509 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1511 i40iw_insert_wqe_hdr(wqe
, header
);
1513 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CEQ_CREATE WQE",
1514 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1517 i40iw_sc_cqp_post_sq(cqp
);
1522 * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete
1523 * @ceq: ceq sc structure
1525 static enum i40iw_status_code
i40iw_sc_cceq_create_done(struct i40iw_sc_ceq
*ceq
)
1527 struct i40iw_sc_cqp
*cqp
;
1529 cqp
= ceq
->dev
->cqp
;
1530 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_CREATE_CEQ
, NULL
);
1534 * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete
1535 * @ceq: ceq sc structure
1537 static enum i40iw_status_code
i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq
*ceq
)
1539 struct i40iw_sc_cqp
*cqp
;
1541 cqp
= ceq
->dev
->cqp
;
1542 cqp
->process_cqp_sds
= i40iw_update_sds_noccq
;
1543 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_DESTROY_CEQ
, NULL
);
1547 * i40iw_sc_cceq_create - create cceq
1548 * @ceq: ceq sc structure
1549 * @scratch: u64 saved to be used during cqp completion
1551 static enum i40iw_status_code
i40iw_sc_cceq_create(struct i40iw_sc_ceq
*ceq
, u64 scratch
)
1553 enum i40iw_status_code ret_code
;
1555 ret_code
= i40iw_sc_ceq_create(ceq
, scratch
, true);
1557 ret_code
= i40iw_sc_cceq_create_done(ceq
);
1562 * i40iw_sc_ceq_destroy - destroy ceq
1563 * @ceq: ceq sc structure
1564 * @scratch: u64 saved to be used during cqp completion
1565 * @post_sq: flag for cqp db to ring
1567 static enum i40iw_status_code
i40iw_sc_ceq_destroy(struct i40iw_sc_ceq
*ceq
,
1571 struct i40iw_sc_cqp
*cqp
;
1575 cqp
= ceq
->dev
->cqp
;
1576 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1578 return I40IW_ERR_RING_FULL
;
1579 set_64bit_val(wqe
, 16, ceq
->elem_cnt
);
1580 set_64bit_val(wqe
, 48, ceq
->first_pm_pbl_idx
);
1581 header
= ceq
->ceq_id
|
1582 LS_64(I40IW_CQP_OP_DESTROY_CEQ
, I40IW_CQPSQ_OPCODE
) |
1583 LS_64(ceq
->pbl_chunk_size
, I40IW_CQPSQ_CEQ_LPBLSIZE
) |
1584 LS_64(ceq
->virtual_map
, I40IW_CQPSQ_CEQ_VMAP
) |
1585 LS_64(ceq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
1586 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1587 i40iw_insert_wqe_hdr(wqe
, header
);
1588 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CEQ_DESTROY WQE",
1589 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1592 i40iw_sc_cqp_post_sq(cqp
);
1597 * i40iw_sc_process_ceq - process ceq
1598 * @dev: sc device struct
1599 * @ceq: ceq sc structure
1601 static void *i40iw_sc_process_ceq(struct i40iw_sc_dev
*dev
, struct i40iw_sc_ceq
*ceq
)
1605 struct i40iw_sc_cq
*cq
= NULL
;
1608 ceqe
= (u64
*)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq
);
1609 get_64bit_val(ceqe
, 0, &temp
);
1610 polarity
= (u8
)RS_64(temp
, I40IW_CEQE_VALID
);
1611 if (polarity
!= ceq
->polarity
)
1614 cq
= (struct i40iw_sc_cq
*)(unsigned long)LS_64_1(temp
, 1);
1616 I40IW_RING_MOVE_TAIL(ceq
->ceq_ring
);
1617 if (I40IW_RING_GETCURRENT_TAIL(ceq
->ceq_ring
) == 0)
1621 i40iw_wr32(dev
->hw
, I40E_PFPE_CQACK
, cq
->cq_uk
.cq_id
);
1623 i40iw_wr32(dev
->hw
, I40E_VFPE_CQACK1
, cq
->cq_uk
.cq_id
);
1629 * i40iw_sc_aeq_init - initialize aeq
1630 * @aeq: aeq structure ptr
1631 * @info: aeq initialization info
1633 static enum i40iw_status_code
i40iw_sc_aeq_init(struct i40iw_sc_aeq
*aeq
,
1634 struct i40iw_aeq_init_info
*info
)
1638 if ((info
->elem_cnt
< I40IW_MIN_AEQ_ENTRIES
) ||
1639 (info
->elem_cnt
> I40IW_MAX_AEQ_ENTRIES
))
1640 return I40IW_ERR_INVALID_SIZE
;
1641 pble_obj_cnt
= info
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
1643 if (info
->virtual_map
&& (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
1644 return I40IW_ERR_INVALID_PBLE_INDEX
;
1646 aeq
->size
= sizeof(*aeq
);
1648 aeq
->aeqe_base
= (struct i40iw_sc_aeqe
*)info
->aeqe_base
;
1649 aeq
->dev
= info
->dev
;
1650 aeq
->elem_cnt
= info
->elem_cnt
;
1652 aeq
->aeq_elem_pa
= info
->aeq_elem_pa
;
1653 I40IW_RING_INIT(aeq
->aeq_ring
, aeq
->elem_cnt
);
1654 info
->dev
->aeq
= aeq
;
1656 aeq
->virtual_map
= info
->virtual_map
;
1657 aeq
->pbl_list
= (aeq
->virtual_map
? info
->pbl_list
: NULL
);
1658 aeq
->pbl_chunk_size
= (aeq
->virtual_map
? info
->pbl_chunk_size
: 0);
1659 aeq
->first_pm_pbl_idx
= (aeq
->virtual_map
? info
->first_pm_pbl_idx
: 0);
1660 info
->dev
->aeq
= aeq
;
1665 * i40iw_sc_aeq_create - create aeq
1666 * @aeq: aeq structure ptr
1667 * @scratch: u64 saved to be used during cqp completion
1668 * @post_sq: flag for cqp db to ring
1670 static enum i40iw_status_code
i40iw_sc_aeq_create(struct i40iw_sc_aeq
*aeq
,
1675 struct i40iw_sc_cqp
*cqp
;
1678 cqp
= aeq
->dev
->cqp
;
1679 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1681 return I40IW_ERR_RING_FULL
;
1682 set_64bit_val(wqe
, 16, aeq
->elem_cnt
);
1683 set_64bit_val(wqe
, 32,
1684 (aeq
->virtual_map
? 0 : aeq
->aeq_elem_pa
));
1685 set_64bit_val(wqe
, 48,
1686 (aeq
->virtual_map
? aeq
->first_pm_pbl_idx
: 0));
1688 header
= LS_64(I40IW_CQP_OP_CREATE_AEQ
, I40IW_CQPSQ_OPCODE
) |
1689 LS_64(aeq
->pbl_chunk_size
, I40IW_CQPSQ_AEQ_LPBLSIZE
) |
1690 LS_64(aeq
->virtual_map
, I40IW_CQPSQ_AEQ_VMAP
) |
1691 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1693 i40iw_insert_wqe_hdr(wqe
, header
);
1694 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "AEQ_CREATE WQE",
1695 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1697 i40iw_sc_cqp_post_sq(cqp
);
1702 * i40iw_sc_aeq_destroy - destroy aeq during close
1703 * @aeq: aeq structure ptr
1704 * @scratch: u64 saved to be used during cqp completion
1705 * @post_sq: flag for cqp db to ring
1707 static enum i40iw_status_code
i40iw_sc_aeq_destroy(struct i40iw_sc_aeq
*aeq
,
1712 struct i40iw_sc_cqp
*cqp
;
1715 cqp
= aeq
->dev
->cqp
;
1716 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1718 return I40IW_ERR_RING_FULL
;
1719 set_64bit_val(wqe
, 16, aeq
->elem_cnt
);
1720 set_64bit_val(wqe
, 48, aeq
->first_pm_pbl_idx
);
1721 header
= LS_64(I40IW_CQP_OP_DESTROY_AEQ
, I40IW_CQPSQ_OPCODE
) |
1722 LS_64(aeq
->pbl_chunk_size
, I40IW_CQPSQ_AEQ_LPBLSIZE
) |
1723 LS_64(aeq
->virtual_map
, I40IW_CQPSQ_AEQ_VMAP
) |
1724 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1725 i40iw_insert_wqe_hdr(wqe
, header
);
1727 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "AEQ_DESTROY WQE",
1728 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1730 i40iw_sc_cqp_post_sq(cqp
);
1735 * i40iw_sc_get_next_aeqe - get next aeq entry
1736 * @aeq: aeq structure ptr
1737 * @info: aeqe info to be returned
1739 static enum i40iw_status_code
i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq
*aeq
,
1740 struct i40iw_aeqe_info
*info
)
1742 u64 temp
, compl_ctx
;
1748 aeqe
= (u64
*)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq
);
1749 get_64bit_val(aeqe
, 0, &compl_ctx
);
1750 get_64bit_val(aeqe
, 8, &temp
);
1751 polarity
= (u8
)RS_64(temp
, I40IW_AEQE_VALID
);
1753 if (aeq
->polarity
!= polarity
)
1754 return I40IW_ERR_QUEUE_EMPTY
;
1756 i40iw_debug_buf(aeq
->dev
, I40IW_DEBUG_WQE
, "AEQ_ENTRY", aeqe
, 16);
1758 ae_src
= (u8
)RS_64(temp
, I40IW_AEQE_AESRC
);
1759 wqe_idx
= (u16
)RS_64(temp
, I40IW_AEQE_WQDESCIDX
);
1760 info
->qp_cq_id
= (u32
)RS_64(temp
, I40IW_AEQE_QPCQID
);
1761 info
->ae_id
= (u16
)RS_64(temp
, I40IW_AEQE_AECODE
);
1762 info
->tcp_state
= (u8
)RS_64(temp
, I40IW_AEQE_TCPSTATE
);
1763 info
->iwarp_state
= (u8
)RS_64(temp
, I40IW_AEQE_IWSTATE
);
1764 info
->q2_data_written
= (u8
)RS_64(temp
, I40IW_AEQE_Q2DATA
);
1765 info
->aeqe_overflow
= (bool)RS_64(temp
, I40IW_AEQE_OVERFLOW
);
1767 case I40IW_AE_SOURCE_RQ
:
1768 case I40IW_AE_SOURCE_RQ_0011
:
1770 info
->wqe_idx
= wqe_idx
;
1771 info
->compl_ctx
= compl_ctx
;
1773 case I40IW_AE_SOURCE_CQ
:
1774 case I40IW_AE_SOURCE_CQ_0110
:
1775 case I40IW_AE_SOURCE_CQ_1010
:
1776 case I40IW_AE_SOURCE_CQ_1110
:
1778 info
->compl_ctx
= LS_64_1(compl_ctx
, 1);
1780 case I40IW_AE_SOURCE_SQ
:
1781 case I40IW_AE_SOURCE_SQ_0111
:
1784 info
->wqe_idx
= wqe_idx
;
1785 info
->compl_ctx
= compl_ctx
;
1787 case I40IW_AE_SOURCE_IN_RR_WR
:
1788 case I40IW_AE_SOURCE_IN_RR_WR_1011
:
1790 info
->compl_ctx
= compl_ctx
;
1791 info
->in_rdrsp_wr
= true;
1793 case I40IW_AE_SOURCE_OUT_RR
:
1794 case I40IW_AE_SOURCE_OUT_RR_1111
:
1796 info
->compl_ctx
= compl_ctx
;
1797 info
->out_rdrsp
= true;
1802 I40IW_RING_MOVE_TAIL(aeq
->aeq_ring
);
1803 if (I40IW_RING_GETCURRENT_TAIL(aeq
->aeq_ring
) == 0)
1809 * i40iw_sc_repost_aeq_entries - repost completed aeq entries
1810 * @dev: sc device struct
1811 * @count: allocate count
1813 static enum i40iw_status_code
i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev
*dev
,
1816 if (count
> I40IW_MAX_AEQ_ALLOCATE_COUNT
)
1817 return I40IW_ERR_INVALID_SIZE
;
1820 i40iw_wr32(dev
->hw
, I40E_PFPE_AEQALLOC
, count
);
1822 i40iw_wr32(dev
->hw
, I40E_VFPE_AEQALLOC1
, count
);
1828 * i40iw_sc_aeq_create_done - create aeq
1829 * @aeq: aeq structure ptr
1831 static enum i40iw_status_code
i40iw_sc_aeq_create_done(struct i40iw_sc_aeq
*aeq
)
1833 struct i40iw_sc_cqp
*cqp
;
1835 cqp
= aeq
->dev
->cqp
;
1836 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_CREATE_AEQ
, NULL
);
1840 * i40iw_sc_aeq_destroy_done - destroy of aeq during close
1841 * @aeq: aeq structure ptr
1843 static enum i40iw_status_code
i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq
*aeq
)
1845 struct i40iw_sc_cqp
*cqp
;
1847 cqp
= aeq
->dev
->cqp
;
1848 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_DESTROY_AEQ
, NULL
);
1852 * i40iw_sc_ccq_init - initialize control cq
1853 * @cq: sc's cq ctruct
1854 * @info: info for control cq initialization
1856 static enum i40iw_status_code
i40iw_sc_ccq_init(struct i40iw_sc_cq
*cq
,
1857 struct i40iw_ccq_init_info
*info
)
1861 if (info
->num_elem
< I40IW_MIN_CQ_SIZE
|| info
->num_elem
> I40IW_MAX_CQ_SIZE
)
1862 return I40IW_ERR_INVALID_SIZE
;
1864 if (info
->ceq_id
> I40IW_MAX_CEQID
)
1865 return I40IW_ERR_INVALID_CEQ_ID
;
1867 pble_obj_cnt
= info
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
1869 if (info
->virtual_map
&& (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
1870 return I40IW_ERR_INVALID_PBLE_INDEX
;
1872 cq
->cq_pa
= info
->cq_pa
;
1873 cq
->cq_uk
.cq_base
= info
->cq_base
;
1874 cq
->shadow_area_pa
= info
->shadow_area_pa
;
1875 cq
->cq_uk
.shadow_area
= info
->shadow_area
;
1876 cq
->shadow_read_threshold
= info
->shadow_read_threshold
;
1877 cq
->dev
= info
->dev
;
1878 cq
->ceq_id
= info
->ceq_id
;
1879 cq
->cq_uk
.cq_size
= info
->num_elem
;
1880 cq
->cq_type
= I40IW_CQ_TYPE_CQP
;
1881 cq
->ceqe_mask
= info
->ceqe_mask
;
1882 I40IW_RING_INIT(cq
->cq_uk
.cq_ring
, info
->num_elem
);
1884 cq
->cq_uk
.cq_id
= 0; /* control cq is id 0 always */
1885 cq
->ceq_id_valid
= info
->ceq_id_valid
;
1886 cq
->tph_en
= info
->tph_en
;
1887 cq
->tph_val
= info
->tph_val
;
1888 cq
->cq_uk
.avoid_mem_cflct
= info
->avoid_mem_cflct
;
1890 cq
->pbl_list
= info
->pbl_list
;
1891 cq
->virtual_map
= info
->virtual_map
;
1892 cq
->pbl_chunk_size
= info
->pbl_chunk_size
;
1893 cq
->first_pm_pbl_idx
= info
->first_pm_pbl_idx
;
1894 cq
->cq_uk
.polarity
= true;
1896 /* following are only for iw cqs so initialize them to zero */
1897 cq
->cq_uk
.cqe_alloc_reg
= NULL
;
1898 info
->dev
->ccq
= cq
;
1903 * i40iw_sc_ccq_create_done - poll cqp for ccq create
1904 * @ccq: ccq sc struct
1906 static enum i40iw_status_code
i40iw_sc_ccq_create_done(struct i40iw_sc_cq
*ccq
)
1908 struct i40iw_sc_cqp
*cqp
;
1910 cqp
= ccq
->dev
->cqp
;
1911 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_CREATE_CQ
, NULL
);
1915 * i40iw_sc_ccq_create - create control cq
1916 * @ccq: ccq sc struct
1917 * @scratch: u64 saved to be used during cqp completion
1918 * @check_overflow: overlow flag for ccq
1919 * @post_sq: flag for cqp db to ring
1921 static enum i40iw_status_code
i40iw_sc_ccq_create(struct i40iw_sc_cq
*ccq
,
1923 bool check_overflow
,
1927 struct i40iw_sc_cqp
*cqp
;
1929 enum i40iw_status_code ret_code
;
1931 cqp
= ccq
->dev
->cqp
;
1932 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1934 return I40IW_ERR_RING_FULL
;
1935 set_64bit_val(wqe
, 0, ccq
->cq_uk
.cq_size
);
1936 set_64bit_val(wqe
, 8, RS_64_1(ccq
, 1));
1937 set_64bit_val(wqe
, 16,
1938 LS_64(ccq
->shadow_read_threshold
, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD
));
1939 set_64bit_val(wqe
, 32, (ccq
->virtual_map
? 0 : ccq
->cq_pa
));
1940 set_64bit_val(wqe
, 40, ccq
->shadow_area_pa
);
1941 set_64bit_val(wqe
, 48,
1942 (ccq
->virtual_map
? ccq
->first_pm_pbl_idx
: 0));
1943 set_64bit_val(wqe
, 56,
1944 LS_64(ccq
->tph_val
, I40IW_CQPSQ_TPHVAL
));
1946 header
= ccq
->cq_uk
.cq_id
|
1947 LS_64((ccq
->ceq_id_valid
? ccq
->ceq_id
: 0), I40IW_CQPSQ_CQ_CEQID
) |
1948 LS_64(I40IW_CQP_OP_CREATE_CQ
, I40IW_CQPSQ_OPCODE
) |
1949 LS_64(ccq
->pbl_chunk_size
, I40IW_CQPSQ_CQ_LPBLSIZE
) |
1950 LS_64(check_overflow
, I40IW_CQPSQ_CQ_CHKOVERFLOW
) |
1951 LS_64(ccq
->virtual_map
, I40IW_CQPSQ_CQ_VIRTMAP
) |
1952 LS_64(ccq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
1953 LS_64(ccq
->ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
1954 LS_64(ccq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
1955 LS_64(ccq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
1956 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1958 i40iw_insert_wqe_hdr(wqe
, header
);
1960 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CCQ_CREATE WQE",
1961 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1964 i40iw_sc_cqp_post_sq(cqp
);
1965 ret_code
= i40iw_sc_ccq_create_done(ccq
);
1969 cqp
->process_cqp_sds
= i40iw_cqp_sds_cmd
;
1975 * i40iw_sc_ccq_destroy - destroy ccq during close
1976 * @ccq: ccq sc struct
1977 * @scratch: u64 saved to be used during cqp completion
1978 * @post_sq: flag for cqp db to ring
1980 static enum i40iw_status_code
i40iw_sc_ccq_destroy(struct i40iw_sc_cq
*ccq
,
1984 struct i40iw_sc_cqp
*cqp
;
1987 enum i40iw_status_code ret_code
= 0;
1988 u32 tail
, val
, error
;
1990 cqp
= ccq
->dev
->cqp
;
1991 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1993 return I40IW_ERR_RING_FULL
;
1994 set_64bit_val(wqe
, 0, ccq
->cq_uk
.cq_size
);
1995 set_64bit_val(wqe
, 8, RS_64_1(ccq
, 1));
1996 set_64bit_val(wqe
, 40, ccq
->shadow_area_pa
);
1998 header
= ccq
->cq_uk
.cq_id
|
1999 LS_64((ccq
->ceq_id_valid
? ccq
->ceq_id
: 0), I40IW_CQPSQ_CQ_CEQID
) |
2000 LS_64(I40IW_CQP_OP_DESTROY_CQ
, I40IW_CQPSQ_OPCODE
) |
2001 LS_64(ccq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
2002 LS_64(ccq
->ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
2003 LS_64(ccq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
2004 LS_64(ccq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
2005 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2007 i40iw_insert_wqe_hdr(wqe
, header
);
2009 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CCQ_DESTROY WQE",
2010 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2012 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
2014 return I40IW_ERR_CQP_COMPL_ERROR
;
2017 i40iw_sc_cqp_post_sq(cqp
);
2018 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, 1000);
2021 cqp
->process_cqp_sds
= i40iw_update_sds_noccq
;
2027 * i40iw_sc_cq_init - initialize completion q
2029 * @info: cq initialization info
2031 static enum i40iw_status_code
i40iw_sc_cq_init(struct i40iw_sc_cq
*cq
,
2032 struct i40iw_cq_init_info
*info
)
2034 u32 __iomem
*cqe_alloc_reg
= NULL
;
2035 enum i40iw_status_code ret_code
;
2039 pble_obj_cnt
= info
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
2041 if (info
->virtual_map
&& (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
2042 return I40IW_ERR_INVALID_PBLE_INDEX
;
2044 cq
->cq_pa
= info
->cq_base_pa
;
2045 cq
->dev
= info
->dev
;
2046 cq
->ceq_id
= info
->ceq_id
;
2047 arm_offset
= (info
->dev
->is_pf
) ? I40E_PFPE_CQARM
: I40E_VFPE_CQARM1
;
2048 if (i40iw_get_hw_addr(cq
->dev
))
2049 cqe_alloc_reg
= (u32 __iomem
*)(i40iw_get_hw_addr(cq
->dev
) +
2051 info
->cq_uk_init_info
.cqe_alloc_reg
= cqe_alloc_reg
;
2052 ret_code
= i40iw_cq_uk_init(&cq
->cq_uk
, &info
->cq_uk_init_info
);
2055 cq
->virtual_map
= info
->virtual_map
;
2056 cq
->pbl_chunk_size
= info
->pbl_chunk_size
;
2057 cq
->ceqe_mask
= info
->ceqe_mask
;
2058 cq
->cq_type
= (info
->type
) ? info
->type
: I40IW_CQ_TYPE_IWARP
;
2060 cq
->shadow_area_pa
= info
->shadow_area_pa
;
2061 cq
->shadow_read_threshold
= info
->shadow_read_threshold
;
2063 cq
->ceq_id_valid
= info
->ceq_id_valid
;
2064 cq
->tph_en
= info
->tph_en
;
2065 cq
->tph_val
= info
->tph_val
;
2067 cq
->first_pm_pbl_idx
= info
->first_pm_pbl_idx
;
2073 * i40iw_sc_cq_create - create completion q
2075 * @scratch: u64 saved to be used during cqp completion
2076 * @check_overflow: flag for overflow check
2077 * @post_sq: flag for cqp db to ring
2079 static enum i40iw_status_code
i40iw_sc_cq_create(struct i40iw_sc_cq
*cq
,
2081 bool check_overflow
,
2085 struct i40iw_sc_cqp
*cqp
;
2088 if (cq
->cq_uk
.cq_id
> I40IW_MAX_CQID
)
2089 return I40IW_ERR_INVALID_CQ_ID
;
2091 if (cq
->ceq_id
> I40IW_MAX_CEQID
)
2092 return I40IW_ERR_INVALID_CEQ_ID
;
2095 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2097 return I40IW_ERR_RING_FULL
;
2099 set_64bit_val(wqe
, 0, cq
->cq_uk
.cq_size
);
2100 set_64bit_val(wqe
, 8, RS_64_1(cq
, 1));
2103 LS_64(cq
->shadow_read_threshold
, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD
));
2105 set_64bit_val(wqe
, 32, (cq
->virtual_map
? 0 : cq
->cq_pa
));
2107 set_64bit_val(wqe
, 40, cq
->shadow_area_pa
);
2108 set_64bit_val(wqe
, 48, (cq
->virtual_map
? cq
->first_pm_pbl_idx
: 0));
2109 set_64bit_val(wqe
, 56, LS_64(cq
->tph_val
, I40IW_CQPSQ_TPHVAL
));
2111 header
= cq
->cq_uk
.cq_id
|
2112 LS_64((cq
->ceq_id_valid
? cq
->ceq_id
: 0), I40IW_CQPSQ_CQ_CEQID
) |
2113 LS_64(I40IW_CQP_OP_CREATE_CQ
, I40IW_CQPSQ_OPCODE
) |
2114 LS_64(cq
->pbl_chunk_size
, I40IW_CQPSQ_CQ_LPBLSIZE
) |
2115 LS_64(check_overflow
, I40IW_CQPSQ_CQ_CHKOVERFLOW
) |
2116 LS_64(cq
->virtual_map
, I40IW_CQPSQ_CQ_VIRTMAP
) |
2117 LS_64(cq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
2118 LS_64(cq
->ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
2119 LS_64(cq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
2120 LS_64(cq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
2121 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2123 i40iw_insert_wqe_hdr(wqe
, header
);
2125 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CQ_CREATE WQE",
2126 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2129 i40iw_sc_cqp_post_sq(cqp
);
2134 * i40iw_sc_cq_destroy - destroy completion q
2136 * @scratch: u64 saved to be used during cqp completion
2137 * @post_sq: flag for cqp db to ring
2139 static enum i40iw_status_code
i40iw_sc_cq_destroy(struct i40iw_sc_cq
*cq
,
2143 struct i40iw_sc_cqp
*cqp
;
2148 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2150 return I40IW_ERR_RING_FULL
;
2151 set_64bit_val(wqe
, 0, cq
->cq_uk
.cq_size
);
2152 set_64bit_val(wqe
, 8, RS_64_1(cq
, 1));
2153 set_64bit_val(wqe
, 40, cq
->shadow_area_pa
);
2154 set_64bit_val(wqe
, 48, (cq
->virtual_map
? cq
->first_pm_pbl_idx
: 0));
2156 header
= cq
->cq_uk
.cq_id
|
2157 LS_64((cq
->ceq_id_valid
? cq
->ceq_id
: 0), I40IW_CQPSQ_CQ_CEQID
) |
2158 LS_64(I40IW_CQP_OP_DESTROY_CQ
, I40IW_CQPSQ_OPCODE
) |
2159 LS_64(cq
->pbl_chunk_size
, I40IW_CQPSQ_CQ_LPBLSIZE
) |
2160 LS_64(cq
->virtual_map
, I40IW_CQPSQ_CQ_VIRTMAP
) |
2161 LS_64(cq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
2162 LS_64(cq
->ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
2163 LS_64(cq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
2164 LS_64(cq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
2165 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2167 i40iw_insert_wqe_hdr(wqe
, header
);
2169 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CQ_DESTROY WQE",
2170 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2173 i40iw_sc_cqp_post_sq(cqp
);
2178 * i40iw_sc_cq_modify - modify a Completion Queue
2180 * @info: modification info struct
2182 * @post_sq: flag to post to sq
2184 static enum i40iw_status_code
i40iw_sc_cq_modify(struct i40iw_sc_cq
*cq
,
2185 struct i40iw_modify_cq_info
*info
,
2189 struct i40iw_sc_cqp
*cqp
;
2192 u32 cq_size
, ceq_id
, first_pm_pbl_idx
;
2194 bool virtual_map
, ceq_id_valid
, check_overflow
;
2197 if (info
->ceq_valid
&& (info
->ceq_id
> I40IW_MAX_CEQID
))
2198 return I40IW_ERR_INVALID_CEQ_ID
;
2200 pble_obj_cnt
= cq
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
2202 if (info
->cq_resize
&& info
->virtual_map
&&
2203 (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
2204 return I40IW_ERR_INVALID_PBLE_INDEX
;
2207 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2209 return I40IW_ERR_RING_FULL
;
2211 cq
->pbl_list
= info
->pbl_list
;
2212 cq
->cq_pa
= info
->cq_pa
;
2213 cq
->first_pm_pbl_idx
= info
->first_pm_pbl_idx
;
2215 cq_size
= info
->cq_resize
? info
->cq_size
: cq
->cq_uk
.cq_size
;
2216 if (info
->ceq_change
) {
2217 ceq_id_valid
= true;
2218 ceq_id
= info
->ceq_id
;
2220 ceq_id_valid
= cq
->ceq_id_valid
;
2221 ceq_id
= ceq_id_valid
? cq
->ceq_id
: 0;
2223 virtual_map
= info
->cq_resize
? info
->virtual_map
: cq
->virtual_map
;
2224 first_pm_pbl_idx
= (info
->cq_resize
?
2225 (info
->virtual_map
? info
->first_pm_pbl_idx
: 0) :
2226 (cq
->virtual_map
? cq
->first_pm_pbl_idx
: 0));
2227 pbl_chunk_size
= (info
->cq_resize
?
2228 (info
->virtual_map
? info
->pbl_chunk_size
: 0) :
2229 (cq
->virtual_map
? cq
->pbl_chunk_size
: 0));
2230 check_overflow
= info
->check_overflow_change
? info
->check_overflow
:
2232 cq
->cq_uk
.cq_size
= cq_size
;
2233 cq
->ceq_id_valid
= ceq_id_valid
;
2234 cq
->ceq_id
= ceq_id
;
2235 cq
->virtual_map
= virtual_map
;
2236 cq
->first_pm_pbl_idx
= first_pm_pbl_idx
;
2237 cq
->pbl_chunk_size
= pbl_chunk_size
;
2238 cq
->check_overflow
= check_overflow
;
2240 set_64bit_val(wqe
, 0, cq_size
);
2241 set_64bit_val(wqe
, 8, RS_64_1(cq
, 1));
2242 set_64bit_val(wqe
, 16,
2243 LS_64(info
->shadow_read_threshold
, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD
));
2244 set_64bit_val(wqe
, 32, (cq
->virtual_map
? 0 : cq
->cq_pa
));
2245 set_64bit_val(wqe
, 40, cq
->shadow_area_pa
);
2246 set_64bit_val(wqe
, 48, (cq
->virtual_map
? first_pm_pbl_idx
: 0));
2247 set_64bit_val(wqe
, 56, LS_64(cq
->tph_val
, I40IW_CQPSQ_TPHVAL
));
2249 header
= cq
->cq_uk
.cq_id
|
2250 LS_64(ceq_id
, I40IW_CQPSQ_CQ_CEQID
) |
2251 LS_64(I40IW_CQP_OP_MODIFY_CQ
, I40IW_CQPSQ_OPCODE
) |
2252 LS_64(info
->cq_resize
, I40IW_CQPSQ_CQ_CQRESIZE
) |
2253 LS_64(pbl_chunk_size
, I40IW_CQPSQ_CQ_LPBLSIZE
) |
2254 LS_64(check_overflow
, I40IW_CQPSQ_CQ_CHKOVERFLOW
) |
2255 LS_64(virtual_map
, I40IW_CQPSQ_CQ_VIRTMAP
) |
2256 LS_64(cq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
2257 LS_64(ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
2258 LS_64(cq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
2259 LS_64(cq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
2260 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2262 i40iw_insert_wqe_hdr(wqe
, header
);
2264 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CQ_MODIFY WQE",
2265 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2268 i40iw_sc_cqp_post_sq(cqp
);
2273 * i40iw_sc_qp_init - initialize qp
2275 * @info: initialization qp info
2277 static enum i40iw_status_code
i40iw_sc_qp_init(struct i40iw_sc_qp
*qp
,
2278 struct i40iw_qp_init_info
*info
)
2280 u32 __iomem
*wqe_alloc_reg
= NULL
;
2281 enum i40iw_status_code ret_code
;
2286 qp
->dev
= info
->pd
->dev
;
2287 qp
->vsi
= info
->vsi
;
2288 qp
->sq_pa
= info
->sq_pa
;
2289 qp
->rq_pa
= info
->rq_pa
;
2290 qp
->hw_host_ctx_pa
= info
->host_ctx_pa
;
2291 qp
->q2_pa
= info
->q2_pa
;
2292 qp
->shadow_area_pa
= info
->shadow_area_pa
;
2294 qp
->q2_buf
= info
->q2
;
2296 qp
->hw_host_ctx
= info
->host_ctx
;
2297 offset
= (qp
->pd
->dev
->is_pf
) ? I40E_PFPE_WQEALLOC
: I40E_VFPE_WQEALLOC1
;
2298 if (i40iw_get_hw_addr(qp
->pd
->dev
))
2299 wqe_alloc_reg
= (u32 __iomem
*)(i40iw_get_hw_addr(qp
->pd
->dev
) +
2302 info
->qp_uk_init_info
.wqe_alloc_reg
= wqe_alloc_reg
;
2303 info
->qp_uk_init_info
.abi_ver
= qp
->pd
->abi_ver
;
2304 ret_code
= i40iw_qp_uk_init(&qp
->qp_uk
, &info
->qp_uk_init_info
);
2307 qp
->virtual_map
= info
->virtual_map
;
2309 pble_obj_cnt
= info
->pd
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
2311 if ((info
->virtual_map
&& (info
->sq_pa
>= pble_obj_cnt
)) ||
2312 (info
->virtual_map
&& (info
->rq_pa
>= pble_obj_cnt
)))
2313 return I40IW_ERR_INVALID_PBLE_INDEX
;
2315 qp
->llp_stream_handle
= (void *)(-1);
2316 qp
->qp_type
= (info
->type
) ? info
->type
: I40IW_QP_TYPE_IWARP
;
2318 qp
->hw_sq_size
= i40iw_get_encoded_wqe_size(qp
->qp_uk
.sq_ring
.size
,
2320 i40iw_debug(qp
->dev
, I40IW_DEBUG_WQE
, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",
2321 __func__
, qp
->hw_sq_size
, qp
->qp_uk
.sq_ring
.size
);
2323 switch (qp
->pd
->abi_ver
) {
2325 ret_code
= i40iw_fragcnt_to_wqesize_rq(qp
->qp_uk
.max_rq_frag_cnt
,
2330 case 5: /* fallthrough until next ABI version */
2332 if (qp
->qp_uk
.max_rq_frag_cnt
> I40IW_MAX_WQ_FRAGMENT_COUNT
)
2333 return I40IW_ERR_INVALID_FRAG_COUNT
;
2334 wqe_size
= I40IW_MAX_WQE_SIZE_RQ
;
2337 qp
->hw_rq_size
= i40iw_get_encoded_wqe_size(qp
->qp_uk
.rq_size
*
2338 (wqe_size
/ I40IW_QP_WQE_MIN_SIZE
), false);
2339 i40iw_debug(qp
->dev
, I40IW_DEBUG_WQE
,
2340 "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
2341 __func__
, qp
->hw_rq_size
, qp
->qp_uk
.rq_size
, wqe_size
);
2342 qp
->sq_tph_val
= info
->sq_tph_val
;
2343 qp
->rq_tph_val
= info
->rq_tph_val
;
2344 qp
->sq_tph_en
= info
->sq_tph_en
;
2345 qp
->rq_tph_en
= info
->rq_tph_en
;
2346 qp
->rcv_tph_en
= info
->rcv_tph_en
;
2347 qp
->xmit_tph_en
= info
->xmit_tph_en
;
2348 qp
->qs_handle
= qp
->vsi
->qos
[qp
->user_pri
].qs_handle
;
2349 qp
->exception_lan_queue
= qp
->pd
->dev
->exception_lan_queue
;
2355 * i40iw_sc_qp_create - create qp
2357 * @info: qp create info
2358 * @scratch: u64 saved to be used during cqp completion
2359 * @post_sq: flag for cqp db to ring
2361 static enum i40iw_status_code
i40iw_sc_qp_create(
2362 struct i40iw_sc_qp
*qp
,
2363 struct i40iw_create_qp_info
*info
,
2367 struct i40iw_sc_cqp
*cqp
;
2371 if ((qp
->qp_uk
.qp_id
< I40IW_MIN_IW_QP_ID
) ||
2372 (qp
->qp_uk
.qp_id
> I40IW_MAX_IW_QP_ID
))
2373 return I40IW_ERR_INVALID_QP_ID
;
2375 cqp
= qp
->pd
->dev
->cqp
;
2376 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2378 return I40IW_ERR_RING_FULL
;
2380 set_64bit_val(wqe
, 16, qp
->hw_host_ctx_pa
);
2382 set_64bit_val(wqe
, 40, qp
->shadow_area_pa
);
2384 header
= qp
->qp_uk
.qp_id
|
2385 LS_64(I40IW_CQP_OP_CREATE_QP
, I40IW_CQPSQ_OPCODE
) |
2386 LS_64((info
->ord_valid
? 1 : 0), I40IW_CQPSQ_QP_ORDVALID
) |
2387 LS_64(info
->tcp_ctx_valid
, I40IW_CQPSQ_QP_TOECTXVALID
) |
2388 LS_64(qp
->qp_type
, I40IW_CQPSQ_QP_QPTYPE
) |
2389 LS_64(qp
->virtual_map
, I40IW_CQPSQ_QP_VQ
) |
2390 LS_64(info
->cq_num_valid
, I40IW_CQPSQ_QP_CQNUMVALID
) |
2391 LS_64(info
->static_rsrc
, I40IW_CQPSQ_QP_STATRSRC
) |
2392 LS_64(info
->arp_cache_idx_valid
, I40IW_CQPSQ_QP_ARPTABIDXVALID
) |
2393 LS_64(info
->next_iwarp_state
, I40IW_CQPSQ_QP_NEXTIWSTATE
) |
2394 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2396 i40iw_insert_wqe_hdr(wqe
, header
);
2397 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QP_CREATE WQE",
2398 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2401 i40iw_sc_cqp_post_sq(cqp
);
2406 * i40iw_sc_qp_modify - modify qp cqp wqe
2408 * @info: modify qp info
2409 * @scratch: u64 saved to be used during cqp completion
2410 * @post_sq: flag for cqp db to ring
2412 static enum i40iw_status_code
i40iw_sc_qp_modify(
2413 struct i40iw_sc_qp
*qp
,
2414 struct i40iw_modify_qp_info
*info
,
2419 struct i40iw_sc_cqp
*cqp
;
2421 u8 term_actions
= 0;
2424 cqp
= qp
->pd
->dev
->cqp
;
2425 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2427 return I40IW_ERR_RING_FULL
;
2428 if (info
->next_iwarp_state
== I40IW_QP_STATE_TERMINATE
) {
2429 if (info
->dont_send_fin
)
2430 term_actions
+= I40IWQP_TERM_SEND_TERM_ONLY
;
2431 if (info
->dont_send_term
)
2432 term_actions
+= I40IWQP_TERM_SEND_FIN_ONLY
;
2433 if ((term_actions
== I40IWQP_TERM_SEND_TERM_AND_FIN
) ||
2434 (term_actions
== I40IWQP_TERM_SEND_TERM_ONLY
))
2435 term_len
= info
->termlen
;
2440 LS_64(term_len
, I40IW_CQPSQ_QP_TERMLEN
));
2442 set_64bit_val(wqe
, 16, qp
->hw_host_ctx_pa
);
2443 set_64bit_val(wqe
, 40, qp
->shadow_area_pa
);
2445 header
= qp
->qp_uk
.qp_id
|
2446 LS_64(I40IW_CQP_OP_MODIFY_QP
, I40IW_CQPSQ_OPCODE
) |
2447 LS_64(info
->ord_valid
, I40IW_CQPSQ_QP_ORDVALID
) |
2448 LS_64(info
->tcp_ctx_valid
, I40IW_CQPSQ_QP_TOECTXVALID
) |
2449 LS_64(info
->cached_var_valid
, I40IW_CQPSQ_QP_CACHEDVARVALID
) |
2450 LS_64(qp
->virtual_map
, I40IW_CQPSQ_QP_VQ
) |
2451 LS_64(info
->cq_num_valid
, I40IW_CQPSQ_QP_CQNUMVALID
) |
2452 LS_64(info
->force_loopback
, I40IW_CQPSQ_QP_FORCELOOPBACK
) |
2453 LS_64(qp
->qp_type
, I40IW_CQPSQ_QP_QPTYPE
) |
2454 LS_64(info
->static_rsrc
, I40IW_CQPSQ_QP_STATRSRC
) |
2455 LS_64(info
->remove_hash_idx
, I40IW_CQPSQ_QP_REMOVEHASHENTRY
) |
2456 LS_64(term_actions
, I40IW_CQPSQ_QP_TERMACT
) |
2457 LS_64(info
->reset_tcp_conn
, I40IW_CQPSQ_QP_RESETCON
) |
2458 LS_64(info
->arp_cache_idx_valid
, I40IW_CQPSQ_QP_ARPTABIDXVALID
) |
2459 LS_64(info
->next_iwarp_state
, I40IW_CQPSQ_QP_NEXTIWSTATE
) |
2460 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2462 i40iw_insert_wqe_hdr(wqe
, header
);
2464 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QP_MODIFY WQE",
2465 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2468 i40iw_sc_cqp_post_sq(cqp
);
2473 * i40iw_sc_qp_destroy - cqp destroy qp
2475 * @scratch: u64 saved to be used during cqp completion
2476 * @remove_hash_idx: flag if to remove hash idx
2477 * @ignore_mw_bnd: memory window bind flag
2478 * @post_sq: flag for cqp db to ring
2480 static enum i40iw_status_code
i40iw_sc_qp_destroy(
2481 struct i40iw_sc_qp
*qp
,
2483 bool remove_hash_idx
,
2488 struct i40iw_sc_cqp
*cqp
;
2491 i40iw_qp_rem_qos(qp
);
2492 cqp
= qp
->pd
->dev
->cqp
;
2493 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2495 return I40IW_ERR_RING_FULL
;
2496 set_64bit_val(wqe
, 16, qp
->hw_host_ctx_pa
);
2497 set_64bit_val(wqe
, 40, qp
->shadow_area_pa
);
2499 header
= qp
->qp_uk
.qp_id
|
2500 LS_64(I40IW_CQP_OP_DESTROY_QP
, I40IW_CQPSQ_OPCODE
) |
2501 LS_64(qp
->qp_type
, I40IW_CQPSQ_QP_QPTYPE
) |
2502 LS_64(ignore_mw_bnd
, I40IW_CQPSQ_QP_IGNOREMWBOUND
) |
2503 LS_64(remove_hash_idx
, I40IW_CQPSQ_QP_REMOVEHASHENTRY
) |
2504 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2506 i40iw_insert_wqe_hdr(wqe
, header
);
2507 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QP_DESTROY WQE",
2508 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2511 i40iw_sc_cqp_post_sq(cqp
);
2516 * i40iw_sc_qp_flush_wqes - flush qp's wqe
2518 * @info: dlush information
2519 * @scratch: u64 saved to be used during cqp completion
2520 * @post_sq: flag for cqp db to ring
2522 static enum i40iw_status_code
i40iw_sc_qp_flush_wqes(
2523 struct i40iw_sc_qp
*qp
,
2524 struct i40iw_qp_flush_info
*info
,
2530 struct i40iw_sc_cqp
*cqp
;
2532 bool flush_sq
= false, flush_rq
= false;
2534 if (info
->rq
&& !qp
->flush_rq
)
2537 if (info
->sq
&& !qp
->flush_sq
)
2540 qp
->flush_sq
|= flush_sq
;
2541 qp
->flush_rq
|= flush_rq
;
2542 if (!flush_sq
&& !flush_rq
) {
2543 if (info
->ae_code
!= I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR
)
2547 cqp
= qp
->pd
->dev
->cqp
;
2548 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2550 return I40IW_ERR_RING_FULL
;
2551 if (info
->userflushcode
) {
2553 temp
|= LS_64(info
->rq_minor_code
, I40IW_CQPSQ_FWQE_RQMNERR
) |
2554 LS_64(info
->rq_major_code
, I40IW_CQPSQ_FWQE_RQMJERR
);
2557 temp
|= LS_64(info
->sq_minor_code
, I40IW_CQPSQ_FWQE_SQMNERR
) |
2558 LS_64(info
->sq_major_code
, I40IW_CQPSQ_FWQE_SQMJERR
);
2561 set_64bit_val(wqe
, 16, temp
);
2563 temp
= (info
->generate_ae
) ?
2564 info
->ae_code
| LS_64(info
->ae_source
, I40IW_CQPSQ_FWQE_AESOURCE
) : 0;
2566 set_64bit_val(wqe
, 8, temp
);
2568 header
= qp
->qp_uk
.qp_id
|
2569 LS_64(I40IW_CQP_OP_FLUSH_WQES
, I40IW_CQPSQ_OPCODE
) |
2570 LS_64(info
->generate_ae
, I40IW_CQPSQ_FWQE_GENERATE_AE
) |
2571 LS_64(info
->userflushcode
, I40IW_CQPSQ_FWQE_USERFLCODE
) |
2572 LS_64(flush_sq
, I40IW_CQPSQ_FWQE_FLUSHSQ
) |
2573 LS_64(flush_rq
, I40IW_CQPSQ_FWQE_FLUSHRQ
) |
2574 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2576 i40iw_insert_wqe_hdr(wqe
, header
);
2578 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QP_FLUSH WQE",
2579 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2582 i40iw_sc_cqp_post_sq(cqp
);
2587 * i40iw_sc_qp_upload_context - upload qp's context
2588 * @dev: sc device struct
2589 * @info: upload context info ptr for return
2590 * @scratch: u64 saved to be used during cqp completion
2591 * @post_sq: flag for cqp db to ring
2593 static enum i40iw_status_code
i40iw_sc_qp_upload_context(
2594 struct i40iw_sc_dev
*dev
,
2595 struct i40iw_upload_context_info
*info
,
2600 struct i40iw_sc_cqp
*cqp
;
2604 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2606 return I40IW_ERR_RING_FULL
;
2607 set_64bit_val(wqe
, 16, info
->buf_pa
);
2609 header
= LS_64(info
->qp_id
, I40IW_CQPSQ_UCTX_QPID
) |
2610 LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT
, I40IW_CQPSQ_OPCODE
) |
2611 LS_64(info
->qp_type
, I40IW_CQPSQ_UCTX_QPTYPE
) |
2612 LS_64(info
->raw_format
, I40IW_CQPSQ_UCTX_RAWFORMAT
) |
2613 LS_64(info
->freeze_qp
, I40IW_CQPSQ_UCTX_FREEZEQP
) |
2614 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2616 i40iw_insert_wqe_hdr(wqe
, header
);
2618 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "QP_UPLOAD_CTX WQE",
2619 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2622 i40iw_sc_cqp_post_sq(cqp
);
2627 * i40iw_sc_qp_setctx - set qp's context
2629 * @qp_ctx: context ptr
2632 static enum i40iw_status_code
i40iw_sc_qp_setctx(
2633 struct i40iw_sc_qp
*qp
,
2635 struct i40iw_qp_host_ctx_info
*info
)
2637 struct i40iwarp_offload_info
*iw
;
2638 struct i40iw_tcp_offload_info
*tcp
;
2639 struct i40iw_sc_vsi
*vsi
;
2640 struct i40iw_sc_dev
*dev
;
2641 u64 qw0
, qw3
, qw7
= 0;
2643 iw
= info
->iwarp_info
;
2644 tcp
= info
->tcp_info
;
2647 if (info
->add_to_qoslist
) {
2648 qp
->user_pri
= info
->user_pri
;
2649 i40iw_qp_add_qos(qp
);
2650 i40iw_debug(qp
->dev
, I40IW_DEBUG_DCB
, "%s qp[%d] UP[%d] qset[%d]\n",
2651 __func__
, qp
->qp_uk
.qp_id
, qp
->user_pri
, qp
->qs_handle
);
2653 qw0
= LS_64(qp
->qp_uk
.rq_wqe_size
, I40IWQPC_RQWQESIZE
) |
2654 LS_64(info
->err_rq_idx_valid
, I40IWQPC_ERR_RQ_IDX_VALID
) |
2655 LS_64(qp
->rcv_tph_en
, I40IWQPC_RCVTPHEN
) |
2656 LS_64(qp
->xmit_tph_en
, I40IWQPC_XMITTPHEN
) |
2657 LS_64(qp
->rq_tph_en
, I40IWQPC_RQTPHEN
) |
2658 LS_64(qp
->sq_tph_en
, I40IWQPC_SQTPHEN
) |
2659 LS_64(info
->push_idx
, I40IWQPC_PPIDX
) |
2660 LS_64(info
->push_mode_en
, I40IWQPC_PMENA
);
2662 set_64bit_val(qp_ctx
, 8, qp
->sq_pa
);
2663 set_64bit_val(qp_ctx
, 16, qp
->rq_pa
);
2665 qw3
= LS_64(qp
->src_mac_addr_idx
, I40IWQPC_SRCMACADDRIDX
) |
2666 LS_64(qp
->hw_rq_size
, I40IWQPC_RQSIZE
) |
2667 LS_64(qp
->hw_sq_size
, I40IWQPC_SQSIZE
);
2669 set_64bit_val(qp_ctx
,
2671 LS_64(info
->err_rq_idx
, I40IWQPC_ERR_RQ_IDX
));
2673 set_64bit_val(qp_ctx
,
2675 LS_64(info
->send_cq_num
, I40IWQPC_TXCQNUM
) |
2676 LS_64(info
->rcv_cq_num
, I40IWQPC_RXCQNUM
));
2678 set_64bit_val(qp_ctx
,
2680 LS_64(info
->qp_compl_ctx
, I40IWQPC_QPCOMPCTX
));
2681 set_64bit_val(qp_ctx
,
2683 LS_64(qp
->sq_tph_val
, I40IWQPC_SQTPHVAL
) |
2684 LS_64(qp
->rq_tph_val
, I40IWQPC_RQTPHVAL
) |
2685 LS_64(qp
->qs_handle
, I40IWQPC_QSHANDLE
) |
2686 LS_64(qp
->exception_lan_queue
, I40IWQPC_EXCEPTION_LAN_QUEUE
));
2688 if (info
->iwarp_info_valid
) {
2689 qw0
|= LS_64(iw
->ddp_ver
, I40IWQPC_DDP_VER
) |
2690 LS_64(iw
->rdmap_ver
, I40IWQPC_RDMAP_VER
);
2692 qw7
|= LS_64(iw
->pd_id
, I40IWQPC_PDIDX
);
2693 set_64bit_val(qp_ctx
,
2695 LS_64(qp
->q2_pa
, I40IWQPC_Q2ADDR
) |
2696 LS_64(vsi
->fcn_id
, I40IWQPC_STAT_INDEX
));
2697 set_64bit_val(qp_ctx
,
2699 LS_64(iw
->last_byte_sent
, I40IWQPC_LASTBYTESENT
));
2701 set_64bit_val(qp_ctx
,
2703 LS_64(iw
->ord_size
, I40IWQPC_ORDSIZE
) |
2704 LS_64(iw
->ird_size
, I40IWQPC_IRDSIZE
) |
2705 LS_64(iw
->wr_rdresp_en
, I40IWQPC_WRRDRSPOK
) |
2706 LS_64(iw
->rd_enable
, I40IWQPC_RDOK
) |
2707 LS_64(iw
->snd_mark_en
, I40IWQPC_SNDMARKERS
) |
2708 LS_64(iw
->bind_en
, I40IWQPC_BINDEN
) |
2709 LS_64(iw
->fast_reg_en
, I40IWQPC_FASTREGEN
) |
2710 LS_64(iw
->priv_mode_en
, I40IWQPC_PRIVEN
) |
2711 LS_64((((vsi
->stats_fcn_id_alloc
) &&
2712 (dev
->is_pf
) && (vsi
->fcn_id
>= I40IW_FIRST_NON_PF_STAT
)) ? 1 : 0),
2713 I40IWQPC_USESTATSINSTANCE
) |
2714 LS_64(1, I40IWQPC_IWARPMODE
) |
2715 LS_64(iw
->rcv_mark_en
, I40IWQPC_RCVMARKERS
) |
2716 LS_64(iw
->align_hdrs
, I40IWQPC_ALIGNHDRS
) |
2717 LS_64(iw
->rcv_no_mpa_crc
, I40IWQPC_RCVNOMPACRC
) |
2718 LS_64(iw
->rcv_mark_offset
, I40IWQPC_RCVMARKOFFSET
) |
2719 LS_64(iw
->snd_mark_offset
, I40IWQPC_SNDMARKOFFSET
));
2721 if (info
->tcp_info_valid
) {
2722 qw0
|= LS_64(tcp
->ipv4
, I40IWQPC_IPV4
) |
2723 LS_64(tcp
->no_nagle
, I40IWQPC_NONAGLE
) |
2724 LS_64(tcp
->insert_vlan_tag
, I40IWQPC_INSERTVLANTAG
) |
2725 LS_64(tcp
->time_stamp
, I40IWQPC_TIMESTAMP
) |
2726 LS_64(tcp
->cwnd_inc_limit
, I40IWQPC_LIMIT
) |
2727 LS_64(tcp
->drop_ooo_seg
, I40IWQPC_DROPOOOSEG
) |
2728 LS_64(tcp
->dup_ack_thresh
, I40IWQPC_DUPACK_THRESH
);
2730 qw3
|= LS_64(tcp
->ttl
, I40IWQPC_TTL
) |
2731 LS_64(tcp
->src_mac_addr_idx
, I40IWQPC_SRCMACADDRIDX
) |
2732 LS_64(tcp
->avoid_stretch_ack
, I40IWQPC_AVOIDSTRETCHACK
) |
2733 LS_64(tcp
->tos
, I40IWQPC_TOS
) |
2734 LS_64(tcp
->src_port
, I40IWQPC_SRCPORTNUM
) |
2735 LS_64(tcp
->dst_port
, I40IWQPC_DESTPORTNUM
);
2737 qp
->src_mac_addr_idx
= tcp
->src_mac_addr_idx
;
2738 set_64bit_val(qp_ctx
,
2740 LS_64(tcp
->dest_ip_addr2
, I40IWQPC_DESTIPADDR2
) |
2741 LS_64(tcp
->dest_ip_addr3
, I40IWQPC_DESTIPADDR3
));
2743 set_64bit_val(qp_ctx
,
2745 LS_64(tcp
->dest_ip_addr0
, I40IWQPC_DESTIPADDR0
) |
2746 LS_64(tcp
->dest_ip_addr1
, I40IWQPC_DESTIPADDR1
));
2748 set_64bit_val(qp_ctx
,
2750 LS_64(tcp
->snd_mss
, I40IWQPC_SNDMSS
) |
2751 LS_64(tcp
->vlan_tag
, I40IWQPC_VLANTAG
) |
2752 LS_64(tcp
->arp_idx
, I40IWQPC_ARPIDX
));
2754 qw7
|= LS_64(tcp
->flow_label
, I40IWQPC_FLOWLABEL
) |
2755 LS_64(tcp
->wscale
, I40IWQPC_WSCALE
) |
2756 LS_64(tcp
->ignore_tcp_opt
, I40IWQPC_IGNORE_TCP_OPT
) |
2757 LS_64(tcp
->ignore_tcp_uns_opt
, I40IWQPC_IGNORE_TCP_UNS_OPT
) |
2758 LS_64(tcp
->tcp_state
, I40IWQPC_TCPSTATE
) |
2759 LS_64(tcp
->rcv_wscale
, I40IWQPC_RCVSCALE
) |
2760 LS_64(tcp
->snd_wscale
, I40IWQPC_SNDSCALE
);
2762 set_64bit_val(qp_ctx
,
2764 LS_64(tcp
->time_stamp_recent
, I40IWQPC_TIMESTAMP_RECENT
) |
2765 LS_64(tcp
->time_stamp_age
, I40IWQPC_TIMESTAMP_AGE
));
2766 set_64bit_val(qp_ctx
,
2768 LS_64(tcp
->snd_nxt
, I40IWQPC_SNDNXT
) |
2769 LS_64(tcp
->snd_wnd
, I40IWQPC_SNDWND
));
2771 set_64bit_val(qp_ctx
,
2773 LS_64(tcp
->rcv_nxt
, I40IWQPC_RCVNXT
) |
2774 LS_64(tcp
->rcv_wnd
, I40IWQPC_RCVWND
));
2775 set_64bit_val(qp_ctx
,
2777 LS_64(tcp
->snd_max
, I40IWQPC_SNDMAX
) |
2778 LS_64(tcp
->snd_una
, I40IWQPC_SNDUNA
));
2779 set_64bit_val(qp_ctx
,
2781 LS_64(tcp
->srtt
, I40IWQPC_SRTT
) |
2782 LS_64(tcp
->rtt_var
, I40IWQPC_RTTVAR
));
2783 set_64bit_val(qp_ctx
,
2785 LS_64(tcp
->ss_thresh
, I40IWQPC_SSTHRESH
) |
2786 LS_64(tcp
->cwnd
, I40IWQPC_CWND
));
2787 set_64bit_val(qp_ctx
,
2789 LS_64(tcp
->snd_wl1
, I40IWQPC_SNDWL1
) |
2790 LS_64(tcp
->snd_wl2
, I40IWQPC_SNDWL2
));
2791 set_64bit_val(qp_ctx
,
2793 LS_64(tcp
->max_snd_window
, I40IWQPC_MAXSNDWND
) |
2794 LS_64(tcp
->rexmit_thresh
, I40IWQPC_REXMIT_THRESH
));
2795 set_64bit_val(qp_ctx
,
2797 LS_64(tcp
->local_ipaddr3
, I40IWQPC_LOCAL_IPADDR3
) |
2798 LS_64(tcp
->local_ipaddr2
, I40IWQPC_LOCAL_IPADDR2
));
2799 set_64bit_val(qp_ctx
,
2801 LS_64(tcp
->local_ipaddr1
, I40IWQPC_LOCAL_IPADDR1
) |
2802 LS_64(tcp
->local_ipaddr0
, I40IWQPC_LOCAL_IPADDR0
));
2805 set_64bit_val(qp_ctx
, 0, qw0
);
2806 set_64bit_val(qp_ctx
, 24, qw3
);
2807 set_64bit_val(qp_ctx
, 56, qw7
);
2809 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_WQE
, "QP_HOST)CTX WQE",
2810 qp_ctx
, I40IW_QP_CTX_SIZE
);
2815 * i40iw_sc_alloc_stag - mr stag alloc
2816 * @dev: sc device struct
2818 * @scratch: u64 saved to be used during cqp completion
2819 * @post_sq: flag for cqp db to ring
2821 static enum i40iw_status_code
i40iw_sc_alloc_stag(
2822 struct i40iw_sc_dev
*dev
,
2823 struct i40iw_allocate_stag_info
*info
,
2828 struct i40iw_sc_cqp
*cqp
;
2830 enum i40iw_page_size page_size
;
2832 page_size
= (info
->page_size
== 0x200000) ? I40IW_PAGE_SIZE_2M
: I40IW_PAGE_SIZE_4K
;
2834 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2836 return I40IW_ERR_RING_FULL
;
2839 LS_64(info
->pd_id
, I40IW_CQPSQ_STAG_PDID
) |
2840 LS_64(info
->total_len
, I40IW_CQPSQ_STAG_STAGLEN
));
2843 LS_64(info
->stag_idx
, I40IW_CQPSQ_STAG_IDX
));
2846 LS_64(info
->hmc_fcn_index
, I40IW_CQPSQ_STAG_HMCFNIDX
));
2848 header
= LS_64(I40IW_CQP_OP_ALLOC_STAG
, I40IW_CQPSQ_OPCODE
) |
2849 LS_64(1, I40IW_CQPSQ_STAG_MR
) |
2850 LS_64(info
->access_rights
, I40IW_CQPSQ_STAG_ARIGHTS
) |
2851 LS_64(info
->chunk_size
, I40IW_CQPSQ_STAG_LPBLSIZE
) |
2852 LS_64(page_size
, I40IW_CQPSQ_STAG_HPAGESIZE
) |
2853 LS_64(info
->remote_access
, I40IW_CQPSQ_STAG_REMACCENABLED
) |
2854 LS_64(info
->use_hmc_fcn_index
, I40IW_CQPSQ_STAG_USEHMCFNIDX
) |
2855 LS_64(info
->use_pf_rid
, I40IW_CQPSQ_STAG_USEPFRID
) |
2856 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2858 i40iw_insert_wqe_hdr(wqe
, header
);
2860 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "ALLOC_STAG WQE",
2861 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2864 i40iw_sc_cqp_post_sq(cqp
);
2869 * i40iw_sc_mr_reg_non_shared - non-shared mr registration
2870 * @dev: sc device struct
2872 * @scratch: u64 saved to be used during cqp completion
2873 * @post_sq: flag for cqp db to ring
2875 static enum i40iw_status_code
i40iw_sc_mr_reg_non_shared(
2876 struct i40iw_sc_dev
*dev
,
2877 struct i40iw_reg_ns_stag_info
*info
,
2883 struct i40iw_sc_cqp
*cqp
;
2888 enum i40iw_page_size page_size
;
2890 page_size
= (info
->page_size
== 0x200000) ? I40IW_PAGE_SIZE_2M
: I40IW_PAGE_SIZE_4K
;
2891 if (info
->access_rights
& (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY
|
2892 I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY
))
2893 remote_access
= true;
2895 remote_access
= false;
2897 pble_obj_cnt
= dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
2899 if (info
->chunk_size
&& (info
->first_pm_pbl_index
>= pble_obj_cnt
))
2900 return I40IW_ERR_INVALID_PBLE_INDEX
;
2903 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2905 return I40IW_ERR_RING_FULL
;
2907 temp
= (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
) ? (uintptr_t)info
->va
: info
->fbo
;
2908 set_64bit_val(wqe
, 0, temp
);
2912 LS_64(info
->total_len
, I40IW_CQPSQ_STAG_STAGLEN
) |
2913 LS_64(info
->pd_id
, I40IW_CQPSQ_STAG_PDID
));
2917 LS_64(info
->stag_key
, I40IW_CQPSQ_STAG_KEY
) |
2918 LS_64(info
->stag_idx
, I40IW_CQPSQ_STAG_IDX
));
2919 if (!info
->chunk_size
) {
2920 set_64bit_val(wqe
, 32, info
->reg_addr_pa
);
2921 set_64bit_val(wqe
, 48, 0);
2923 set_64bit_val(wqe
, 32, 0);
2924 set_64bit_val(wqe
, 48, info
->first_pm_pbl_index
);
2926 set_64bit_val(wqe
, 40, info
->hmc_fcn_index
);
2927 set_64bit_val(wqe
, 56, 0);
2929 addr_type
= (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
) ? 1 : 0;
2930 header
= LS_64(I40IW_CQP_OP_REG_MR
, I40IW_CQPSQ_OPCODE
) |
2931 LS_64(1, I40IW_CQPSQ_STAG_MR
) |
2932 LS_64(info
->chunk_size
, I40IW_CQPSQ_STAG_LPBLSIZE
) |
2933 LS_64(page_size
, I40IW_CQPSQ_STAG_HPAGESIZE
) |
2934 LS_64(info
->access_rights
, I40IW_CQPSQ_STAG_ARIGHTS
) |
2935 LS_64(remote_access
, I40IW_CQPSQ_STAG_REMACCENABLED
) |
2936 LS_64(addr_type
, I40IW_CQPSQ_STAG_VABASEDTO
) |
2937 LS_64(info
->use_hmc_fcn_index
, I40IW_CQPSQ_STAG_USEHMCFNIDX
) |
2938 LS_64(info
->use_pf_rid
, I40IW_CQPSQ_STAG_USEPFRID
) |
2939 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2941 i40iw_insert_wqe_hdr(wqe
, header
);
2943 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "MR_REG_NS WQE",
2944 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2947 i40iw_sc_cqp_post_sq(cqp
);
2952 * i40iw_sc_mr_reg_shared - registered shared memory region
2953 * @dev: sc device struct
2954 * @info: info for shared memory registeration
2955 * @scratch: u64 saved to be used during cqp completion
2956 * @post_sq: flag for cqp db to ring
2958 static enum i40iw_status_code
i40iw_sc_mr_reg_shared(
2959 struct i40iw_sc_dev
*dev
,
2960 struct i40iw_register_shared_stag
*info
,
2965 struct i40iw_sc_cqp
*cqp
;
2966 u64 temp
, va64
, fbo
, header
;
2971 if (info
->access_rights
& (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY
|
2972 I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY
))
2973 remote_access
= true;
2975 remote_access
= false;
2977 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2979 return I40IW_ERR_RING_FULL
;
2980 va64
= (uintptr_t)(info
->va
);
2981 va32
= (u32
)(va64
& 0x00000000FFFFFFFF);
2982 fbo
= (u64
)(va32
& (4096 - 1));
2986 (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
? (uintptr_t)info
->va
: fbo
));
2990 LS_64(info
->pd_id
, I40IW_CQPSQ_STAG_PDID
));
2991 temp
= LS_64(info
->new_stag_key
, I40IW_CQPSQ_STAG_KEY
) |
2992 LS_64(info
->new_stag_idx
, I40IW_CQPSQ_STAG_IDX
) |
2993 LS_64(info
->parent_stag_idx
, I40IW_CQPSQ_STAG_PARENTSTAGIDX
);
2994 set_64bit_val(wqe
, 16, temp
);
2996 addr_type
= (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
) ? 1 : 0;
2997 header
= LS_64(I40IW_CQP_OP_REG_SMR
, I40IW_CQPSQ_OPCODE
) |
2998 LS_64(1, I40IW_CQPSQ_STAG_MR
) |
2999 LS_64(info
->access_rights
, I40IW_CQPSQ_STAG_ARIGHTS
) |
3000 LS_64(remote_access
, I40IW_CQPSQ_STAG_REMACCENABLED
) |
3001 LS_64(addr_type
, I40IW_CQPSQ_STAG_VABASEDTO
) |
3002 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3004 i40iw_insert_wqe_hdr(wqe
, header
);
3006 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "MR_REG_SHARED WQE",
3007 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3010 i40iw_sc_cqp_post_sq(cqp
);
3015 * i40iw_sc_dealloc_stag - deallocate stag
3016 * @dev: sc device struct
3017 * @info: dealloc stag info
3018 * @scratch: u64 saved to be used during cqp completion
3019 * @post_sq: flag for cqp db to ring
3021 static enum i40iw_status_code
i40iw_sc_dealloc_stag(
3022 struct i40iw_sc_dev
*dev
,
3023 struct i40iw_dealloc_stag_info
*info
,
3029 struct i40iw_sc_cqp
*cqp
;
3032 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3034 return I40IW_ERR_RING_FULL
;
3037 LS_64(info
->pd_id
, I40IW_CQPSQ_STAG_PDID
));
3040 LS_64(info
->stag_idx
, I40IW_CQPSQ_STAG_IDX
));
3042 header
= LS_64(I40IW_CQP_OP_DEALLOC_STAG
, I40IW_CQPSQ_OPCODE
) |
3043 LS_64(info
->mr
, I40IW_CQPSQ_STAG_MR
) |
3044 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3046 i40iw_insert_wqe_hdr(wqe
, header
);
3048 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "DEALLOC_STAG WQE",
3049 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3052 i40iw_sc_cqp_post_sq(cqp
);
3057 * i40iw_sc_query_stag - query hardware for stag
3058 * @dev: sc device struct
3059 * @scratch: u64 saved to be used during cqp completion
3060 * @stag_index: stag index for query
3061 * @post_sq: flag for cqp db to ring
3063 static enum i40iw_status_code
i40iw_sc_query_stag(struct i40iw_sc_dev
*dev
,
3070 struct i40iw_sc_cqp
*cqp
;
3073 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3075 return I40IW_ERR_RING_FULL
;
3078 LS_64(stag_index
, I40IW_CQPSQ_QUERYSTAG_IDX
));
3080 header
= LS_64(I40IW_CQP_OP_QUERY_STAG
, I40IW_CQPSQ_OPCODE
) |
3081 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3083 i40iw_insert_wqe_hdr(wqe
, header
);
3085 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "QUERY_STAG WQE",
3086 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3089 i40iw_sc_cqp_post_sq(cqp
);
3094 * i40iw_sc_mw_alloc - mw allocate
3095 * @dev: sc device struct
3096 * @scratch: u64 saved to be used during cqp completion
3097 * @mw_stag_index:stag index
3098 * @pd_id: pd is for this mw
3099 * @post_sq: flag for cqp db to ring
3101 static enum i40iw_status_code
i40iw_sc_mw_alloc(
3102 struct i40iw_sc_dev
*dev
,
3109 struct i40iw_sc_cqp
*cqp
;
3113 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3115 return I40IW_ERR_RING_FULL
;
3116 set_64bit_val(wqe
, 8, LS_64(pd_id
, I40IW_CQPSQ_STAG_PDID
));
3119 LS_64(mw_stag_index
, I40IW_CQPSQ_STAG_IDX
));
3121 header
= LS_64(I40IW_CQP_OP_ALLOC_STAG
, I40IW_CQPSQ_OPCODE
) |
3122 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3124 i40iw_insert_wqe_hdr(wqe
, header
);
3126 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "MW_ALLOC WQE",
3127 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3130 i40iw_sc_cqp_post_sq(cqp
);
3135 * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
3137 * @info: fast mr info
3138 * @post_sq: flag for cqp db to ring
3140 enum i40iw_status_code
i40iw_sc_mr_fast_register(
3141 struct i40iw_sc_qp
*qp
,
3142 struct i40iw_fast_reg_stag_info
*info
,
3148 enum i40iw_page_size page_size
;
3150 page_size
= (info
->page_size
== 0x200000) ? I40IW_PAGE_SIZE_2M
: I40IW_PAGE_SIZE_4K
;
3151 wqe
= i40iw_qp_get_next_send_wqe(&qp
->qp_uk
, &wqe_idx
, I40IW_QP_WQE_MIN_SIZE
,
3154 return I40IW_ERR_QP_TOOMANY_WRS_POSTED
;
3156 i40iw_debug(qp
->dev
, I40IW_DEBUG_MR
, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
3157 __func__
, info
->wr_id
, wqe_idx
,
3158 &qp
->qp_uk
.sq_wrtrk_array
[wqe_idx
].wrid
);
3159 temp
= (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
) ? (uintptr_t)info
->va
: info
->fbo
;
3160 set_64bit_val(wqe
, 0, temp
);
3162 temp
= RS_64(info
->first_pm_pbl_index
>> 16, I40IWQPSQ_FIRSTPMPBLIDXHI
);
3165 LS_64(temp
, I40IWQPSQ_FIRSTPMPBLIDXHI
) |
3166 LS_64(info
->reg_addr_pa
>> I40IWQPSQ_PBLADDR_SHIFT
, I40IWQPSQ_PBLADDR
));
3171 LS_64(info
->first_pm_pbl_index
, I40IWQPSQ_FIRSTPMPBLIDXLO
));
3173 header
= LS_64(info
->stag_key
, I40IWQPSQ_STAGKEY
) |
3174 LS_64(info
->stag_idx
, I40IWQPSQ_STAGINDEX
) |
3175 LS_64(I40IWQP_OP_FAST_REGISTER
, I40IWQPSQ_OPCODE
) |
3176 LS_64(info
->chunk_size
, I40IWQPSQ_LPBLSIZE
) |
3177 LS_64(page_size
, I40IWQPSQ_HPAGESIZE
) |
3178 LS_64(info
->access_rights
, I40IWQPSQ_STAGRIGHTS
) |
3179 LS_64(info
->addr_type
, I40IWQPSQ_VABASEDTO
) |
3180 LS_64(info
->read_fence
, I40IWQPSQ_READFENCE
) |
3181 LS_64(info
->local_fence
, I40IWQPSQ_LOCALFENCE
) |
3182 LS_64(info
->signaled
, I40IWQPSQ_SIGCOMPL
) |
3183 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3185 i40iw_insert_wqe_hdr(wqe
, header
);
3187 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_WQE
, "FAST_REG WQE",
3188 wqe
, I40IW_QP_WQE_MIN_SIZE
);
3191 i40iw_qp_post_wr(&qp
->qp_uk
);
3196 * i40iw_sc_send_lsmm - send last streaming mode message
3198 * @lsmm_buf: buffer with lsmm message
3199 * @size: size of lsmm buffer
3200 * @stag: stag of lsmm buffer
3202 static void i40iw_sc_send_lsmm(struct i40iw_sc_qp
*qp
,
3209 struct i40iw_qp_uk
*qp_uk
;
3212 wqe
= qp_uk
->sq_base
->elem
;
3214 set_64bit_val(wqe
, 0, (uintptr_t)lsmm_buf
);
3216 set_64bit_val(wqe
, 8, (size
| LS_64(stag
, I40IWQPSQ_FRAG_STAG
)));
3218 set_64bit_val(wqe
, 16, 0);
3220 header
= LS_64(I40IWQP_OP_RDMA_SEND
, I40IWQPSQ_OPCODE
) |
3221 LS_64(1, I40IWQPSQ_STREAMMODE
) |
3222 LS_64(1, I40IWQPSQ_WAITFORRCVPDU
) |
3223 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3225 i40iw_insert_wqe_hdr(wqe
, header
);
3227 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_QP
, "SEND_LSMM WQE",
3228 wqe
, I40IW_QP_WQE_MIN_SIZE
);
3232 * i40iw_sc_send_lsmm_nostag - for privilege qp
3234 * @lsmm_buf: buffer with lsmm message
3235 * @size: size of lsmm buffer
3237 static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp
*qp
,
3243 struct i40iw_qp_uk
*qp_uk
;
3246 wqe
= qp_uk
->sq_base
->elem
;
3248 set_64bit_val(wqe
, 0, (uintptr_t)lsmm_buf
);
3250 set_64bit_val(wqe
, 8, size
);
3252 set_64bit_val(wqe
, 16, 0);
3254 header
= LS_64(I40IWQP_OP_RDMA_SEND
, I40IWQPSQ_OPCODE
) |
3255 LS_64(1, I40IWQPSQ_STREAMMODE
) |
3256 LS_64(1, I40IWQPSQ_WAITFORRCVPDU
) |
3257 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3259 i40iw_insert_wqe_hdr(wqe
, header
);
3261 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_WQE
, "SEND_LSMM_NOSTAG WQE",
3262 wqe
, I40IW_QP_WQE_MIN_SIZE
);
3266 * i40iw_sc_send_rtt - send last read0 or write0
3268 * @read: Do read0 or write0
3270 static void i40iw_sc_send_rtt(struct i40iw_sc_qp
*qp
, bool read
)
3274 struct i40iw_qp_uk
*qp_uk
;
3277 wqe
= qp_uk
->sq_base
->elem
;
3279 set_64bit_val(wqe
, 0, 0);
3280 set_64bit_val(wqe
, 8, 0);
3281 set_64bit_val(wqe
, 16, 0);
3283 header
= LS_64(0x1234, I40IWQPSQ_REMSTAG
) |
3284 LS_64(I40IWQP_OP_RDMA_READ
, I40IWQPSQ_OPCODE
) |
3285 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3286 set_64bit_val(wqe
, 8, ((u64
)0xabcd << 32));
3288 header
= LS_64(I40IWQP_OP_RDMA_WRITE
, I40IWQPSQ_OPCODE
) |
3289 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3292 i40iw_insert_wqe_hdr(wqe
, header
);
3294 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_WQE
, "RTR WQE",
3295 wqe
, I40IW_QP_WQE_MIN_SIZE
);
3299 * i40iw_sc_post_wqe0 - send wqe with opcode
3301 * @opcode: opcode to use for wqe0
3303 static enum i40iw_status_code
i40iw_sc_post_wqe0(struct i40iw_sc_qp
*qp
, u8 opcode
)
3307 struct i40iw_qp_uk
*qp_uk
;
3310 wqe
= qp_uk
->sq_base
->elem
;
3313 return I40IW_ERR_QP_TOOMANY_WRS_POSTED
;
3315 case I40IWQP_OP_NOP
:
3316 set_64bit_val(wqe
, 0, 0);
3317 set_64bit_val(wqe
, 8, 0);
3318 set_64bit_val(wqe
, 16, 0);
3319 header
= LS_64(I40IWQP_OP_NOP
, I40IWQPSQ_OPCODE
) |
3320 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3322 i40iw_insert_wqe_hdr(wqe
, header
);
3324 case I40IWQP_OP_RDMA_SEND
:
3325 set_64bit_val(wqe
, 0, 0);
3326 set_64bit_val(wqe
, 8, 0);
3327 set_64bit_val(wqe
, 16, 0);
3328 header
= LS_64(I40IWQP_OP_RDMA_SEND
, I40IWQPSQ_OPCODE
) |
3329 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
) |
3330 LS_64(1, I40IWQPSQ_STREAMMODE
) |
3331 LS_64(1, I40IWQPSQ_WAITFORRCVPDU
);
3333 i40iw_insert_wqe_hdr(wqe
, header
);
3336 i40iw_debug(qp
->dev
, I40IW_DEBUG_QP
, "%s: Invalid WQE zero opcode\n",
3344 * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
3345 * @dev : ptr to i40iw_dev struct
3346 * @hmc_fn_id: hmc function id
3348 enum i40iw_status_code
i40iw_sc_init_iw_hmc(struct i40iw_sc_dev
*dev
, u8 hmc_fn_id
)
3350 struct i40iw_hmc_info
*hmc_info
;
3351 struct i40iw_dma_mem query_fpm_mem
;
3352 struct i40iw_virt_mem virt_mem
;
3353 struct i40iw_vfdev
*vf_dev
= NULL
;
3355 enum i40iw_status_code ret_code
= 0;
3356 bool poll_registers
= true;
3360 if (hmc_fn_id
>= I40IW_MAX_VF_FPM_ID
||
3361 (dev
->hmc_fn_id
!= hmc_fn_id
&& hmc_fn_id
< I40IW_FIRST_VF_FPM_ID
))
3362 return I40IW_ERR_INVALID_HMCFN_ID
;
3364 i40iw_debug(dev
, I40IW_DEBUG_HMC
, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id
,
3366 if (hmc_fn_id
== dev
->hmc_fn_id
) {
3367 hmc_info
= dev
->hmc_info
;
3368 query_fpm_mem
.pa
= dev
->fpm_query_buf_pa
;
3369 query_fpm_mem
.va
= dev
->fpm_query_buf
;
3371 vf_dev
= i40iw_vfdev_from_fpm(dev
, hmc_fn_id
);
3373 return I40IW_ERR_INVALID_VF_ID
;
3375 hmc_info
= &vf_dev
->hmc_info
;
3376 iw_vf_idx
= vf_dev
->iw_vf_idx
;
3377 i40iw_debug(dev
, I40IW_DEBUG_HMC
, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev
,
3378 hmc_info
, hmc_info
->hmc_obj
);
3379 if (!vf_dev
->fpm_query_buf
) {
3380 if (!dev
->vf_fpm_query_buf
[iw_vf_idx
].va
) {
3381 ret_code
= i40iw_alloc_query_fpm_buf(dev
,
3382 &dev
->vf_fpm_query_buf
[iw_vf_idx
]);
3386 vf_dev
->fpm_query_buf
= dev
->vf_fpm_query_buf
[iw_vf_idx
].va
;
3387 vf_dev
->fpm_query_buf_pa
= dev
->vf_fpm_query_buf
[iw_vf_idx
].pa
;
3389 query_fpm_mem
.pa
= vf_dev
->fpm_query_buf_pa
;
3390 query_fpm_mem
.va
= vf_dev
->fpm_query_buf
;
3392 * It is HARDWARE specific:
3393 * this call is done by PF for VF and
3394 * i40iw_sc_query_fpm_values needs ccq poll
3395 * because PF ccq is already created.
3397 poll_registers
= false;
3400 hmc_info
->hmc_fn_id
= hmc_fn_id
;
3402 if (hmc_fn_id
!= dev
->hmc_fn_id
) {
3404 i40iw_cqp_query_fpm_values_cmd(dev
, &query_fpm_mem
, hmc_fn_id
);
3406 wait_type
= poll_registers
? (u8
)I40IW_CQP_WAIT_POLL_REGS
:
3407 (u8
)I40IW_CQP_WAIT_POLL_CQ
;
3409 ret_code
= i40iw_sc_query_fpm_values(
3412 hmc_info
->hmc_fn_id
,
3420 /* parse the fpm_query_buf and fill hmc obj info */
3422 i40iw_sc_parse_fpm_query_buf((u64
*)query_fpm_mem
.va
,
3424 &dev
->hmc_fpm_misc
);
3427 i40iw_debug_buf(dev
, I40IW_DEBUG_HMC
, "QUERY FPM BUFFER",
3428 query_fpm_mem
.va
, I40IW_QUERY_FPM_BUF_SIZE
);
3430 if (hmc_fn_id
!= dev
->hmc_fn_id
) {
3431 i40iw_cqp_commit_fpm_values_cmd(dev
, &query_fpm_mem
, hmc_fn_id
);
3433 /* parse the fpm_commit_buf and fill hmc obj info */
3434 i40iw_sc_parse_fpm_commit_buf((u64
*)query_fpm_mem
.va
, hmc_info
->hmc_obj
, &hmc_info
->sd_table
.sd_cnt
);
3435 mem_size
= sizeof(struct i40iw_hmc_sd_entry
) *
3436 (hmc_info
->sd_table
.sd_cnt
+ hmc_info
->first_sd_index
);
3437 ret_code
= i40iw_allocate_virt_mem(dev
->hw
, &virt_mem
, mem_size
);
3440 hmc_info
->sd_table
.sd_entry
= virt_mem
.va
;
3447 * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
3448 * populates fpm base address in hmc_info
3449 * @dev : ptr to i40iw_dev struct
3450 * @hmc_fn_id: hmc function id
3452 static enum i40iw_status_code
i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev
*dev
,
3455 struct i40iw_hmc_info
*hmc_info
;
3456 struct i40iw_hmc_obj_info
*obj_info
;
3458 struct i40iw_dma_mem commit_fpm_mem
;
3460 enum i40iw_status_code ret_code
= 0;
3461 bool poll_registers
= true;
3464 if (hmc_fn_id
>= I40IW_MAX_VF_FPM_ID
||
3465 (dev
->hmc_fn_id
!= hmc_fn_id
&& hmc_fn_id
< I40IW_FIRST_VF_FPM_ID
))
3466 return I40IW_ERR_INVALID_HMCFN_ID
;
3468 if (hmc_fn_id
== dev
->hmc_fn_id
) {
3469 hmc_info
= dev
->hmc_info
;
3471 hmc_info
= i40iw_vf_hmcinfo_from_fpm(dev
, hmc_fn_id
);
3472 poll_registers
= false;
3475 return I40IW_ERR_BAD_PTR
;
3477 obj_info
= hmc_info
->hmc_obj
;
3478 buf
= dev
->fpm_commit_buf
;
3480 /* copy cnt values in commit buf */
3481 for (i
= I40IW_HMC_IW_QP
, j
= 0; i
<= I40IW_HMC_IW_PBLE
;
3483 set_64bit_val(buf
, j
, (u64
)obj_info
[i
].cnt
);
3485 set_64bit_val(buf
, 40, 0); /* APBVT rsvd */
3487 commit_fpm_mem
.pa
= dev
->fpm_commit_buf_pa
;
3488 commit_fpm_mem
.va
= dev
->fpm_commit_buf
;
3489 wait_type
= poll_registers
? (u8
)I40IW_CQP_WAIT_POLL_REGS
:
3490 (u8
)I40IW_CQP_WAIT_POLL_CQ
;
3491 ret_code
= i40iw_sc_commit_fpm_values(
3494 hmc_info
->hmc_fn_id
,
3499 /* parse the fpm_commit_buf and fill hmc obj info */
3501 ret_code
= i40iw_sc_parse_fpm_commit_buf(dev
->fpm_commit_buf
,
3503 &hmc_info
->sd_table
.sd_cnt
);
3505 i40iw_debug_buf(dev
, I40IW_DEBUG_HMC
, "COMMIT FPM BUFFER",
3506 commit_fpm_mem
.va
, I40IW_COMMIT_FPM_BUF_SIZE
);
3512 * cqp_sds_wqe_fill - fill cqp wqe doe sd
3513 * @cqp: struct for cqp hw
3514 * @info; sd info for wqe
3515 * @scratch: u64 saved to be used during cqp completion
3517 static enum i40iw_status_code
cqp_sds_wqe_fill(struct i40iw_sc_cqp
*cqp
,
3518 struct i40iw_update_sds_info
*info
,
3524 int mem_entries
, wqe_entries
;
3525 struct i40iw_dma_mem
*sdbuf
= &cqp
->sdbuf
;
3527 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3529 return I40IW_ERR_RING_FULL
;
3531 I40IW_CQP_INIT_WQE(wqe
);
3532 wqe_entries
= (info
->cnt
> 3) ? 3 : info
->cnt
;
3533 mem_entries
= info
->cnt
- wqe_entries
;
3535 header
= LS_64(I40IW_CQP_OP_UPDATE_PE_SDS
, I40IW_CQPSQ_OPCODE
) |
3536 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
) |
3537 LS_64(mem_entries
, I40IW_CQPSQ_UPESD_ENTRY_COUNT
);
3540 memcpy(sdbuf
->va
, &info
->entry
[3], (mem_entries
<< 4));
3545 data
|= LS_64(info
->hmc_fn_id
, I40IW_CQPSQ_UPESD_HMCFNID
);
3547 set_64bit_val(wqe
, 16, data
);
3549 switch (wqe_entries
) {
3551 set_64bit_val(wqe
, 48,
3552 (LS_64(info
->entry
[2].cmd
, I40IW_CQPSQ_UPESD_SDCMD
) |
3553 LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID
)));
3555 set_64bit_val(wqe
, 56, info
->entry
[2].data
);
3558 set_64bit_val(wqe
, 32,
3559 (LS_64(info
->entry
[1].cmd
, I40IW_CQPSQ_UPESD_SDCMD
) |
3560 LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID
)));
3562 set_64bit_val(wqe
, 40, info
->entry
[1].data
);
3565 set_64bit_val(wqe
, 0,
3566 LS_64(info
->entry
[0].cmd
, I40IW_CQPSQ_UPESD_SDCMD
));
3568 set_64bit_val(wqe
, 8, info
->entry
[0].data
);
3574 i40iw_insert_wqe_hdr(wqe
, header
);
3576 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "UPDATE_PE_SDS WQE",
3577 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3582 * i40iw_update_pe_sds - cqp wqe for sd
3583 * @dev: ptr to i40iw_dev struct
3584 * @info: sd info for sd's
3585 * @scratch: u64 saved to be used during cqp completion
3587 static enum i40iw_status_code
i40iw_update_pe_sds(struct i40iw_sc_dev
*dev
,
3588 struct i40iw_update_sds_info
*info
,
3591 struct i40iw_sc_cqp
*cqp
= dev
->cqp
;
3592 enum i40iw_status_code ret_code
;
3594 ret_code
= cqp_sds_wqe_fill(cqp
, info
, scratch
);
3596 i40iw_sc_cqp_post_sq(cqp
);
3602 * i40iw_update_sds_noccq - update sd before ccq created
3603 * @dev: sc device struct
3604 * @info: sd info for sd's
3606 enum i40iw_status_code
i40iw_update_sds_noccq(struct i40iw_sc_dev
*dev
,
3607 struct i40iw_update_sds_info
*info
)
3609 u32 error
, val
, tail
;
3610 struct i40iw_sc_cqp
*cqp
= dev
->cqp
;
3611 enum i40iw_status_code ret_code
;
3613 ret_code
= cqp_sds_wqe_fill(cqp
, info
, 0);
3616 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
3618 return I40IW_ERR_CQP_COMPL_ERROR
;
3620 i40iw_sc_cqp_post_sq(cqp
);
3621 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, I40IW_DONE_COUNT
);
3627 * i40iw_sc_suspend_qp - suspend qp for param change
3628 * @cqp: struct for cqp hw
3630 * @scratch: u64 saved to be used during cqp completion
3632 enum i40iw_status_code
i40iw_sc_suspend_qp(struct i40iw_sc_cqp
*cqp
,
3633 struct i40iw_sc_qp
*qp
,
3639 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3641 return I40IW_ERR_RING_FULL
;
3642 header
= LS_64(qp
->qp_uk
.qp_id
, I40IW_CQPSQ_SUSPENDQP_QPID
) |
3643 LS_64(I40IW_CQP_OP_SUSPEND_QP
, I40IW_CQPSQ_OPCODE
) |
3644 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3646 i40iw_insert_wqe_hdr(wqe
, header
);
3648 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "SUSPEND_QP WQE",
3649 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3651 i40iw_sc_cqp_post_sq(cqp
);
3656 * i40iw_sc_resume_qp - resume qp after suspend
3657 * @cqp: struct for cqp hw
3659 * @scratch: u64 saved to be used during cqp completion
3661 enum i40iw_status_code
i40iw_sc_resume_qp(struct i40iw_sc_cqp
*cqp
,
3662 struct i40iw_sc_qp
*qp
,
3668 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3670 return I40IW_ERR_RING_FULL
;
3673 LS_64(qp
->qs_handle
, I40IW_CQPSQ_RESUMEQP_QSHANDLE
));
3675 header
= LS_64(qp
->qp_uk
.qp_id
, I40IW_CQPSQ_RESUMEQP_QPID
) |
3676 LS_64(I40IW_CQP_OP_RESUME_QP
, I40IW_CQPSQ_OPCODE
) |
3677 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3679 i40iw_insert_wqe_hdr(wqe
, header
);
3681 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "RESUME_QP WQE",
3682 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3684 i40iw_sc_cqp_post_sq(cqp
);
3689 * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
3690 * @cqp: struct for cqp hw
3691 * @scratch: u64 saved to be used during cqp completion
3692 * @hmc_fn_id: hmc function id
3693 * @post_sq: flag for cqp db to ring
3694 * @poll_registers: flag to poll register for cqp completion
3696 enum i40iw_status_code
i40iw_sc_static_hmc_pages_allocated(
3697 struct i40iw_sc_cqp
*cqp
,
3701 bool poll_registers
)
3705 u32 tail
, val
, error
;
3706 enum i40iw_status_code ret_code
= 0;
3708 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3710 return I40IW_ERR_RING_FULL
;
3713 LS_64(hmc_fn_id
, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID
));
3715 header
= LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED
, I40IW_CQPSQ_OPCODE
) |
3716 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3718 i40iw_insert_wqe_hdr(wqe
, header
);
3720 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "SHMC_PAGES_ALLOCATED WQE",
3721 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3722 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
3724 ret_code
= I40IW_ERR_CQP_COMPL_ERROR
;
3728 i40iw_sc_cqp_post_sq(cqp
);
3730 /* check for cqp sq tail update */
3731 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, 1000);
3733 ret_code
= i40iw_sc_poll_for_cqp_op_done(cqp
,
3734 I40IW_CQP_OP_SHMC_PAGES_ALLOCATED
,
3742 * i40iw_ring_full - check if cqp ring is full
3743 * @cqp: struct for cqp hw
3745 static bool i40iw_ring_full(struct i40iw_sc_cqp
*cqp
)
3747 return I40IW_RING_FULL_ERR(cqp
->sq_ring
);
3751 * i40iw_est_sd - returns approximate number of SDs for HMC
3752 * @dev: sc device struct
3753 * @hmc_info: hmc structure, size and count for HMC objects
3755 static u64
i40iw_est_sd(struct i40iw_sc_dev
*dev
, struct i40iw_hmc_info
*hmc_info
)
3761 for (i
= I40IW_HMC_IW_QP
; i
< I40IW_HMC_IW_PBLE
; i
++)
3762 size
+= hmc_info
->hmc_obj
[i
].cnt
* hmc_info
->hmc_obj
[i
].size
;
3765 size
+= hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
* hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].size
;
3767 if (size
& 0x1FFFFF)
3768 sd
= (size
>> 21) + 1; /* add 1 for remainder */
3773 /* 2MB alignment for VF PBLE HMC */
3774 size
= hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
* hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].size
;
3775 if (size
& 0x1FFFFF)
3776 sd
+= (size
>> 21) + 1; /* add 1 for remainder */
3785 * i40iw_config_fpm_values - configure HMC objects
3786 * @dev: sc device struct
3787 * @qp_count: desired qp count
3789 enum i40iw_status_code
i40iw_config_fpm_values(struct i40iw_sc_dev
*dev
, u32 qp_count
)
3791 struct i40iw_virt_mem virt_mem
;
3793 u32 qpwantedoriginal
, qpwanted
, mrwanted
, pblewanted
;
3798 struct i40iw_hmc_info
*hmc_info
;
3799 struct i40iw_hmc_fpm_misc
*hmc_fpm_misc
;
3800 enum i40iw_status_code ret_code
= 0;
3802 hmc_info
= dev
->hmc_info
;
3803 hmc_fpm_misc
= &dev
->hmc_fpm_misc
;
3805 ret_code
= i40iw_sc_init_iw_hmc(dev
, dev
->hmc_fn_id
);
3807 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3808 "i40iw_sc_init_iw_hmc returned error_code = %d\n",
3813 for (i
= I40IW_HMC_IW_QP
; i
< I40IW_HMC_IW_MAX
; i
++)
3814 hmc_info
->hmc_obj
[i
].cnt
= hmc_info
->hmc_obj
[i
].max_cnt
;
3815 sd_needed
= i40iw_est_sd(dev
, hmc_info
);
3816 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3817 "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n",
3818 __func__
, sd_needed
, hmc_info
->first_sd_index
);
3819 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3820 "%s: sd count %d where max sd is %d\n",
3821 __func__
, hmc_info
->sd_table
.sd_cnt
,
3822 hmc_fpm_misc
->max_sds
);
3824 qpwanted
= min(qp_count
, hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].max_cnt
);
3825 qpwantedoriginal
= qpwanted
;
3826 mrwanted
= hmc_info
->hmc_obj
[I40IW_HMC_IW_MR
].max_cnt
;
3827 pblewanted
= hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].max_cnt
;
3829 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3830 "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",
3831 qp_count
, hmc_fpm_misc
->max_sds
,
3832 hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].max_cnt
,
3833 hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].max_cnt
,
3834 hmc_info
->hmc_obj
[I40IW_HMC_IW_MR
].max_cnt
,
3835 hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].max_cnt
);
3839 hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].cnt
= qpwanted
;
3840 hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
=
3841 min(2 * qpwanted
, hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
);
3842 hmc_info
->hmc_obj
[I40IW_HMC_IW_SRQ
].cnt
= 0x00; /* Reserved */
3843 hmc_info
->hmc_obj
[I40IW_HMC_IW_HTE
].cnt
=
3844 qpwanted
* hmc_fpm_misc
->ht_multiplier
;
3845 hmc_info
->hmc_obj
[I40IW_HMC_IW_ARP
].cnt
=
3846 hmc_info
->hmc_obj
[I40IW_HMC_IW_ARP
].max_cnt
;
3847 hmc_info
->hmc_obj
[I40IW_HMC_IW_APBVT_ENTRY
].cnt
= 1;
3848 hmc_info
->hmc_obj
[I40IW_HMC_IW_MR
].cnt
= mrwanted
;
3850 hmc_info
->hmc_obj
[I40IW_HMC_IW_XF
].cnt
= I40IW_MAX_WQ_ENTRIES
* qpwanted
;
3851 hmc_info
->hmc_obj
[I40IW_HMC_IW_Q1
].cnt
= 4 * I40IW_MAX_IRD_SIZE
* qpwanted
;
3852 hmc_info
->hmc_obj
[I40IW_HMC_IW_XFFL
].cnt
=
3853 hmc_info
->hmc_obj
[I40IW_HMC_IW_XF
].cnt
/ hmc_fpm_misc
->xf_block_size
;
3854 hmc_info
->hmc_obj
[I40IW_HMC_IW_Q1FL
].cnt
=
3855 hmc_info
->hmc_obj
[I40IW_HMC_IW_Q1
].cnt
/ hmc_fpm_misc
->q1_block_size
;
3856 hmc_info
->hmc_obj
[I40IW_HMC_IW_TIMER
].cnt
=
3857 ((qpwanted
) / 512 + 1) * hmc_fpm_misc
->timer_bucket
;
3858 hmc_info
->hmc_obj
[I40IW_HMC_IW_FSIMC
].cnt
= 0x00;
3859 hmc_info
->hmc_obj
[I40IW_HMC_IW_FSIAV
].cnt
= 0x00;
3860 hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
= pblewanted
;
3862 /* How much memory is needed for all the objects. */
3863 sd_needed
= i40iw_est_sd(dev
, hmc_info
);
3864 if ((loop_count
> 1000) ||
3865 ((!(loop_count
% 10)) &&
3866 (qpwanted
> qpwantedoriginal
* 2 / 3))) {
3867 if (qpwanted
> FPM_MULTIPLIER
) {
3868 qpwanted
-= FPM_MULTIPLIER
;
3870 while (powerof2
< qpwanted
)
3873 qpwanted
= powerof2
;
3878 if (mrwanted
> FPM_MULTIPLIER
* 10)
3879 mrwanted
-= FPM_MULTIPLIER
* 10;
3880 if (pblewanted
> FPM_MULTIPLIER
* 1000)
3881 pblewanted
-= FPM_MULTIPLIER
* 1000;
3882 } while (sd_needed
> hmc_fpm_misc
->max_sds
&& loop_count
< 2000);
3884 sd_needed
= i40iw_est_sd(dev
, hmc_info
);
3886 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3887 "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
3888 loop_count
, sd_needed
,
3889 hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].cnt
,
3890 hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
,
3891 hmc_info
->hmc_obj
[I40IW_HMC_IW_MR
].cnt
,
3892 hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
);
3894 ret_code
= i40iw_sc_configure_iw_fpm(dev
, dev
->hmc_fn_id
);
3896 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3897 "configure_iw_fpm returned error_code[x%08X]\n",
3898 i40iw_rd32(dev
->hw
, dev
->is_pf
? I40E_PFPE_CQPERRCODES
: I40E_VFPE_CQPERRCODES1
));
3902 mem_size
= sizeof(struct i40iw_hmc_sd_entry
) *
3903 (hmc_info
->sd_table
.sd_cnt
+ hmc_info
->first_sd_index
+ 1);
3904 ret_code
= i40iw_allocate_virt_mem(dev
->hw
, &virt_mem
, mem_size
);
3906 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3907 "%s: failed to allocate memory for sd_entry buffer\n",
3911 hmc_info
->sd_table
.sd_entry
= virt_mem
.va
;
3917 * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available
3919 * @pcmdinfo: cqp command info
3921 static enum i40iw_status_code
i40iw_exec_cqp_cmd(struct i40iw_sc_dev
*dev
,
3922 struct cqp_commands_info
*pcmdinfo
)
3924 enum i40iw_status_code status
;
3925 struct i40iw_dma_mem values_mem
;
3927 dev
->cqp_cmd_stats
[pcmdinfo
->cqp_cmd
]++;
3928 switch (pcmdinfo
->cqp_cmd
) {
3929 case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY
:
3930 status
= i40iw_sc_del_local_mac_ipaddr_entry(
3931 pcmdinfo
->in
.u
.del_local_mac_ipaddr_entry
.cqp
,
3932 pcmdinfo
->in
.u
.del_local_mac_ipaddr_entry
.scratch
,
3933 pcmdinfo
->in
.u
.del_local_mac_ipaddr_entry
.entry_idx
,
3934 pcmdinfo
->in
.u
.del_local_mac_ipaddr_entry
.ignore_ref_count
,
3937 case OP_CEQ_DESTROY
:
3938 status
= i40iw_sc_ceq_destroy(pcmdinfo
->in
.u
.ceq_destroy
.ceq
,
3939 pcmdinfo
->in
.u
.ceq_destroy
.scratch
,
3942 case OP_AEQ_DESTROY
:
3943 status
= i40iw_sc_aeq_destroy(pcmdinfo
->in
.u
.aeq_destroy
.aeq
,
3944 pcmdinfo
->in
.u
.aeq_destroy
.scratch
,
3948 case OP_DELETE_ARP_CACHE_ENTRY
:
3949 status
= i40iw_sc_del_arp_cache_entry(
3950 pcmdinfo
->in
.u
.del_arp_cache_entry
.cqp
,
3951 pcmdinfo
->in
.u
.del_arp_cache_entry
.scratch
,
3952 pcmdinfo
->in
.u
.del_arp_cache_entry
.arp_index
,
3955 case OP_MANAGE_APBVT_ENTRY
:
3956 status
= i40iw_sc_manage_apbvt_entry(
3957 pcmdinfo
->in
.u
.manage_apbvt_entry
.cqp
,
3958 &pcmdinfo
->in
.u
.manage_apbvt_entry
.info
,
3959 pcmdinfo
->in
.u
.manage_apbvt_entry
.scratch
,
3963 status
= i40iw_sc_ceq_create(pcmdinfo
->in
.u
.ceq_create
.ceq
,
3964 pcmdinfo
->in
.u
.ceq_create
.scratch
,
3968 status
= i40iw_sc_aeq_create(pcmdinfo
->in
.u
.aeq_create
.aeq
,
3969 pcmdinfo
->in
.u
.aeq_create
.scratch
,
3972 case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY
:
3973 status
= i40iw_sc_alloc_local_mac_ipaddr_entry(
3974 pcmdinfo
->in
.u
.alloc_local_mac_ipaddr_entry
.cqp
,
3975 pcmdinfo
->in
.u
.alloc_local_mac_ipaddr_entry
.scratch
,
3978 case OP_ADD_LOCAL_MAC_IPADDR_ENTRY
:
3979 status
= i40iw_sc_add_local_mac_ipaddr_entry(
3980 pcmdinfo
->in
.u
.add_local_mac_ipaddr_entry
.cqp
,
3981 &pcmdinfo
->in
.u
.add_local_mac_ipaddr_entry
.info
,
3982 pcmdinfo
->in
.u
.add_local_mac_ipaddr_entry
.scratch
,
3985 case OP_MANAGE_QHASH_TABLE_ENTRY
:
3986 status
= i40iw_sc_manage_qhash_table_entry(
3987 pcmdinfo
->in
.u
.manage_qhash_table_entry
.cqp
,
3988 &pcmdinfo
->in
.u
.manage_qhash_table_entry
.info
,
3989 pcmdinfo
->in
.u
.manage_qhash_table_entry
.scratch
,
3994 status
= i40iw_sc_qp_modify(
3995 pcmdinfo
->in
.u
.qp_modify
.qp
,
3996 &pcmdinfo
->in
.u
.qp_modify
.info
,
3997 pcmdinfo
->in
.u
.qp_modify
.scratch
,
4001 case OP_QP_UPLOAD_CONTEXT
:
4002 status
= i40iw_sc_qp_upload_context(
4003 pcmdinfo
->in
.u
.qp_upload_context
.dev
,
4004 &pcmdinfo
->in
.u
.qp_upload_context
.info
,
4005 pcmdinfo
->in
.u
.qp_upload_context
.scratch
,
4010 status
= i40iw_sc_cq_create(
4011 pcmdinfo
->in
.u
.cq_create
.cq
,
4012 pcmdinfo
->in
.u
.cq_create
.scratch
,
4013 pcmdinfo
->in
.u
.cq_create
.check_overflow
,
4017 status
= i40iw_sc_cq_destroy(
4018 pcmdinfo
->in
.u
.cq_destroy
.cq
,
4019 pcmdinfo
->in
.u
.cq_destroy
.scratch
,
4024 status
= i40iw_sc_qp_create(
4025 pcmdinfo
->in
.u
.qp_create
.qp
,
4026 &pcmdinfo
->in
.u
.qp_create
.info
,
4027 pcmdinfo
->in
.u
.qp_create
.scratch
,
4031 status
= i40iw_sc_qp_destroy(
4032 pcmdinfo
->in
.u
.qp_destroy
.qp
,
4033 pcmdinfo
->in
.u
.qp_destroy
.scratch
,
4034 pcmdinfo
->in
.u
.qp_destroy
.remove_hash_idx
,
4035 pcmdinfo
->in
.u
.qp_destroy
.
4041 status
= i40iw_sc_alloc_stag(
4042 pcmdinfo
->in
.u
.alloc_stag
.dev
,
4043 &pcmdinfo
->in
.u
.alloc_stag
.info
,
4044 pcmdinfo
->in
.u
.alloc_stag
.scratch
,
4047 case OP_MR_REG_NON_SHARED
:
4048 status
= i40iw_sc_mr_reg_non_shared(
4049 pcmdinfo
->in
.u
.mr_reg_non_shared
.dev
,
4050 &pcmdinfo
->in
.u
.mr_reg_non_shared
.info
,
4051 pcmdinfo
->in
.u
.mr_reg_non_shared
.scratch
,
4055 case OP_DEALLOC_STAG
:
4056 status
= i40iw_sc_dealloc_stag(
4057 pcmdinfo
->in
.u
.dealloc_stag
.dev
,
4058 &pcmdinfo
->in
.u
.dealloc_stag
.info
,
4059 pcmdinfo
->in
.u
.dealloc_stag
.scratch
,
4064 status
= i40iw_sc_mw_alloc(
4065 pcmdinfo
->in
.u
.mw_alloc
.dev
,
4066 pcmdinfo
->in
.u
.mw_alloc
.scratch
,
4067 pcmdinfo
->in
.u
.mw_alloc
.mw_stag_index
,
4068 pcmdinfo
->in
.u
.mw_alloc
.pd_id
,
4072 case OP_QP_FLUSH_WQES
:
4073 status
= i40iw_sc_qp_flush_wqes(
4074 pcmdinfo
->in
.u
.qp_flush_wqes
.qp
,
4075 &pcmdinfo
->in
.u
.qp_flush_wqes
.info
,
4076 pcmdinfo
->in
.u
.qp_flush_wqes
.
4077 scratch
, pcmdinfo
->post_sq
);
4079 case OP_ADD_ARP_CACHE_ENTRY
:
4080 status
= i40iw_sc_add_arp_cache_entry(
4081 pcmdinfo
->in
.u
.add_arp_cache_entry
.cqp
,
4082 &pcmdinfo
->in
.u
.add_arp_cache_entry
.info
,
4083 pcmdinfo
->in
.u
.add_arp_cache_entry
.scratch
,
4086 case OP_MANAGE_PUSH_PAGE
:
4087 status
= i40iw_sc_manage_push_page(
4088 pcmdinfo
->in
.u
.manage_push_page
.cqp
,
4089 &pcmdinfo
->in
.u
.manage_push_page
.info
,
4090 pcmdinfo
->in
.u
.manage_push_page
.scratch
,
4093 case OP_UPDATE_PE_SDS
:
4094 /* case I40IW_CQP_OP_UPDATE_PE_SDS */
4095 status
= i40iw_update_pe_sds(
4096 pcmdinfo
->in
.u
.update_pe_sds
.dev
,
4097 &pcmdinfo
->in
.u
.update_pe_sds
.info
,
4098 pcmdinfo
->in
.u
.update_pe_sds
.
4102 case OP_MANAGE_HMC_PM_FUNC_TABLE
:
4103 status
= i40iw_sc_manage_hmc_pm_func_table(
4104 pcmdinfo
->in
.u
.manage_hmc_pm
.dev
->cqp
,
4105 pcmdinfo
->in
.u
.manage_hmc_pm
.scratch
,
4106 (u8
)pcmdinfo
->in
.u
.manage_hmc_pm
.info
.vf_id
,
4107 pcmdinfo
->in
.u
.manage_hmc_pm
.info
.free_fcn
,
4111 status
= i40iw_sc_suspend_qp(
4112 pcmdinfo
->in
.u
.suspend_resume
.cqp
,
4113 pcmdinfo
->in
.u
.suspend_resume
.qp
,
4114 pcmdinfo
->in
.u
.suspend_resume
.scratch
);
4117 status
= i40iw_sc_resume_qp(
4118 pcmdinfo
->in
.u
.suspend_resume
.cqp
,
4119 pcmdinfo
->in
.u
.suspend_resume
.qp
,
4120 pcmdinfo
->in
.u
.suspend_resume
.scratch
);
4122 case OP_MANAGE_VF_PBLE_BP
:
4123 status
= i40iw_manage_vf_pble_bp(
4124 pcmdinfo
->in
.u
.manage_vf_pble_bp
.cqp
,
4125 &pcmdinfo
->in
.u
.manage_vf_pble_bp
.info
,
4126 pcmdinfo
->in
.u
.manage_vf_pble_bp
.scratch
, true);
4128 case OP_QUERY_FPM_VALUES
:
4129 values_mem
.pa
= pcmdinfo
->in
.u
.query_fpm_values
.fpm_values_pa
;
4130 values_mem
.va
= pcmdinfo
->in
.u
.query_fpm_values
.fpm_values_va
;
4131 status
= i40iw_sc_query_fpm_values(
4132 pcmdinfo
->in
.u
.query_fpm_values
.cqp
,
4133 pcmdinfo
->in
.u
.query_fpm_values
.scratch
,
4134 pcmdinfo
->in
.u
.query_fpm_values
.hmc_fn_id
,
4135 &values_mem
, true, I40IW_CQP_WAIT_EVENT
);
4137 case OP_COMMIT_FPM_VALUES
:
4138 values_mem
.pa
= pcmdinfo
->in
.u
.commit_fpm_values
.fpm_values_pa
;
4139 values_mem
.va
= pcmdinfo
->in
.u
.commit_fpm_values
.fpm_values_va
;
4140 status
= i40iw_sc_commit_fpm_values(
4141 pcmdinfo
->in
.u
.commit_fpm_values
.cqp
,
4142 pcmdinfo
->in
.u
.commit_fpm_values
.scratch
,
4143 pcmdinfo
->in
.u
.commit_fpm_values
.hmc_fn_id
,
4146 I40IW_CQP_WAIT_EVENT
);
4149 status
= I40IW_NOT_SUPPORTED
;
4157 * i40iw_process_cqp_cmd - process all cqp commands
4158 * @dev: sc device struct
4159 * @pcmdinfo: cqp command info
4161 enum i40iw_status_code
i40iw_process_cqp_cmd(struct i40iw_sc_dev
*dev
,
4162 struct cqp_commands_info
*pcmdinfo
)
4164 enum i40iw_status_code status
= 0;
4165 unsigned long flags
;
4167 spin_lock_irqsave(&dev
->cqp_lock
, flags
);
4168 if (list_empty(&dev
->cqp_cmd_head
) && !i40iw_ring_full(dev
->cqp
))
4169 status
= i40iw_exec_cqp_cmd(dev
, pcmdinfo
);
4171 list_add_tail(&pcmdinfo
->cqp_cmd_entry
, &dev
->cqp_cmd_head
);
4172 spin_unlock_irqrestore(&dev
->cqp_lock
, flags
);
4177 * i40iw_process_bh - called from tasklet for cqp list
4178 * @dev: sc device struct
4180 enum i40iw_status_code
i40iw_process_bh(struct i40iw_sc_dev
*dev
)
4182 enum i40iw_status_code status
= 0;
4183 struct cqp_commands_info
*pcmdinfo
;
4184 unsigned long flags
;
4186 spin_lock_irqsave(&dev
->cqp_lock
, flags
);
4187 while (!list_empty(&dev
->cqp_cmd_head
) && !i40iw_ring_full(dev
->cqp
)) {
4188 pcmdinfo
= (struct cqp_commands_info
*)i40iw_remove_head(&dev
->cqp_cmd_head
);
4190 status
= i40iw_exec_cqp_cmd(dev
, pcmdinfo
);
4194 spin_unlock_irqrestore(&dev
->cqp_lock
, flags
);
4199 * i40iw_iwarp_opcode - determine if incoming is rdma layer
4200 * @info: aeq info for the packet
4201 * @pkt: packet for error
4203 static u32
i40iw_iwarp_opcode(struct i40iw_aeqe_info
*info
, u8
*pkt
)
4206 u32 opcode
= 0xffffffff;
4208 if (info
->q2_data_written
) {
4209 mpa
= (__be16
*)pkt
;
4210 opcode
= ntohs(mpa
[1]) & 0xf;
4216 * i40iw_locate_mpa - return pointer to mpa in the pkt
4217 * @pkt: packet with data
4219 static u8
*i40iw_locate_mpa(u8
*pkt
)
4221 /* skip over ethernet header */
4222 pkt
+= I40IW_MAC_HLEN
;
4224 /* Skip over IP and TCP headers */
4225 pkt
+= 4 * (pkt
[0] & 0x0f);
4226 pkt
+= 4 * ((pkt
[12] >> 4) & 0x0f);
4231 * i40iw_setup_termhdr - termhdr for terminate pkt
4232 * @qp: sc qp ptr for pkt
4234 * @opcode: flush opcode for termhdr
4235 * @layer_etype: error layer + error type
4236 * @err: error cod ein the header
4238 static void i40iw_setup_termhdr(struct i40iw_sc_qp
*qp
,
4239 struct i40iw_terminate_hdr
*hdr
,
4240 enum i40iw_flush_opcode opcode
,
4244 qp
->flush_code
= opcode
;
4245 hdr
->layer_etype
= layer_etype
;
4246 hdr
->error_code
= err
;
4250 * i40iw_bld_terminate_hdr - build terminate message header
4251 * @qp: qp associated with received terminate AE
4252 * @info: the struct contiaing AE information
4254 static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp
*qp
,
4255 struct i40iw_aeqe_info
*info
)
4257 u8
*pkt
= qp
->q2_buf
+ Q2_BAD_FRAME_OFFSET
;
4262 struct i40iw_terminate_hdr
*termhdr
;
4264 termhdr
= (struct i40iw_terminate_hdr
*)qp
->q2_buf
;
4265 memset(termhdr
, 0, Q2_BAD_FRAME_OFFSET
);
4267 if (info
->q2_data_written
) {
4268 /* Use data from offending packet to fill in ddp & rdma hdrs */
4269 pkt
= i40iw_locate_mpa(pkt
);
4270 ddp_seg_len
= ntohs(*(__be16
*)pkt
);
4273 termhdr
->hdrct
= DDP_LEN_FLAG
;
4274 if (pkt
[2] & 0x80) {
4276 if (ddp_seg_len
>= TERM_DDP_LEN_TAGGED
) {
4277 copy_len
+= TERM_DDP_LEN_TAGGED
;
4278 termhdr
->hdrct
|= DDP_HDR_FLAG
;
4281 if (ddp_seg_len
>= TERM_DDP_LEN_UNTAGGED
) {
4282 copy_len
+= TERM_DDP_LEN_UNTAGGED
;
4283 termhdr
->hdrct
|= DDP_HDR_FLAG
;
4286 if (ddp_seg_len
>= (TERM_DDP_LEN_UNTAGGED
+ TERM_RDMA_LEN
)) {
4287 if ((pkt
[3] & RDMA_OPCODE_MASK
) == RDMA_READ_REQ_OPCODE
) {
4288 copy_len
+= TERM_RDMA_LEN
;
4289 termhdr
->hdrct
|= RDMA_HDR_FLAG
;
4296 opcode
= i40iw_iwarp_opcode(info
, pkt
);
4298 switch (info
->ae_id
) {
4299 case I40IW_AE_AMP_UNALLOCATED_STAG
:
4300 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4301 if (opcode
== I40IW_OP_TYPE_RDMA_WRITE
)
4302 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_PROT_ERR
,
4303 (LAYER_DDP
<< 4) | DDP_TAGGED_BUFFER
, DDP_TAGGED_INV_STAG
);
4305 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4306 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_INV_STAG
);
4308 case I40IW_AE_AMP_BOUNDS_VIOLATION
:
4309 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4310 if (info
->q2_data_written
)
4311 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_PROT_ERR
,
4312 (LAYER_DDP
<< 4) | DDP_TAGGED_BUFFER
, DDP_TAGGED_BOUNDS
);
4314 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4315 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_INV_BOUNDS
);
4317 case I40IW_AE_AMP_BAD_PD
:
4319 case I40IW_OP_TYPE_RDMA_WRITE
:
4320 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_PROT_ERR
,
4321 (LAYER_DDP
<< 4) | DDP_TAGGED_BUFFER
, DDP_TAGGED_UNASSOC_STAG
);
4323 case I40IW_OP_TYPE_SEND_INV
:
4324 case I40IW_OP_TYPE_SEND_SOL_INV
:
4325 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4326 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_CANT_INV_STAG
);
4329 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4330 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_UNASSOC_STAG
);
4333 case I40IW_AE_AMP_INVALID_STAG
:
4334 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4335 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4336 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_INV_STAG
);
4338 case I40IW_AE_AMP_BAD_QP
:
4339 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_QP_OP_ERR
,
4340 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_QN
);
4342 case I40IW_AE_AMP_BAD_STAG_KEY
:
4343 case I40IW_AE_AMP_BAD_STAG_INDEX
:
4344 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4346 case I40IW_OP_TYPE_SEND_INV
:
4347 case I40IW_OP_TYPE_SEND_SOL_INV
:
4348 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_OP_ERR
,
4349 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_CANT_INV_STAG
);
4352 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4353 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_INV_STAG
);
4356 case I40IW_AE_AMP_RIGHTS_VIOLATION
:
4357 case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS
:
4358 case I40IW_AE_PRIV_OPERATION_DENIED
:
4359 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4360 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4361 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_ACCESS
);
4363 case I40IW_AE_AMP_TO_WRAP
:
4364 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4365 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4366 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_TO_WRAP
);
4368 case I40IW_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH
:
4369 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_LEN_ERR
,
4370 (LAYER_MPA
<< 4) | DDP_LLP
, MPA_MARKER
);
4372 case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR
:
4373 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4374 (LAYER_MPA
<< 4) | DDP_LLP
, MPA_CRC
);
4376 case I40IW_AE_LLP_SEGMENT_TOO_LARGE
:
4377 case I40IW_AE_LLP_SEGMENT_TOO_SMALL
:
4378 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_LEN_ERR
,
4379 (LAYER_DDP
<< 4) | DDP_CATASTROPHIC
, DDP_CATASTROPHIC_LOCAL
);
4381 case I40IW_AE_LCE_QP_CATASTROPHIC
:
4382 case I40IW_AE_DDP_NO_L_BIT
:
4383 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_FATAL_ERR
,
4384 (LAYER_DDP
<< 4) | DDP_CATASTROPHIC
, DDP_CATASTROPHIC_LOCAL
);
4386 case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN
:
4387 case I40IW_AE_DDP_INVALID_MSN_RANGE_IS_NOT_VALID
:
4388 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4389 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_MSN_RANGE
);
4391 case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER
:
4392 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4393 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_LEN_ERR
,
4394 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_TOO_LONG
);
4396 case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION
:
4398 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4399 (LAYER_DDP
<< 4) | DDP_TAGGED_BUFFER
, DDP_TAGGED_INV_DDP_VER
);
4401 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4402 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_DDP_VER
);
4404 case I40IW_AE_DDP_UBE_INVALID_MO
:
4405 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4406 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_MO
);
4408 case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE
:
4409 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_OP_ERR
,
4410 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_MSN_NO_BUF
);
4412 case I40IW_AE_DDP_UBE_INVALID_QN
:
4413 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4414 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_QN
);
4416 case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION
:
4417 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4418 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_INV_RDMAP_VER
);
4420 case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE
:
4421 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_QP_OP_ERR
,
4422 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_UNEXPECTED_OP
);
4425 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_FATAL_ERR
,
4426 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_UNSPECIFIED
);
4431 memcpy(termhdr
+ 1, pkt
, copy_len
);
4433 return sizeof(struct i40iw_terminate_hdr
) + copy_len
;
4437 * i40iw_terminate_send_fin() - Send fin for terminate message
4438 * @qp: qp associated with received terminate AE
4440 void i40iw_terminate_send_fin(struct i40iw_sc_qp
*qp
)
4442 /* Send the fin only */
4443 i40iw_term_modify_qp(qp
,
4444 I40IW_QP_STATE_TERMINATE
,
4445 I40IWQP_TERM_SEND_FIN_ONLY
,
4450 * i40iw_terminate_connection() - Bad AE and send terminate to remote QP
4451 * @qp: qp associated with received terminate AE
4452 * @info: the struct contiaing AE information
4454 void i40iw_terminate_connection(struct i40iw_sc_qp
*qp
, struct i40iw_aeqe_info
*info
)
4458 if (qp
->term_flags
& I40IW_TERM_SENT
)
4459 return; /* Sanity check */
4461 /* Eventtype can change from bld_terminate_hdr */
4462 qp
->eventtype
= TERM_EVENT_QP_FATAL
;
4463 termlen
= i40iw_bld_terminate_hdr(qp
, info
);
4464 i40iw_terminate_start_timer(qp
);
4465 qp
->term_flags
|= I40IW_TERM_SENT
;
4466 i40iw_term_modify_qp(qp
, I40IW_QP_STATE_TERMINATE
,
4467 I40IWQP_TERM_SEND_TERM_ONLY
, termlen
);
4471 * i40iw_terminate_received - handle terminate received AE
4472 * @qp: qp associated with received terminate AE
4473 * @info: the struct contiaing AE information
4475 void i40iw_terminate_received(struct i40iw_sc_qp
*qp
, struct i40iw_aeqe_info
*info
)
4477 u8
*pkt
= qp
->q2_buf
+ Q2_BAD_FRAME_OFFSET
;
4482 struct i40iw_terminate_hdr
*termhdr
;
4484 mpa
= (__be32
*)i40iw_locate_mpa(pkt
);
4485 if (info
->q2_data_written
) {
4486 /* did not validate the frame - do it now */
4487 ddp_ctl
= (ntohl(mpa
[0]) >> 8) & 0xff;
4488 rdma_ctl
= ntohl(mpa
[0]) & 0xff;
4489 if ((ddp_ctl
& 0xc0) != 0x40)
4490 aeq_id
= I40IW_AE_LCE_QP_CATASTROPHIC
;
4491 else if ((ddp_ctl
& 0x03) != 1)
4492 aeq_id
= I40IW_AE_DDP_UBE_INVALID_DDP_VERSION
;
4493 else if (ntohl(mpa
[2]) != 2)
4494 aeq_id
= I40IW_AE_DDP_UBE_INVALID_QN
;
4495 else if (ntohl(mpa
[3]) != 1)
4496 aeq_id
= I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN
;
4497 else if (ntohl(mpa
[4]) != 0)
4498 aeq_id
= I40IW_AE_DDP_UBE_INVALID_MO
;
4499 else if ((rdma_ctl
& 0xc0) != 0x40)
4500 aeq_id
= I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION
;
4502 info
->ae_id
= aeq_id
;
4504 /* Bad terminate recvd - send back a terminate */
4505 i40iw_terminate_connection(qp
, info
);
4510 qp
->term_flags
|= I40IW_TERM_RCVD
;
4511 qp
->eventtype
= TERM_EVENT_QP_FATAL
;
4512 termhdr
= (struct i40iw_terminate_hdr
*)&mpa
[5];
4513 if (termhdr
->layer_etype
== RDMAP_REMOTE_PROT
||
4514 termhdr
->layer_etype
== RDMAP_REMOTE_OP
) {
4515 i40iw_terminate_done(qp
, 0);
4517 i40iw_terminate_start_timer(qp
);
4518 i40iw_terminate_send_fin(qp
);
4523 * i40iw_sc_vsi_init - Initialize virtual device
4524 * @vsi: pointer to the vsi structure
4525 * @info: parameters to initialize vsi
4527 void i40iw_sc_vsi_init(struct i40iw_sc_vsi
*vsi
, struct i40iw_vsi_init_info
*info
)
4531 vsi
->dev
= info
->dev
;
4532 vsi
->back_vsi
= info
->back_vsi
;
4533 vsi
->mss
= info
->params
->mss
;
4534 i40iw_fill_qos_list(info
->params
->qs_handle_list
);
4536 for (i
= 0; i
< I40IW_MAX_USER_PRIORITY
; i
++) {
4537 vsi
->qos
[i
].qs_handle
= info
->params
->qs_handle_list
[i
];
4538 i40iw_debug(vsi
->dev
, I40IW_DEBUG_DCB
, "qset[%d]: %d\n", i
,
4539 vsi
->qos
[i
].qs_handle
);
4540 spin_lock_init(&vsi
->qos
[i
].lock
);
4541 INIT_LIST_HEAD(&vsi
->qos
[i
].qplist
);
4546 * i40iw_hw_stats_init - Initiliaze HW stats table
4547 * @stats: pestat struct
4548 * @fcn_idx: PCI fn id
4549 * @is_pf: Is it a PF?
4551 * Populate the HW stats table with register offset addr for each
4552 * stats. And start the perioidic stats timer.
4554 void i40iw_hw_stats_init(struct i40iw_vsi_pestat
*stats
, u8 fcn_idx
, bool is_pf
)
4556 u32 stats_reg_offset
;
4558 struct i40iw_dev_hw_stats_offsets
*stats_table
=
4559 &stats
->hw_stats_offsets
;
4560 struct i40iw_dev_hw_stats
*last_rd_stats
= &stats
->last_read_hw_stats
;
4563 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4RXDISCARD
] =
4564 I40E_GLPES_PFIP4RXDISCARD(fcn_idx
);
4565 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4RXTRUNC
] =
4566 I40E_GLPES_PFIP4RXTRUNC(fcn_idx
);
4567 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4TXNOROUTE
] =
4568 I40E_GLPES_PFIP4TXNOROUTE(fcn_idx
);
4569 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6RXDISCARD
] =
4570 I40E_GLPES_PFIP6RXDISCARD(fcn_idx
);
4571 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6RXTRUNC
] =
4572 I40E_GLPES_PFIP6RXTRUNC(fcn_idx
);
4573 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6TXNOROUTE
] =
4574 I40E_GLPES_PFIP6TXNOROUTE(fcn_idx
);
4575 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRTXSEG
] =
4576 I40E_GLPES_PFTCPRTXSEG(fcn_idx
);
4577 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRXOPTERR
] =
4578 I40E_GLPES_PFTCPRXOPTERR(fcn_idx
);
4579 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRXPROTOERR
] =
4580 I40E_GLPES_PFTCPRXPROTOERR(fcn_idx
);
4582 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXOCTS
] =
4583 I40E_GLPES_PFIP4RXOCTSLO(fcn_idx
);
4584 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXPKTS
] =
4585 I40E_GLPES_PFIP4RXPKTSLO(fcn_idx
);
4586 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXFRAGS
] =
4587 I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx
);
4588 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXMCPKTS
] =
4589 I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx
);
4590 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXOCTS
] =
4591 I40E_GLPES_PFIP4TXOCTSLO(fcn_idx
);
4592 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXPKTS
] =
4593 I40E_GLPES_PFIP4TXPKTSLO(fcn_idx
);
4594 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXFRAGS
] =
4595 I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx
);
4596 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXMCPKTS
] =
4597 I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx
);
4598 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXOCTS
] =
4599 I40E_GLPES_PFIP6RXOCTSLO(fcn_idx
);
4600 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXPKTS
] =
4601 I40E_GLPES_PFIP6RXPKTSLO(fcn_idx
);
4602 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXFRAGS
] =
4603 I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx
);
4604 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXMCPKTS
] =
4605 I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx
);
4606 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXOCTS
] =
4607 I40E_GLPES_PFIP6TXOCTSLO(fcn_idx
);
4608 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXPKTS
] =
4609 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx
);
4610 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXPKTS
] =
4611 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx
);
4612 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXFRAGS
] =
4613 I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx
);
4614 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_TCPRXSEGS
] =
4615 I40E_GLPES_PFTCPRXSEGSLO(fcn_idx
);
4616 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_TCPTXSEG
] =
4617 I40E_GLPES_PFTCPTXSEGLO(fcn_idx
);
4618 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXRDS
] =
4619 I40E_GLPES_PFRDMARXRDSLO(fcn_idx
);
4620 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXSNDS
] =
4621 I40E_GLPES_PFRDMARXSNDSLO(fcn_idx
);
4622 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXWRS
] =
4623 I40E_GLPES_PFRDMARXWRSLO(fcn_idx
);
4624 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXRDS
] =
4625 I40E_GLPES_PFRDMATXRDSLO(fcn_idx
);
4626 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXSNDS
] =
4627 I40E_GLPES_PFRDMATXSNDSLO(fcn_idx
);
4628 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXWRS
] =
4629 I40E_GLPES_PFRDMATXWRSLO(fcn_idx
);
4630 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMAVBND
] =
4631 I40E_GLPES_PFRDMAVBNDLO(fcn_idx
);
4632 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMAVINV
] =
4633 I40E_GLPES_PFRDMAVINVLO(fcn_idx
);
4635 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4RXDISCARD
] =
4636 I40E_GLPES_VFIP4RXDISCARD(fcn_idx
);
4637 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4RXTRUNC
] =
4638 I40E_GLPES_VFIP4RXTRUNC(fcn_idx
);
4639 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4TXNOROUTE
] =
4640 I40E_GLPES_VFIP4TXNOROUTE(fcn_idx
);
4641 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6RXDISCARD
] =
4642 I40E_GLPES_VFIP6RXDISCARD(fcn_idx
);
4643 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6RXTRUNC
] =
4644 I40E_GLPES_VFIP6RXTRUNC(fcn_idx
);
4645 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6TXNOROUTE
] =
4646 I40E_GLPES_VFIP6TXNOROUTE(fcn_idx
);
4647 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRTXSEG
] =
4648 I40E_GLPES_VFTCPRTXSEG(fcn_idx
);
4649 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRXOPTERR
] =
4650 I40E_GLPES_VFTCPRXOPTERR(fcn_idx
);
4651 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRXPROTOERR
] =
4652 I40E_GLPES_VFTCPRXPROTOERR(fcn_idx
);
4654 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXOCTS
] =
4655 I40E_GLPES_VFIP4RXOCTSLO(fcn_idx
);
4656 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXPKTS
] =
4657 I40E_GLPES_VFIP4RXPKTSLO(fcn_idx
);
4658 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXFRAGS
] =
4659 I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx
);
4660 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXMCPKTS
] =
4661 I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx
);
4662 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXOCTS
] =
4663 I40E_GLPES_VFIP4TXOCTSLO(fcn_idx
);
4664 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXPKTS
] =
4665 I40E_GLPES_VFIP4TXPKTSLO(fcn_idx
);
4666 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXFRAGS
] =
4667 I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx
);
4668 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXMCPKTS
] =
4669 I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx
);
4670 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXOCTS
] =
4671 I40E_GLPES_VFIP6RXOCTSLO(fcn_idx
);
4672 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXPKTS
] =
4673 I40E_GLPES_VFIP6RXPKTSLO(fcn_idx
);
4674 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXFRAGS
] =
4675 I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx
);
4676 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXMCPKTS
] =
4677 I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx
);
4678 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXOCTS
] =
4679 I40E_GLPES_VFIP6TXOCTSLO(fcn_idx
);
4680 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXPKTS
] =
4681 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx
);
4682 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXPKTS
] =
4683 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx
);
4684 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXFRAGS
] =
4685 I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx
);
4686 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_TCPRXSEGS
] =
4687 I40E_GLPES_VFTCPRXSEGSLO(fcn_idx
);
4688 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_TCPTXSEG
] =
4689 I40E_GLPES_VFTCPTXSEGLO(fcn_idx
);
4690 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXRDS
] =
4691 I40E_GLPES_VFRDMARXRDSLO(fcn_idx
);
4692 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXSNDS
] =
4693 I40E_GLPES_VFRDMARXSNDSLO(fcn_idx
);
4694 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXWRS
] =
4695 I40E_GLPES_VFRDMARXWRSLO(fcn_idx
);
4696 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXRDS
] =
4697 I40E_GLPES_VFRDMATXRDSLO(fcn_idx
);
4698 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXSNDS
] =
4699 I40E_GLPES_VFRDMATXSNDSLO(fcn_idx
);
4700 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXWRS
] =
4701 I40E_GLPES_VFRDMATXWRSLO(fcn_idx
);
4702 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMAVBND
] =
4703 I40E_GLPES_VFRDMAVBNDLO(fcn_idx
);
4704 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMAVINV
] =
4705 I40E_GLPES_VFRDMAVINVLO(fcn_idx
);
4708 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_64
;
4710 stats_reg_offset
= stats_table
->stats_offset_64
[stats_index
];
4711 last_rd_stats
->stats_value_64
[stats_index
] =
4712 readq(stats
->hw
->hw_addr
+ stats_reg_offset
);
4715 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_32
;
4717 stats_reg_offset
= stats_table
->stats_offset_32
[stats_index
];
4718 last_rd_stats
->stats_value_32
[stats_index
] =
4719 i40iw_rd32(stats
->hw
, stats_reg_offset
);
4724 * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
4725 * @stat: pestat struct
4726 * @index: index in HW stats table which contains offset reg-addr
4727 * @value: hw stats value
4729 void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat
*stats
,
4730 enum i40iw_hw_stats_index_32b index
,
4733 struct i40iw_dev_hw_stats_offsets
*stats_table
=
4734 &stats
->hw_stats_offsets
;
4735 struct i40iw_dev_hw_stats
*last_rd_stats
= &stats
->last_read_hw_stats
;
4736 struct i40iw_dev_hw_stats
*hw_stats
= &stats
->hw_stats
;
4737 u64 new_stats_value
= 0;
4738 u32 stats_reg_offset
= stats_table
->stats_offset_32
[index
];
4740 new_stats_value
= i40iw_rd32(stats
->hw
, stats_reg_offset
);
4742 if (new_stats_value
< last_rd_stats
->stats_value_32
[index
])
4743 hw_stats
->stats_value_32
[index
] += new_stats_value
;
4745 hw_stats
->stats_value_32
[index
] +=
4746 new_stats_value
- last_rd_stats
->stats_value_32
[index
];
4747 last_rd_stats
->stats_value_32
[index
] = new_stats_value
;
4748 *value
= hw_stats
->stats_value_32
[index
];
4752 * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
4753 * @stats: pestat struct
4754 * @index: index in HW stats table which contains offset reg-addr
4755 * @value: hw stats value
4757 void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat
*stats
,
4758 enum i40iw_hw_stats_index_64b index
,
4761 struct i40iw_dev_hw_stats_offsets
*stats_table
=
4762 &stats
->hw_stats_offsets
;
4763 struct i40iw_dev_hw_stats
*last_rd_stats
= &stats
->last_read_hw_stats
;
4764 struct i40iw_dev_hw_stats
*hw_stats
= &stats
->hw_stats
;
4765 u64 new_stats_value
= 0;
4766 u32 stats_reg_offset
= stats_table
->stats_offset_64
[index
];
4768 new_stats_value
= readq(stats
->hw
->hw_addr
+ stats_reg_offset
);
4770 if (new_stats_value
< last_rd_stats
->stats_value_64
[index
])
4771 hw_stats
->stats_value_64
[index
] += new_stats_value
;
4773 hw_stats
->stats_value_64
[index
] +=
4774 new_stats_value
- last_rd_stats
->stats_value_64
[index
];
4775 last_rd_stats
->stats_value_64
[index
] = new_stats_value
;
4776 *value
= hw_stats
->stats_value_64
[index
];
4780 * i40iw_hw_stats_read_all - read all HW stat counters
4781 * @stats: pestat struct
4782 * @stats_values: hw stats structure
4784 * Read all the HW stat counters and populates hw_stats structure
4785 * of passed-in vsi's pestat as well as copy created in stat_values.
4787 void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat
*stats
,
4788 struct i40iw_dev_hw_stats
*stats_values
)
4791 unsigned long flags
;
4793 spin_lock_irqsave(&stats
->lock
, flags
);
4795 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_32
;
4797 i40iw_hw_stats_read_32(stats
, stats_index
,
4798 &stats_values
->stats_value_32
[stats_index
]);
4799 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_64
;
4801 i40iw_hw_stats_read_64(stats
, stats_index
,
4802 &stats_values
->stats_value_64
[stats_index
]);
4803 spin_unlock_irqrestore(&stats
->lock
, flags
);
4807 * i40iw_hw_stats_refresh_all - Update all HW stats structs
4808 * @stats: pestat struct
4810 * Read all the HW stats counters to refresh values in hw_stats structure
4811 * of passed-in dev's pestat
4813 void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat
*stats
)
4817 unsigned long flags
;
4819 spin_lock_irqsave(&stats
->lock
, flags
);
4821 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_32
;
4823 i40iw_hw_stats_read_32(stats
, stats_index
, &stats_value
);
4824 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_64
;
4826 i40iw_hw_stats_read_64(stats
, stats_index
, &stats_value
);
4827 spin_unlock_irqrestore(&stats
->lock
, flags
);
4831 * i40iw_get_fcn_id - Return the function id
4832 * @dev: pointer to the device
4834 static u8
i40iw_get_fcn_id(struct i40iw_sc_dev
*dev
)
4836 u8 fcn_id
= I40IW_INVALID_FCN_ID
;
4839 for (i
= I40IW_FIRST_NON_PF_STAT
; i
< I40IW_MAX_STATS_COUNT
; i
++)
4840 if (!dev
->fcn_id_array
[i
]) {
4842 dev
->fcn_id_array
[i
] = true;
4849 * i40iw_vsi_stats_init - Initialize the vsi statistics
4850 * @vsi: pointer to the vsi structure
4851 * @info: The info structure used for initialization
4853 enum i40iw_status_code
i40iw_vsi_stats_init(struct i40iw_sc_vsi
*vsi
, struct i40iw_vsi_stats_info
*info
)
4855 u8 fcn_id
= info
->fcn_id
;
4857 if (info
->alloc_fcn_id
)
4858 fcn_id
= i40iw_get_fcn_id(vsi
->dev
);
4860 if (fcn_id
== I40IW_INVALID_FCN_ID
)
4861 return I40IW_ERR_NOT_READY
;
4863 vsi
->pestat
= info
->pestat
;
4864 vsi
->pestat
->hw
= vsi
->dev
->hw
;
4866 if (info
->stats_initialize
) {
4867 i40iw_hw_stats_init(vsi
->pestat
, fcn_id
, true);
4868 spin_lock_init(&vsi
->pestat
->lock
);
4869 i40iw_hw_stats_start_timer(vsi
);
4871 vsi
->stats_fcn_id_alloc
= info
->alloc_fcn_id
;
4872 vsi
->fcn_id
= fcn_id
;
4873 return I40IW_SUCCESS
;
4877 * i40iw_vsi_stats_free - Free the vsi stats
4878 * @vsi: pointer to the vsi structure
4880 void i40iw_vsi_stats_free(struct i40iw_sc_vsi
*vsi
)
4882 u8 fcn_id
= vsi
->fcn_id
;
4884 if (vsi
->stats_fcn_id_alloc
&& fcn_id
< I40IW_MAX_STATS_COUNT
)
4885 vsi
->dev
->fcn_id_array
[fcn_id
] = false;
4886 i40iw_hw_stats_stop_timer(vsi
);
4889 static struct i40iw_cqp_ops iw_cqp_ops
= {
4890 .cqp_init
= i40iw_sc_cqp_init
,
4891 .cqp_create
= i40iw_sc_cqp_create
,
4892 .cqp_post_sq
= i40iw_sc_cqp_post_sq
,
4893 .cqp_get_next_send_wqe
= i40iw_sc_cqp_get_next_send_wqe
,
4894 .cqp_destroy
= i40iw_sc_cqp_destroy
,
4895 .poll_for_cqp_op_done
= i40iw_sc_poll_for_cqp_op_done
4898 static struct i40iw_ccq_ops iw_ccq_ops
= {
4899 .ccq_init
= i40iw_sc_ccq_init
,
4900 .ccq_create
= i40iw_sc_ccq_create
,
4901 .ccq_destroy
= i40iw_sc_ccq_destroy
,
4902 .ccq_create_done
= i40iw_sc_ccq_create_done
,
4903 .ccq_get_cqe_info
= i40iw_sc_ccq_get_cqe_info
,
4904 .ccq_arm
= i40iw_sc_ccq_arm
4907 static struct i40iw_ceq_ops iw_ceq_ops
= {
4908 .ceq_init
= i40iw_sc_ceq_init
,
4909 .ceq_create
= i40iw_sc_ceq_create
,
4910 .cceq_create_done
= i40iw_sc_cceq_create_done
,
4911 .cceq_destroy_done
= i40iw_sc_cceq_destroy_done
,
4912 .cceq_create
= i40iw_sc_cceq_create
,
4913 .ceq_destroy
= i40iw_sc_ceq_destroy
,
4914 .process_ceq
= i40iw_sc_process_ceq
4917 static struct i40iw_aeq_ops iw_aeq_ops
= {
4918 .aeq_init
= i40iw_sc_aeq_init
,
4919 .aeq_create
= i40iw_sc_aeq_create
,
4920 .aeq_destroy
= i40iw_sc_aeq_destroy
,
4921 .get_next_aeqe
= i40iw_sc_get_next_aeqe
,
4922 .repost_aeq_entries
= i40iw_sc_repost_aeq_entries
,
4923 .aeq_create_done
= i40iw_sc_aeq_create_done
,
4924 .aeq_destroy_done
= i40iw_sc_aeq_destroy_done
4928 static struct i40iw_pd_ops iw_pd_ops
= {
4929 .pd_init
= i40iw_sc_pd_init
,
4932 static struct i40iw_priv_qp_ops iw_priv_qp_ops
= {
4933 .qp_init
= i40iw_sc_qp_init
,
4934 .qp_create
= i40iw_sc_qp_create
,
4935 .qp_modify
= i40iw_sc_qp_modify
,
4936 .qp_destroy
= i40iw_sc_qp_destroy
,
4937 .qp_flush_wqes
= i40iw_sc_qp_flush_wqes
,
4938 .qp_upload_context
= i40iw_sc_qp_upload_context
,
4939 .qp_setctx
= i40iw_sc_qp_setctx
,
4940 .qp_send_lsmm
= i40iw_sc_send_lsmm
,
4941 .qp_send_lsmm_nostag
= i40iw_sc_send_lsmm_nostag
,
4942 .qp_send_rtt
= i40iw_sc_send_rtt
,
4943 .qp_post_wqe0
= i40iw_sc_post_wqe0
,
4944 .iw_mr_fast_register
= i40iw_sc_mr_fast_register
4947 static struct i40iw_priv_cq_ops iw_priv_cq_ops
= {
4948 .cq_init
= i40iw_sc_cq_init
,
4949 .cq_create
= i40iw_sc_cq_create
,
4950 .cq_destroy
= i40iw_sc_cq_destroy
,
4951 .cq_modify
= i40iw_sc_cq_modify
,
4954 static struct i40iw_mr_ops iw_mr_ops
= {
4955 .alloc_stag
= i40iw_sc_alloc_stag
,
4956 .mr_reg_non_shared
= i40iw_sc_mr_reg_non_shared
,
4957 .mr_reg_shared
= i40iw_sc_mr_reg_shared
,
4958 .dealloc_stag
= i40iw_sc_dealloc_stag
,
4959 .query_stag
= i40iw_sc_query_stag
,
4960 .mw_alloc
= i40iw_sc_mw_alloc
4963 static struct i40iw_cqp_misc_ops iw_cqp_misc_ops
= {
4964 .manage_push_page
= i40iw_sc_manage_push_page
,
4965 .manage_hmc_pm_func_table
= i40iw_sc_manage_hmc_pm_func_table
,
4966 .set_hmc_resource_profile
= i40iw_sc_set_hmc_resource_profile
,
4967 .commit_fpm_values
= i40iw_sc_commit_fpm_values
,
4968 .query_fpm_values
= i40iw_sc_query_fpm_values
,
4969 .static_hmc_pages_allocated
= i40iw_sc_static_hmc_pages_allocated
,
4970 .add_arp_cache_entry
= i40iw_sc_add_arp_cache_entry
,
4971 .del_arp_cache_entry
= i40iw_sc_del_arp_cache_entry
,
4972 .query_arp_cache_entry
= i40iw_sc_query_arp_cache_entry
,
4973 .manage_apbvt_entry
= i40iw_sc_manage_apbvt_entry
,
4974 .manage_qhash_table_entry
= i40iw_sc_manage_qhash_table_entry
,
4975 .alloc_local_mac_ipaddr_table_entry
= i40iw_sc_alloc_local_mac_ipaddr_entry
,
4976 .add_local_mac_ipaddr_entry
= i40iw_sc_add_local_mac_ipaddr_entry
,
4977 .del_local_mac_ipaddr_entry
= i40iw_sc_del_local_mac_ipaddr_entry
,
4978 .cqp_nop
= i40iw_sc_cqp_nop
,
4979 .commit_fpm_values_done
= i40iw_sc_commit_fpm_values_done
,
4980 .query_fpm_values_done
= i40iw_sc_query_fpm_values_done
,
4981 .manage_hmc_pm_func_table_done
= i40iw_sc_manage_hmc_pm_func_table_done
,
4982 .update_suspend_qp
= i40iw_sc_suspend_qp
,
4983 .update_resume_qp
= i40iw_sc_resume_qp
4986 static struct i40iw_hmc_ops iw_hmc_ops
= {
4987 .init_iw_hmc
= i40iw_sc_init_iw_hmc
,
4988 .parse_fpm_query_buf
= i40iw_sc_parse_fpm_query_buf
,
4989 .configure_iw_fpm
= i40iw_sc_configure_iw_fpm
,
4990 .parse_fpm_commit_buf
= i40iw_sc_parse_fpm_commit_buf
,
4991 .create_hmc_object
= i40iw_sc_create_hmc_obj
,
4992 .del_hmc_object
= i40iw_sc_del_hmc_obj
4996 * i40iw_device_init - Initialize IWARP device
4997 * @dev: IWARP device pointer
4998 * @info: IWARP init info
5000 enum i40iw_status_code
i40iw_device_init(struct i40iw_sc_dev
*dev
,
5001 struct i40iw_device_init_info
*info
)
5006 enum i40iw_status_code ret_code
= 0;
5009 spin_lock_init(&dev
->cqp_lock
);
5010 INIT_LIST_HEAD(&dev
->cqp_cmd_head
); /* for the cqp commands backlog. */
5012 i40iw_device_init_uk(&dev
->dev_uk
);
5014 dev
->debug_mask
= info
->debug_mask
;
5016 dev
->hmc_fn_id
= info
->hmc_fn_id
;
5017 dev
->exception_lan_queue
= info
->exception_lan_queue
;
5018 dev
->is_pf
= info
->is_pf
;
5020 dev
->fpm_query_buf_pa
= info
->fpm_query_buf_pa
;
5021 dev
->fpm_query_buf
= info
->fpm_query_buf
;
5023 dev
->fpm_commit_buf_pa
= info
->fpm_commit_buf_pa
;
5024 dev
->fpm_commit_buf
= info
->fpm_commit_buf
;
5027 dev
->hw
->hw_addr
= info
->bar0
;
5030 val
= i40iw_rd32(dev
->hw
, I40E_GLPCI_DREVID
);
5031 dev
->hw_rev
= (u8
)RS_32(val
, I40E_GLPCI_DREVID_DEFAULT_REVID
);
5033 val
= i40iw_rd32(dev
->hw
, I40E_GLPCI_LBARCTRL
);
5034 db_size
= (u8
)RS_32(val
, I40E_GLPCI_LBARCTRL_PE_DB_SIZE
);
5035 if ((db_size
!= I40IW_PE_DB_SIZE_4M
) &&
5036 (db_size
!= I40IW_PE_DB_SIZE_8M
)) {
5037 i40iw_debug(dev
, I40IW_DEBUG_DEV
,
5038 "%s: PE doorbell is not enabled in CSR val 0x%x\n",
5040 ret_code
= I40IW_ERR_PE_DOORBELL_NOT_ENABLED
;
5043 dev
->db_addr
= dev
->hw
->hw_addr
+ I40IW_DB_ADDR_OFFSET
;
5044 dev
->vchnl_if
.vchnl_recv
= i40iw_vchnl_recv_pf
;
5046 dev
->db_addr
= dev
->hw
->hw_addr
+ I40IW_VF_DB_ADDR_OFFSET
;
5049 dev
->cqp_ops
= &iw_cqp_ops
;
5050 dev
->ccq_ops
= &iw_ccq_ops
;
5051 dev
->ceq_ops
= &iw_ceq_ops
;
5052 dev
->aeq_ops
= &iw_aeq_ops
;
5053 dev
->cqp_misc_ops
= &iw_cqp_misc_ops
;
5054 dev
->iw_pd_ops
= &iw_pd_ops
;
5055 dev
->iw_priv_qp_ops
= &iw_priv_qp_ops
;
5056 dev
->iw_priv_cq_ops
= &iw_priv_cq_ops
;
5057 dev
->mr_ops
= &iw_mr_ops
;
5058 dev
->hmc_ops
= &iw_hmc_ops
;
5059 dev
->vchnl_if
.vchnl_send
= info
->vchnl_send
;
5060 if (dev
->vchnl_if
.vchnl_send
)
5061 dev
->vchnl_up
= true;
5063 dev
->vchnl_up
= false;
5065 dev
->vchnl_if
.vchnl_recv
= i40iw_vchnl_recv_vf
;
5066 ret_code
= i40iw_vchnl_vf_get_ver(dev
, &vchnl_ver
);
5068 i40iw_debug(dev
, I40IW_DEBUG_DEV
,
5069 "%s: Get Channel version rc = 0x%0x, version is %u\n",
5070 __func__
, ret_code
, vchnl_ver
);
5071 ret_code
= i40iw_vchnl_vf_get_hmc_fcn(dev
, &hmc_fcn
);
5073 i40iw_debug(dev
, I40IW_DEBUG_DEV
,
5074 "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n",
5075 __func__
, ret_code
, hmc_fcn
);
5076 dev
->hmc_fn_id
= (u8
)hmc_fcn
;
5080 dev
->iw_vf_cqp_ops
= &iw_vf_cqp_ops
;