1 /*******************************************************************************
3 * Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenFabrics.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 *******************************************************************************/
35 #include "i40iw_osdep.h"
36 #include "i40iw_register.h"
37 #include "i40iw_status.h"
38 #include "i40iw_hmc.h"
41 #include "i40iw_type.h"
44 #include "i40iw_virtchnl.h"
47 * i40iw_insert_wqe_hdr - write wqe header
48 * @wqe: cqp wqe for header
49 * @header: header for the cqp wqe
51 static inline void i40iw_insert_wqe_hdr(u64
*wqe
, u64 header
)
53 wmb(); /* make sure WQE is populated before polarity is set */
54 set_64bit_val(wqe
, 24, header
);
58 * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
59 * @cqp: struct for cqp hw
60 * @val: cqp tail register value
61 * @tail:wqtail register value
62 * @error: cqp processing err
64 static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp
*cqp
,
69 if (cqp
->dev
->is_pf
) {
70 *val
= i40iw_rd32(cqp
->dev
->hw
, I40E_PFPE_CQPTAIL
);
71 *tail
= RS_32(*val
, I40E_PFPE_CQPTAIL_WQTAIL
);
72 *error
= RS_32(*val
, I40E_PFPE_CQPTAIL_CQP_OP_ERR
);
74 *val
= i40iw_rd32(cqp
->dev
->hw
, I40E_VFPE_CQPTAIL1
);
75 *tail
= RS_32(*val
, I40E_VFPE_CQPTAIL_WQTAIL
);
76 *error
= RS_32(*val
, I40E_VFPE_CQPTAIL_CQP_OP_ERR
);
81 * i40iw_cqp_poll_registers - poll cqp registers
82 * @cqp: struct for cqp hw
83 * @tail:wqtail register value
84 * @count: how many times to try for completion
86 static enum i40iw_status_code
i40iw_cqp_poll_registers(
87 struct i40iw_sc_cqp
*cqp
,
92 u32 newtail
, error
, val
;
96 i40iw_get_cqp_reg_info(cqp
, &val
, &newtail
, &error
);
98 error
= (cqp
->dev
->is_pf
) ?
99 i40iw_rd32(cqp
->dev
->hw
, I40E_PFPE_CQPERRCODES
) :
100 i40iw_rd32(cqp
->dev
->hw
, I40E_VFPE_CQPERRCODES1
);
101 return I40IW_ERR_CQP_COMPL_ERROR
;
103 if (newtail
!= tail
) {
105 I40IW_RING_MOVE_TAIL(cqp
->sq_ring
);
106 cqp
->dev
->cqp_cmd_stats
[OP_COMPLETED_COMMANDS
]++;
109 udelay(I40IW_SLEEP_COUNT
);
111 return I40IW_ERR_TIMEOUT
;
115 * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer
116 * @buf: ptr to fpm commit buffer
117 * @info: ptr to i40iw_hmc_obj_info struct
118 * @sd: number of SDs for HMC objects
120 * parses fpm commit info and copy base value
121 * of hmc objects in hmc_info
123 static enum i40iw_status_code
i40iw_sc_parse_fpm_commit_buf(
125 struct i40iw_hmc_obj_info
*info
,
135 /* copy base values in obj_info */
136 for (i
= I40IW_HMC_IW_QP
, j
= 0;
137 i
<= I40IW_HMC_IW_PBLE
; i
++, j
+= 8) {
138 get_64bit_val(buf
, j
, &temp
);
139 info
[i
].base
= RS_64_1(temp
, 32) * 512;
140 if (info
[i
].base
> base
) {
148 size
= info
[k
].cnt
* info
[k
].size
+ info
[k
].base
;
150 *sd
= (u32
)((size
>> 21) + 1); /* add 1 for remainder */
152 *sd
= (u32
)(size
>> 21);
158 * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
159 * @buf: ptr to fpm query buffer
160 * @info: ptr to i40iw_hmc_obj_info struct
161 * @hmc_fpm_misc: ptr to fpm data
163 * parses fpm query buffer and copy max_cnt and
164 * size value of hmc objects in hmc_info
166 static enum i40iw_status_code
i40iw_sc_parse_fpm_query_buf(
168 struct i40iw_hmc_info
*hmc_info
,
169 struct i40iw_hmc_fpm_misc
*hmc_fpm_misc
)
172 struct i40iw_hmc_obj_info
*obj_info
;
176 obj_info
= hmc_info
->hmc_obj
;
178 get_64bit_val(buf
, 0, &temp
);
179 hmc_info
->first_sd_index
= (u16
)RS_64(temp
, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX
);
180 max_pe_sds
= (u16
)RS_64(temp
, I40IW_QUERY_FPM_MAX_PE_SDS
);
182 /* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */
183 if (hmc_info
->hmc_fn_id
>= I40IW_FIRST_VF_FPM_ID
)
185 hmc_fpm_misc
->max_sds
= max_pe_sds
;
186 hmc_info
->sd_table
.sd_cnt
= max_pe_sds
+ hmc_info
->first_sd_index
;
188 for (i
= I40IW_HMC_IW_QP
, j
= 8;
189 i
<= I40IW_HMC_IW_ARP
; i
++, j
+= 8) {
190 get_64bit_val(buf
, j
, &temp
);
191 if (i
== I40IW_HMC_IW_QP
)
192 obj_info
[i
].max_cnt
= (u32
)RS_64(temp
, I40IW_QUERY_FPM_MAX_QPS
);
193 else if (i
== I40IW_HMC_IW_CQ
)
194 obj_info
[i
].max_cnt
= (u32
)RS_64(temp
, I40IW_QUERY_FPM_MAX_CQS
);
196 obj_info
[i
].max_cnt
= (u32
)temp
;
198 size
= (u32
)RS_64_1(temp
, 32);
199 obj_info
[i
].size
= ((u64
)1 << size
);
201 for (i
= I40IW_HMC_IW_MR
, j
= 48;
202 i
<= I40IW_HMC_IW_PBLE
; i
++, j
+= 8) {
203 get_64bit_val(buf
, j
, &temp
);
204 obj_info
[i
].max_cnt
= (u32
)temp
;
205 size
= (u32
)RS_64_1(temp
, 32);
206 obj_info
[i
].size
= LS_64_1(1, size
);
209 get_64bit_val(buf
, 120, &temp
);
210 hmc_fpm_misc
->max_ceqs
= (u8
)RS_64(temp
, I40IW_QUERY_FPM_MAX_CEQS
);
211 get_64bit_val(buf
, 120, &temp
);
212 hmc_fpm_misc
->ht_multiplier
= RS_64(temp
, I40IW_QUERY_FPM_HTMULTIPLIER
);
213 get_64bit_val(buf
, 120, &temp
);
214 hmc_fpm_misc
->timer_bucket
= RS_64(temp
, I40IW_QUERY_FPM_TIMERBUCKET
);
215 get_64bit_val(buf
, 64, &temp
);
216 hmc_fpm_misc
->xf_block_size
= RS_64(temp
, I40IW_QUERY_FPM_XFBLOCKSIZE
);
217 if (!hmc_fpm_misc
->xf_block_size
)
218 return I40IW_ERR_INVALID_SIZE
;
219 get_64bit_val(buf
, 80, &temp
);
220 hmc_fpm_misc
->q1_block_size
= RS_64(temp
, I40IW_QUERY_FPM_Q1BLOCKSIZE
);
221 if (!hmc_fpm_misc
->q1_block_size
)
222 return I40IW_ERR_INVALID_SIZE
;
227 * i40iw_fill_qos_list - Change all unknown qs handles to available ones
228 * @qs_list: list of qs_handles to be fixed with valid qs_handles
230 static void i40iw_fill_qos_list(u16
*qs_list
)
232 u16 qshandle
= qs_list
[0];
235 for (i
= 0; i
< I40IW_MAX_USER_PRIORITY
; i
++) {
236 if (qs_list
[i
] == QS_HANDLE_UNKNOWN
)
237 qs_list
[i
] = qshandle
;
239 qshandle
= qs_list
[i
];
244 * i40iw_qp_from_entry - Given entry, get to the qp structure
245 * @entry: Points to list of qp structure
247 static struct i40iw_sc_qp
*i40iw_qp_from_entry(struct list_head
*entry
)
252 return (struct i40iw_sc_qp
*)((char *)entry
- offsetof(struct i40iw_sc_qp
, list
));
256 * i40iw_get_qp - get the next qp from the list given current qp
257 * @head: Listhead of qp's
260 static struct i40iw_sc_qp
*i40iw_get_qp(struct list_head
*head
, struct i40iw_sc_qp
*qp
)
262 struct list_head
*entry
= NULL
;
263 struct list_head
*lastentry
;
265 if (list_empty(head
))
271 lastentry
= &qp
->list
;
272 entry
= (lastentry
!= head
) ? lastentry
->next
: NULL
;
275 return i40iw_qp_from_entry(entry
);
279 * i40iw_change_l2params - given the new l2 parameters, change all qp
280 * @vsi: pointer to the vsi structure
281 * @l2params: New paramaters from l2
283 void i40iw_change_l2params(struct i40iw_sc_vsi
*vsi
, struct i40iw_l2params
*l2params
)
285 struct i40iw_sc_dev
*dev
= vsi
->dev
;
286 struct i40iw_sc_qp
*qp
= NULL
;
287 bool qs_handle_change
= false;
288 bool mss_change
= false;
293 if (vsi
->mss
!= l2params
->mss
) {
295 vsi
->mss
= l2params
->mss
;
298 i40iw_fill_qos_list(l2params
->qs_handle_list
);
299 for (i
= 0; i
< I40IW_MAX_USER_PRIORITY
; i
++) {
300 qs_handle
= l2params
->qs_handle_list
[i
];
301 if (vsi
->qos
[i
].qs_handle
!= qs_handle
)
302 qs_handle_change
= true;
303 else if (!mss_change
)
304 continue; /* no MSS nor qs handle change */
305 spin_lock_irqsave(&vsi
->qos
[i
].lock
, flags
);
306 qp
= i40iw_get_qp(&vsi
->qos
[i
].qplist
, qp
);
309 i40iw_qp_mss_modify(dev
, qp
);
310 if (qs_handle_change
) {
311 qp
->qs_handle
= qs_handle
;
312 /* issue cqp suspend command */
313 i40iw_qp_suspend_resume(dev
, qp
, true);
315 qp
= i40iw_get_qp(&vsi
->qos
[i
].qplist
, qp
);
317 spin_unlock_irqrestore(&vsi
->qos
[i
].lock
, flags
);
318 vsi
->qos
[i
].qs_handle
= qs_handle
;
323 * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp
324 * @qp: qp to be removed from qos
326 static void i40iw_qp_rem_qos(struct i40iw_sc_qp
*qp
)
328 struct i40iw_sc_vsi
*vsi
= qp
->vsi
;
333 spin_lock_irqsave(&vsi
->qos
[qp
->user_pri
].lock
, flags
);
335 spin_unlock_irqrestore(&vsi
->qos
[qp
->user_pri
].lock
, flags
);
339 * i40iw_qp_add_qos - called during setctx fot qp to be added to qos
340 * @qp: qp to be added to qos
342 void i40iw_qp_add_qos(struct i40iw_sc_qp
*qp
)
344 struct i40iw_sc_vsi
*vsi
= qp
->vsi
;
349 spin_lock_irqsave(&vsi
->qos
[qp
->user_pri
].lock
, flags
);
350 qp
->qs_handle
= vsi
->qos
[qp
->user_pri
].qs_handle
;
351 list_add(&qp
->list
, &vsi
->qos
[qp
->user_pri
].qplist
);
352 qp
->on_qoslist
= true;
353 spin_unlock_irqrestore(&vsi
->qos
[qp
->user_pri
].lock
, flags
);
357 * i40iw_sc_pd_init - initialize sc pd struct
358 * @dev: sc device struct
360 * @pd_id: pd_id for allocated pd
361 * @abi_ver: ABI version from user context, -1 if not valid
363 static void i40iw_sc_pd_init(struct i40iw_sc_dev
*dev
,
364 struct i40iw_sc_pd
*pd
,
368 pd
->size
= sizeof(*pd
);
370 pd
->abi_ver
= abi_ver
;
375 * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size
376 * @wqsize: size of the wq (sq, rq, srq) to encoded_size
377 * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
379 u8
i40iw_get_encoded_wqe_size(u32 wqsize
, bool cqpsq
)
383 /* cqp sq's hw coded value starts from 1 for size of 4
384 * while it starts from 0 for qp' wq's.
395 * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair
396 * @cqp: IWARP control queue pair pointer
397 * @info: IWARP control queue pair init info pointer
399 * Initializes the object and context buffers for a control Queue Pair.
401 static enum i40iw_status_code
i40iw_sc_cqp_init(struct i40iw_sc_cqp
*cqp
,
402 struct i40iw_cqp_init_info
*info
)
406 if ((info
->sq_size
> I40IW_CQP_SW_SQSIZE_2048
) ||
407 (info
->sq_size
< I40IW_CQP_SW_SQSIZE_4
) ||
408 ((info
->sq_size
& (info
->sq_size
- 1))))
409 return I40IW_ERR_INVALID_SIZE
;
411 hw_sq_size
= i40iw_get_encoded_wqe_size(info
->sq_size
, true);
412 cqp
->size
= sizeof(*cqp
);
413 cqp
->sq_size
= info
->sq_size
;
414 cqp
->hw_sq_size
= hw_sq_size
;
415 cqp
->sq_base
= info
->sq
;
416 cqp
->host_ctx
= info
->host_ctx
;
417 cqp
->sq_pa
= info
->sq_pa
;
418 cqp
->host_ctx_pa
= info
->host_ctx_pa
;
419 cqp
->dev
= info
->dev
;
420 cqp
->struct_ver
= info
->struct_ver
;
421 cqp
->scratch_array
= info
->scratch_array
;
423 cqp
->en_datacenter_tcp
= info
->en_datacenter_tcp
;
424 cqp
->enabled_vf_count
= info
->enabled_vf_count
;
425 cqp
->hmc_profile
= info
->hmc_profile
;
426 info
->dev
->cqp
= cqp
;
428 I40IW_RING_INIT(cqp
->sq_ring
, cqp
->sq_size
);
429 cqp
->dev
->cqp_cmd_stats
[OP_REQUESTED_COMMANDS
] = 0;
430 cqp
->dev
->cqp_cmd_stats
[OP_COMPLETED_COMMANDS
] = 0;
432 i40iw_debug(cqp
->dev
, I40IW_DEBUG_WQE
,
433 "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",
434 __func__
, cqp
->sq_size
, cqp
->hw_sq_size
,
435 cqp
->sq_base
, cqp
->sq_pa
, cqp
, cqp
->polarity
);
440 * i40iw_sc_cqp_create - create cqp during bringup
441 * @cqp: struct for cqp hw
442 * @maj_err: If error, major err number
443 * @min_err: If error, minor err number
445 static enum i40iw_status_code
i40iw_sc_cqp_create(struct i40iw_sc_cqp
*cqp
,
450 u32 cnt
= 0, p1
, p2
, val
= 0, err_code
;
451 enum i40iw_status_code ret_code
;
456 ret_code
= i40iw_allocate_dma_mem(cqp
->dev
->hw
,
459 I40IW_SD_BUF_ALIGNMENT
);
464 temp
= LS_64(cqp
->hw_sq_size
, I40IW_CQPHC_SQSIZE
) |
465 LS_64(cqp
->struct_ver
, I40IW_CQPHC_SVER
);
467 set_64bit_val(cqp
->host_ctx
, 0, temp
);
468 set_64bit_val(cqp
->host_ctx
, 8, cqp
->sq_pa
);
469 temp
= LS_64(cqp
->enabled_vf_count
, I40IW_CQPHC_ENABLED_VFS
) |
470 LS_64(cqp
->hmc_profile
, I40IW_CQPHC_HMC_PROFILE
);
471 set_64bit_val(cqp
->host_ctx
, 16, temp
);
472 set_64bit_val(cqp
->host_ctx
, 24, (uintptr_t)cqp
);
473 set_64bit_val(cqp
->host_ctx
, 32, 0);
474 set_64bit_val(cqp
->host_ctx
, 40, 0);
475 set_64bit_val(cqp
->host_ctx
, 48, 0);
476 set_64bit_val(cqp
->host_ctx
, 56, 0);
478 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CQP_HOST_CTX",
479 cqp
->host_ctx
, I40IW_CQP_CTX_SIZE
* 8);
481 p1
= RS_32_1(cqp
->host_ctx_pa
, 32);
482 p2
= (u32
)cqp
->host_ctx_pa
;
484 if (cqp
->dev
->is_pf
) {
485 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CCQPHIGH
, p1
);
486 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CCQPLOW
, p2
);
488 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CCQPHIGH1
, p1
);
489 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CCQPLOW1
, p2
);
492 if (cnt
++ > I40IW_DONE_COUNT
) {
493 i40iw_free_dma_mem(cqp
->dev
->hw
, &cqp
->sdbuf
);
494 ret_code
= I40IW_ERR_TIMEOUT
;
496 * read PFPE_CQPERRORCODES register to get the minor
497 * and major error code
500 err_code
= i40iw_rd32(cqp
->dev
->hw
, I40E_PFPE_CQPERRCODES
);
502 err_code
= i40iw_rd32(cqp
->dev
->hw
, I40E_VFPE_CQPERRCODES1
);
503 *min_err
= RS_32(err_code
, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE
);
504 *maj_err
= RS_32(err_code
, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE
);
507 udelay(I40IW_SLEEP_COUNT
);
509 val
= i40iw_rd32(cqp
->dev
->hw
, I40E_PFPE_CCQPSTATUS
);
511 val
= i40iw_rd32(cqp
->dev
->hw
, I40E_VFPE_CCQPSTATUS1
);
516 cqp
->process_cqp_sds
= i40iw_update_sds_noccq
;
521 * i40iw_sc_cqp_post_sq - post of cqp's sq
522 * @cqp: struct for cqp hw
524 void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp
*cqp
)
527 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CQPDB
, I40IW_RING_GETCURRENT_HEAD(cqp
->sq_ring
));
529 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CQPDB1
, I40IW_RING_GETCURRENT_HEAD(cqp
->sq_ring
));
531 i40iw_debug(cqp
->dev
,
533 "%s: HEAD_TAIL[%04d,%04d,%04d]\n",
541 * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
542 * @cqp: struct for cqp hw
543 * @wqe_idx: we index of cqp ring
545 u64
*i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp
*cqp
, u64 scratch
)
549 enum i40iw_status_code ret_code
;
551 if (I40IW_RING_FULL_ERR(cqp
->sq_ring
)) {
552 i40iw_debug(cqp
->dev
,
554 "%s: ring is full head %x tail %x size %x\n",
561 I40IW_ATOMIC_RING_MOVE_HEAD(cqp
->sq_ring
, wqe_idx
, ret_code
);
562 cqp
->dev
->cqp_cmd_stats
[OP_REQUESTED_COMMANDS
]++;
566 cqp
->polarity
= !cqp
->polarity
;
568 wqe
= cqp
->sq_base
[wqe_idx
].elem
;
569 cqp
->scratch_array
[wqe_idx
] = scratch
;
570 I40IW_CQP_INIT_WQE(wqe
);
576 * i40iw_sc_cqp_destroy - destroy cqp during close
577 * @cqp: struct for cqp hw
579 static enum i40iw_status_code
i40iw_sc_cqp_destroy(struct i40iw_sc_cqp
*cqp
)
581 u32 cnt
= 0, val
= 1;
582 enum i40iw_status_code ret_code
= 0;
585 if (cqp
->dev
->is_pf
) {
586 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CCQPHIGH
, 0);
587 i40iw_wr32(cqp
->dev
->hw
, I40E_PFPE_CCQPLOW
, 0);
588 cqpstat_addr
= I40E_PFPE_CCQPSTATUS
;
590 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CCQPHIGH1
, 0);
591 i40iw_wr32(cqp
->dev
->hw
, I40E_VFPE_CCQPLOW1
, 0);
592 cqpstat_addr
= I40E_VFPE_CCQPSTATUS1
;
595 if (cnt
++ > I40IW_DONE_COUNT
) {
596 ret_code
= I40IW_ERR_TIMEOUT
;
599 udelay(I40IW_SLEEP_COUNT
);
600 val
= i40iw_rd32(cqp
->dev
->hw
, cqpstat_addr
);
603 i40iw_free_dma_mem(cqp
->dev
->hw
, &cqp
->sdbuf
);
608 * i40iw_sc_ccq_arm - enable intr for control cq
609 * @ccq: ccq sc struct
611 static void i40iw_sc_ccq_arm(struct i40iw_sc_cq
*ccq
)
618 /* write to cq doorbell shadow area */
619 /* arm next se should always be zero */
620 get_64bit_val(ccq
->cq_uk
.shadow_area
, 32, &temp_val
);
622 sw_cq_sel
= (u16
)RS_64(temp_val
, I40IW_CQ_DBSA_SW_CQ_SELECT
);
623 arm_next_se
= (u8
)RS_64(temp_val
, I40IW_CQ_DBSA_ARM_NEXT_SE
);
625 arm_seq_num
= (u8
)RS_64(temp_val
, I40IW_CQ_DBSA_ARM_SEQ_NUM
);
628 temp_val
= LS_64(arm_seq_num
, I40IW_CQ_DBSA_ARM_SEQ_NUM
) |
629 LS_64(sw_cq_sel
, I40IW_CQ_DBSA_SW_CQ_SELECT
) |
630 LS_64(arm_next_se
, I40IW_CQ_DBSA_ARM_NEXT_SE
) |
631 LS_64(1, I40IW_CQ_DBSA_ARM_NEXT
);
633 set_64bit_val(ccq
->cq_uk
.shadow_area
, 32, temp_val
);
635 wmb(); /* make sure shadow area is updated before arming */
638 i40iw_wr32(ccq
->dev
->hw
, I40E_PFPE_CQARM
, ccq
->cq_uk
.cq_id
);
640 i40iw_wr32(ccq
->dev
->hw
, I40E_VFPE_CQARM1
, ccq
->cq_uk
.cq_id
);
644 * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry
645 * @ccq: ccq sc struct
646 * @info: completion q entry to return
648 static enum i40iw_status_code
i40iw_sc_ccq_get_cqe_info(
649 struct i40iw_sc_cq
*ccq
,
650 struct i40iw_ccq_cqe_info
*info
)
652 u64 qp_ctx
, temp
, temp1
;
654 struct i40iw_sc_cqp
*cqp
;
657 enum i40iw_status_code ret_code
= 0;
659 if (ccq
->cq_uk
.avoid_mem_cflct
)
660 cqe
= (u64
*)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq
->cq_uk
);
662 cqe
= (u64
*)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq
->cq_uk
);
664 get_64bit_val(cqe
, 24, &temp
);
665 polarity
= (u8
)RS_64(temp
, I40IW_CQ_VALID
);
666 if (polarity
!= ccq
->cq_uk
.polarity
)
667 return I40IW_ERR_QUEUE_EMPTY
;
669 get_64bit_val(cqe
, 8, &qp_ctx
);
670 cqp
= (struct i40iw_sc_cqp
*)(unsigned long)qp_ctx
;
671 info
->error
= (bool)RS_64(temp
, I40IW_CQ_ERROR
);
672 info
->min_err_code
= (u16
)RS_64(temp
, I40IW_CQ_MINERR
);
674 info
->maj_err_code
= (u16
)RS_64(temp
, I40IW_CQ_MAJERR
);
675 info
->min_err_code
= (u16
)RS_64(temp
, I40IW_CQ_MINERR
);
677 wqe_idx
= (u32
)RS_64(temp
, I40IW_CQ_WQEIDX
);
678 info
->scratch
= cqp
->scratch_array
[wqe_idx
];
680 get_64bit_val(cqe
, 16, &temp1
);
681 info
->op_ret_val
= (u32
)RS_64(temp1
, I40IW_CCQ_OPRETVAL
);
682 get_64bit_val(cqp
->sq_base
[wqe_idx
].elem
, 24, &temp1
);
683 info
->op_code
= (u8
)RS_64(temp1
, I40IW_CQPSQ_OPCODE
);
686 /* move the head for cq */
687 I40IW_RING_MOVE_HEAD(ccq
->cq_uk
.cq_ring
, ret_code
);
688 if (I40IW_RING_GETCURRENT_HEAD(ccq
->cq_uk
.cq_ring
) == 0)
689 ccq
->cq_uk
.polarity
^= 1;
691 /* update cq tail in cq shadow memory also */
692 I40IW_RING_MOVE_TAIL(ccq
->cq_uk
.cq_ring
);
693 set_64bit_val(ccq
->cq_uk
.shadow_area
,
695 I40IW_RING_GETCURRENT_HEAD(ccq
->cq_uk
.cq_ring
));
696 wmb(); /* write shadow area before tail */
697 I40IW_RING_MOVE_TAIL(cqp
->sq_ring
);
698 ccq
->dev
->cqp_cmd_stats
[OP_COMPLETED_COMMANDS
]++;
704 * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
705 * @cqp: struct for cqp hw
706 * @op_code: cqp opcode for completion
707 * @info: completion q entry to return
709 static enum i40iw_status_code
i40iw_sc_poll_for_cqp_op_done(
710 struct i40iw_sc_cqp
*cqp
,
712 struct i40iw_ccq_cqe_info
*compl_info
)
714 struct i40iw_ccq_cqe_info info
;
715 struct i40iw_sc_cq
*ccq
;
716 enum i40iw_status_code ret_code
= 0;
719 memset(&info
, 0, sizeof(info
));
722 if (cnt
++ > I40IW_DONE_COUNT
)
723 return I40IW_ERR_TIMEOUT
;
725 if (i40iw_sc_ccq_get_cqe_info(ccq
, &info
)) {
726 udelay(I40IW_SLEEP_COUNT
);
731 ret_code
= I40IW_ERR_CQP_COMPL_ERROR
;
734 /* check if opcode is cq create */
735 if (op_code
!= info
.op_code
) {
736 i40iw_debug(cqp
->dev
, I40IW_DEBUG_WQE
,
737 "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",
738 __func__
, op_code
, info
.op_code
);
740 /* success, exit out of the loop */
741 if (op_code
== info
.op_code
)
746 memcpy(compl_info
, &info
, sizeof(*compl_info
));
752 * i40iw_sc_manage_push_page - Handle push page
753 * @cqp: struct for cqp hw
754 * @info: push page info
755 * @scratch: u64 saved to be used during cqp completion
756 * @post_sq: flag for cqp db to ring
758 static enum i40iw_status_code
i40iw_sc_manage_push_page(
759 struct i40iw_sc_cqp
*cqp
,
760 struct i40iw_cqp_manage_push_page_info
*info
,
767 if (info
->push_idx
>= I40IW_MAX_PUSH_PAGE_COUNT
)
768 return I40IW_ERR_INVALID_PUSH_PAGE_INDEX
;
770 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
772 return I40IW_ERR_RING_FULL
;
774 set_64bit_val(wqe
, 16, info
->qs_handle
);
776 header
= LS_64(info
->push_idx
, I40IW_CQPSQ_MPP_PPIDX
) |
777 LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES
, I40IW_CQPSQ_OPCODE
) |
778 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
) |
779 LS_64(info
->free_page
, I40IW_CQPSQ_MPP_FREE_PAGE
);
781 i40iw_insert_wqe_hdr(wqe
, header
);
783 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_PUSH_PAGES WQE",
784 wqe
, I40IW_CQP_WQE_SIZE
* 8);
787 i40iw_sc_cqp_post_sq(cqp
);
792 * i40iw_sc_manage_hmc_pm_func_table - manage of function table
793 * @cqp: struct for cqp hw
794 * @scratch: u64 saved to be used during cqp completion
795 * @vf_index: vf index for cqp
796 * @free_pm_fcn: function number
797 * @post_sq: flag for cqp db to ring
799 static enum i40iw_status_code
i40iw_sc_manage_hmc_pm_func_table(
800 struct i40iw_sc_cqp
*cqp
,
809 if (vf_index
>= I40IW_MAX_VF_PER_PF
)
810 return I40IW_ERR_INVALID_VF_ID
;
811 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
813 return I40IW_ERR_RING_FULL
;
815 header
= LS_64(vf_index
, I40IW_CQPSQ_MHMC_VFIDX
) |
816 LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE
, I40IW_CQPSQ_OPCODE
) |
817 LS_64(free_pm_fcn
, I40IW_CQPSQ_MHMC_FREEPMFN
) |
818 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
820 i40iw_insert_wqe_hdr(wqe
, header
);
821 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_HMC_PM_FUNC_TABLE WQE",
822 wqe
, I40IW_CQP_WQE_SIZE
* 8);
824 i40iw_sc_cqp_post_sq(cqp
);
829 * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile
830 * @cqp: struct for cqp hw
831 * @scratch: u64 saved to be used during cqp completion
832 * @hmc_profile_type: type of profile to set
833 * @vf_num: vf number for profile
834 * @post_sq: flag for cqp db to ring
835 * @poll_registers: flag to poll register for cqp completion
837 static enum i40iw_status_code
i40iw_sc_set_hmc_resource_profile(
838 struct i40iw_sc_cqp
*cqp
,
841 u8 vf_num
, bool post_sq
,
846 u32 val
, tail
, error
;
847 enum i40iw_status_code ret_code
= 0;
849 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
851 return I40IW_ERR_RING_FULL
;
853 set_64bit_val(wqe
, 16,
854 (LS_64(hmc_profile_type
, I40IW_CQPSQ_SHMCRP_HMC_PROFILE
) |
855 LS_64(vf_num
, I40IW_CQPSQ_SHMCRP_VFNUM
)));
857 header
= LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE
, I40IW_CQPSQ_OPCODE
) |
858 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
860 i40iw_insert_wqe_hdr(wqe
, header
);
862 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_HMC_PM_FUNC_TABLE WQE",
863 wqe
, I40IW_CQP_WQE_SIZE
* 8);
865 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
867 return I40IW_ERR_CQP_COMPL_ERROR
;
870 i40iw_sc_cqp_post_sq(cqp
);
872 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, 1000000);
874 ret_code
= i40iw_sc_poll_for_cqp_op_done(cqp
,
875 I40IW_CQP_OP_SHMC_PAGES_ALLOCATED
,
883 * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
884 * @cqp: struct for cqp hw
886 static enum i40iw_status_code
i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp
*cqp
)
888 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE
, NULL
);
892 * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
893 * @cqp: struct for cqp hw
895 static enum i40iw_status_code
i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp
*cqp
)
897 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_COMMIT_FPM_VALUES
, NULL
);
901 * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values
902 * @cqp: struct for cqp hw
903 * @scratch: u64 saved to be used during cqp completion
904 * @hmc_fn_id: hmc function id
905 * @commit_fpm_mem; Memory for fpm values
906 * @post_sq: flag for cqp db to ring
907 * @wait_type: poll ccq or cqp registers for cqp completion
909 static enum i40iw_status_code
i40iw_sc_commit_fpm_values(
910 struct i40iw_sc_cqp
*cqp
,
913 struct i40iw_dma_mem
*commit_fpm_mem
,
919 u32 tail
, val
, error
;
920 enum i40iw_status_code ret_code
= 0;
922 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
924 return I40IW_ERR_RING_FULL
;
926 set_64bit_val(wqe
, 16, hmc_fn_id
);
927 set_64bit_val(wqe
, 32, commit_fpm_mem
->pa
);
929 header
= LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES
, I40IW_CQPSQ_OPCODE
) |
930 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
932 i40iw_insert_wqe_hdr(wqe
, header
);
934 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "COMMIT_FPM_VALUES WQE",
935 wqe
, I40IW_CQP_WQE_SIZE
* 8);
937 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
939 return I40IW_ERR_CQP_COMPL_ERROR
;
942 i40iw_sc_cqp_post_sq(cqp
);
944 if (wait_type
== I40IW_CQP_WAIT_POLL_REGS
)
945 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, I40IW_DONE_COUNT
);
946 else if (wait_type
== I40IW_CQP_WAIT_POLL_CQ
)
947 ret_code
= i40iw_sc_commit_fpm_values_done(cqp
);
954 * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
955 * @cqp: struct for cqp hw
957 static enum i40iw_status_code
i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp
*cqp
)
959 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_QUERY_FPM_VALUES
, NULL
);
963 * i40iw_sc_query_fpm_values - cqp wqe query fpm values
964 * @cqp: struct for cqp hw
965 * @scratch: u64 saved to be used during cqp completion
966 * @hmc_fn_id: hmc function id
967 * @query_fpm_mem: memory for return fpm values
968 * @post_sq: flag for cqp db to ring
969 * @wait_type: poll ccq or cqp registers for cqp completion
971 static enum i40iw_status_code
i40iw_sc_query_fpm_values(
972 struct i40iw_sc_cqp
*cqp
,
975 struct i40iw_dma_mem
*query_fpm_mem
,
981 u32 tail
, val
, error
;
982 enum i40iw_status_code ret_code
= 0;
984 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
986 return I40IW_ERR_RING_FULL
;
988 set_64bit_val(wqe
, 16, hmc_fn_id
);
989 set_64bit_val(wqe
, 32, query_fpm_mem
->pa
);
991 header
= LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES
, I40IW_CQPSQ_OPCODE
) |
992 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
994 i40iw_insert_wqe_hdr(wqe
, header
);
996 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QUERY_FPM WQE",
997 wqe
, I40IW_CQP_WQE_SIZE
* 8);
999 /* read the tail from CQP_TAIL register */
1000 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
1003 return I40IW_ERR_CQP_COMPL_ERROR
;
1006 i40iw_sc_cqp_post_sq(cqp
);
1007 if (wait_type
== I40IW_CQP_WAIT_POLL_REGS
)
1008 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, I40IW_DONE_COUNT
);
1009 else if (wait_type
== I40IW_CQP_WAIT_POLL_CQ
)
1010 ret_code
= i40iw_sc_query_fpm_values_done(cqp
);
1017 * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry
1018 * @cqp: struct for cqp hw
1019 * @info: arp entry information
1020 * @scratch: u64 saved to be used during cqp completion
1021 * @post_sq: flag for cqp db to ring
1023 static enum i40iw_status_code
i40iw_sc_add_arp_cache_entry(
1024 struct i40iw_sc_cqp
*cqp
,
1025 struct i40iw_add_arp_cache_entry_info
*info
,
1032 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1034 return I40IW_ERR_RING_FULL
;
1035 set_64bit_val(wqe
, 8, info
->reach_max
);
1037 temp
= info
->mac_addr
[5] |
1038 LS_64_1(info
->mac_addr
[4], 8) |
1039 LS_64_1(info
->mac_addr
[3], 16) |
1040 LS_64_1(info
->mac_addr
[2], 24) |
1041 LS_64_1(info
->mac_addr
[1], 32) |
1042 LS_64_1(info
->mac_addr
[0], 40);
1044 set_64bit_val(wqe
, 16, temp
);
1046 header
= info
->arp_index
|
1047 LS_64(I40IW_CQP_OP_MANAGE_ARP
, I40IW_CQPSQ_OPCODE
) |
1048 LS_64((info
->permanent
? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT
) |
1049 LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID
) |
1050 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1052 i40iw_insert_wqe_hdr(wqe
, header
);
1054 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "ARP_CACHE_ENTRY WQE",
1055 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1058 i40iw_sc_cqp_post_sq(cqp
);
1063 * i40iw_sc_del_arp_cache_entry - dele arp cache entry
1064 * @cqp: struct for cqp hw
1065 * @scratch: u64 saved to be used during cqp completion
1066 * @arp_index: arp index to delete arp entry
1067 * @post_sq: flag for cqp db to ring
1069 static enum i40iw_status_code
i40iw_sc_del_arp_cache_entry(
1070 struct i40iw_sc_cqp
*cqp
,
1078 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1080 return I40IW_ERR_RING_FULL
;
1082 header
= arp_index
|
1083 LS_64(I40IW_CQP_OP_MANAGE_ARP
, I40IW_CQPSQ_OPCODE
) |
1084 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1085 i40iw_insert_wqe_hdr(wqe
, header
);
1087 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "ARP_CACHE_DEL_ENTRY WQE",
1088 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1091 i40iw_sc_cqp_post_sq(cqp
);
1096 * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
1097 * @cqp: struct for cqp hw
1098 * @scratch: u64 saved to be used during cqp completion
1099 * @arp_index: arp index to delete arp entry
1100 * @post_sq: flag for cqp db to ring
1102 static enum i40iw_status_code
i40iw_sc_query_arp_cache_entry(
1103 struct i40iw_sc_cqp
*cqp
,
1111 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1113 return I40IW_ERR_RING_FULL
;
1115 header
= arp_index
|
1116 LS_64(I40IW_CQP_OP_MANAGE_ARP
, I40IW_CQPSQ_OPCODE
) |
1117 LS_64(1, I40IW_CQPSQ_MAT_QUERY
) |
1118 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1120 i40iw_insert_wqe_hdr(wqe
, header
);
1122 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QUERY_ARP_CACHE_ENTRY WQE",
1123 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1126 i40iw_sc_cqp_post_sq(cqp
);
1131 * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries
1132 * @cqp: struct for cqp hw
1133 * @info: info for apbvt entry to add or delete
1134 * @scratch: u64 saved to be used during cqp completion
1135 * @post_sq: flag for cqp db to ring
1137 static enum i40iw_status_code
i40iw_sc_manage_apbvt_entry(
1138 struct i40iw_sc_cqp
*cqp
,
1139 struct i40iw_apbvt_info
*info
,
1146 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1148 return I40IW_ERR_RING_FULL
;
1150 set_64bit_val(wqe
, 16, info
->port
);
1152 header
= LS_64(I40IW_CQP_OP_MANAGE_APBVT
, I40IW_CQPSQ_OPCODE
) |
1153 LS_64(info
->add
, I40IW_CQPSQ_MAPT_ADDPORT
) |
1154 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1156 i40iw_insert_wqe_hdr(wqe
, header
);
1158 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_APBVT WQE",
1159 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1162 i40iw_sc_cqp_post_sq(cqp
);
1167 * i40iw_sc_manage_qhash_table_entry - manage quad hash entries
1168 * @cqp: struct for cqp hw
1169 * @info: info for quad hash to manage
1170 * @scratch: u64 saved to be used during cqp completion
1171 * @post_sq: flag for cqp db to ring
1173 * This is called before connection establishment is started. For passive connections, when
1174 * listener is created, it will call with entry type of I40IW_QHASH_TYPE_TCP_SYN with local
1175 * ip address and tcp port. When SYN is received (passive connections) or
1176 * sent (active connections), this routine is called with entry type of
1177 * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.
1179 * When iwarp connection is done and its state moves to RTS, the quad hash entry in
1180 * the hardware will point to iwarp's qp number and requires no calls from the driver.
1182 static enum i40iw_status_code
i40iw_sc_manage_qhash_table_entry(
1183 struct i40iw_sc_cqp
*cqp
,
1184 struct i40iw_qhash_table_info
*info
,
1192 struct i40iw_sc_vsi
*vsi
= info
->vsi
;
1194 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1196 return I40IW_ERR_RING_FULL
;
1198 temp
= info
->mac_addr
[5] |
1199 LS_64_1(info
->mac_addr
[4], 8) |
1200 LS_64_1(info
->mac_addr
[3], 16) |
1201 LS_64_1(info
->mac_addr
[2], 24) |
1202 LS_64_1(info
->mac_addr
[1], 32) |
1203 LS_64_1(info
->mac_addr
[0], 40);
1205 set_64bit_val(wqe
, 0, temp
);
1207 qw1
= LS_64(info
->qp_num
, I40IW_CQPSQ_QHASH_QPN
) |
1208 LS_64(info
->dest_port
, I40IW_CQPSQ_QHASH_DEST_PORT
);
1209 if (info
->ipv4_valid
) {
1212 LS_64(info
->dest_ip
[0], I40IW_CQPSQ_QHASH_ADDR3
));
1216 LS_64(info
->dest_ip
[0], I40IW_CQPSQ_QHASH_ADDR0
) |
1217 LS_64(info
->dest_ip
[1], I40IW_CQPSQ_QHASH_ADDR1
));
1221 LS_64(info
->dest_ip
[2], I40IW_CQPSQ_QHASH_ADDR2
) |
1222 LS_64(info
->dest_ip
[3], I40IW_CQPSQ_QHASH_ADDR3
));
1224 qw2
= LS_64(vsi
->qos
[info
->user_pri
].qs_handle
, I40IW_CQPSQ_QHASH_QS_HANDLE
);
1225 if (info
->vlan_valid
)
1226 qw2
|= LS_64(info
->vlan_id
, I40IW_CQPSQ_QHASH_VLANID
);
1227 set_64bit_val(wqe
, 16, qw2
);
1228 if (info
->entry_type
== I40IW_QHASH_TYPE_TCP_ESTABLISHED
) {
1229 qw1
|= LS_64(info
->src_port
, I40IW_CQPSQ_QHASH_SRC_PORT
);
1230 if (!info
->ipv4_valid
) {
1233 LS_64(info
->src_ip
[0], I40IW_CQPSQ_QHASH_ADDR0
) |
1234 LS_64(info
->src_ip
[1], I40IW_CQPSQ_QHASH_ADDR1
));
1237 LS_64(info
->src_ip
[2], I40IW_CQPSQ_QHASH_ADDR2
) |
1238 LS_64(info
->src_ip
[3], I40IW_CQPSQ_QHASH_ADDR3
));
1242 LS_64(info
->src_ip
[0], I40IW_CQPSQ_QHASH_ADDR3
));
1246 set_64bit_val(wqe
, 8, qw1
);
1247 temp
= LS_64(cqp
->polarity
, I40IW_CQPSQ_QHASH_WQEVALID
) |
1248 LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY
, I40IW_CQPSQ_QHASH_OPCODE
) |
1249 LS_64(info
->manage
, I40IW_CQPSQ_QHASH_MANAGE
) |
1250 LS_64(info
->ipv4_valid
, I40IW_CQPSQ_QHASH_IPV4VALID
) |
1251 LS_64(info
->vlan_valid
, I40IW_CQPSQ_QHASH_VLANVALID
) |
1252 LS_64(info
->entry_type
, I40IW_CQPSQ_QHASH_ENTRYTYPE
);
1254 i40iw_insert_wqe_hdr(wqe
, temp
);
1256 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "MANAGE_QHASH WQE",
1257 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1260 i40iw_sc_cqp_post_sq(cqp
);
1265 * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry
1266 * @cqp: struct for cqp hw
1267 * @scratch: u64 saved to be used during cqp completion
1268 * @post_sq: flag for cqp db to ring
1270 static enum i40iw_status_code
i40iw_sc_alloc_local_mac_ipaddr_entry(
1271 struct i40iw_sc_cqp
*cqp
,
1278 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1280 return I40IW_ERR_RING_FULL
;
1281 header
= LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY
, I40IW_CQPSQ_OPCODE
) |
1282 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1284 i40iw_insert_wqe_hdr(wqe
, header
);
1285 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "ALLOCATE_LOCAL_MAC_IPADDR WQE",
1286 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1288 i40iw_sc_cqp_post_sq(cqp
);
1293 * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry
1294 * @cqp: struct for cqp hw
1295 * @info:mac addr info
1296 * @scratch: u64 saved to be used during cqp completion
1297 * @post_sq: flag for cqp db to ring
1299 static enum i40iw_status_code
i40iw_sc_add_local_mac_ipaddr_entry(
1300 struct i40iw_sc_cqp
*cqp
,
1301 struct i40iw_local_mac_ipaddr_entry_info
*info
,
1308 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1310 return I40IW_ERR_RING_FULL
;
1311 temp
= info
->mac_addr
[5] |
1312 LS_64_1(info
->mac_addr
[4], 8) |
1313 LS_64_1(info
->mac_addr
[3], 16) |
1314 LS_64_1(info
->mac_addr
[2], 24) |
1315 LS_64_1(info
->mac_addr
[1], 32) |
1316 LS_64_1(info
->mac_addr
[0], 40);
1318 set_64bit_val(wqe
, 32, temp
);
1320 header
= LS_64(info
->entry_idx
, I40IW_CQPSQ_MLIPA_IPTABLEIDX
) |
1321 LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE
, I40IW_CQPSQ_OPCODE
) |
1322 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1324 i40iw_insert_wqe_hdr(wqe
, header
);
1326 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "ADD_LOCAL_MAC_IPADDR WQE",
1327 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1330 i40iw_sc_cqp_post_sq(cqp
);
1335 * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac
1336 * @cqp: struct for cqp hw
1337 * @scratch: u64 saved to be used during cqp completion
1338 * @entry_idx: index of mac entry
1339 * @ ignore_ref_count: to force mac adde delete
1340 * @post_sq: flag for cqp db to ring
1342 static enum i40iw_status_code
i40iw_sc_del_local_mac_ipaddr_entry(
1343 struct i40iw_sc_cqp
*cqp
,
1346 u8 ignore_ref_count
,
1352 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1354 return I40IW_ERR_RING_FULL
;
1355 header
= LS_64(entry_idx
, I40IW_CQPSQ_MLIPA_IPTABLEIDX
) |
1356 LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE
, I40IW_CQPSQ_OPCODE
) |
1357 LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY
) |
1358 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
) |
1359 LS_64(ignore_ref_count
, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT
);
1361 i40iw_insert_wqe_hdr(wqe
, header
);
1363 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "DEL_LOCAL_MAC_IPADDR WQE",
1364 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1367 i40iw_sc_cqp_post_sq(cqp
);
1372 * i40iw_sc_cqp_nop - send a nop wqe
1373 * @cqp: struct for cqp hw
1374 * @scratch: u64 saved to be used during cqp completion
1375 * @post_sq: flag for cqp db to ring
1377 static enum i40iw_status_code
i40iw_sc_cqp_nop(struct i40iw_sc_cqp
*cqp
,
1384 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1386 return I40IW_ERR_RING_FULL
;
1387 header
= LS_64(I40IW_CQP_OP_NOP
, I40IW_CQPSQ_OPCODE
) |
1388 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1389 i40iw_insert_wqe_hdr(wqe
, header
);
1390 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "NOP WQE",
1391 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1394 i40iw_sc_cqp_post_sq(cqp
);
1399 * i40iw_sc_ceq_init - initialize ceq
1400 * @ceq: ceq sc structure
1401 * @info: ceq initialization info
1403 static enum i40iw_status_code
i40iw_sc_ceq_init(struct i40iw_sc_ceq
*ceq
,
1404 struct i40iw_ceq_init_info
*info
)
1408 if ((info
->elem_cnt
< I40IW_MIN_CEQ_ENTRIES
) ||
1409 (info
->elem_cnt
> I40IW_MAX_CEQ_ENTRIES
))
1410 return I40IW_ERR_INVALID_SIZE
;
1412 if (info
->ceq_id
>= I40IW_MAX_CEQID
)
1413 return I40IW_ERR_INVALID_CEQ_ID
;
1415 pble_obj_cnt
= info
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
1417 if (info
->virtual_map
&& (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
1418 return I40IW_ERR_INVALID_PBLE_INDEX
;
1420 ceq
->size
= sizeof(*ceq
);
1421 ceq
->ceqe_base
= (struct i40iw_ceqe
*)info
->ceqe_base
;
1422 ceq
->ceq_id
= info
->ceq_id
;
1423 ceq
->dev
= info
->dev
;
1424 ceq
->elem_cnt
= info
->elem_cnt
;
1425 ceq
->ceq_elem_pa
= info
->ceqe_pa
;
1426 ceq
->virtual_map
= info
->virtual_map
;
1428 ceq
->pbl_chunk_size
= (ceq
->virtual_map
? info
->pbl_chunk_size
: 0);
1429 ceq
->first_pm_pbl_idx
= (ceq
->virtual_map
? info
->first_pm_pbl_idx
: 0);
1430 ceq
->pbl_list
= (ceq
->virtual_map
? info
->pbl_list
: NULL
);
1432 ceq
->tph_en
= info
->tph_en
;
1433 ceq
->tph_val
= info
->tph_val
;
1435 I40IW_RING_INIT(ceq
->ceq_ring
, ceq
->elem_cnt
);
1436 ceq
->dev
->ceq
[info
->ceq_id
] = ceq
;
1442 * i40iw_sc_ceq_create - create ceq wqe
1443 * @ceq: ceq sc structure
1444 * @scratch: u64 saved to be used during cqp completion
1445 * @post_sq: flag for cqp db to ring
1447 static enum i40iw_status_code
i40iw_sc_ceq_create(struct i40iw_sc_ceq
*ceq
,
1451 struct i40iw_sc_cqp
*cqp
;
1455 cqp
= ceq
->dev
->cqp
;
1456 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1458 return I40IW_ERR_RING_FULL
;
1459 set_64bit_val(wqe
, 16, ceq
->elem_cnt
);
1460 set_64bit_val(wqe
, 32, (ceq
->virtual_map
? 0 : ceq
->ceq_elem_pa
));
1461 set_64bit_val(wqe
, 48, (ceq
->virtual_map
? ceq
->first_pm_pbl_idx
: 0));
1462 set_64bit_val(wqe
, 56, LS_64(ceq
->tph_val
, I40IW_CQPSQ_TPHVAL
));
1464 header
= ceq
->ceq_id
|
1465 LS_64(I40IW_CQP_OP_CREATE_CEQ
, I40IW_CQPSQ_OPCODE
) |
1466 LS_64(ceq
->pbl_chunk_size
, I40IW_CQPSQ_CEQ_LPBLSIZE
) |
1467 LS_64(ceq
->virtual_map
, I40IW_CQPSQ_CEQ_VMAP
) |
1468 LS_64(ceq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
1469 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1471 i40iw_insert_wqe_hdr(wqe
, header
);
1473 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CEQ_CREATE WQE",
1474 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1477 i40iw_sc_cqp_post_sq(cqp
);
1482 * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete
1483 * @ceq: ceq sc structure
1485 static enum i40iw_status_code
i40iw_sc_cceq_create_done(struct i40iw_sc_ceq
*ceq
)
1487 struct i40iw_sc_cqp
*cqp
;
1489 cqp
= ceq
->dev
->cqp
;
1490 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_CREATE_CEQ
, NULL
);
1494 * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete
1495 * @ceq: ceq sc structure
1497 static enum i40iw_status_code
i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq
*ceq
)
1499 struct i40iw_sc_cqp
*cqp
;
1501 cqp
= ceq
->dev
->cqp
;
1502 cqp
->process_cqp_sds
= i40iw_update_sds_noccq
;
1503 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_DESTROY_CEQ
, NULL
);
1507 * i40iw_sc_cceq_create - create cceq
1508 * @ceq: ceq sc structure
1509 * @scratch: u64 saved to be used during cqp completion
1511 static enum i40iw_status_code
i40iw_sc_cceq_create(struct i40iw_sc_ceq
*ceq
, u64 scratch
)
1513 enum i40iw_status_code ret_code
;
1515 ret_code
= i40iw_sc_ceq_create(ceq
, scratch
, true);
1517 ret_code
= i40iw_sc_cceq_create_done(ceq
);
1522 * i40iw_sc_ceq_destroy - destroy ceq
1523 * @ceq: ceq sc structure
1524 * @scratch: u64 saved to be used during cqp completion
1525 * @post_sq: flag for cqp db to ring
1527 static enum i40iw_status_code
i40iw_sc_ceq_destroy(struct i40iw_sc_ceq
*ceq
,
1531 struct i40iw_sc_cqp
*cqp
;
1535 cqp
= ceq
->dev
->cqp
;
1536 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1538 return I40IW_ERR_RING_FULL
;
1539 set_64bit_val(wqe
, 16, ceq
->elem_cnt
);
1540 set_64bit_val(wqe
, 48, ceq
->first_pm_pbl_idx
);
1541 header
= ceq
->ceq_id
|
1542 LS_64(I40IW_CQP_OP_DESTROY_CEQ
, I40IW_CQPSQ_OPCODE
) |
1543 LS_64(ceq
->pbl_chunk_size
, I40IW_CQPSQ_CEQ_LPBLSIZE
) |
1544 LS_64(ceq
->virtual_map
, I40IW_CQPSQ_CEQ_VMAP
) |
1545 LS_64(ceq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
1546 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1547 i40iw_insert_wqe_hdr(wqe
, header
);
1548 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CEQ_DESTROY WQE",
1549 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1552 i40iw_sc_cqp_post_sq(cqp
);
1557 * i40iw_sc_process_ceq - process ceq
1558 * @dev: sc device struct
1559 * @ceq: ceq sc structure
1561 static void *i40iw_sc_process_ceq(struct i40iw_sc_dev
*dev
, struct i40iw_sc_ceq
*ceq
)
1565 struct i40iw_sc_cq
*cq
= NULL
;
1568 ceqe
= (u64
*)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq
);
1569 get_64bit_val(ceqe
, 0, &temp
);
1570 polarity
= (u8
)RS_64(temp
, I40IW_CEQE_VALID
);
1571 if (polarity
!= ceq
->polarity
)
1574 cq
= (struct i40iw_sc_cq
*)(unsigned long)LS_64_1(temp
, 1);
1576 I40IW_RING_MOVE_TAIL(ceq
->ceq_ring
);
1577 if (I40IW_RING_GETCURRENT_TAIL(ceq
->ceq_ring
) == 0)
1581 i40iw_wr32(dev
->hw
, I40E_PFPE_CQACK
, cq
->cq_uk
.cq_id
);
1583 i40iw_wr32(dev
->hw
, I40E_VFPE_CQACK1
, cq
->cq_uk
.cq_id
);
1589 * i40iw_sc_aeq_init - initialize aeq
1590 * @aeq: aeq structure ptr
1591 * @info: aeq initialization info
1593 static enum i40iw_status_code
i40iw_sc_aeq_init(struct i40iw_sc_aeq
*aeq
,
1594 struct i40iw_aeq_init_info
*info
)
1598 if ((info
->elem_cnt
< I40IW_MIN_AEQ_ENTRIES
) ||
1599 (info
->elem_cnt
> I40IW_MAX_AEQ_ENTRIES
))
1600 return I40IW_ERR_INVALID_SIZE
;
1601 pble_obj_cnt
= info
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
1603 if (info
->virtual_map
&& (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
1604 return I40IW_ERR_INVALID_PBLE_INDEX
;
1606 aeq
->size
= sizeof(*aeq
);
1608 aeq
->aeqe_base
= (struct i40iw_sc_aeqe
*)info
->aeqe_base
;
1609 aeq
->dev
= info
->dev
;
1610 aeq
->elem_cnt
= info
->elem_cnt
;
1612 aeq
->aeq_elem_pa
= info
->aeq_elem_pa
;
1613 I40IW_RING_INIT(aeq
->aeq_ring
, aeq
->elem_cnt
);
1614 info
->dev
->aeq
= aeq
;
1616 aeq
->virtual_map
= info
->virtual_map
;
1617 aeq
->pbl_list
= (aeq
->virtual_map
? info
->pbl_list
: NULL
);
1618 aeq
->pbl_chunk_size
= (aeq
->virtual_map
? info
->pbl_chunk_size
: 0);
1619 aeq
->first_pm_pbl_idx
= (aeq
->virtual_map
? info
->first_pm_pbl_idx
: 0);
1620 info
->dev
->aeq
= aeq
;
1625 * i40iw_sc_aeq_create - create aeq
1626 * @aeq: aeq structure ptr
1627 * @scratch: u64 saved to be used during cqp completion
1628 * @post_sq: flag for cqp db to ring
1630 static enum i40iw_status_code
i40iw_sc_aeq_create(struct i40iw_sc_aeq
*aeq
,
1635 struct i40iw_sc_cqp
*cqp
;
1638 cqp
= aeq
->dev
->cqp
;
1639 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1641 return I40IW_ERR_RING_FULL
;
1642 set_64bit_val(wqe
, 16, aeq
->elem_cnt
);
1643 set_64bit_val(wqe
, 32,
1644 (aeq
->virtual_map
? 0 : aeq
->aeq_elem_pa
));
1645 set_64bit_val(wqe
, 48,
1646 (aeq
->virtual_map
? aeq
->first_pm_pbl_idx
: 0));
1648 header
= LS_64(I40IW_CQP_OP_CREATE_AEQ
, I40IW_CQPSQ_OPCODE
) |
1649 LS_64(aeq
->pbl_chunk_size
, I40IW_CQPSQ_AEQ_LPBLSIZE
) |
1650 LS_64(aeq
->virtual_map
, I40IW_CQPSQ_AEQ_VMAP
) |
1651 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1653 i40iw_insert_wqe_hdr(wqe
, header
);
1654 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "AEQ_CREATE WQE",
1655 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1657 i40iw_sc_cqp_post_sq(cqp
);
1662 * i40iw_sc_aeq_destroy - destroy aeq during close
1663 * @aeq: aeq structure ptr
1664 * @scratch: u64 saved to be used during cqp completion
1665 * @post_sq: flag for cqp db to ring
1667 static enum i40iw_status_code
i40iw_sc_aeq_destroy(struct i40iw_sc_aeq
*aeq
,
1672 struct i40iw_sc_cqp
*cqp
;
1675 cqp
= aeq
->dev
->cqp
;
1676 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1678 return I40IW_ERR_RING_FULL
;
1679 set_64bit_val(wqe
, 16, aeq
->elem_cnt
);
1680 set_64bit_val(wqe
, 48, aeq
->first_pm_pbl_idx
);
1681 header
= LS_64(I40IW_CQP_OP_DESTROY_AEQ
, I40IW_CQPSQ_OPCODE
) |
1682 LS_64(aeq
->pbl_chunk_size
, I40IW_CQPSQ_AEQ_LPBLSIZE
) |
1683 LS_64(aeq
->virtual_map
, I40IW_CQPSQ_AEQ_VMAP
) |
1684 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1685 i40iw_insert_wqe_hdr(wqe
, header
);
1687 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "AEQ_DESTROY WQE",
1688 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1690 i40iw_sc_cqp_post_sq(cqp
);
1695 * i40iw_sc_get_next_aeqe - get next aeq entry
1696 * @aeq: aeq structure ptr
1697 * @info: aeqe info to be returned
1699 static enum i40iw_status_code
i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq
*aeq
,
1700 struct i40iw_aeqe_info
*info
)
1702 u64 temp
, compl_ctx
;
1708 aeqe
= (u64
*)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq
);
1709 get_64bit_val(aeqe
, 0, &compl_ctx
);
1710 get_64bit_val(aeqe
, 8, &temp
);
1711 polarity
= (u8
)RS_64(temp
, I40IW_AEQE_VALID
);
1713 if (aeq
->polarity
!= polarity
)
1714 return I40IW_ERR_QUEUE_EMPTY
;
1716 i40iw_debug_buf(aeq
->dev
, I40IW_DEBUG_WQE
, "AEQ_ENTRY", aeqe
, 16);
1718 ae_src
= (u8
)RS_64(temp
, I40IW_AEQE_AESRC
);
1719 wqe_idx
= (u16
)RS_64(temp
, I40IW_AEQE_WQDESCIDX
);
1720 info
->qp_cq_id
= (u32
)RS_64(temp
, I40IW_AEQE_QPCQID
);
1721 info
->ae_id
= (u16
)RS_64(temp
, I40IW_AEQE_AECODE
);
1722 info
->tcp_state
= (u8
)RS_64(temp
, I40IW_AEQE_TCPSTATE
);
1723 info
->iwarp_state
= (u8
)RS_64(temp
, I40IW_AEQE_IWSTATE
);
1724 info
->q2_data_written
= (u8
)RS_64(temp
, I40IW_AEQE_Q2DATA
);
1725 info
->aeqe_overflow
= (bool)RS_64(temp
, I40IW_AEQE_OVERFLOW
);
1727 case I40IW_AE_SOURCE_RQ
:
1728 case I40IW_AE_SOURCE_RQ_0011
:
1730 info
->wqe_idx
= wqe_idx
;
1731 info
->compl_ctx
= compl_ctx
;
1733 case I40IW_AE_SOURCE_CQ
:
1734 case I40IW_AE_SOURCE_CQ_0110
:
1735 case I40IW_AE_SOURCE_CQ_1010
:
1736 case I40IW_AE_SOURCE_CQ_1110
:
1738 info
->compl_ctx
= LS_64_1(compl_ctx
, 1);
1740 case I40IW_AE_SOURCE_SQ
:
1741 case I40IW_AE_SOURCE_SQ_0111
:
1744 info
->wqe_idx
= wqe_idx
;
1745 info
->compl_ctx
= compl_ctx
;
1747 case I40IW_AE_SOURCE_IN_RR_WR
:
1748 case I40IW_AE_SOURCE_IN_RR_WR_1011
:
1750 info
->compl_ctx
= compl_ctx
;
1751 info
->in_rdrsp_wr
= true;
1753 case I40IW_AE_SOURCE_OUT_RR
:
1754 case I40IW_AE_SOURCE_OUT_RR_1111
:
1756 info
->compl_ctx
= compl_ctx
;
1757 info
->out_rdrsp
= true;
1762 I40IW_RING_MOVE_TAIL(aeq
->aeq_ring
);
1763 if (I40IW_RING_GETCURRENT_TAIL(aeq
->aeq_ring
) == 0)
1769 * i40iw_sc_repost_aeq_entries - repost completed aeq entries
1770 * @dev: sc device struct
1771 * @count: allocate count
1773 static enum i40iw_status_code
i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev
*dev
,
1776 if (count
> I40IW_MAX_AEQ_ALLOCATE_COUNT
)
1777 return I40IW_ERR_INVALID_SIZE
;
1780 i40iw_wr32(dev
->hw
, I40E_PFPE_AEQALLOC
, count
);
1782 i40iw_wr32(dev
->hw
, I40E_VFPE_AEQALLOC1
, count
);
1788 * i40iw_sc_aeq_create_done - create aeq
1789 * @aeq: aeq structure ptr
1791 static enum i40iw_status_code
i40iw_sc_aeq_create_done(struct i40iw_sc_aeq
*aeq
)
1793 struct i40iw_sc_cqp
*cqp
;
1795 cqp
= aeq
->dev
->cqp
;
1796 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_CREATE_AEQ
, NULL
);
1800 * i40iw_sc_aeq_destroy_done - destroy of aeq during close
1801 * @aeq: aeq structure ptr
1803 static enum i40iw_status_code
i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq
*aeq
)
1805 struct i40iw_sc_cqp
*cqp
;
1807 cqp
= aeq
->dev
->cqp
;
1808 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_DESTROY_AEQ
, NULL
);
1812 * i40iw_sc_ccq_init - initialize control cq
1813 * @cq: sc's cq ctruct
1814 * @info: info for control cq initialization
1816 static enum i40iw_status_code
i40iw_sc_ccq_init(struct i40iw_sc_cq
*cq
,
1817 struct i40iw_ccq_init_info
*info
)
1821 if (info
->num_elem
< I40IW_MIN_CQ_SIZE
|| info
->num_elem
> I40IW_MAX_CQ_SIZE
)
1822 return I40IW_ERR_INVALID_SIZE
;
1824 if (info
->ceq_id
> I40IW_MAX_CEQID
)
1825 return I40IW_ERR_INVALID_CEQ_ID
;
1827 pble_obj_cnt
= info
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
1829 if (info
->virtual_map
&& (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
1830 return I40IW_ERR_INVALID_PBLE_INDEX
;
1832 cq
->cq_pa
= info
->cq_pa
;
1833 cq
->cq_uk
.cq_base
= info
->cq_base
;
1834 cq
->shadow_area_pa
= info
->shadow_area_pa
;
1835 cq
->cq_uk
.shadow_area
= info
->shadow_area
;
1836 cq
->shadow_read_threshold
= info
->shadow_read_threshold
;
1837 cq
->dev
= info
->dev
;
1838 cq
->ceq_id
= info
->ceq_id
;
1839 cq
->cq_uk
.cq_size
= info
->num_elem
;
1840 cq
->cq_type
= I40IW_CQ_TYPE_CQP
;
1841 cq
->ceqe_mask
= info
->ceqe_mask
;
1842 I40IW_RING_INIT(cq
->cq_uk
.cq_ring
, info
->num_elem
);
1844 cq
->cq_uk
.cq_id
= 0; /* control cq is id 0 always */
1845 cq
->ceq_id_valid
= info
->ceq_id_valid
;
1846 cq
->tph_en
= info
->tph_en
;
1847 cq
->tph_val
= info
->tph_val
;
1848 cq
->cq_uk
.avoid_mem_cflct
= info
->avoid_mem_cflct
;
1850 cq
->pbl_list
= info
->pbl_list
;
1851 cq
->virtual_map
= info
->virtual_map
;
1852 cq
->pbl_chunk_size
= info
->pbl_chunk_size
;
1853 cq
->first_pm_pbl_idx
= info
->first_pm_pbl_idx
;
1854 cq
->cq_uk
.polarity
= true;
1856 /* following are only for iw cqs so initialize them to zero */
1857 cq
->cq_uk
.cqe_alloc_reg
= NULL
;
1858 info
->dev
->ccq
= cq
;
1863 * i40iw_sc_ccq_create_done - poll cqp for ccq create
1864 * @ccq: ccq sc struct
1866 static enum i40iw_status_code
i40iw_sc_ccq_create_done(struct i40iw_sc_cq
*ccq
)
1868 struct i40iw_sc_cqp
*cqp
;
1870 cqp
= ccq
->dev
->cqp
;
1871 return i40iw_sc_poll_for_cqp_op_done(cqp
, I40IW_CQP_OP_CREATE_CQ
, NULL
);
1875 * i40iw_sc_ccq_create - create control cq
1876 * @ccq: ccq sc struct
1877 * @scratch: u64 saved to be used during cqp completion
1878 * @check_overflow: overlow flag for ccq
1879 * @post_sq: flag for cqp db to ring
1881 static enum i40iw_status_code
i40iw_sc_ccq_create(struct i40iw_sc_cq
*ccq
,
1883 bool check_overflow
,
1887 struct i40iw_sc_cqp
*cqp
;
1889 enum i40iw_status_code ret_code
;
1891 cqp
= ccq
->dev
->cqp
;
1892 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1894 return I40IW_ERR_RING_FULL
;
1895 set_64bit_val(wqe
, 0, ccq
->cq_uk
.cq_size
);
1896 set_64bit_val(wqe
, 8, RS_64_1(ccq
, 1));
1897 set_64bit_val(wqe
, 16,
1898 LS_64(ccq
->shadow_read_threshold
, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD
));
1899 set_64bit_val(wqe
, 32, (ccq
->virtual_map
? 0 : ccq
->cq_pa
));
1900 set_64bit_val(wqe
, 40, ccq
->shadow_area_pa
);
1901 set_64bit_val(wqe
, 48,
1902 (ccq
->virtual_map
? ccq
->first_pm_pbl_idx
: 0));
1903 set_64bit_val(wqe
, 56,
1904 LS_64(ccq
->tph_val
, I40IW_CQPSQ_TPHVAL
));
1906 header
= ccq
->cq_uk
.cq_id
|
1907 LS_64((ccq
->ceq_id_valid
? ccq
->ceq_id
: 0), I40IW_CQPSQ_CQ_CEQID
) |
1908 LS_64(I40IW_CQP_OP_CREATE_CQ
, I40IW_CQPSQ_OPCODE
) |
1909 LS_64(ccq
->pbl_chunk_size
, I40IW_CQPSQ_CQ_LPBLSIZE
) |
1910 LS_64(check_overflow
, I40IW_CQPSQ_CQ_CHKOVERFLOW
) |
1911 LS_64(ccq
->virtual_map
, I40IW_CQPSQ_CQ_VIRTMAP
) |
1912 LS_64(ccq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
1913 LS_64(ccq
->ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
1914 LS_64(ccq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
1915 LS_64(ccq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
1916 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1918 i40iw_insert_wqe_hdr(wqe
, header
);
1920 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CCQ_CREATE WQE",
1921 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1924 i40iw_sc_cqp_post_sq(cqp
);
1925 ret_code
= i40iw_sc_ccq_create_done(ccq
);
1929 cqp
->process_cqp_sds
= i40iw_cqp_sds_cmd
;
1935 * i40iw_sc_ccq_destroy - destroy ccq during close
1936 * @ccq: ccq sc struct
1937 * @scratch: u64 saved to be used during cqp completion
1938 * @post_sq: flag for cqp db to ring
1940 static enum i40iw_status_code
i40iw_sc_ccq_destroy(struct i40iw_sc_cq
*ccq
,
1944 struct i40iw_sc_cqp
*cqp
;
1947 enum i40iw_status_code ret_code
= 0;
1948 u32 tail
, val
, error
;
1950 cqp
= ccq
->dev
->cqp
;
1951 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
1953 return I40IW_ERR_RING_FULL
;
1954 set_64bit_val(wqe
, 0, ccq
->cq_uk
.cq_size
);
1955 set_64bit_val(wqe
, 8, RS_64_1(ccq
, 1));
1956 set_64bit_val(wqe
, 40, ccq
->shadow_area_pa
);
1958 header
= ccq
->cq_uk
.cq_id
|
1959 LS_64((ccq
->ceq_id_valid
? ccq
->ceq_id
: 0), I40IW_CQPSQ_CQ_CEQID
) |
1960 LS_64(I40IW_CQP_OP_DESTROY_CQ
, I40IW_CQPSQ_OPCODE
) |
1961 LS_64(ccq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
1962 LS_64(ccq
->ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
1963 LS_64(ccq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
1964 LS_64(ccq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
1965 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
1967 i40iw_insert_wqe_hdr(wqe
, header
);
1969 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CCQ_DESTROY WQE",
1970 wqe
, I40IW_CQP_WQE_SIZE
* 8);
1972 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
1974 return I40IW_ERR_CQP_COMPL_ERROR
;
1977 i40iw_sc_cqp_post_sq(cqp
);
1978 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, 1000);
1985 * i40iw_sc_cq_init - initialize completion q
1987 * @info: cq initialization info
1989 static enum i40iw_status_code
i40iw_sc_cq_init(struct i40iw_sc_cq
*cq
,
1990 struct i40iw_cq_init_info
*info
)
1992 u32 __iomem
*cqe_alloc_reg
= NULL
;
1993 enum i40iw_status_code ret_code
;
1997 pble_obj_cnt
= info
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
1999 if (info
->virtual_map
&& (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
2000 return I40IW_ERR_INVALID_PBLE_INDEX
;
2002 cq
->cq_pa
= info
->cq_base_pa
;
2003 cq
->dev
= info
->dev
;
2004 cq
->ceq_id
= info
->ceq_id
;
2005 arm_offset
= (info
->dev
->is_pf
) ? I40E_PFPE_CQARM
: I40E_VFPE_CQARM1
;
2006 if (i40iw_get_hw_addr(cq
->dev
))
2007 cqe_alloc_reg
= (u32 __iomem
*)(i40iw_get_hw_addr(cq
->dev
) +
2009 info
->cq_uk_init_info
.cqe_alloc_reg
= cqe_alloc_reg
;
2010 ret_code
= i40iw_cq_uk_init(&cq
->cq_uk
, &info
->cq_uk_init_info
);
2013 cq
->virtual_map
= info
->virtual_map
;
2014 cq
->pbl_chunk_size
= info
->pbl_chunk_size
;
2015 cq
->ceqe_mask
= info
->ceqe_mask
;
2016 cq
->cq_type
= (info
->type
) ? info
->type
: I40IW_CQ_TYPE_IWARP
;
2018 cq
->shadow_area_pa
= info
->shadow_area_pa
;
2019 cq
->shadow_read_threshold
= info
->shadow_read_threshold
;
2021 cq
->ceq_id_valid
= info
->ceq_id_valid
;
2022 cq
->tph_en
= info
->tph_en
;
2023 cq
->tph_val
= info
->tph_val
;
2025 cq
->first_pm_pbl_idx
= info
->first_pm_pbl_idx
;
2031 * i40iw_sc_cq_create - create completion q
2033 * @scratch: u64 saved to be used during cqp completion
2034 * @check_overflow: flag for overflow check
2035 * @post_sq: flag for cqp db to ring
2037 static enum i40iw_status_code
i40iw_sc_cq_create(struct i40iw_sc_cq
*cq
,
2039 bool check_overflow
,
2043 struct i40iw_sc_cqp
*cqp
;
2046 if (cq
->cq_uk
.cq_id
> I40IW_MAX_CQID
)
2047 return I40IW_ERR_INVALID_CQ_ID
;
2049 if (cq
->ceq_id
> I40IW_MAX_CEQID
)
2050 return I40IW_ERR_INVALID_CEQ_ID
;
2053 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2055 return I40IW_ERR_RING_FULL
;
2057 set_64bit_val(wqe
, 0, cq
->cq_uk
.cq_size
);
2058 set_64bit_val(wqe
, 8, RS_64_1(cq
, 1));
2061 LS_64(cq
->shadow_read_threshold
, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD
));
2063 set_64bit_val(wqe
, 32, (cq
->virtual_map
? 0 : cq
->cq_pa
));
2065 set_64bit_val(wqe
, 40, cq
->shadow_area_pa
);
2066 set_64bit_val(wqe
, 48, (cq
->virtual_map
? cq
->first_pm_pbl_idx
: 0));
2067 set_64bit_val(wqe
, 56, LS_64(cq
->tph_val
, I40IW_CQPSQ_TPHVAL
));
2069 header
= cq
->cq_uk
.cq_id
|
2070 LS_64((cq
->ceq_id_valid
? cq
->ceq_id
: 0), I40IW_CQPSQ_CQ_CEQID
) |
2071 LS_64(I40IW_CQP_OP_CREATE_CQ
, I40IW_CQPSQ_OPCODE
) |
2072 LS_64(cq
->pbl_chunk_size
, I40IW_CQPSQ_CQ_LPBLSIZE
) |
2073 LS_64(check_overflow
, I40IW_CQPSQ_CQ_CHKOVERFLOW
) |
2074 LS_64(cq
->virtual_map
, I40IW_CQPSQ_CQ_VIRTMAP
) |
2075 LS_64(cq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
2076 LS_64(cq
->ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
2077 LS_64(cq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
2078 LS_64(cq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
2079 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2081 i40iw_insert_wqe_hdr(wqe
, header
);
2083 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CQ_CREATE WQE",
2084 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2087 i40iw_sc_cqp_post_sq(cqp
);
2092 * i40iw_sc_cq_destroy - destroy completion q
2094 * @scratch: u64 saved to be used during cqp completion
2095 * @post_sq: flag for cqp db to ring
2097 static enum i40iw_status_code
i40iw_sc_cq_destroy(struct i40iw_sc_cq
*cq
,
2101 struct i40iw_sc_cqp
*cqp
;
2106 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2108 return I40IW_ERR_RING_FULL
;
2109 set_64bit_val(wqe
, 0, cq
->cq_uk
.cq_size
);
2110 set_64bit_val(wqe
, 8, RS_64_1(cq
, 1));
2111 set_64bit_val(wqe
, 40, cq
->shadow_area_pa
);
2112 set_64bit_val(wqe
, 48, (cq
->virtual_map
? cq
->first_pm_pbl_idx
: 0));
2114 header
= cq
->cq_uk
.cq_id
|
2115 LS_64((cq
->ceq_id_valid
? cq
->ceq_id
: 0), I40IW_CQPSQ_CQ_CEQID
) |
2116 LS_64(I40IW_CQP_OP_DESTROY_CQ
, I40IW_CQPSQ_OPCODE
) |
2117 LS_64(cq
->pbl_chunk_size
, I40IW_CQPSQ_CQ_LPBLSIZE
) |
2118 LS_64(cq
->virtual_map
, I40IW_CQPSQ_CQ_VIRTMAP
) |
2119 LS_64(cq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
2120 LS_64(cq
->ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
2121 LS_64(cq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
2122 LS_64(cq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
2123 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2125 i40iw_insert_wqe_hdr(wqe
, header
);
2127 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CQ_DESTROY WQE",
2128 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2131 i40iw_sc_cqp_post_sq(cqp
);
2136 * i40iw_sc_cq_modify - modify a Completion Queue
2138 * @info: modification info struct
2140 * @post_sq: flag to post to sq
2142 static enum i40iw_status_code
i40iw_sc_cq_modify(struct i40iw_sc_cq
*cq
,
2143 struct i40iw_modify_cq_info
*info
,
2147 struct i40iw_sc_cqp
*cqp
;
2150 u32 cq_size
, ceq_id
, first_pm_pbl_idx
;
2152 bool virtual_map
, ceq_id_valid
, check_overflow
;
2155 if (info
->ceq_valid
&& (info
->ceq_id
> I40IW_MAX_CEQID
))
2156 return I40IW_ERR_INVALID_CEQ_ID
;
2158 pble_obj_cnt
= cq
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
2160 if (info
->cq_resize
&& info
->virtual_map
&&
2161 (info
->first_pm_pbl_idx
>= pble_obj_cnt
))
2162 return I40IW_ERR_INVALID_PBLE_INDEX
;
2165 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2167 return I40IW_ERR_RING_FULL
;
2169 cq
->pbl_list
= info
->pbl_list
;
2170 cq
->cq_pa
= info
->cq_pa
;
2171 cq
->first_pm_pbl_idx
= info
->first_pm_pbl_idx
;
2173 cq_size
= info
->cq_resize
? info
->cq_size
: cq
->cq_uk
.cq_size
;
2174 if (info
->ceq_change
) {
2175 ceq_id_valid
= true;
2176 ceq_id
= info
->ceq_id
;
2178 ceq_id_valid
= cq
->ceq_id_valid
;
2179 ceq_id
= ceq_id_valid
? cq
->ceq_id
: 0;
2181 virtual_map
= info
->cq_resize
? info
->virtual_map
: cq
->virtual_map
;
2182 first_pm_pbl_idx
= (info
->cq_resize
?
2183 (info
->virtual_map
? info
->first_pm_pbl_idx
: 0) :
2184 (cq
->virtual_map
? cq
->first_pm_pbl_idx
: 0));
2185 pbl_chunk_size
= (info
->cq_resize
?
2186 (info
->virtual_map
? info
->pbl_chunk_size
: 0) :
2187 (cq
->virtual_map
? cq
->pbl_chunk_size
: 0));
2188 check_overflow
= info
->check_overflow_change
? info
->check_overflow
:
2190 cq
->cq_uk
.cq_size
= cq_size
;
2191 cq
->ceq_id_valid
= ceq_id_valid
;
2192 cq
->ceq_id
= ceq_id
;
2193 cq
->virtual_map
= virtual_map
;
2194 cq
->first_pm_pbl_idx
= first_pm_pbl_idx
;
2195 cq
->pbl_chunk_size
= pbl_chunk_size
;
2196 cq
->check_overflow
= check_overflow
;
2198 set_64bit_val(wqe
, 0, cq_size
);
2199 set_64bit_val(wqe
, 8, RS_64_1(cq
, 1));
2200 set_64bit_val(wqe
, 16,
2201 LS_64(info
->shadow_read_threshold
, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD
));
2202 set_64bit_val(wqe
, 32, (cq
->virtual_map
? 0 : cq
->cq_pa
));
2203 set_64bit_val(wqe
, 40, cq
->shadow_area_pa
);
2204 set_64bit_val(wqe
, 48, (cq
->virtual_map
? first_pm_pbl_idx
: 0));
2205 set_64bit_val(wqe
, 56, LS_64(cq
->tph_val
, I40IW_CQPSQ_TPHVAL
));
2207 header
= cq
->cq_uk
.cq_id
|
2208 LS_64(ceq_id
, I40IW_CQPSQ_CQ_CEQID
) |
2209 LS_64(I40IW_CQP_OP_MODIFY_CQ
, I40IW_CQPSQ_OPCODE
) |
2210 LS_64(info
->cq_resize
, I40IW_CQPSQ_CQ_CQRESIZE
) |
2211 LS_64(pbl_chunk_size
, I40IW_CQPSQ_CQ_LPBLSIZE
) |
2212 LS_64(check_overflow
, I40IW_CQPSQ_CQ_CHKOVERFLOW
) |
2213 LS_64(virtual_map
, I40IW_CQPSQ_CQ_VIRTMAP
) |
2214 LS_64(cq
->ceqe_mask
, I40IW_CQPSQ_CQ_ENCEQEMASK
) |
2215 LS_64(ceq_id_valid
, I40IW_CQPSQ_CQ_CEQIDVALID
) |
2216 LS_64(cq
->tph_en
, I40IW_CQPSQ_TPHEN
) |
2217 LS_64(cq
->cq_uk
.avoid_mem_cflct
, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT
) |
2218 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2220 i40iw_insert_wqe_hdr(wqe
, header
);
2222 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "CQ_MODIFY WQE",
2223 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2226 i40iw_sc_cqp_post_sq(cqp
);
2231 * i40iw_sc_qp_init - initialize qp
2233 * @info: initialization qp info
2235 static enum i40iw_status_code
i40iw_sc_qp_init(struct i40iw_sc_qp
*qp
,
2236 struct i40iw_qp_init_info
*info
)
2238 u32 __iomem
*wqe_alloc_reg
= NULL
;
2239 enum i40iw_status_code ret_code
;
2244 qp
->dev
= info
->pd
->dev
;
2245 qp
->vsi
= info
->vsi
;
2246 qp
->sq_pa
= info
->sq_pa
;
2247 qp
->rq_pa
= info
->rq_pa
;
2248 qp
->hw_host_ctx_pa
= info
->host_ctx_pa
;
2249 qp
->q2_pa
= info
->q2_pa
;
2250 qp
->shadow_area_pa
= info
->shadow_area_pa
;
2252 qp
->q2_buf
= info
->q2
;
2254 qp
->hw_host_ctx
= info
->host_ctx
;
2255 offset
= (qp
->pd
->dev
->is_pf
) ? I40E_PFPE_WQEALLOC
: I40E_VFPE_WQEALLOC1
;
2256 if (i40iw_get_hw_addr(qp
->pd
->dev
))
2257 wqe_alloc_reg
= (u32 __iomem
*)(i40iw_get_hw_addr(qp
->pd
->dev
) +
2260 info
->qp_uk_init_info
.wqe_alloc_reg
= wqe_alloc_reg
;
2261 info
->qp_uk_init_info
.abi_ver
= qp
->pd
->abi_ver
;
2262 ret_code
= i40iw_qp_uk_init(&qp
->qp_uk
, &info
->qp_uk_init_info
);
2265 qp
->virtual_map
= info
->virtual_map
;
2267 pble_obj_cnt
= info
->pd
->dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
2269 if ((info
->virtual_map
&& (info
->sq_pa
>= pble_obj_cnt
)) ||
2270 (info
->virtual_map
&& (info
->rq_pa
>= pble_obj_cnt
)))
2271 return I40IW_ERR_INVALID_PBLE_INDEX
;
2273 qp
->llp_stream_handle
= (void *)(-1);
2274 qp
->qp_type
= (info
->type
) ? info
->type
: I40IW_QP_TYPE_IWARP
;
2276 qp
->hw_sq_size
= i40iw_get_encoded_wqe_size(qp
->qp_uk
.sq_ring
.size
,
2278 i40iw_debug(qp
->dev
, I40IW_DEBUG_WQE
, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",
2279 __func__
, qp
->hw_sq_size
, qp
->qp_uk
.sq_ring
.size
);
2281 switch (qp
->pd
->abi_ver
) {
2283 ret_code
= i40iw_fragcnt_to_wqesize_rq(qp
->qp_uk
.max_rq_frag_cnt
,
2288 case 5: /* fallthrough until next ABI version */
2290 if (qp
->qp_uk
.max_rq_frag_cnt
> I40IW_MAX_WQ_FRAGMENT_COUNT
)
2291 return I40IW_ERR_INVALID_FRAG_COUNT
;
2292 wqe_size
= I40IW_MAX_WQE_SIZE_RQ
;
2295 qp
->hw_rq_size
= i40iw_get_encoded_wqe_size(qp
->qp_uk
.rq_size
*
2296 (wqe_size
/ I40IW_QP_WQE_MIN_SIZE
), false);
2297 i40iw_debug(qp
->dev
, I40IW_DEBUG_WQE
,
2298 "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
2299 __func__
, qp
->hw_rq_size
, qp
->qp_uk
.rq_size
, wqe_size
);
2300 qp
->sq_tph_val
= info
->sq_tph_val
;
2301 qp
->rq_tph_val
= info
->rq_tph_val
;
2302 qp
->sq_tph_en
= info
->sq_tph_en
;
2303 qp
->rq_tph_en
= info
->rq_tph_en
;
2304 qp
->rcv_tph_en
= info
->rcv_tph_en
;
2305 qp
->xmit_tph_en
= info
->xmit_tph_en
;
2306 qp
->qs_handle
= qp
->vsi
->qos
[qp
->user_pri
].qs_handle
;
2307 qp
->exception_lan_queue
= qp
->pd
->dev
->exception_lan_queue
;
2313 * i40iw_sc_qp_create - create qp
2315 * @info: qp create info
2316 * @scratch: u64 saved to be used during cqp completion
2317 * @post_sq: flag for cqp db to ring
2319 static enum i40iw_status_code
i40iw_sc_qp_create(
2320 struct i40iw_sc_qp
*qp
,
2321 struct i40iw_create_qp_info
*info
,
2325 struct i40iw_sc_cqp
*cqp
;
2329 if ((qp
->qp_uk
.qp_id
< I40IW_MIN_IW_QP_ID
) ||
2330 (qp
->qp_uk
.qp_id
> I40IW_MAX_IW_QP_ID
))
2331 return I40IW_ERR_INVALID_QP_ID
;
2333 cqp
= qp
->pd
->dev
->cqp
;
2334 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2336 return I40IW_ERR_RING_FULL
;
2338 set_64bit_val(wqe
, 16, qp
->hw_host_ctx_pa
);
2340 set_64bit_val(wqe
, 40, qp
->shadow_area_pa
);
2342 header
= qp
->qp_uk
.qp_id
|
2343 LS_64(I40IW_CQP_OP_CREATE_QP
, I40IW_CQPSQ_OPCODE
) |
2344 LS_64((info
->ord_valid
? 1 : 0), I40IW_CQPSQ_QP_ORDVALID
) |
2345 LS_64(info
->tcp_ctx_valid
, I40IW_CQPSQ_QP_TOECTXVALID
) |
2346 LS_64(qp
->qp_type
, I40IW_CQPSQ_QP_QPTYPE
) |
2347 LS_64(qp
->virtual_map
, I40IW_CQPSQ_QP_VQ
) |
2348 LS_64(info
->cq_num_valid
, I40IW_CQPSQ_QP_CQNUMVALID
) |
2349 LS_64(info
->static_rsrc
, I40IW_CQPSQ_QP_STATRSRC
) |
2350 LS_64(info
->arp_cache_idx_valid
, I40IW_CQPSQ_QP_ARPTABIDXVALID
) |
2351 LS_64(info
->next_iwarp_state
, I40IW_CQPSQ_QP_NEXTIWSTATE
) |
2352 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2354 i40iw_insert_wqe_hdr(wqe
, header
);
2355 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QP_CREATE WQE",
2356 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2359 i40iw_sc_cqp_post_sq(cqp
);
2364 * i40iw_sc_qp_modify - modify qp cqp wqe
2366 * @info: modify qp info
2367 * @scratch: u64 saved to be used during cqp completion
2368 * @post_sq: flag for cqp db to ring
2370 static enum i40iw_status_code
i40iw_sc_qp_modify(
2371 struct i40iw_sc_qp
*qp
,
2372 struct i40iw_modify_qp_info
*info
,
2377 struct i40iw_sc_cqp
*cqp
;
2379 u8 term_actions
= 0;
2382 cqp
= qp
->pd
->dev
->cqp
;
2383 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2385 return I40IW_ERR_RING_FULL
;
2386 if (info
->next_iwarp_state
== I40IW_QP_STATE_TERMINATE
) {
2387 if (info
->dont_send_fin
)
2388 term_actions
+= I40IWQP_TERM_SEND_TERM_ONLY
;
2389 if (info
->dont_send_term
)
2390 term_actions
+= I40IWQP_TERM_SEND_FIN_ONLY
;
2391 if ((term_actions
== I40IWQP_TERM_SEND_TERM_AND_FIN
) ||
2392 (term_actions
== I40IWQP_TERM_SEND_TERM_ONLY
))
2393 term_len
= info
->termlen
;
2398 LS_64(info
->new_mss
, I40IW_CQPSQ_QP_NEWMSS
) |
2399 LS_64(term_len
, I40IW_CQPSQ_QP_TERMLEN
));
2401 set_64bit_val(wqe
, 16, qp
->hw_host_ctx_pa
);
2402 set_64bit_val(wqe
, 40, qp
->shadow_area_pa
);
2404 header
= qp
->qp_uk
.qp_id
|
2405 LS_64(I40IW_CQP_OP_MODIFY_QP
, I40IW_CQPSQ_OPCODE
) |
2406 LS_64(info
->ord_valid
, I40IW_CQPSQ_QP_ORDVALID
) |
2407 LS_64(info
->tcp_ctx_valid
, I40IW_CQPSQ_QP_TOECTXVALID
) |
2408 LS_64(info
->cached_var_valid
, I40IW_CQPSQ_QP_CACHEDVARVALID
) |
2409 LS_64(qp
->virtual_map
, I40IW_CQPSQ_QP_VQ
) |
2410 LS_64(info
->cq_num_valid
, I40IW_CQPSQ_QP_CQNUMVALID
) |
2411 LS_64(info
->force_loopback
, I40IW_CQPSQ_QP_FORCELOOPBACK
) |
2412 LS_64(qp
->qp_type
, I40IW_CQPSQ_QP_QPTYPE
) |
2413 LS_64(info
->mss_change
, I40IW_CQPSQ_QP_MSSCHANGE
) |
2414 LS_64(info
->static_rsrc
, I40IW_CQPSQ_QP_STATRSRC
) |
2415 LS_64(info
->remove_hash_idx
, I40IW_CQPSQ_QP_REMOVEHASHENTRY
) |
2416 LS_64(term_actions
, I40IW_CQPSQ_QP_TERMACT
) |
2417 LS_64(info
->reset_tcp_conn
, I40IW_CQPSQ_QP_RESETCON
) |
2418 LS_64(info
->arp_cache_idx_valid
, I40IW_CQPSQ_QP_ARPTABIDXVALID
) |
2419 LS_64(info
->next_iwarp_state
, I40IW_CQPSQ_QP_NEXTIWSTATE
) |
2420 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2422 i40iw_insert_wqe_hdr(wqe
, header
);
2424 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QP_MODIFY WQE",
2425 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2428 i40iw_sc_cqp_post_sq(cqp
);
2433 * i40iw_sc_qp_destroy - cqp destroy qp
2435 * @scratch: u64 saved to be used during cqp completion
2436 * @remove_hash_idx: flag if to remove hash idx
2437 * @ignore_mw_bnd: memory window bind flag
2438 * @post_sq: flag for cqp db to ring
2440 static enum i40iw_status_code
i40iw_sc_qp_destroy(
2441 struct i40iw_sc_qp
*qp
,
2443 bool remove_hash_idx
,
2448 struct i40iw_sc_cqp
*cqp
;
2451 i40iw_qp_rem_qos(qp
);
2452 cqp
= qp
->pd
->dev
->cqp
;
2453 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2455 return I40IW_ERR_RING_FULL
;
2456 set_64bit_val(wqe
, 16, qp
->hw_host_ctx_pa
);
2457 set_64bit_val(wqe
, 40, qp
->shadow_area_pa
);
2459 header
= qp
->qp_uk
.qp_id
|
2460 LS_64(I40IW_CQP_OP_DESTROY_QP
, I40IW_CQPSQ_OPCODE
) |
2461 LS_64(qp
->qp_type
, I40IW_CQPSQ_QP_QPTYPE
) |
2462 LS_64(ignore_mw_bnd
, I40IW_CQPSQ_QP_IGNOREMWBOUND
) |
2463 LS_64(remove_hash_idx
, I40IW_CQPSQ_QP_REMOVEHASHENTRY
) |
2464 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2466 i40iw_insert_wqe_hdr(wqe
, header
);
2467 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QP_DESTROY WQE",
2468 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2471 i40iw_sc_cqp_post_sq(cqp
);
2476 * i40iw_sc_qp_flush_wqes - flush qp's wqe
2478 * @info: dlush information
2479 * @scratch: u64 saved to be used during cqp completion
2480 * @post_sq: flag for cqp db to ring
2482 static enum i40iw_status_code
i40iw_sc_qp_flush_wqes(
2483 struct i40iw_sc_qp
*qp
,
2484 struct i40iw_qp_flush_info
*info
,
2490 struct i40iw_sc_cqp
*cqp
;
2492 bool flush_sq
= false, flush_rq
= false;
2494 if (info
->rq
&& !qp
->flush_rq
)
2497 if (info
->sq
&& !qp
->flush_sq
)
2500 qp
->flush_sq
|= flush_sq
;
2501 qp
->flush_rq
|= flush_rq
;
2502 if (!flush_sq
&& !flush_rq
) {
2503 if (info
->ae_code
!= I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR
)
2507 cqp
= qp
->pd
->dev
->cqp
;
2508 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2510 return I40IW_ERR_RING_FULL
;
2511 if (info
->userflushcode
) {
2513 temp
|= LS_64(info
->rq_minor_code
, I40IW_CQPSQ_FWQE_RQMNERR
) |
2514 LS_64(info
->rq_major_code
, I40IW_CQPSQ_FWQE_RQMJERR
);
2517 temp
|= LS_64(info
->sq_minor_code
, I40IW_CQPSQ_FWQE_SQMNERR
) |
2518 LS_64(info
->sq_major_code
, I40IW_CQPSQ_FWQE_SQMJERR
);
2521 set_64bit_val(wqe
, 16, temp
);
2523 temp
= (info
->generate_ae
) ?
2524 info
->ae_code
| LS_64(info
->ae_source
, I40IW_CQPSQ_FWQE_AESOURCE
) : 0;
2526 set_64bit_val(wqe
, 8, temp
);
2528 header
= qp
->qp_uk
.qp_id
|
2529 LS_64(I40IW_CQP_OP_FLUSH_WQES
, I40IW_CQPSQ_OPCODE
) |
2530 LS_64(info
->generate_ae
, I40IW_CQPSQ_FWQE_GENERATE_AE
) |
2531 LS_64(info
->userflushcode
, I40IW_CQPSQ_FWQE_USERFLCODE
) |
2532 LS_64(flush_sq
, I40IW_CQPSQ_FWQE_FLUSHSQ
) |
2533 LS_64(flush_rq
, I40IW_CQPSQ_FWQE_FLUSHRQ
) |
2534 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2536 i40iw_insert_wqe_hdr(wqe
, header
);
2538 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "QP_FLUSH WQE",
2539 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2542 i40iw_sc_cqp_post_sq(cqp
);
2547 * i40iw_sc_qp_upload_context - upload qp's context
2548 * @dev: sc device struct
2549 * @info: upload context info ptr for return
2550 * @scratch: u64 saved to be used during cqp completion
2551 * @post_sq: flag for cqp db to ring
2553 static enum i40iw_status_code
i40iw_sc_qp_upload_context(
2554 struct i40iw_sc_dev
*dev
,
2555 struct i40iw_upload_context_info
*info
,
2560 struct i40iw_sc_cqp
*cqp
;
2564 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2566 return I40IW_ERR_RING_FULL
;
2567 set_64bit_val(wqe
, 16, info
->buf_pa
);
2569 header
= LS_64(info
->qp_id
, I40IW_CQPSQ_UCTX_QPID
) |
2570 LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT
, I40IW_CQPSQ_OPCODE
) |
2571 LS_64(info
->qp_type
, I40IW_CQPSQ_UCTX_QPTYPE
) |
2572 LS_64(info
->raw_format
, I40IW_CQPSQ_UCTX_RAWFORMAT
) |
2573 LS_64(info
->freeze_qp
, I40IW_CQPSQ_UCTX_FREEZEQP
) |
2574 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2576 i40iw_insert_wqe_hdr(wqe
, header
);
2578 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "QP_UPLOAD_CTX WQE",
2579 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2582 i40iw_sc_cqp_post_sq(cqp
);
2587 * i40iw_sc_qp_setctx - set qp's context
2589 * @qp_ctx: context ptr
2592 static enum i40iw_status_code
i40iw_sc_qp_setctx(
2593 struct i40iw_sc_qp
*qp
,
2595 struct i40iw_qp_host_ctx_info
*info
)
2597 struct i40iwarp_offload_info
*iw
;
2598 struct i40iw_tcp_offload_info
*tcp
;
2599 struct i40iw_sc_vsi
*vsi
;
2600 struct i40iw_sc_dev
*dev
;
2601 u64 qw0
, qw3
, qw7
= 0;
2603 iw
= info
->iwarp_info
;
2604 tcp
= info
->tcp_info
;
2607 if (info
->add_to_qoslist
) {
2608 qp
->user_pri
= info
->user_pri
;
2609 i40iw_qp_add_qos(qp
);
2610 i40iw_debug(qp
->dev
, I40IW_DEBUG_DCB
, "%s qp[%d] UP[%d] qset[%d]\n",
2611 __func__
, qp
->qp_uk
.qp_id
, qp
->user_pri
, qp
->qs_handle
);
2613 qw0
= LS_64(qp
->qp_uk
.rq_wqe_size
, I40IWQPC_RQWQESIZE
) |
2614 LS_64(info
->err_rq_idx_valid
, I40IWQPC_ERR_RQ_IDX_VALID
) |
2615 LS_64(qp
->rcv_tph_en
, I40IWQPC_RCVTPHEN
) |
2616 LS_64(qp
->xmit_tph_en
, I40IWQPC_XMITTPHEN
) |
2617 LS_64(qp
->rq_tph_en
, I40IWQPC_RQTPHEN
) |
2618 LS_64(qp
->sq_tph_en
, I40IWQPC_SQTPHEN
) |
2619 LS_64(info
->push_idx
, I40IWQPC_PPIDX
) |
2620 LS_64(info
->push_mode_en
, I40IWQPC_PMENA
);
2622 set_64bit_val(qp_ctx
, 8, qp
->sq_pa
);
2623 set_64bit_val(qp_ctx
, 16, qp
->rq_pa
);
2625 qw3
= LS_64(qp
->src_mac_addr_idx
, I40IWQPC_SRCMACADDRIDX
) |
2626 LS_64(qp
->hw_rq_size
, I40IWQPC_RQSIZE
) |
2627 LS_64(qp
->hw_sq_size
, I40IWQPC_SQSIZE
);
2629 set_64bit_val(qp_ctx
,
2631 LS_64(info
->err_rq_idx
, I40IWQPC_ERR_RQ_IDX
));
2633 set_64bit_val(qp_ctx
,
2635 LS_64(info
->send_cq_num
, I40IWQPC_TXCQNUM
) |
2636 LS_64(info
->rcv_cq_num
, I40IWQPC_RXCQNUM
));
2638 set_64bit_val(qp_ctx
,
2640 LS_64(info
->qp_compl_ctx
, I40IWQPC_QPCOMPCTX
));
2641 set_64bit_val(qp_ctx
,
2643 LS_64(qp
->sq_tph_val
, I40IWQPC_SQTPHVAL
) |
2644 LS_64(qp
->rq_tph_val
, I40IWQPC_RQTPHVAL
) |
2645 LS_64(qp
->qs_handle
, I40IWQPC_QSHANDLE
) |
2646 LS_64(qp
->exception_lan_queue
, I40IWQPC_EXCEPTION_LAN_QUEUE
));
2648 if (info
->iwarp_info_valid
) {
2649 qw0
|= LS_64(iw
->ddp_ver
, I40IWQPC_DDP_VER
) |
2650 LS_64(iw
->rdmap_ver
, I40IWQPC_RDMAP_VER
);
2652 qw7
|= LS_64(iw
->pd_id
, I40IWQPC_PDIDX
);
2653 set_64bit_val(qp_ctx
,
2655 LS_64(qp
->q2_pa
, I40IWQPC_Q2ADDR
) |
2656 LS_64(vsi
->fcn_id
, I40IWQPC_STAT_INDEX
));
2657 set_64bit_val(qp_ctx
,
2659 LS_64(iw
->last_byte_sent
, I40IWQPC_LASTBYTESENT
));
2661 set_64bit_val(qp_ctx
,
2663 LS_64(iw
->ord_size
, I40IWQPC_ORDSIZE
) |
2664 LS_64(iw
->ird_size
, I40IWQPC_IRDSIZE
) |
2665 LS_64(iw
->wr_rdresp_en
, I40IWQPC_WRRDRSPOK
) |
2666 LS_64(iw
->rd_enable
, I40IWQPC_RDOK
) |
2667 LS_64(iw
->snd_mark_en
, I40IWQPC_SNDMARKERS
) |
2668 LS_64(iw
->bind_en
, I40IWQPC_BINDEN
) |
2669 LS_64(iw
->fast_reg_en
, I40IWQPC_FASTREGEN
) |
2670 LS_64(iw
->priv_mode_en
, I40IWQPC_PRIVEN
) |
2671 LS_64((((vsi
->stats_fcn_id_alloc
) &&
2672 (dev
->is_pf
) && (vsi
->fcn_id
>= I40IW_FIRST_NON_PF_STAT
)) ? 1 : 0),
2673 I40IWQPC_USESTATSINSTANCE
) |
2674 LS_64(1, I40IWQPC_IWARPMODE
) |
2675 LS_64(iw
->rcv_mark_en
, I40IWQPC_RCVMARKERS
) |
2676 LS_64(iw
->align_hdrs
, I40IWQPC_ALIGNHDRS
) |
2677 LS_64(iw
->rcv_no_mpa_crc
, I40IWQPC_RCVNOMPACRC
) |
2678 LS_64(iw
->rcv_mark_offset
, I40IWQPC_RCVMARKOFFSET
) |
2679 LS_64(iw
->snd_mark_offset
, I40IWQPC_SNDMARKOFFSET
));
2681 if (info
->tcp_info_valid
) {
2682 qw0
|= LS_64(tcp
->ipv4
, I40IWQPC_IPV4
) |
2683 LS_64(tcp
->no_nagle
, I40IWQPC_NONAGLE
) |
2684 LS_64(tcp
->insert_vlan_tag
, I40IWQPC_INSERTVLANTAG
) |
2685 LS_64(tcp
->time_stamp
, I40IWQPC_TIMESTAMP
) |
2686 LS_64(tcp
->cwnd_inc_limit
, I40IWQPC_LIMIT
) |
2687 LS_64(tcp
->drop_ooo_seg
, I40IWQPC_DROPOOOSEG
) |
2688 LS_64(tcp
->dup_ack_thresh
, I40IWQPC_DUPACK_THRESH
);
2690 qw3
|= LS_64(tcp
->ttl
, I40IWQPC_TTL
) |
2691 LS_64(tcp
->src_mac_addr_idx
, I40IWQPC_SRCMACADDRIDX
) |
2692 LS_64(tcp
->avoid_stretch_ack
, I40IWQPC_AVOIDSTRETCHACK
) |
2693 LS_64(tcp
->tos
, I40IWQPC_TOS
) |
2694 LS_64(tcp
->src_port
, I40IWQPC_SRCPORTNUM
) |
2695 LS_64(tcp
->dst_port
, I40IWQPC_DESTPORTNUM
);
2697 qp
->src_mac_addr_idx
= tcp
->src_mac_addr_idx
;
2698 set_64bit_val(qp_ctx
,
2700 LS_64(tcp
->dest_ip_addr2
, I40IWQPC_DESTIPADDR2
) |
2701 LS_64(tcp
->dest_ip_addr3
, I40IWQPC_DESTIPADDR3
));
2703 set_64bit_val(qp_ctx
,
2705 LS_64(tcp
->dest_ip_addr0
, I40IWQPC_DESTIPADDR0
) |
2706 LS_64(tcp
->dest_ip_addr1
, I40IWQPC_DESTIPADDR1
));
2708 set_64bit_val(qp_ctx
,
2710 LS_64(tcp
->snd_mss
, I40IWQPC_SNDMSS
) |
2711 LS_64(tcp
->vlan_tag
, I40IWQPC_VLANTAG
) |
2712 LS_64(tcp
->arp_idx
, I40IWQPC_ARPIDX
));
2714 qw7
|= LS_64(tcp
->flow_label
, I40IWQPC_FLOWLABEL
) |
2715 LS_64(tcp
->wscale
, I40IWQPC_WSCALE
) |
2716 LS_64(tcp
->ignore_tcp_opt
, I40IWQPC_IGNORE_TCP_OPT
) |
2717 LS_64(tcp
->ignore_tcp_uns_opt
, I40IWQPC_IGNORE_TCP_UNS_OPT
) |
2718 LS_64(tcp
->tcp_state
, I40IWQPC_TCPSTATE
) |
2719 LS_64(tcp
->rcv_wscale
, I40IWQPC_RCVSCALE
) |
2720 LS_64(tcp
->snd_wscale
, I40IWQPC_SNDSCALE
);
2722 set_64bit_val(qp_ctx
,
2724 LS_64(tcp
->time_stamp_recent
, I40IWQPC_TIMESTAMP_RECENT
) |
2725 LS_64(tcp
->time_stamp_age
, I40IWQPC_TIMESTAMP_AGE
));
2726 set_64bit_val(qp_ctx
,
2728 LS_64(tcp
->snd_nxt
, I40IWQPC_SNDNXT
) |
2729 LS_64(tcp
->snd_wnd
, I40IWQPC_SNDWND
));
2731 set_64bit_val(qp_ctx
,
2733 LS_64(tcp
->rcv_nxt
, I40IWQPC_RCVNXT
) |
2734 LS_64(tcp
->rcv_wnd
, I40IWQPC_RCVWND
));
2735 set_64bit_val(qp_ctx
,
2737 LS_64(tcp
->snd_max
, I40IWQPC_SNDMAX
) |
2738 LS_64(tcp
->snd_una
, I40IWQPC_SNDUNA
));
2739 set_64bit_val(qp_ctx
,
2741 LS_64(tcp
->srtt
, I40IWQPC_SRTT
) |
2742 LS_64(tcp
->rtt_var
, I40IWQPC_RTTVAR
));
2743 set_64bit_val(qp_ctx
,
2745 LS_64(tcp
->ss_thresh
, I40IWQPC_SSTHRESH
) |
2746 LS_64(tcp
->cwnd
, I40IWQPC_CWND
));
2747 set_64bit_val(qp_ctx
,
2749 LS_64(tcp
->snd_wl1
, I40IWQPC_SNDWL1
) |
2750 LS_64(tcp
->snd_wl2
, I40IWQPC_SNDWL2
));
2751 set_64bit_val(qp_ctx
,
2753 LS_64(tcp
->max_snd_window
, I40IWQPC_MAXSNDWND
) |
2754 LS_64(tcp
->rexmit_thresh
, I40IWQPC_REXMIT_THRESH
));
2755 set_64bit_val(qp_ctx
,
2757 LS_64(tcp
->local_ipaddr3
, I40IWQPC_LOCAL_IPADDR3
) |
2758 LS_64(tcp
->local_ipaddr2
, I40IWQPC_LOCAL_IPADDR2
));
2759 set_64bit_val(qp_ctx
,
2761 LS_64(tcp
->local_ipaddr1
, I40IWQPC_LOCAL_IPADDR1
) |
2762 LS_64(tcp
->local_ipaddr0
, I40IWQPC_LOCAL_IPADDR0
));
2765 set_64bit_val(qp_ctx
, 0, qw0
);
2766 set_64bit_val(qp_ctx
, 24, qw3
);
2767 set_64bit_val(qp_ctx
, 56, qw7
);
2769 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_WQE
, "QP_HOST)CTX WQE",
2770 qp_ctx
, I40IW_QP_CTX_SIZE
);
2775 * i40iw_sc_alloc_stag - mr stag alloc
2776 * @dev: sc device struct
2778 * @scratch: u64 saved to be used during cqp completion
2779 * @post_sq: flag for cqp db to ring
2781 static enum i40iw_status_code
i40iw_sc_alloc_stag(
2782 struct i40iw_sc_dev
*dev
,
2783 struct i40iw_allocate_stag_info
*info
,
2788 struct i40iw_sc_cqp
*cqp
;
2790 enum i40iw_page_size page_size
;
2792 page_size
= (info
->page_size
== 0x200000) ? I40IW_PAGE_SIZE_2M
: I40IW_PAGE_SIZE_4K
;
2794 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2796 return I40IW_ERR_RING_FULL
;
2799 LS_64(info
->pd_id
, I40IW_CQPSQ_STAG_PDID
) |
2800 LS_64(info
->total_len
, I40IW_CQPSQ_STAG_STAGLEN
));
2803 LS_64(info
->stag_idx
, I40IW_CQPSQ_STAG_IDX
));
2806 LS_64(info
->hmc_fcn_index
, I40IW_CQPSQ_STAG_HMCFNIDX
));
2808 header
= LS_64(I40IW_CQP_OP_ALLOC_STAG
, I40IW_CQPSQ_OPCODE
) |
2809 LS_64(1, I40IW_CQPSQ_STAG_MR
) |
2810 LS_64(info
->access_rights
, I40IW_CQPSQ_STAG_ARIGHTS
) |
2811 LS_64(info
->chunk_size
, I40IW_CQPSQ_STAG_LPBLSIZE
) |
2812 LS_64(page_size
, I40IW_CQPSQ_STAG_HPAGESIZE
) |
2813 LS_64(info
->remote_access
, I40IW_CQPSQ_STAG_REMACCENABLED
) |
2814 LS_64(info
->use_hmc_fcn_index
, I40IW_CQPSQ_STAG_USEHMCFNIDX
) |
2815 LS_64(info
->use_pf_rid
, I40IW_CQPSQ_STAG_USEPFRID
) |
2816 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2818 i40iw_insert_wqe_hdr(wqe
, header
);
2820 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "ALLOC_STAG WQE",
2821 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2824 i40iw_sc_cqp_post_sq(cqp
);
2829 * i40iw_sc_mr_reg_non_shared - non-shared mr registration
2830 * @dev: sc device struct
2832 * @scratch: u64 saved to be used during cqp completion
2833 * @post_sq: flag for cqp db to ring
2835 static enum i40iw_status_code
i40iw_sc_mr_reg_non_shared(
2836 struct i40iw_sc_dev
*dev
,
2837 struct i40iw_reg_ns_stag_info
*info
,
2843 struct i40iw_sc_cqp
*cqp
;
2848 enum i40iw_page_size page_size
;
2850 page_size
= (info
->page_size
== 0x200000) ? I40IW_PAGE_SIZE_2M
: I40IW_PAGE_SIZE_4K
;
2851 if (info
->access_rights
& (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY
|
2852 I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY
))
2853 remote_access
= true;
2855 remote_access
= false;
2857 pble_obj_cnt
= dev
->hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
;
2859 if (info
->chunk_size
&& (info
->first_pm_pbl_index
>= pble_obj_cnt
))
2860 return I40IW_ERR_INVALID_PBLE_INDEX
;
2863 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2865 return I40IW_ERR_RING_FULL
;
2867 temp
= (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
) ? (uintptr_t)info
->va
: info
->fbo
;
2868 set_64bit_val(wqe
, 0, temp
);
2872 LS_64(info
->total_len
, I40IW_CQPSQ_STAG_STAGLEN
) |
2873 LS_64(info
->pd_id
, I40IW_CQPSQ_STAG_PDID
));
2877 LS_64(info
->stag_key
, I40IW_CQPSQ_STAG_KEY
) |
2878 LS_64(info
->stag_idx
, I40IW_CQPSQ_STAG_IDX
));
2879 if (!info
->chunk_size
) {
2880 set_64bit_val(wqe
, 32, info
->reg_addr_pa
);
2881 set_64bit_val(wqe
, 48, 0);
2883 set_64bit_val(wqe
, 32, 0);
2884 set_64bit_val(wqe
, 48, info
->first_pm_pbl_index
);
2886 set_64bit_val(wqe
, 40, info
->hmc_fcn_index
);
2887 set_64bit_val(wqe
, 56, 0);
2889 addr_type
= (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
) ? 1 : 0;
2890 header
= LS_64(I40IW_CQP_OP_REG_MR
, I40IW_CQPSQ_OPCODE
) |
2891 LS_64(1, I40IW_CQPSQ_STAG_MR
) |
2892 LS_64(info
->chunk_size
, I40IW_CQPSQ_STAG_LPBLSIZE
) |
2893 LS_64(page_size
, I40IW_CQPSQ_STAG_HPAGESIZE
) |
2894 LS_64(info
->access_rights
, I40IW_CQPSQ_STAG_ARIGHTS
) |
2895 LS_64(remote_access
, I40IW_CQPSQ_STAG_REMACCENABLED
) |
2896 LS_64(addr_type
, I40IW_CQPSQ_STAG_VABASEDTO
) |
2897 LS_64(info
->use_hmc_fcn_index
, I40IW_CQPSQ_STAG_USEHMCFNIDX
) |
2898 LS_64(info
->use_pf_rid
, I40IW_CQPSQ_STAG_USEPFRID
) |
2899 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2901 i40iw_insert_wqe_hdr(wqe
, header
);
2903 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "MR_REG_NS WQE",
2904 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2907 i40iw_sc_cqp_post_sq(cqp
);
2912 * i40iw_sc_mr_reg_shared - registered shared memory region
2913 * @dev: sc device struct
2914 * @info: info for shared memory registeration
2915 * @scratch: u64 saved to be used during cqp completion
2916 * @post_sq: flag for cqp db to ring
2918 static enum i40iw_status_code
i40iw_sc_mr_reg_shared(
2919 struct i40iw_sc_dev
*dev
,
2920 struct i40iw_register_shared_stag
*info
,
2925 struct i40iw_sc_cqp
*cqp
;
2926 u64 temp
, va64
, fbo
, header
;
2931 if (info
->access_rights
& (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY
|
2932 I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY
))
2933 remote_access
= true;
2935 remote_access
= false;
2937 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2939 return I40IW_ERR_RING_FULL
;
2940 va64
= (uintptr_t)(info
->va
);
2941 va32
= (u32
)(va64
& 0x00000000FFFFFFFF);
2942 fbo
= (u64
)(va32
& (4096 - 1));
2946 (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
? (uintptr_t)info
->va
: fbo
));
2950 LS_64(info
->pd_id
, I40IW_CQPSQ_STAG_PDID
));
2951 temp
= LS_64(info
->new_stag_key
, I40IW_CQPSQ_STAG_KEY
) |
2952 LS_64(info
->new_stag_idx
, I40IW_CQPSQ_STAG_IDX
) |
2953 LS_64(info
->parent_stag_idx
, I40IW_CQPSQ_STAG_PARENTSTAGIDX
);
2954 set_64bit_val(wqe
, 16, temp
);
2956 addr_type
= (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
) ? 1 : 0;
2957 header
= LS_64(I40IW_CQP_OP_REG_SMR
, I40IW_CQPSQ_OPCODE
) |
2958 LS_64(1, I40IW_CQPSQ_STAG_MR
) |
2959 LS_64(info
->access_rights
, I40IW_CQPSQ_STAG_ARIGHTS
) |
2960 LS_64(remote_access
, I40IW_CQPSQ_STAG_REMACCENABLED
) |
2961 LS_64(addr_type
, I40IW_CQPSQ_STAG_VABASEDTO
) |
2962 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
2964 i40iw_insert_wqe_hdr(wqe
, header
);
2966 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "MR_REG_SHARED WQE",
2967 wqe
, I40IW_CQP_WQE_SIZE
* 8);
2970 i40iw_sc_cqp_post_sq(cqp
);
2975 * i40iw_sc_dealloc_stag - deallocate stag
2976 * @dev: sc device struct
2977 * @info: dealloc stag info
2978 * @scratch: u64 saved to be used during cqp completion
2979 * @post_sq: flag for cqp db to ring
2981 static enum i40iw_status_code
i40iw_sc_dealloc_stag(
2982 struct i40iw_sc_dev
*dev
,
2983 struct i40iw_dealloc_stag_info
*info
,
2989 struct i40iw_sc_cqp
*cqp
;
2992 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
2994 return I40IW_ERR_RING_FULL
;
2997 LS_64(info
->pd_id
, I40IW_CQPSQ_STAG_PDID
));
3000 LS_64(info
->stag_idx
, I40IW_CQPSQ_STAG_IDX
));
3002 header
= LS_64(I40IW_CQP_OP_DEALLOC_STAG
, I40IW_CQPSQ_OPCODE
) |
3003 LS_64(info
->mr
, I40IW_CQPSQ_STAG_MR
) |
3004 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3006 i40iw_insert_wqe_hdr(wqe
, header
);
3008 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "DEALLOC_STAG WQE",
3009 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3012 i40iw_sc_cqp_post_sq(cqp
);
3017 * i40iw_sc_query_stag - query hardware for stag
3018 * @dev: sc device struct
3019 * @scratch: u64 saved to be used during cqp completion
3020 * @stag_index: stag index for query
3021 * @post_sq: flag for cqp db to ring
3023 static enum i40iw_status_code
i40iw_sc_query_stag(struct i40iw_sc_dev
*dev
,
3030 struct i40iw_sc_cqp
*cqp
;
3033 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3035 return I40IW_ERR_RING_FULL
;
3038 LS_64(stag_index
, I40IW_CQPSQ_QUERYSTAG_IDX
));
3040 header
= LS_64(I40IW_CQP_OP_QUERY_STAG
, I40IW_CQPSQ_OPCODE
) |
3041 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3043 i40iw_insert_wqe_hdr(wqe
, header
);
3045 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "QUERY_STAG WQE",
3046 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3049 i40iw_sc_cqp_post_sq(cqp
);
3054 * i40iw_sc_mw_alloc - mw allocate
3055 * @dev: sc device struct
3056 * @scratch: u64 saved to be used during cqp completion
3057 * @mw_stag_index:stag index
3058 * @pd_id: pd is for this mw
3059 * @post_sq: flag for cqp db to ring
3061 static enum i40iw_status_code
i40iw_sc_mw_alloc(
3062 struct i40iw_sc_dev
*dev
,
3069 struct i40iw_sc_cqp
*cqp
;
3073 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3075 return I40IW_ERR_RING_FULL
;
3076 set_64bit_val(wqe
, 8, LS_64(pd_id
, I40IW_CQPSQ_STAG_PDID
));
3079 LS_64(mw_stag_index
, I40IW_CQPSQ_STAG_IDX
));
3081 header
= LS_64(I40IW_CQP_OP_ALLOC_STAG
, I40IW_CQPSQ_OPCODE
) |
3082 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3084 i40iw_insert_wqe_hdr(wqe
, header
);
3086 i40iw_debug_buf(dev
, I40IW_DEBUG_WQE
, "MW_ALLOC WQE",
3087 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3090 i40iw_sc_cqp_post_sq(cqp
);
3095 * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
3097 * @info: fast mr info
3098 * @post_sq: flag for cqp db to ring
3100 enum i40iw_status_code
i40iw_sc_mr_fast_register(
3101 struct i40iw_sc_qp
*qp
,
3102 struct i40iw_fast_reg_stag_info
*info
,
3108 enum i40iw_page_size page_size
;
3110 page_size
= (info
->page_size
== 0x200000) ? I40IW_PAGE_SIZE_2M
: I40IW_PAGE_SIZE_4K
;
3111 wqe
= i40iw_qp_get_next_send_wqe(&qp
->qp_uk
, &wqe_idx
, I40IW_QP_WQE_MIN_SIZE
,
3114 return I40IW_ERR_QP_TOOMANY_WRS_POSTED
;
3116 i40iw_debug(qp
->dev
, I40IW_DEBUG_MR
, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
3117 __func__
, info
->wr_id
, wqe_idx
,
3118 &qp
->qp_uk
.sq_wrtrk_array
[wqe_idx
].wrid
);
3119 temp
= (info
->addr_type
== I40IW_ADDR_TYPE_VA_BASED
) ? (uintptr_t)info
->va
: info
->fbo
;
3120 set_64bit_val(wqe
, 0, temp
);
3122 temp
= RS_64(info
->first_pm_pbl_index
>> 16, I40IWQPSQ_FIRSTPMPBLIDXHI
);
3125 LS_64(temp
, I40IWQPSQ_FIRSTPMPBLIDXHI
) |
3126 LS_64(info
->reg_addr_pa
>> I40IWQPSQ_PBLADDR_SHIFT
, I40IWQPSQ_PBLADDR
));
3131 LS_64(info
->first_pm_pbl_index
, I40IWQPSQ_FIRSTPMPBLIDXLO
));
3133 header
= LS_64(info
->stag_key
, I40IWQPSQ_STAGKEY
) |
3134 LS_64(info
->stag_idx
, I40IWQPSQ_STAGINDEX
) |
3135 LS_64(I40IWQP_OP_FAST_REGISTER
, I40IWQPSQ_OPCODE
) |
3136 LS_64(info
->chunk_size
, I40IWQPSQ_LPBLSIZE
) |
3137 LS_64(page_size
, I40IWQPSQ_HPAGESIZE
) |
3138 LS_64(info
->access_rights
, I40IWQPSQ_STAGRIGHTS
) |
3139 LS_64(info
->addr_type
, I40IWQPSQ_VABASEDTO
) |
3140 LS_64(info
->read_fence
, I40IWQPSQ_READFENCE
) |
3141 LS_64(info
->local_fence
, I40IWQPSQ_LOCALFENCE
) |
3142 LS_64(info
->signaled
, I40IWQPSQ_SIGCOMPL
) |
3143 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3145 i40iw_insert_wqe_hdr(wqe
, header
);
3147 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_WQE
, "FAST_REG WQE",
3148 wqe
, I40IW_QP_WQE_MIN_SIZE
);
3151 i40iw_qp_post_wr(&qp
->qp_uk
);
3156 * i40iw_sc_send_lsmm - send last streaming mode message
3158 * @lsmm_buf: buffer with lsmm message
3159 * @size: size of lsmm buffer
3160 * @stag: stag of lsmm buffer
3162 static void i40iw_sc_send_lsmm(struct i40iw_sc_qp
*qp
,
3169 struct i40iw_qp_uk
*qp_uk
;
3172 wqe
= qp_uk
->sq_base
->elem
;
3174 set_64bit_val(wqe
, 0, (uintptr_t)lsmm_buf
);
3176 set_64bit_val(wqe
, 8, (size
| LS_64(stag
, I40IWQPSQ_FRAG_STAG
)));
3178 set_64bit_val(wqe
, 16, 0);
3180 header
= LS_64(I40IWQP_OP_RDMA_SEND
, I40IWQPSQ_OPCODE
) |
3181 LS_64(1, I40IWQPSQ_STREAMMODE
) |
3182 LS_64(1, I40IWQPSQ_WAITFORRCVPDU
) |
3183 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3185 i40iw_insert_wqe_hdr(wqe
, header
);
3187 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_QP
, "SEND_LSMM WQE",
3188 wqe
, I40IW_QP_WQE_MIN_SIZE
);
3192 * i40iw_sc_send_lsmm_nostag - for privilege qp
3194 * @lsmm_buf: buffer with lsmm message
3195 * @size: size of lsmm buffer
3197 static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp
*qp
,
3203 struct i40iw_qp_uk
*qp_uk
;
3206 wqe
= qp_uk
->sq_base
->elem
;
3208 set_64bit_val(wqe
, 0, (uintptr_t)lsmm_buf
);
3210 set_64bit_val(wqe
, 8, size
);
3212 set_64bit_val(wqe
, 16, 0);
3214 header
= LS_64(I40IWQP_OP_RDMA_SEND
, I40IWQPSQ_OPCODE
) |
3215 LS_64(1, I40IWQPSQ_STREAMMODE
) |
3216 LS_64(1, I40IWQPSQ_WAITFORRCVPDU
) |
3217 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3219 i40iw_insert_wqe_hdr(wqe
, header
);
3221 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_WQE
, "SEND_LSMM_NOSTAG WQE",
3222 wqe
, I40IW_QP_WQE_MIN_SIZE
);
3226 * i40iw_sc_send_rtt - send last read0 or write0
3228 * @read: Do read0 or write0
3230 static void i40iw_sc_send_rtt(struct i40iw_sc_qp
*qp
, bool read
)
3234 struct i40iw_qp_uk
*qp_uk
;
3237 wqe
= qp_uk
->sq_base
->elem
;
3239 set_64bit_val(wqe
, 0, 0);
3240 set_64bit_val(wqe
, 8, 0);
3241 set_64bit_val(wqe
, 16, 0);
3243 header
= LS_64(0x1234, I40IWQPSQ_REMSTAG
) |
3244 LS_64(I40IWQP_OP_RDMA_READ
, I40IWQPSQ_OPCODE
) |
3245 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3246 set_64bit_val(wqe
, 8, ((u64
)0xabcd << 32));
3248 header
= LS_64(I40IWQP_OP_RDMA_WRITE
, I40IWQPSQ_OPCODE
) |
3249 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3252 i40iw_insert_wqe_hdr(wqe
, header
);
3254 i40iw_debug_buf(qp
->dev
, I40IW_DEBUG_WQE
, "RTR WQE",
3255 wqe
, I40IW_QP_WQE_MIN_SIZE
);
3259 * i40iw_sc_post_wqe0 - send wqe with opcode
3261 * @opcode: opcode to use for wqe0
3263 static enum i40iw_status_code
i40iw_sc_post_wqe0(struct i40iw_sc_qp
*qp
, u8 opcode
)
3267 struct i40iw_qp_uk
*qp_uk
;
3270 wqe
= qp_uk
->sq_base
->elem
;
3273 return I40IW_ERR_QP_TOOMANY_WRS_POSTED
;
3275 case I40IWQP_OP_NOP
:
3276 set_64bit_val(wqe
, 0, 0);
3277 set_64bit_val(wqe
, 8, 0);
3278 set_64bit_val(wqe
, 16, 0);
3279 header
= LS_64(I40IWQP_OP_NOP
, I40IWQPSQ_OPCODE
) |
3280 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
);
3282 i40iw_insert_wqe_hdr(wqe
, header
);
3284 case I40IWQP_OP_RDMA_SEND
:
3285 set_64bit_val(wqe
, 0, 0);
3286 set_64bit_val(wqe
, 8, 0);
3287 set_64bit_val(wqe
, 16, 0);
3288 header
= LS_64(I40IWQP_OP_RDMA_SEND
, I40IWQPSQ_OPCODE
) |
3289 LS_64(qp
->qp_uk
.swqe_polarity
, I40IWQPSQ_VALID
) |
3290 LS_64(1, I40IWQPSQ_STREAMMODE
) |
3291 LS_64(1, I40IWQPSQ_WAITFORRCVPDU
);
3293 i40iw_insert_wqe_hdr(wqe
, header
);
3296 i40iw_debug(qp
->dev
, I40IW_DEBUG_QP
, "%s: Invalid WQE zero opcode\n",
3304 * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
3305 * @dev : ptr to i40iw_dev struct
3306 * @hmc_fn_id: hmc function id
3308 enum i40iw_status_code
i40iw_sc_init_iw_hmc(struct i40iw_sc_dev
*dev
, u8 hmc_fn_id
)
3310 struct i40iw_hmc_info
*hmc_info
;
3311 struct i40iw_dma_mem query_fpm_mem
;
3312 struct i40iw_virt_mem virt_mem
;
3313 struct i40iw_vfdev
*vf_dev
= NULL
;
3315 enum i40iw_status_code ret_code
= 0;
3316 bool poll_registers
= true;
3320 if (hmc_fn_id
>= I40IW_MAX_VF_FPM_ID
||
3321 (dev
->hmc_fn_id
!= hmc_fn_id
&& hmc_fn_id
< I40IW_FIRST_VF_FPM_ID
))
3322 return I40IW_ERR_INVALID_HMCFN_ID
;
3324 i40iw_debug(dev
, I40IW_DEBUG_HMC
, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id
,
3326 if (hmc_fn_id
== dev
->hmc_fn_id
) {
3327 hmc_info
= dev
->hmc_info
;
3328 query_fpm_mem
.pa
= dev
->fpm_query_buf_pa
;
3329 query_fpm_mem
.va
= dev
->fpm_query_buf
;
3331 vf_dev
= i40iw_vfdev_from_fpm(dev
, hmc_fn_id
);
3333 return I40IW_ERR_INVALID_VF_ID
;
3335 hmc_info
= &vf_dev
->hmc_info
;
3336 iw_vf_idx
= vf_dev
->iw_vf_idx
;
3337 i40iw_debug(dev
, I40IW_DEBUG_HMC
, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev
,
3338 hmc_info
, hmc_info
->hmc_obj
);
3339 if (!vf_dev
->fpm_query_buf
) {
3340 if (!dev
->vf_fpm_query_buf
[iw_vf_idx
].va
) {
3341 ret_code
= i40iw_alloc_query_fpm_buf(dev
,
3342 &dev
->vf_fpm_query_buf
[iw_vf_idx
]);
3346 vf_dev
->fpm_query_buf
= dev
->vf_fpm_query_buf
[iw_vf_idx
].va
;
3347 vf_dev
->fpm_query_buf_pa
= dev
->vf_fpm_query_buf
[iw_vf_idx
].pa
;
3349 query_fpm_mem
.pa
= vf_dev
->fpm_query_buf_pa
;
3350 query_fpm_mem
.va
= vf_dev
->fpm_query_buf
;
3352 * It is HARDWARE specific:
3353 * this call is done by PF for VF and
3354 * i40iw_sc_query_fpm_values needs ccq poll
3355 * because PF ccq is already created.
3357 poll_registers
= false;
3360 hmc_info
->hmc_fn_id
= hmc_fn_id
;
3362 if (hmc_fn_id
!= dev
->hmc_fn_id
) {
3364 i40iw_cqp_query_fpm_values_cmd(dev
, &query_fpm_mem
, hmc_fn_id
);
3366 wait_type
= poll_registers
? (u8
)I40IW_CQP_WAIT_POLL_REGS
:
3367 (u8
)I40IW_CQP_WAIT_POLL_CQ
;
3369 ret_code
= i40iw_sc_query_fpm_values(
3372 hmc_info
->hmc_fn_id
,
3380 /* parse the fpm_query_buf and fill hmc obj info */
3382 i40iw_sc_parse_fpm_query_buf((u64
*)query_fpm_mem
.va
,
3384 &dev
->hmc_fpm_misc
);
3387 i40iw_debug_buf(dev
, I40IW_DEBUG_HMC
, "QUERY FPM BUFFER",
3388 query_fpm_mem
.va
, I40IW_QUERY_FPM_BUF_SIZE
);
3390 if (hmc_fn_id
!= dev
->hmc_fn_id
) {
3391 i40iw_cqp_commit_fpm_values_cmd(dev
, &query_fpm_mem
, hmc_fn_id
);
3393 /* parse the fpm_commit_buf and fill hmc obj info */
3394 i40iw_sc_parse_fpm_commit_buf((u64
*)query_fpm_mem
.va
, hmc_info
->hmc_obj
, &hmc_info
->sd_table
.sd_cnt
);
3395 mem_size
= sizeof(struct i40iw_hmc_sd_entry
) *
3396 (hmc_info
->sd_table
.sd_cnt
+ hmc_info
->first_sd_index
);
3397 ret_code
= i40iw_allocate_virt_mem(dev
->hw
, &virt_mem
, mem_size
);
3400 hmc_info
->sd_table
.sd_entry
= virt_mem
.va
;
3403 /* fill size of objects which are fixed */
3404 hmc_info
->hmc_obj
[I40IW_HMC_IW_XFFL
].size
= 4;
3405 hmc_info
->hmc_obj
[I40IW_HMC_IW_Q1FL
].size
= 4;
3406 hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].size
= 8;
3407 hmc_info
->hmc_obj
[I40IW_HMC_IW_APBVT_ENTRY
].size
= 8192;
3408 hmc_info
->hmc_obj
[I40IW_HMC_IW_APBVT_ENTRY
].max_cnt
= 1;
3414 * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
3415 * populates fpm base address in hmc_info
3416 * @dev : ptr to i40iw_dev struct
3417 * @hmc_fn_id: hmc function id
3419 static enum i40iw_status_code
i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev
*dev
,
3422 struct i40iw_hmc_info
*hmc_info
;
3423 struct i40iw_hmc_obj_info
*obj_info
;
3425 struct i40iw_dma_mem commit_fpm_mem
;
3427 enum i40iw_status_code ret_code
= 0;
3428 bool poll_registers
= true;
3431 if (hmc_fn_id
>= I40IW_MAX_VF_FPM_ID
||
3432 (dev
->hmc_fn_id
!= hmc_fn_id
&& hmc_fn_id
< I40IW_FIRST_VF_FPM_ID
))
3433 return I40IW_ERR_INVALID_HMCFN_ID
;
3435 if (hmc_fn_id
== dev
->hmc_fn_id
) {
3436 hmc_info
= dev
->hmc_info
;
3438 hmc_info
= i40iw_vf_hmcinfo_from_fpm(dev
, hmc_fn_id
);
3439 poll_registers
= false;
3442 return I40IW_ERR_BAD_PTR
;
3444 obj_info
= hmc_info
->hmc_obj
;
3445 buf
= dev
->fpm_commit_buf
;
3447 /* copy cnt values in commit buf */
3448 for (i
= I40IW_HMC_IW_QP
, j
= 0; i
<= I40IW_HMC_IW_PBLE
;
3450 set_64bit_val(buf
, j
, (u64
)obj_info
[i
].cnt
);
3452 set_64bit_val(buf
, 40, 0); /* APBVT rsvd */
3454 commit_fpm_mem
.pa
= dev
->fpm_commit_buf_pa
;
3455 commit_fpm_mem
.va
= dev
->fpm_commit_buf
;
3456 wait_type
= poll_registers
? (u8
)I40IW_CQP_WAIT_POLL_REGS
:
3457 (u8
)I40IW_CQP_WAIT_POLL_CQ
;
3458 ret_code
= i40iw_sc_commit_fpm_values(
3461 hmc_info
->hmc_fn_id
,
3466 /* parse the fpm_commit_buf and fill hmc obj info */
3468 ret_code
= i40iw_sc_parse_fpm_commit_buf(dev
->fpm_commit_buf
,
3470 &hmc_info
->sd_table
.sd_cnt
);
3472 i40iw_debug_buf(dev
, I40IW_DEBUG_HMC
, "COMMIT FPM BUFFER",
3473 commit_fpm_mem
.va
, I40IW_COMMIT_FPM_BUF_SIZE
);
3479 * cqp_sds_wqe_fill - fill cqp wqe doe sd
3480 * @cqp: struct for cqp hw
3481 * @info; sd info for wqe
3482 * @scratch: u64 saved to be used during cqp completion
3484 static enum i40iw_status_code
cqp_sds_wqe_fill(struct i40iw_sc_cqp
*cqp
,
3485 struct i40iw_update_sds_info
*info
,
3491 int mem_entries
, wqe_entries
;
3492 struct i40iw_dma_mem
*sdbuf
= &cqp
->sdbuf
;
3494 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3496 return I40IW_ERR_RING_FULL
;
3498 I40IW_CQP_INIT_WQE(wqe
);
3499 wqe_entries
= (info
->cnt
> 3) ? 3 : info
->cnt
;
3500 mem_entries
= info
->cnt
- wqe_entries
;
3502 header
= LS_64(I40IW_CQP_OP_UPDATE_PE_SDS
, I40IW_CQPSQ_OPCODE
) |
3503 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
) |
3504 LS_64(mem_entries
, I40IW_CQPSQ_UPESD_ENTRY_COUNT
);
3507 memcpy(sdbuf
->va
, &info
->entry
[3], (mem_entries
<< 4));
3512 data
|= LS_64(info
->hmc_fn_id
, I40IW_CQPSQ_UPESD_HMCFNID
);
3514 set_64bit_val(wqe
, 16, data
);
3516 switch (wqe_entries
) {
3518 set_64bit_val(wqe
, 48,
3519 (LS_64(info
->entry
[2].cmd
, I40IW_CQPSQ_UPESD_SDCMD
) |
3520 LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID
)));
3522 set_64bit_val(wqe
, 56, info
->entry
[2].data
);
3525 set_64bit_val(wqe
, 32,
3526 (LS_64(info
->entry
[1].cmd
, I40IW_CQPSQ_UPESD_SDCMD
) |
3527 LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID
)));
3529 set_64bit_val(wqe
, 40, info
->entry
[1].data
);
3532 set_64bit_val(wqe
, 0,
3533 LS_64(info
->entry
[0].cmd
, I40IW_CQPSQ_UPESD_SDCMD
));
3535 set_64bit_val(wqe
, 8, info
->entry
[0].data
);
3541 i40iw_insert_wqe_hdr(wqe
, header
);
3543 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "UPDATE_PE_SDS WQE",
3544 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3549 * i40iw_update_pe_sds - cqp wqe for sd
3550 * @dev: ptr to i40iw_dev struct
3551 * @info: sd info for sd's
3552 * @scratch: u64 saved to be used during cqp completion
3554 static enum i40iw_status_code
i40iw_update_pe_sds(struct i40iw_sc_dev
*dev
,
3555 struct i40iw_update_sds_info
*info
,
3558 struct i40iw_sc_cqp
*cqp
= dev
->cqp
;
3559 enum i40iw_status_code ret_code
;
3561 ret_code
= cqp_sds_wqe_fill(cqp
, info
, scratch
);
3563 i40iw_sc_cqp_post_sq(cqp
);
3569 * i40iw_update_sds_noccq - update sd before ccq created
3570 * @dev: sc device struct
3571 * @info: sd info for sd's
3573 enum i40iw_status_code
i40iw_update_sds_noccq(struct i40iw_sc_dev
*dev
,
3574 struct i40iw_update_sds_info
*info
)
3576 u32 error
, val
, tail
;
3577 struct i40iw_sc_cqp
*cqp
= dev
->cqp
;
3578 enum i40iw_status_code ret_code
;
3580 ret_code
= cqp_sds_wqe_fill(cqp
, info
, 0);
3583 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
3585 return I40IW_ERR_CQP_COMPL_ERROR
;
3587 i40iw_sc_cqp_post_sq(cqp
);
3588 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, I40IW_DONE_COUNT
);
3594 * i40iw_sc_suspend_qp - suspend qp for param change
3595 * @cqp: struct for cqp hw
3597 * @scratch: u64 saved to be used during cqp completion
3599 enum i40iw_status_code
i40iw_sc_suspend_qp(struct i40iw_sc_cqp
*cqp
,
3600 struct i40iw_sc_qp
*qp
,
3606 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3608 return I40IW_ERR_RING_FULL
;
3609 header
= LS_64(qp
->qp_uk
.qp_id
, I40IW_CQPSQ_SUSPENDQP_QPID
) |
3610 LS_64(I40IW_CQP_OP_SUSPEND_QP
, I40IW_CQPSQ_OPCODE
) |
3611 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3613 i40iw_insert_wqe_hdr(wqe
, header
);
3615 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "SUSPEND_QP WQE",
3616 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3618 i40iw_sc_cqp_post_sq(cqp
);
3623 * i40iw_sc_resume_qp - resume qp after suspend
3624 * @cqp: struct for cqp hw
3626 * @scratch: u64 saved to be used during cqp completion
3628 enum i40iw_status_code
i40iw_sc_resume_qp(struct i40iw_sc_cqp
*cqp
,
3629 struct i40iw_sc_qp
*qp
,
3635 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3637 return I40IW_ERR_RING_FULL
;
3640 LS_64(qp
->qs_handle
, I40IW_CQPSQ_RESUMEQP_QSHANDLE
));
3642 header
= LS_64(qp
->qp_uk
.qp_id
, I40IW_CQPSQ_RESUMEQP_QPID
) |
3643 LS_64(I40IW_CQP_OP_RESUME_QP
, I40IW_CQPSQ_OPCODE
) |
3644 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3646 i40iw_insert_wqe_hdr(wqe
, header
);
3648 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "RESUME_QP WQE",
3649 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3651 i40iw_sc_cqp_post_sq(cqp
);
3656 * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
3657 * @cqp: struct for cqp hw
3658 * @scratch: u64 saved to be used during cqp completion
3659 * @hmc_fn_id: hmc function id
3660 * @post_sq: flag for cqp db to ring
3661 * @poll_registers: flag to poll register for cqp completion
3663 enum i40iw_status_code
i40iw_sc_static_hmc_pages_allocated(
3664 struct i40iw_sc_cqp
*cqp
,
3668 bool poll_registers
)
3672 u32 tail
, val
, error
;
3673 enum i40iw_status_code ret_code
= 0;
3675 wqe
= i40iw_sc_cqp_get_next_send_wqe(cqp
, scratch
);
3677 return I40IW_ERR_RING_FULL
;
3680 LS_64(hmc_fn_id
, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID
));
3682 header
= LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED
, I40IW_CQPSQ_OPCODE
) |
3683 LS_64(cqp
->polarity
, I40IW_CQPSQ_WQEVALID
);
3685 i40iw_insert_wqe_hdr(wqe
, header
);
3687 i40iw_debug_buf(cqp
->dev
, I40IW_DEBUG_WQE
, "SHMC_PAGES_ALLOCATED WQE",
3688 wqe
, I40IW_CQP_WQE_SIZE
* 8);
3689 i40iw_get_cqp_reg_info(cqp
, &val
, &tail
, &error
);
3691 ret_code
= I40IW_ERR_CQP_COMPL_ERROR
;
3695 i40iw_sc_cqp_post_sq(cqp
);
3697 /* check for cqp sq tail update */
3698 ret_code
= i40iw_cqp_poll_registers(cqp
, tail
, 1000);
3700 ret_code
= i40iw_sc_poll_for_cqp_op_done(cqp
,
3701 I40IW_CQP_OP_SHMC_PAGES_ALLOCATED
,
3709 * i40iw_ring_full - check if cqp ring is full
3710 * @cqp: struct for cqp hw
3712 static bool i40iw_ring_full(struct i40iw_sc_cqp
*cqp
)
3714 return I40IW_RING_FULL_ERR(cqp
->sq_ring
);
3718 * i40iw_est_sd - returns approximate number of SDs for HMC
3719 * @dev: sc device struct
3720 * @hmc_info: hmc structure, size and count for HMC objects
3722 static u64
i40iw_est_sd(struct i40iw_sc_dev
*dev
, struct i40iw_hmc_info
*hmc_info
)
3728 for (i
= I40IW_HMC_IW_QP
; i
< I40IW_HMC_IW_PBLE
; i
++)
3729 size
+= hmc_info
->hmc_obj
[i
].cnt
* hmc_info
->hmc_obj
[i
].size
;
3732 size
+= hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
* hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].size
;
3734 if (size
& 0x1FFFFF)
3735 sd
= (size
>> 21) + 1; /* add 1 for remainder */
3740 /* 2MB alignment for VF PBLE HMC */
3741 size
= hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
* hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].size
;
3742 if (size
& 0x1FFFFF)
3743 sd
+= (size
>> 21) + 1; /* add 1 for remainder */
3752 * i40iw_config_fpm_values - configure HMC objects
3753 * @dev: sc device struct
3754 * @qp_count: desired qp count
3756 enum i40iw_status_code
i40iw_config_fpm_values(struct i40iw_sc_dev
*dev
, u32 qp_count
)
3758 struct i40iw_virt_mem virt_mem
;
3760 u32 qpwantedoriginal
, qpwanted
, mrwanted
, pblewanted
;
3765 struct i40iw_hmc_info
*hmc_info
;
3766 struct i40iw_hmc_fpm_misc
*hmc_fpm_misc
;
3767 enum i40iw_status_code ret_code
= 0;
3769 hmc_info
= dev
->hmc_info
;
3770 hmc_fpm_misc
= &dev
->hmc_fpm_misc
;
3772 ret_code
= i40iw_sc_init_iw_hmc(dev
, dev
->hmc_fn_id
);
3774 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3775 "i40iw_sc_init_iw_hmc returned error_code = %d\n",
3780 for (i
= I40IW_HMC_IW_QP
; i
< I40IW_HMC_IW_MAX
; i
++)
3781 hmc_info
->hmc_obj
[i
].cnt
= hmc_info
->hmc_obj
[i
].max_cnt
;
3782 sd_needed
= i40iw_est_sd(dev
, hmc_info
);
3783 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3784 "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n",
3785 __func__
, sd_needed
, hmc_info
->first_sd_index
);
3786 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3787 "%s: sd count %d where max sd is %d\n",
3788 __func__
, hmc_info
->sd_table
.sd_cnt
,
3789 hmc_fpm_misc
->max_sds
);
3791 qpwanted
= min(qp_count
, hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].max_cnt
);
3792 qpwantedoriginal
= qpwanted
;
3793 mrwanted
= hmc_info
->hmc_obj
[I40IW_HMC_IW_MR
].max_cnt
;
3794 pblewanted
= hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].max_cnt
;
3796 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3797 "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",
3798 qp_count
, hmc_fpm_misc
->max_sds
,
3799 hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].max_cnt
,
3800 hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].max_cnt
,
3801 hmc_info
->hmc_obj
[I40IW_HMC_IW_MR
].max_cnt
,
3802 hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].max_cnt
);
3806 hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].cnt
= qpwanted
;
3807 hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
=
3808 min(2 * qpwanted
, hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
);
3809 hmc_info
->hmc_obj
[I40IW_HMC_IW_SRQ
].cnt
= 0x00; /* Reserved */
3810 hmc_info
->hmc_obj
[I40IW_HMC_IW_HTE
].cnt
=
3811 qpwanted
* hmc_fpm_misc
->ht_multiplier
;
3812 hmc_info
->hmc_obj
[I40IW_HMC_IW_ARP
].cnt
=
3813 hmc_info
->hmc_obj
[I40IW_HMC_IW_ARP
].max_cnt
;
3814 hmc_info
->hmc_obj
[I40IW_HMC_IW_APBVT_ENTRY
].cnt
= 1;
3815 hmc_info
->hmc_obj
[I40IW_HMC_IW_MR
].cnt
= mrwanted
;
3817 hmc_info
->hmc_obj
[I40IW_HMC_IW_XF
].cnt
= I40IW_MAX_WQ_ENTRIES
* qpwanted
;
3818 hmc_info
->hmc_obj
[I40IW_HMC_IW_Q1
].cnt
= 4 * I40IW_MAX_IRD_SIZE
* qpwanted
;
3819 hmc_info
->hmc_obj
[I40IW_HMC_IW_XFFL
].cnt
=
3820 hmc_info
->hmc_obj
[I40IW_HMC_IW_XF
].cnt
/ hmc_fpm_misc
->xf_block_size
;
3821 hmc_info
->hmc_obj
[I40IW_HMC_IW_Q1FL
].cnt
=
3822 hmc_info
->hmc_obj
[I40IW_HMC_IW_Q1
].cnt
/ hmc_fpm_misc
->q1_block_size
;
3823 hmc_info
->hmc_obj
[I40IW_HMC_IW_TIMER
].cnt
=
3824 ((qpwanted
) / 512 + 1) * hmc_fpm_misc
->timer_bucket
;
3825 hmc_info
->hmc_obj
[I40IW_HMC_IW_FSIMC
].cnt
= 0x00;
3826 hmc_info
->hmc_obj
[I40IW_HMC_IW_FSIAV
].cnt
= 0x00;
3827 hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
= pblewanted
;
3829 /* How much memory is needed for all the objects. */
3830 sd_needed
= i40iw_est_sd(dev
, hmc_info
);
3831 if ((loop_count
> 1000) ||
3832 ((!(loop_count
% 10)) &&
3833 (qpwanted
> qpwantedoriginal
* 2 / 3))) {
3834 if (qpwanted
> FPM_MULTIPLIER
) {
3835 qpwanted
-= FPM_MULTIPLIER
;
3837 while (powerof2
< qpwanted
)
3840 qpwanted
= powerof2
;
3845 if (mrwanted
> FPM_MULTIPLIER
* 10)
3846 mrwanted
-= FPM_MULTIPLIER
* 10;
3847 if (pblewanted
> FPM_MULTIPLIER
* 1000)
3848 pblewanted
-= FPM_MULTIPLIER
* 1000;
3849 } while (sd_needed
> hmc_fpm_misc
->max_sds
&& loop_count
< 2000);
3851 sd_needed
= i40iw_est_sd(dev
, hmc_info
);
3853 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3854 "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
3855 loop_count
, sd_needed
,
3856 hmc_info
->hmc_obj
[I40IW_HMC_IW_QP
].cnt
,
3857 hmc_info
->hmc_obj
[I40IW_HMC_IW_CQ
].cnt
,
3858 hmc_info
->hmc_obj
[I40IW_HMC_IW_MR
].cnt
,
3859 hmc_info
->hmc_obj
[I40IW_HMC_IW_PBLE
].cnt
);
3861 ret_code
= i40iw_sc_configure_iw_fpm(dev
, dev
->hmc_fn_id
);
3863 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3864 "configure_iw_fpm returned error_code[x%08X]\n",
3865 i40iw_rd32(dev
->hw
, dev
->is_pf
? I40E_PFPE_CQPERRCODES
: I40E_VFPE_CQPERRCODES1
));
3869 mem_size
= sizeof(struct i40iw_hmc_sd_entry
) *
3870 (hmc_info
->sd_table
.sd_cnt
+ hmc_info
->first_sd_index
+ 1);
3871 ret_code
= i40iw_allocate_virt_mem(dev
->hw
, &virt_mem
, mem_size
);
3873 i40iw_debug(dev
, I40IW_DEBUG_HMC
,
3874 "%s: failed to allocate memory for sd_entry buffer\n",
3878 hmc_info
->sd_table
.sd_entry
= virt_mem
.va
;
3884 * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available
3886 * @pcmdinfo: cqp command info
3888 static enum i40iw_status_code
i40iw_exec_cqp_cmd(struct i40iw_sc_dev
*dev
,
3889 struct cqp_commands_info
*pcmdinfo
)
3891 enum i40iw_status_code status
;
3892 struct i40iw_dma_mem values_mem
;
3894 dev
->cqp_cmd_stats
[pcmdinfo
->cqp_cmd
]++;
3895 switch (pcmdinfo
->cqp_cmd
) {
3896 case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY
:
3897 status
= i40iw_sc_del_local_mac_ipaddr_entry(
3898 pcmdinfo
->in
.u
.del_local_mac_ipaddr_entry
.cqp
,
3899 pcmdinfo
->in
.u
.del_local_mac_ipaddr_entry
.scratch
,
3900 pcmdinfo
->in
.u
.del_local_mac_ipaddr_entry
.entry_idx
,
3901 pcmdinfo
->in
.u
.del_local_mac_ipaddr_entry
.ignore_ref_count
,
3904 case OP_CEQ_DESTROY
:
3905 status
= i40iw_sc_ceq_destroy(pcmdinfo
->in
.u
.ceq_destroy
.ceq
,
3906 pcmdinfo
->in
.u
.ceq_destroy
.scratch
,
3909 case OP_AEQ_DESTROY
:
3910 status
= i40iw_sc_aeq_destroy(pcmdinfo
->in
.u
.aeq_destroy
.aeq
,
3911 pcmdinfo
->in
.u
.aeq_destroy
.scratch
,
3915 case OP_DELETE_ARP_CACHE_ENTRY
:
3916 status
= i40iw_sc_del_arp_cache_entry(
3917 pcmdinfo
->in
.u
.del_arp_cache_entry
.cqp
,
3918 pcmdinfo
->in
.u
.del_arp_cache_entry
.scratch
,
3919 pcmdinfo
->in
.u
.del_arp_cache_entry
.arp_index
,
3922 case OP_MANAGE_APBVT_ENTRY
:
3923 status
= i40iw_sc_manage_apbvt_entry(
3924 pcmdinfo
->in
.u
.manage_apbvt_entry
.cqp
,
3925 &pcmdinfo
->in
.u
.manage_apbvt_entry
.info
,
3926 pcmdinfo
->in
.u
.manage_apbvt_entry
.scratch
,
3930 status
= i40iw_sc_ceq_create(pcmdinfo
->in
.u
.ceq_create
.ceq
,
3931 pcmdinfo
->in
.u
.ceq_create
.scratch
,
3935 status
= i40iw_sc_aeq_create(pcmdinfo
->in
.u
.aeq_create
.aeq
,
3936 pcmdinfo
->in
.u
.aeq_create
.scratch
,
3939 case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY
:
3940 status
= i40iw_sc_alloc_local_mac_ipaddr_entry(
3941 pcmdinfo
->in
.u
.alloc_local_mac_ipaddr_entry
.cqp
,
3942 pcmdinfo
->in
.u
.alloc_local_mac_ipaddr_entry
.scratch
,
3945 case OP_ADD_LOCAL_MAC_IPADDR_ENTRY
:
3946 status
= i40iw_sc_add_local_mac_ipaddr_entry(
3947 pcmdinfo
->in
.u
.add_local_mac_ipaddr_entry
.cqp
,
3948 &pcmdinfo
->in
.u
.add_local_mac_ipaddr_entry
.info
,
3949 pcmdinfo
->in
.u
.add_local_mac_ipaddr_entry
.scratch
,
3952 case OP_MANAGE_QHASH_TABLE_ENTRY
:
3953 status
= i40iw_sc_manage_qhash_table_entry(
3954 pcmdinfo
->in
.u
.manage_qhash_table_entry
.cqp
,
3955 &pcmdinfo
->in
.u
.manage_qhash_table_entry
.info
,
3956 pcmdinfo
->in
.u
.manage_qhash_table_entry
.scratch
,
3961 status
= i40iw_sc_qp_modify(
3962 pcmdinfo
->in
.u
.qp_modify
.qp
,
3963 &pcmdinfo
->in
.u
.qp_modify
.info
,
3964 pcmdinfo
->in
.u
.qp_modify
.scratch
,
3968 case OP_QP_UPLOAD_CONTEXT
:
3969 status
= i40iw_sc_qp_upload_context(
3970 pcmdinfo
->in
.u
.qp_upload_context
.dev
,
3971 &pcmdinfo
->in
.u
.qp_upload_context
.info
,
3972 pcmdinfo
->in
.u
.qp_upload_context
.scratch
,
3977 status
= i40iw_sc_cq_create(
3978 pcmdinfo
->in
.u
.cq_create
.cq
,
3979 pcmdinfo
->in
.u
.cq_create
.scratch
,
3980 pcmdinfo
->in
.u
.cq_create
.check_overflow
,
3984 status
= i40iw_sc_cq_destroy(
3985 pcmdinfo
->in
.u
.cq_destroy
.cq
,
3986 pcmdinfo
->in
.u
.cq_destroy
.scratch
,
3991 status
= i40iw_sc_qp_create(
3992 pcmdinfo
->in
.u
.qp_create
.qp
,
3993 &pcmdinfo
->in
.u
.qp_create
.info
,
3994 pcmdinfo
->in
.u
.qp_create
.scratch
,
3998 status
= i40iw_sc_qp_destroy(
3999 pcmdinfo
->in
.u
.qp_destroy
.qp
,
4000 pcmdinfo
->in
.u
.qp_destroy
.scratch
,
4001 pcmdinfo
->in
.u
.qp_destroy
.remove_hash_idx
,
4002 pcmdinfo
->in
.u
.qp_destroy
.
4008 status
= i40iw_sc_alloc_stag(
4009 pcmdinfo
->in
.u
.alloc_stag
.dev
,
4010 &pcmdinfo
->in
.u
.alloc_stag
.info
,
4011 pcmdinfo
->in
.u
.alloc_stag
.scratch
,
4014 case OP_MR_REG_NON_SHARED
:
4015 status
= i40iw_sc_mr_reg_non_shared(
4016 pcmdinfo
->in
.u
.mr_reg_non_shared
.dev
,
4017 &pcmdinfo
->in
.u
.mr_reg_non_shared
.info
,
4018 pcmdinfo
->in
.u
.mr_reg_non_shared
.scratch
,
4022 case OP_DEALLOC_STAG
:
4023 status
= i40iw_sc_dealloc_stag(
4024 pcmdinfo
->in
.u
.dealloc_stag
.dev
,
4025 &pcmdinfo
->in
.u
.dealloc_stag
.info
,
4026 pcmdinfo
->in
.u
.dealloc_stag
.scratch
,
4031 status
= i40iw_sc_mw_alloc(
4032 pcmdinfo
->in
.u
.mw_alloc
.dev
,
4033 pcmdinfo
->in
.u
.mw_alloc
.scratch
,
4034 pcmdinfo
->in
.u
.mw_alloc
.mw_stag_index
,
4035 pcmdinfo
->in
.u
.mw_alloc
.pd_id
,
4039 case OP_QP_FLUSH_WQES
:
4040 status
= i40iw_sc_qp_flush_wqes(
4041 pcmdinfo
->in
.u
.qp_flush_wqes
.qp
,
4042 &pcmdinfo
->in
.u
.qp_flush_wqes
.info
,
4043 pcmdinfo
->in
.u
.qp_flush_wqes
.
4044 scratch
, pcmdinfo
->post_sq
);
4046 case OP_ADD_ARP_CACHE_ENTRY
:
4047 status
= i40iw_sc_add_arp_cache_entry(
4048 pcmdinfo
->in
.u
.add_arp_cache_entry
.cqp
,
4049 &pcmdinfo
->in
.u
.add_arp_cache_entry
.info
,
4050 pcmdinfo
->in
.u
.add_arp_cache_entry
.scratch
,
4053 case OP_MANAGE_PUSH_PAGE
:
4054 status
= i40iw_sc_manage_push_page(
4055 pcmdinfo
->in
.u
.manage_push_page
.cqp
,
4056 &pcmdinfo
->in
.u
.manage_push_page
.info
,
4057 pcmdinfo
->in
.u
.manage_push_page
.scratch
,
4060 case OP_UPDATE_PE_SDS
:
4061 /* case I40IW_CQP_OP_UPDATE_PE_SDS */
4062 status
= i40iw_update_pe_sds(
4063 pcmdinfo
->in
.u
.update_pe_sds
.dev
,
4064 &pcmdinfo
->in
.u
.update_pe_sds
.info
,
4065 pcmdinfo
->in
.u
.update_pe_sds
.
4069 case OP_MANAGE_HMC_PM_FUNC_TABLE
:
4070 status
= i40iw_sc_manage_hmc_pm_func_table(
4071 pcmdinfo
->in
.u
.manage_hmc_pm
.dev
->cqp
,
4072 pcmdinfo
->in
.u
.manage_hmc_pm
.scratch
,
4073 (u8
)pcmdinfo
->in
.u
.manage_hmc_pm
.info
.vf_id
,
4074 pcmdinfo
->in
.u
.manage_hmc_pm
.info
.free_fcn
,
4078 status
= i40iw_sc_suspend_qp(
4079 pcmdinfo
->in
.u
.suspend_resume
.cqp
,
4080 pcmdinfo
->in
.u
.suspend_resume
.qp
,
4081 pcmdinfo
->in
.u
.suspend_resume
.scratch
);
4084 status
= i40iw_sc_resume_qp(
4085 pcmdinfo
->in
.u
.suspend_resume
.cqp
,
4086 pcmdinfo
->in
.u
.suspend_resume
.qp
,
4087 pcmdinfo
->in
.u
.suspend_resume
.scratch
);
4089 case OP_MANAGE_VF_PBLE_BP
:
4090 status
= i40iw_manage_vf_pble_bp(
4091 pcmdinfo
->in
.u
.manage_vf_pble_bp
.cqp
,
4092 &pcmdinfo
->in
.u
.manage_vf_pble_bp
.info
,
4093 pcmdinfo
->in
.u
.manage_vf_pble_bp
.scratch
, true);
4095 case OP_QUERY_FPM_VALUES
:
4096 values_mem
.pa
= pcmdinfo
->in
.u
.query_fpm_values
.fpm_values_pa
;
4097 values_mem
.va
= pcmdinfo
->in
.u
.query_fpm_values
.fpm_values_va
;
4098 status
= i40iw_sc_query_fpm_values(
4099 pcmdinfo
->in
.u
.query_fpm_values
.cqp
,
4100 pcmdinfo
->in
.u
.query_fpm_values
.scratch
,
4101 pcmdinfo
->in
.u
.query_fpm_values
.hmc_fn_id
,
4102 &values_mem
, true, I40IW_CQP_WAIT_EVENT
);
4104 case OP_COMMIT_FPM_VALUES
:
4105 values_mem
.pa
= pcmdinfo
->in
.u
.commit_fpm_values
.fpm_values_pa
;
4106 values_mem
.va
= pcmdinfo
->in
.u
.commit_fpm_values
.fpm_values_va
;
4107 status
= i40iw_sc_commit_fpm_values(
4108 pcmdinfo
->in
.u
.commit_fpm_values
.cqp
,
4109 pcmdinfo
->in
.u
.commit_fpm_values
.scratch
,
4110 pcmdinfo
->in
.u
.commit_fpm_values
.hmc_fn_id
,
4113 I40IW_CQP_WAIT_EVENT
);
4116 status
= I40IW_NOT_SUPPORTED
;
4124 * i40iw_process_cqp_cmd - process all cqp commands
4125 * @dev: sc device struct
4126 * @pcmdinfo: cqp command info
4128 enum i40iw_status_code
i40iw_process_cqp_cmd(struct i40iw_sc_dev
*dev
,
4129 struct cqp_commands_info
*pcmdinfo
)
4131 enum i40iw_status_code status
= 0;
4132 unsigned long flags
;
4134 spin_lock_irqsave(&dev
->cqp_lock
, flags
);
4135 if (list_empty(&dev
->cqp_cmd_head
) && !i40iw_ring_full(dev
->cqp
))
4136 status
= i40iw_exec_cqp_cmd(dev
, pcmdinfo
);
4138 list_add_tail(&pcmdinfo
->cqp_cmd_entry
, &dev
->cqp_cmd_head
);
4139 spin_unlock_irqrestore(&dev
->cqp_lock
, flags
);
4144 * i40iw_process_bh - called from tasklet for cqp list
4145 * @dev: sc device struct
4147 enum i40iw_status_code
i40iw_process_bh(struct i40iw_sc_dev
*dev
)
4149 enum i40iw_status_code status
= 0;
4150 struct cqp_commands_info
*pcmdinfo
;
4151 unsigned long flags
;
4153 spin_lock_irqsave(&dev
->cqp_lock
, flags
);
4154 while (!list_empty(&dev
->cqp_cmd_head
) && !i40iw_ring_full(dev
->cqp
)) {
4155 pcmdinfo
= (struct cqp_commands_info
*)i40iw_remove_head(&dev
->cqp_cmd_head
);
4157 status
= i40iw_exec_cqp_cmd(dev
, pcmdinfo
);
4161 spin_unlock_irqrestore(&dev
->cqp_lock
, flags
);
4166 * i40iw_iwarp_opcode - determine if incoming is rdma layer
4167 * @info: aeq info for the packet
4168 * @pkt: packet for error
4170 static u32
i40iw_iwarp_opcode(struct i40iw_aeqe_info
*info
, u8
*pkt
)
4173 u32 opcode
= 0xffffffff;
4175 if (info
->q2_data_written
) {
4176 mpa
= (__be16
*)pkt
;
4177 opcode
= ntohs(mpa
[1]) & 0xf;
4183 * i40iw_locate_mpa - return pointer to mpa in the pkt
4184 * @pkt: packet with data
4186 static u8
*i40iw_locate_mpa(u8
*pkt
)
4188 /* skip over ethernet header */
4189 pkt
+= I40IW_MAC_HLEN
;
4191 /* Skip over IP and TCP headers */
4192 pkt
+= 4 * (pkt
[0] & 0x0f);
4193 pkt
+= 4 * ((pkt
[12] >> 4) & 0x0f);
4198 * i40iw_setup_termhdr - termhdr for terminate pkt
4199 * @qp: sc qp ptr for pkt
4201 * @opcode: flush opcode for termhdr
4202 * @layer_etype: error layer + error type
4203 * @err: error cod ein the header
4205 static void i40iw_setup_termhdr(struct i40iw_sc_qp
*qp
,
4206 struct i40iw_terminate_hdr
*hdr
,
4207 enum i40iw_flush_opcode opcode
,
4211 qp
->flush_code
= opcode
;
4212 hdr
->layer_etype
= layer_etype
;
4213 hdr
->error_code
= err
;
4217 * i40iw_bld_terminate_hdr - build terminate message header
4218 * @qp: qp associated with received terminate AE
4219 * @info: the struct contiaing AE information
4221 static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp
*qp
,
4222 struct i40iw_aeqe_info
*info
)
4224 u8
*pkt
= qp
->q2_buf
+ Q2_BAD_FRAME_OFFSET
;
4229 struct i40iw_terminate_hdr
*termhdr
;
4231 termhdr
= (struct i40iw_terminate_hdr
*)qp
->q2_buf
;
4232 memset(termhdr
, 0, Q2_BAD_FRAME_OFFSET
);
4234 if (info
->q2_data_written
) {
4235 /* Use data from offending packet to fill in ddp & rdma hdrs */
4236 pkt
= i40iw_locate_mpa(pkt
);
4237 ddp_seg_len
= ntohs(*(__be16
*)pkt
);
4240 termhdr
->hdrct
= DDP_LEN_FLAG
;
4241 if (pkt
[2] & 0x80) {
4243 if (ddp_seg_len
>= TERM_DDP_LEN_TAGGED
) {
4244 copy_len
+= TERM_DDP_LEN_TAGGED
;
4245 termhdr
->hdrct
|= DDP_HDR_FLAG
;
4248 if (ddp_seg_len
>= TERM_DDP_LEN_UNTAGGED
) {
4249 copy_len
+= TERM_DDP_LEN_UNTAGGED
;
4250 termhdr
->hdrct
|= DDP_HDR_FLAG
;
4253 if (ddp_seg_len
>= (TERM_DDP_LEN_UNTAGGED
+ TERM_RDMA_LEN
)) {
4254 if ((pkt
[3] & RDMA_OPCODE_MASK
) == RDMA_READ_REQ_OPCODE
) {
4255 copy_len
+= TERM_RDMA_LEN
;
4256 termhdr
->hdrct
|= RDMA_HDR_FLAG
;
4263 opcode
= i40iw_iwarp_opcode(info
, pkt
);
4265 switch (info
->ae_id
) {
4266 case I40IW_AE_AMP_UNALLOCATED_STAG
:
4267 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4268 if (opcode
== I40IW_OP_TYPE_RDMA_WRITE
)
4269 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_PROT_ERR
,
4270 (LAYER_DDP
<< 4) | DDP_TAGGED_BUFFER
, DDP_TAGGED_INV_STAG
);
4272 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4273 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_INV_STAG
);
4275 case I40IW_AE_AMP_BOUNDS_VIOLATION
:
4276 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4277 if (info
->q2_data_written
)
4278 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_PROT_ERR
,
4279 (LAYER_DDP
<< 4) | DDP_TAGGED_BUFFER
, DDP_TAGGED_BOUNDS
);
4281 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4282 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_INV_BOUNDS
);
4284 case I40IW_AE_AMP_BAD_PD
:
4286 case I40IW_OP_TYPE_RDMA_WRITE
:
4287 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_PROT_ERR
,
4288 (LAYER_DDP
<< 4) | DDP_TAGGED_BUFFER
, DDP_TAGGED_UNASSOC_STAG
);
4290 case I40IW_OP_TYPE_SEND_INV
:
4291 case I40IW_OP_TYPE_SEND_SOL_INV
:
4292 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4293 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_CANT_INV_STAG
);
4296 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4297 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_UNASSOC_STAG
);
4300 case I40IW_AE_AMP_INVALID_STAG
:
4301 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4302 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4303 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_INV_STAG
);
4305 case I40IW_AE_AMP_BAD_QP
:
4306 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_QP_OP_ERR
,
4307 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_QN
);
4309 case I40IW_AE_AMP_BAD_STAG_KEY
:
4310 case I40IW_AE_AMP_BAD_STAG_INDEX
:
4311 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4313 case I40IW_OP_TYPE_SEND_INV
:
4314 case I40IW_OP_TYPE_SEND_SOL_INV
:
4315 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_OP_ERR
,
4316 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_CANT_INV_STAG
);
4319 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4320 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_INV_STAG
);
4323 case I40IW_AE_AMP_RIGHTS_VIOLATION
:
4324 case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS
:
4325 case I40IW_AE_PRIV_OPERATION_DENIED
:
4326 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4327 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4328 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_ACCESS
);
4330 case I40IW_AE_AMP_TO_WRAP
:
4331 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4332 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_ACCESS_ERR
,
4333 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_PROT
, RDMAP_TO_WRAP
);
4335 case I40IW_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH
:
4336 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_LEN_ERR
,
4337 (LAYER_MPA
<< 4) | DDP_LLP
, MPA_MARKER
);
4339 case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR
:
4340 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4341 (LAYER_MPA
<< 4) | DDP_LLP
, MPA_CRC
);
4343 case I40IW_AE_LLP_SEGMENT_TOO_LARGE
:
4344 case I40IW_AE_LLP_SEGMENT_TOO_SMALL
:
4345 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_LEN_ERR
,
4346 (LAYER_DDP
<< 4) | DDP_CATASTROPHIC
, DDP_CATASTROPHIC_LOCAL
);
4348 case I40IW_AE_LCE_QP_CATASTROPHIC
:
4349 case I40IW_AE_DDP_NO_L_BIT
:
4350 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_FATAL_ERR
,
4351 (LAYER_DDP
<< 4) | DDP_CATASTROPHIC
, DDP_CATASTROPHIC_LOCAL
);
4353 case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN
:
4354 case I40IW_AE_DDP_INVALID_MSN_RANGE_IS_NOT_VALID
:
4355 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4356 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_MSN_RANGE
);
4358 case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER
:
4359 qp
->eventtype
= TERM_EVENT_QP_ACCESS_ERR
;
4360 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_LEN_ERR
,
4361 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_TOO_LONG
);
4363 case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION
:
4365 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4366 (LAYER_DDP
<< 4) | DDP_TAGGED_BUFFER
, DDP_TAGGED_INV_DDP_VER
);
4368 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4369 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_DDP_VER
);
4371 case I40IW_AE_DDP_UBE_INVALID_MO
:
4372 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4373 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_MO
);
4375 case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE
:
4376 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_REM_OP_ERR
,
4377 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_MSN_NO_BUF
);
4379 case I40IW_AE_DDP_UBE_INVALID_QN
:
4380 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4381 (LAYER_DDP
<< 4) | DDP_UNTAGGED_BUFFER
, DDP_UNTAGGED_INV_QN
);
4383 case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION
:
4384 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_GENERAL_ERR
,
4385 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_INV_RDMAP_VER
);
4387 case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE
:
4388 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_LOC_QP_OP_ERR
,
4389 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_UNEXPECTED_OP
);
4392 i40iw_setup_termhdr(qp
, termhdr
, FLUSH_FATAL_ERR
,
4393 (LAYER_RDMA
<< 4) | RDMAP_REMOTE_OP
, RDMAP_UNSPECIFIED
);
4398 memcpy(termhdr
+ 1, pkt
, copy_len
);
4400 return sizeof(struct i40iw_terminate_hdr
) + copy_len
;
4404 * i40iw_terminate_send_fin() - Send fin for terminate message
4405 * @qp: qp associated with received terminate AE
4407 void i40iw_terminate_send_fin(struct i40iw_sc_qp
*qp
)
4409 /* Send the fin only */
4410 i40iw_term_modify_qp(qp
,
4411 I40IW_QP_STATE_TERMINATE
,
4412 I40IWQP_TERM_SEND_FIN_ONLY
,
4417 * i40iw_terminate_connection() - Bad AE and send terminate to remote QP
4418 * @qp: qp associated with received terminate AE
4419 * @info: the struct contiaing AE information
4421 void i40iw_terminate_connection(struct i40iw_sc_qp
*qp
, struct i40iw_aeqe_info
*info
)
4425 if (qp
->term_flags
& I40IW_TERM_SENT
)
4426 return; /* Sanity check */
4428 /* Eventtype can change from bld_terminate_hdr */
4429 qp
->eventtype
= TERM_EVENT_QP_FATAL
;
4430 termlen
= i40iw_bld_terminate_hdr(qp
, info
);
4431 i40iw_terminate_start_timer(qp
);
4432 qp
->term_flags
|= I40IW_TERM_SENT
;
4433 i40iw_term_modify_qp(qp
, I40IW_QP_STATE_TERMINATE
,
4434 I40IWQP_TERM_SEND_TERM_ONLY
, termlen
);
4438 * i40iw_terminate_received - handle terminate received AE
4439 * @qp: qp associated with received terminate AE
4440 * @info: the struct contiaing AE information
4442 void i40iw_terminate_received(struct i40iw_sc_qp
*qp
, struct i40iw_aeqe_info
*info
)
4444 u8
*pkt
= qp
->q2_buf
+ Q2_BAD_FRAME_OFFSET
;
4449 struct i40iw_terminate_hdr
*termhdr
;
4451 mpa
= (__be32
*)i40iw_locate_mpa(pkt
);
4452 if (info
->q2_data_written
) {
4453 /* did not validate the frame - do it now */
4454 ddp_ctl
= (ntohl(mpa
[0]) >> 8) & 0xff;
4455 rdma_ctl
= ntohl(mpa
[0]) & 0xff;
4456 if ((ddp_ctl
& 0xc0) != 0x40)
4457 aeq_id
= I40IW_AE_LCE_QP_CATASTROPHIC
;
4458 else if ((ddp_ctl
& 0x03) != 1)
4459 aeq_id
= I40IW_AE_DDP_UBE_INVALID_DDP_VERSION
;
4460 else if (ntohl(mpa
[2]) != 2)
4461 aeq_id
= I40IW_AE_DDP_UBE_INVALID_QN
;
4462 else if (ntohl(mpa
[3]) != 1)
4463 aeq_id
= I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN
;
4464 else if (ntohl(mpa
[4]) != 0)
4465 aeq_id
= I40IW_AE_DDP_UBE_INVALID_MO
;
4466 else if ((rdma_ctl
& 0xc0) != 0x40)
4467 aeq_id
= I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION
;
4469 info
->ae_id
= aeq_id
;
4471 /* Bad terminate recvd - send back a terminate */
4472 i40iw_terminate_connection(qp
, info
);
4477 qp
->term_flags
|= I40IW_TERM_RCVD
;
4478 qp
->eventtype
= TERM_EVENT_QP_FATAL
;
4479 termhdr
= (struct i40iw_terminate_hdr
*)&mpa
[5];
4480 if (termhdr
->layer_etype
== RDMAP_REMOTE_PROT
||
4481 termhdr
->layer_etype
== RDMAP_REMOTE_OP
) {
4482 i40iw_terminate_done(qp
, 0);
4484 i40iw_terminate_start_timer(qp
);
4485 i40iw_terminate_send_fin(qp
);
4490 * i40iw_sc_vsi_init - Initialize virtual device
4491 * @vsi: pointer to the vsi structure
4492 * @info: parameters to initialize vsi
4494 void i40iw_sc_vsi_init(struct i40iw_sc_vsi
*vsi
, struct i40iw_vsi_init_info
*info
)
4498 vsi
->dev
= info
->dev
;
4499 vsi
->back_vsi
= info
->back_vsi
;
4500 vsi
->mss
= info
->params
->mss
;
4501 i40iw_fill_qos_list(info
->params
->qs_handle_list
);
4503 for (i
= 0; i
< I40IW_MAX_USER_PRIORITY
; i
++) {
4504 vsi
->qos
[i
].qs_handle
= info
->params
->qs_handle_list
[i
];
4505 i40iw_debug(vsi
->dev
, I40IW_DEBUG_DCB
, "qset[%d]: %d\n", i
,
4506 vsi
->qos
[i
].qs_handle
);
4507 spin_lock_init(&vsi
->qos
[i
].lock
);
4508 INIT_LIST_HEAD(&vsi
->qos
[i
].qplist
);
4513 * i40iw_hw_stats_init - Initiliaze HW stats table
4514 * @stats: pestat struct
4515 * @fcn_idx: PCI fn id
4516 * @is_pf: Is it a PF?
4518 * Populate the HW stats table with register offset addr for each
4519 * stats. And start the perioidic stats timer.
4521 void i40iw_hw_stats_init(struct i40iw_vsi_pestat
*stats
, u8 fcn_idx
, bool is_pf
)
4523 u32 stats_reg_offset
;
4525 struct i40iw_dev_hw_stats_offsets
*stats_table
=
4526 &stats
->hw_stats_offsets
;
4527 struct i40iw_dev_hw_stats
*last_rd_stats
= &stats
->last_read_hw_stats
;
4530 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4RXDISCARD
] =
4531 I40E_GLPES_PFIP4RXDISCARD(fcn_idx
);
4532 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4RXTRUNC
] =
4533 I40E_GLPES_PFIP4RXTRUNC(fcn_idx
);
4534 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4TXNOROUTE
] =
4535 I40E_GLPES_PFIP4TXNOROUTE(fcn_idx
);
4536 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6RXDISCARD
] =
4537 I40E_GLPES_PFIP6RXDISCARD(fcn_idx
);
4538 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6RXTRUNC
] =
4539 I40E_GLPES_PFIP6RXTRUNC(fcn_idx
);
4540 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6TXNOROUTE
] =
4541 I40E_GLPES_PFIP6TXNOROUTE(fcn_idx
);
4542 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRTXSEG
] =
4543 I40E_GLPES_PFTCPRTXSEG(fcn_idx
);
4544 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRXOPTERR
] =
4545 I40E_GLPES_PFTCPRXOPTERR(fcn_idx
);
4546 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRXPROTOERR
] =
4547 I40E_GLPES_PFTCPRXPROTOERR(fcn_idx
);
4549 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXOCTS
] =
4550 I40E_GLPES_PFIP4RXOCTSLO(fcn_idx
);
4551 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXPKTS
] =
4552 I40E_GLPES_PFIP4RXPKTSLO(fcn_idx
);
4553 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXFRAGS
] =
4554 I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx
);
4555 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXMCPKTS
] =
4556 I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx
);
4557 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXOCTS
] =
4558 I40E_GLPES_PFIP4TXOCTSLO(fcn_idx
);
4559 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXPKTS
] =
4560 I40E_GLPES_PFIP4TXPKTSLO(fcn_idx
);
4561 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXFRAGS
] =
4562 I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx
);
4563 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXMCPKTS
] =
4564 I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx
);
4565 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXOCTS
] =
4566 I40E_GLPES_PFIP6RXOCTSLO(fcn_idx
);
4567 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXPKTS
] =
4568 I40E_GLPES_PFIP6RXPKTSLO(fcn_idx
);
4569 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXFRAGS
] =
4570 I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx
);
4571 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXMCPKTS
] =
4572 I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx
);
4573 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXOCTS
] =
4574 I40E_GLPES_PFIP6TXOCTSLO(fcn_idx
);
4575 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXPKTS
] =
4576 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx
);
4577 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXPKTS
] =
4578 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx
);
4579 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXFRAGS
] =
4580 I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx
);
4581 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_TCPRXSEGS
] =
4582 I40E_GLPES_PFTCPRXSEGSLO(fcn_idx
);
4583 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_TCPTXSEG
] =
4584 I40E_GLPES_PFTCPTXSEGLO(fcn_idx
);
4585 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXRDS
] =
4586 I40E_GLPES_PFRDMARXRDSLO(fcn_idx
);
4587 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXSNDS
] =
4588 I40E_GLPES_PFRDMARXSNDSLO(fcn_idx
);
4589 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXWRS
] =
4590 I40E_GLPES_PFRDMARXWRSLO(fcn_idx
);
4591 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXRDS
] =
4592 I40E_GLPES_PFRDMATXRDSLO(fcn_idx
);
4593 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXSNDS
] =
4594 I40E_GLPES_PFRDMATXSNDSLO(fcn_idx
);
4595 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXWRS
] =
4596 I40E_GLPES_PFRDMATXWRSLO(fcn_idx
);
4597 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMAVBND
] =
4598 I40E_GLPES_PFRDMAVBNDLO(fcn_idx
);
4599 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMAVINV
] =
4600 I40E_GLPES_PFRDMAVINVLO(fcn_idx
);
4602 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4RXDISCARD
] =
4603 I40E_GLPES_VFIP4RXDISCARD(fcn_idx
);
4604 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4RXTRUNC
] =
4605 I40E_GLPES_VFIP4RXTRUNC(fcn_idx
);
4606 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP4TXNOROUTE
] =
4607 I40E_GLPES_VFIP4TXNOROUTE(fcn_idx
);
4608 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6RXDISCARD
] =
4609 I40E_GLPES_VFIP6RXDISCARD(fcn_idx
);
4610 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6RXTRUNC
] =
4611 I40E_GLPES_VFIP6RXTRUNC(fcn_idx
);
4612 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_IP6TXNOROUTE
] =
4613 I40E_GLPES_VFIP6TXNOROUTE(fcn_idx
);
4614 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRTXSEG
] =
4615 I40E_GLPES_VFTCPRTXSEG(fcn_idx
);
4616 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRXOPTERR
] =
4617 I40E_GLPES_VFTCPRXOPTERR(fcn_idx
);
4618 stats_table
->stats_offset_32
[I40IW_HW_STAT_INDEX_TCPRXPROTOERR
] =
4619 I40E_GLPES_VFTCPRXPROTOERR(fcn_idx
);
4621 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXOCTS
] =
4622 I40E_GLPES_VFIP4RXOCTSLO(fcn_idx
);
4623 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXPKTS
] =
4624 I40E_GLPES_VFIP4RXPKTSLO(fcn_idx
);
4625 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXFRAGS
] =
4626 I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx
);
4627 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4RXMCPKTS
] =
4628 I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx
);
4629 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXOCTS
] =
4630 I40E_GLPES_VFIP4TXOCTSLO(fcn_idx
);
4631 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXPKTS
] =
4632 I40E_GLPES_VFIP4TXPKTSLO(fcn_idx
);
4633 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXFRAGS
] =
4634 I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx
);
4635 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP4TXMCPKTS
] =
4636 I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx
);
4637 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXOCTS
] =
4638 I40E_GLPES_VFIP6RXOCTSLO(fcn_idx
);
4639 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXPKTS
] =
4640 I40E_GLPES_VFIP6RXPKTSLO(fcn_idx
);
4641 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXFRAGS
] =
4642 I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx
);
4643 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6RXMCPKTS
] =
4644 I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx
);
4645 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXOCTS
] =
4646 I40E_GLPES_VFIP6TXOCTSLO(fcn_idx
);
4647 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXPKTS
] =
4648 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx
);
4649 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXPKTS
] =
4650 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx
);
4651 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_IP6TXFRAGS
] =
4652 I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx
);
4653 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_TCPRXSEGS
] =
4654 I40E_GLPES_VFTCPRXSEGSLO(fcn_idx
);
4655 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_TCPTXSEG
] =
4656 I40E_GLPES_VFTCPTXSEGLO(fcn_idx
);
4657 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXRDS
] =
4658 I40E_GLPES_VFRDMARXRDSLO(fcn_idx
);
4659 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXSNDS
] =
4660 I40E_GLPES_VFRDMARXSNDSLO(fcn_idx
);
4661 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMARXWRS
] =
4662 I40E_GLPES_VFRDMARXWRSLO(fcn_idx
);
4663 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXRDS
] =
4664 I40E_GLPES_VFRDMATXRDSLO(fcn_idx
);
4665 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXSNDS
] =
4666 I40E_GLPES_VFRDMATXSNDSLO(fcn_idx
);
4667 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMATXWRS
] =
4668 I40E_GLPES_VFRDMATXWRSLO(fcn_idx
);
4669 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMAVBND
] =
4670 I40E_GLPES_VFRDMAVBNDLO(fcn_idx
);
4671 stats_table
->stats_offset_64
[I40IW_HW_STAT_INDEX_RDMAVINV
] =
4672 I40E_GLPES_VFRDMAVINVLO(fcn_idx
);
4675 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_64
;
4677 stats_reg_offset
= stats_table
->stats_offset_64
[stats_index
];
4678 last_rd_stats
->stats_value_64
[stats_index
] =
4679 readq(stats
->hw
->hw_addr
+ stats_reg_offset
);
4682 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_32
;
4684 stats_reg_offset
= stats_table
->stats_offset_32
[stats_index
];
4685 last_rd_stats
->stats_value_32
[stats_index
] =
4686 i40iw_rd32(stats
->hw
, stats_reg_offset
);
4691 * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
4692 * @stat: pestat struct
4693 * @index: index in HW stats table which contains offset reg-addr
4694 * @value: hw stats value
4696 void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat
*stats
,
4697 enum i40iw_hw_stats_index_32b index
,
4700 struct i40iw_dev_hw_stats_offsets
*stats_table
=
4701 &stats
->hw_stats_offsets
;
4702 struct i40iw_dev_hw_stats
*last_rd_stats
= &stats
->last_read_hw_stats
;
4703 struct i40iw_dev_hw_stats
*hw_stats
= &stats
->hw_stats
;
4704 u64 new_stats_value
= 0;
4705 u32 stats_reg_offset
= stats_table
->stats_offset_32
[index
];
4707 new_stats_value
= i40iw_rd32(stats
->hw
, stats_reg_offset
);
4709 if (new_stats_value
< last_rd_stats
->stats_value_32
[index
])
4710 hw_stats
->stats_value_32
[index
] += new_stats_value
;
4712 hw_stats
->stats_value_32
[index
] +=
4713 new_stats_value
- last_rd_stats
->stats_value_32
[index
];
4714 last_rd_stats
->stats_value_32
[index
] = new_stats_value
;
4715 *value
= hw_stats
->stats_value_32
[index
];
4719 * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
4720 * @stats: pestat struct
4721 * @index: index in HW stats table which contains offset reg-addr
4722 * @value: hw stats value
4724 void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat
*stats
,
4725 enum i40iw_hw_stats_index_64b index
,
4728 struct i40iw_dev_hw_stats_offsets
*stats_table
=
4729 &stats
->hw_stats_offsets
;
4730 struct i40iw_dev_hw_stats
*last_rd_stats
= &stats
->last_read_hw_stats
;
4731 struct i40iw_dev_hw_stats
*hw_stats
= &stats
->hw_stats
;
4732 u64 new_stats_value
= 0;
4733 u32 stats_reg_offset
= stats_table
->stats_offset_64
[index
];
4735 new_stats_value
= readq(stats
->hw
->hw_addr
+ stats_reg_offset
);
4737 if (new_stats_value
< last_rd_stats
->stats_value_64
[index
])
4738 hw_stats
->stats_value_64
[index
] += new_stats_value
;
4740 hw_stats
->stats_value_64
[index
] +=
4741 new_stats_value
- last_rd_stats
->stats_value_64
[index
];
4742 last_rd_stats
->stats_value_64
[index
] = new_stats_value
;
4743 *value
= hw_stats
->stats_value_64
[index
];
4747 * i40iw_hw_stats_read_all - read all HW stat counters
4748 * @stats: pestat struct
4749 * @stats_values: hw stats structure
4751 * Read all the HW stat counters and populates hw_stats structure
4752 * of passed-in vsi's pestat as well as copy created in stat_values.
4754 void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat
*stats
,
4755 struct i40iw_dev_hw_stats
*stats_values
)
4758 unsigned long flags
;
4760 spin_lock_irqsave(&stats
->lock
, flags
);
4762 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_32
;
4764 i40iw_hw_stats_read_32(stats
, stats_index
,
4765 &stats_values
->stats_value_32
[stats_index
]);
4766 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_64
;
4768 i40iw_hw_stats_read_64(stats
, stats_index
,
4769 &stats_values
->stats_value_64
[stats_index
]);
4770 spin_unlock_irqrestore(&stats
->lock
, flags
);
4774 * i40iw_hw_stats_refresh_all - Update all HW stats structs
4775 * @stats: pestat struct
4777 * Read all the HW stats counters to refresh values in hw_stats structure
4778 * of passed-in dev's pestat
4780 void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat
*stats
)
4784 unsigned long flags
;
4786 spin_lock_irqsave(&stats
->lock
, flags
);
4788 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_32
;
4790 i40iw_hw_stats_read_32(stats
, stats_index
, &stats_value
);
4791 for (stats_index
= 0; stats_index
< I40IW_HW_STAT_INDEX_MAX_64
;
4793 i40iw_hw_stats_read_64(stats
, stats_index
, &stats_value
);
4794 spin_unlock_irqrestore(&stats
->lock
, flags
);
4798 * i40iw_get_fcn_id - Return the function id
4799 * @dev: pointer to the device
4801 static u8
i40iw_get_fcn_id(struct i40iw_sc_dev
*dev
)
4803 u8 fcn_id
= I40IW_INVALID_FCN_ID
;
4806 for (i
= I40IW_FIRST_NON_PF_STAT
; i
< I40IW_MAX_STATS_COUNT
; i
++)
4807 if (!dev
->fcn_id_array
[i
]) {
4809 dev
->fcn_id_array
[i
] = true;
4816 * i40iw_vsi_stats_init - Initialize the vsi statistics
4817 * @vsi: pointer to the vsi structure
4818 * @info: The info structure used for initialization
4820 enum i40iw_status_code
i40iw_vsi_stats_init(struct i40iw_sc_vsi
*vsi
, struct i40iw_vsi_stats_info
*info
)
4822 u8 fcn_id
= info
->fcn_id
;
4824 if (info
->alloc_fcn_id
)
4825 fcn_id
= i40iw_get_fcn_id(vsi
->dev
);
4827 if (fcn_id
== I40IW_INVALID_FCN_ID
)
4828 return I40IW_ERR_NOT_READY
;
4830 vsi
->pestat
= info
->pestat
;
4831 vsi
->pestat
->hw
= vsi
->dev
->hw
;
4833 if (info
->stats_initialize
) {
4834 i40iw_hw_stats_init(vsi
->pestat
, fcn_id
, true);
4835 spin_lock_init(&vsi
->pestat
->lock
);
4836 i40iw_hw_stats_start_timer(vsi
);
4838 vsi
->stats_fcn_id_alloc
= info
->alloc_fcn_id
;
4839 vsi
->fcn_id
= fcn_id
;
4840 return I40IW_SUCCESS
;
4844 * i40iw_vsi_stats_free - Free the vsi stats
4845 * @vsi: pointer to the vsi structure
4847 void i40iw_vsi_stats_free(struct i40iw_sc_vsi
*vsi
)
4849 u8 fcn_id
= vsi
->fcn_id
;
4851 if ((vsi
->stats_fcn_id_alloc
) && (fcn_id
!= I40IW_INVALID_FCN_ID
))
4852 vsi
->dev
->fcn_id_array
[fcn_id
] = false;
4853 i40iw_hw_stats_stop_timer(vsi
);
4856 static struct i40iw_cqp_ops iw_cqp_ops
= {
4857 .cqp_init
= i40iw_sc_cqp_init
,
4858 .cqp_create
= i40iw_sc_cqp_create
,
4859 .cqp_post_sq
= i40iw_sc_cqp_post_sq
,
4860 .cqp_get_next_send_wqe
= i40iw_sc_cqp_get_next_send_wqe
,
4861 .cqp_destroy
= i40iw_sc_cqp_destroy
,
4862 .poll_for_cqp_op_done
= i40iw_sc_poll_for_cqp_op_done
4865 static struct i40iw_ccq_ops iw_ccq_ops
= {
4866 .ccq_init
= i40iw_sc_ccq_init
,
4867 .ccq_create
= i40iw_sc_ccq_create
,
4868 .ccq_destroy
= i40iw_sc_ccq_destroy
,
4869 .ccq_create_done
= i40iw_sc_ccq_create_done
,
4870 .ccq_get_cqe_info
= i40iw_sc_ccq_get_cqe_info
,
4871 .ccq_arm
= i40iw_sc_ccq_arm
4874 static struct i40iw_ceq_ops iw_ceq_ops
= {
4875 .ceq_init
= i40iw_sc_ceq_init
,
4876 .ceq_create
= i40iw_sc_ceq_create
,
4877 .cceq_create_done
= i40iw_sc_cceq_create_done
,
4878 .cceq_destroy_done
= i40iw_sc_cceq_destroy_done
,
4879 .cceq_create
= i40iw_sc_cceq_create
,
4880 .ceq_destroy
= i40iw_sc_ceq_destroy
,
4881 .process_ceq
= i40iw_sc_process_ceq
4884 static struct i40iw_aeq_ops iw_aeq_ops
= {
4885 .aeq_init
= i40iw_sc_aeq_init
,
4886 .aeq_create
= i40iw_sc_aeq_create
,
4887 .aeq_destroy
= i40iw_sc_aeq_destroy
,
4888 .get_next_aeqe
= i40iw_sc_get_next_aeqe
,
4889 .repost_aeq_entries
= i40iw_sc_repost_aeq_entries
,
4890 .aeq_create_done
= i40iw_sc_aeq_create_done
,
4891 .aeq_destroy_done
= i40iw_sc_aeq_destroy_done
4895 static struct i40iw_pd_ops iw_pd_ops
= {
4896 .pd_init
= i40iw_sc_pd_init
,
4899 static struct i40iw_priv_qp_ops iw_priv_qp_ops
= {
4900 .qp_init
= i40iw_sc_qp_init
,
4901 .qp_create
= i40iw_sc_qp_create
,
4902 .qp_modify
= i40iw_sc_qp_modify
,
4903 .qp_destroy
= i40iw_sc_qp_destroy
,
4904 .qp_flush_wqes
= i40iw_sc_qp_flush_wqes
,
4905 .qp_upload_context
= i40iw_sc_qp_upload_context
,
4906 .qp_setctx
= i40iw_sc_qp_setctx
,
4907 .qp_send_lsmm
= i40iw_sc_send_lsmm
,
4908 .qp_send_lsmm_nostag
= i40iw_sc_send_lsmm_nostag
,
4909 .qp_send_rtt
= i40iw_sc_send_rtt
,
4910 .qp_post_wqe0
= i40iw_sc_post_wqe0
,
4911 .iw_mr_fast_register
= i40iw_sc_mr_fast_register
4914 static struct i40iw_priv_cq_ops iw_priv_cq_ops
= {
4915 .cq_init
= i40iw_sc_cq_init
,
4916 .cq_create
= i40iw_sc_cq_create
,
4917 .cq_destroy
= i40iw_sc_cq_destroy
,
4918 .cq_modify
= i40iw_sc_cq_modify
,
4921 static struct i40iw_mr_ops iw_mr_ops
= {
4922 .alloc_stag
= i40iw_sc_alloc_stag
,
4923 .mr_reg_non_shared
= i40iw_sc_mr_reg_non_shared
,
4924 .mr_reg_shared
= i40iw_sc_mr_reg_shared
,
4925 .dealloc_stag
= i40iw_sc_dealloc_stag
,
4926 .query_stag
= i40iw_sc_query_stag
,
4927 .mw_alloc
= i40iw_sc_mw_alloc
4930 static struct i40iw_cqp_misc_ops iw_cqp_misc_ops
= {
4931 .manage_push_page
= i40iw_sc_manage_push_page
,
4932 .manage_hmc_pm_func_table
= i40iw_sc_manage_hmc_pm_func_table
,
4933 .set_hmc_resource_profile
= i40iw_sc_set_hmc_resource_profile
,
4934 .commit_fpm_values
= i40iw_sc_commit_fpm_values
,
4935 .query_fpm_values
= i40iw_sc_query_fpm_values
,
4936 .static_hmc_pages_allocated
= i40iw_sc_static_hmc_pages_allocated
,
4937 .add_arp_cache_entry
= i40iw_sc_add_arp_cache_entry
,
4938 .del_arp_cache_entry
= i40iw_sc_del_arp_cache_entry
,
4939 .query_arp_cache_entry
= i40iw_sc_query_arp_cache_entry
,
4940 .manage_apbvt_entry
= i40iw_sc_manage_apbvt_entry
,
4941 .manage_qhash_table_entry
= i40iw_sc_manage_qhash_table_entry
,
4942 .alloc_local_mac_ipaddr_table_entry
= i40iw_sc_alloc_local_mac_ipaddr_entry
,
4943 .add_local_mac_ipaddr_entry
= i40iw_sc_add_local_mac_ipaddr_entry
,
4944 .del_local_mac_ipaddr_entry
= i40iw_sc_del_local_mac_ipaddr_entry
,
4945 .cqp_nop
= i40iw_sc_cqp_nop
,
4946 .commit_fpm_values_done
= i40iw_sc_commit_fpm_values_done
,
4947 .query_fpm_values_done
= i40iw_sc_query_fpm_values_done
,
4948 .manage_hmc_pm_func_table_done
= i40iw_sc_manage_hmc_pm_func_table_done
,
4949 .update_suspend_qp
= i40iw_sc_suspend_qp
,
4950 .update_resume_qp
= i40iw_sc_resume_qp
4953 static struct i40iw_hmc_ops iw_hmc_ops
= {
4954 .init_iw_hmc
= i40iw_sc_init_iw_hmc
,
4955 .parse_fpm_query_buf
= i40iw_sc_parse_fpm_query_buf
,
4956 .configure_iw_fpm
= i40iw_sc_configure_iw_fpm
,
4957 .parse_fpm_commit_buf
= i40iw_sc_parse_fpm_commit_buf
,
4958 .create_hmc_object
= i40iw_sc_create_hmc_obj
,
4959 .del_hmc_object
= i40iw_sc_del_hmc_obj
4963 * i40iw_device_init - Initialize IWARP device
4964 * @dev: IWARP device pointer
4965 * @info: IWARP init info
4967 enum i40iw_status_code
i40iw_device_init(struct i40iw_sc_dev
*dev
,
4968 struct i40iw_device_init_info
*info
)
4973 enum i40iw_status_code ret_code
= 0;
4976 spin_lock_init(&dev
->cqp_lock
);
4977 INIT_LIST_HEAD(&dev
->cqp_cmd_head
); /* for the cqp commands backlog. */
4979 i40iw_device_init_uk(&dev
->dev_uk
);
4981 dev
->debug_mask
= info
->debug_mask
;
4983 dev
->hmc_fn_id
= info
->hmc_fn_id
;
4984 dev
->exception_lan_queue
= info
->exception_lan_queue
;
4985 dev
->is_pf
= info
->is_pf
;
4987 dev
->fpm_query_buf_pa
= info
->fpm_query_buf_pa
;
4988 dev
->fpm_query_buf
= info
->fpm_query_buf
;
4990 dev
->fpm_commit_buf_pa
= info
->fpm_commit_buf_pa
;
4991 dev
->fpm_commit_buf
= info
->fpm_commit_buf
;
4994 dev
->hw
->hw_addr
= info
->bar0
;
4997 val
= i40iw_rd32(dev
->hw
, I40E_GLPCI_DREVID
);
4998 dev
->hw_rev
= (u8
)RS_32(val
, I40E_GLPCI_DREVID_DEFAULT_REVID
);
5000 val
= i40iw_rd32(dev
->hw
, I40E_GLPCI_LBARCTRL
);
5001 db_size
= (u8
)RS_32(val
, I40E_GLPCI_LBARCTRL_PE_DB_SIZE
);
5002 if ((db_size
!= I40IW_PE_DB_SIZE_4M
) &&
5003 (db_size
!= I40IW_PE_DB_SIZE_8M
)) {
5004 i40iw_debug(dev
, I40IW_DEBUG_DEV
,
5005 "%s: PE doorbell is not enabled in CSR val 0x%x\n",
5007 ret_code
= I40IW_ERR_PE_DOORBELL_NOT_ENABLED
;
5010 dev
->db_addr
= dev
->hw
->hw_addr
+ I40IW_DB_ADDR_OFFSET
;
5011 dev
->vchnl_if
.vchnl_recv
= i40iw_vchnl_recv_pf
;
5013 dev
->db_addr
= dev
->hw
->hw_addr
+ I40IW_VF_DB_ADDR_OFFSET
;
5016 dev
->cqp_ops
= &iw_cqp_ops
;
5017 dev
->ccq_ops
= &iw_ccq_ops
;
5018 dev
->ceq_ops
= &iw_ceq_ops
;
5019 dev
->aeq_ops
= &iw_aeq_ops
;
5020 dev
->cqp_misc_ops
= &iw_cqp_misc_ops
;
5021 dev
->iw_pd_ops
= &iw_pd_ops
;
5022 dev
->iw_priv_qp_ops
= &iw_priv_qp_ops
;
5023 dev
->iw_priv_cq_ops
= &iw_priv_cq_ops
;
5024 dev
->mr_ops
= &iw_mr_ops
;
5025 dev
->hmc_ops
= &iw_hmc_ops
;
5026 dev
->vchnl_if
.vchnl_send
= info
->vchnl_send
;
5027 if (dev
->vchnl_if
.vchnl_send
)
5028 dev
->vchnl_up
= true;
5030 dev
->vchnl_up
= false;
5032 dev
->vchnl_if
.vchnl_recv
= i40iw_vchnl_recv_vf
;
5033 ret_code
= i40iw_vchnl_vf_get_ver(dev
, &vchnl_ver
);
5035 i40iw_debug(dev
, I40IW_DEBUG_DEV
,
5036 "%s: Get Channel version rc = 0x%0x, version is %u\n",
5037 __func__
, ret_code
, vchnl_ver
);
5038 ret_code
= i40iw_vchnl_vf_get_hmc_fcn(dev
, &hmc_fcn
);
5040 i40iw_debug(dev
, I40IW_DEBUG_DEV
,
5041 "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n",
5042 __func__
, ret_code
, hmc_fcn
);
5043 dev
->hmc_fn_id
= (u8
)hmc_fcn
;
5047 dev
->iw_vf_cqp_ops
= &iw_vf_cqp_ops
;