1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Cavium, Inc
4 #ifndef _OTX_CRYPTODEV_HW_ACCESS_H_
5 #define _OTX_CRYPTODEV_HW_ACCESS_H_
9 #include <rte_branch_prediction.h>
10 #include <rte_cryptodev.h>
11 #include <rte_cycles.h>
13 #include <rte_memory.h>
14 #include <rte_prefetch.h>
16 #include "cpt_common.h"
17 #include "cpt_hw_types.h"
18 #include "cpt_mcode_defines.h"
19 #include "cpt_pmd_logs.h"
21 #define CPT_INTR_POLL_INTERVAL_MS (50)
23 /* Default command queue length */
24 #define DEFAULT_CMD_QCHUNKS 2
25 #define DEFAULT_CMD_QCHUNK_SIZE 1023
26 #define DEFAULT_CMD_QLEN \
27 (DEFAULT_CMD_QCHUNK_SIZE * DEFAULT_CMD_QCHUNKS)
29 #define CPT_CSR_REG_BASE(cpt) ((cpt)->reg_base)
31 /* Read hw register */
32 #define CPT_READ_CSR(__hw_addr, __offset) \
33 rte_read64_relaxed((uint8_t *)__hw_addr + __offset)
35 /* Write hw register */
36 #define CPT_WRITE_CSR(__hw_addr, __offset, __val) \
37 rte_write64_relaxed((__val), ((uint8_t *)__hw_addr + __offset))
43 struct rte_mempool
*sess_mp
;
44 struct rte_mempool
*sess_mp_priv
;
45 struct cpt_qp_meta_info meta_info
;
48 struct command_chunk
{
49 /** 128-byte aligned real_vaddr */
51 /** 128-byte aligned real_dma_addr */
56 * Command queue structure
58 struct command_queue
{
59 /** Command queue host write idx */
61 /** Command queue chunk */
63 /** Command queue head; instructions are inserted here */
65 /** Command chunk list head */
66 struct command_chunk chead
[DEFAULT_CMD_QCHUNKS
];
70 * CPT VF device structure
74 struct cpt_instance instance
;
75 /** Register start address */
77 /** Command queue information */
78 struct command_queue cqueue
;
79 /** Pending queue information */
80 struct pending_queue pqueue
;
82 /** Below fields are accessed only in control path */
84 /** Env specific pdev representing the pci dev */
86 /** Calculated queue size */
88 /** Device index (0...CPT_MAX_VQ_NUM)*/
90 /** VF type of cpt_vf_type_t (SE_TYPE(2) or AE_TYPE(1) */
92 /** VF group (0 - 8) */
94 /** Operating node: Bits (46:44) in BAR0 address */
97 /** VF-PF mailbox communication */
101 /** Flag if not acked */
106 } __rte_cache_aligned
;
109 * CPT Registers map for 81xx
113 #define CPTX_VQX_CTL(a, b) (0x0000100ll + 0x1000000000ll * \
114 ((a) & 0x0) + 0x100000ll * (b))
115 #define CPTX_VQX_SADDR(a, b) (0x0000200ll + 0x1000000000ll * \
116 ((a) & 0x0) + 0x100000ll * (b))
117 #define CPTX_VQX_DONE_WAIT(a, b) (0x0000400ll + 0x1000000000ll * \
118 ((a) & 0x0) + 0x100000ll * (b))
119 #define CPTX_VQX_INPROG(a, b) (0x0000410ll + 0x1000000000ll * \
120 ((a) & 0x0) + 0x100000ll * (b))
121 #define CPTX_VQX_DONE(a, b) (0x0000420ll + 0x1000000000ll * \
122 ((a) & 0x1) + 0x100000ll * (b))
123 #define CPTX_VQX_DONE_ACK(a, b) (0x0000440ll + 0x1000000000ll * \
124 ((a) & 0x1) + 0x100000ll * (b))
125 #define CPTX_VQX_DONE_INT_W1S(a, b) (0x0000460ll + 0x1000000000ll * \
126 ((a) & 0x1) + 0x100000ll * (b))
127 #define CPTX_VQX_DONE_INT_W1C(a, b) (0x0000468ll + 0x1000000000ll * \
128 ((a) & 0x1) + 0x100000ll * (b))
129 #define CPTX_VQX_DONE_ENA_W1S(a, b) (0x0000470ll + 0x1000000000ll * \
130 ((a) & 0x1) + 0x100000ll * (b))
131 #define CPTX_VQX_DONE_ENA_W1C(a, b) (0x0000478ll + 0x1000000000ll * \
132 ((a) & 0x1) + 0x100000ll * (b))
133 #define CPTX_VQX_MISC_INT(a, b) (0x0000500ll + 0x1000000000ll * \
134 ((a) & 0x1) + 0x100000ll * (b))
135 #define CPTX_VQX_MISC_INT_W1S(a, b) (0x0000508ll + 0x1000000000ll * \
136 ((a) & 0x1) + 0x100000ll * (b))
137 #define CPTX_VQX_MISC_ENA_W1S(a, b) (0x0000510ll + 0x1000000000ll * \
138 ((a) & 0x1) + 0x100000ll * (b))
139 #define CPTX_VQX_MISC_ENA_W1C(a, b) (0x0000518ll + 0x1000000000ll * \
140 ((a) & 0x1) + 0x100000ll * (b))
141 #define CPTX_VQX_DOORBELL(a, b) (0x0000600ll + 0x1000000000ll * \
142 ((a) & 0x1) + 0x100000ll * (b))
143 #define CPTX_VFX_PF_MBOXX(a, b, c) (0x0001000ll + 0x1000000000ll * \
144 ((a) & 0x1) + 0x100000ll * (b) + \
147 /* VF HAL functions */
150 otx_cpt_poll_misc(struct cpt_vf
*cptvf
);
153 otx_cpt_hw_init(struct cpt_vf
*cptvf
, void *pdev
, void *reg_base
, char *name
);
156 otx_cpt_deinit_device(void *dev
);
159 otx_cpt_get_resource(const struct rte_cryptodev
*dev
, uint8_t group
,
160 struct cpt_instance
**instance
, uint16_t qp_id
);
163 otx_cpt_put_resource(struct cpt_instance
*instance
);
166 otx_cpt_start_device(void *cptvf
);
169 otx_cpt_stop_device(void *cptvf
);
171 /* Write to VQX_DOORBELL register
173 static __rte_always_inline
void
174 otx_cpt_write_vq_doorbell(struct cpt_vf
*cptvf
, uint32_t val
)
176 cptx_vqx_doorbell_t vqx_dbell
;
179 vqx_dbell
.s
.dbell_cnt
= val
* 8; /* Num of Instructions * 8 words */
180 CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf
),
181 CPTX_VQX_DOORBELL(0, 0), vqx_dbell
.u
);
184 static __rte_always_inline
uint32_t
185 otx_cpt_read_vq_doorbell(struct cpt_vf
*cptvf
)
187 cptx_vqx_doorbell_t vqx_dbell
;
189 vqx_dbell
.u
= CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf
),
190 CPTX_VQX_DOORBELL(0, 0));
191 return vqx_dbell
.s
.dbell_cnt
;
194 static __rte_always_inline
void
195 otx_cpt_ring_dbell(struct cpt_instance
*instance
, uint16_t count
)
197 struct cpt_vf
*cptvf
= (struct cpt_vf
*)instance
;
198 /* Memory barrier to flush pending writes */
200 otx_cpt_write_vq_doorbell(cptvf
, count
);
203 static __rte_always_inline
void *
204 get_cpt_inst(struct command_queue
*cqueue
)
206 CPT_LOG_DP_DEBUG("CPT queue idx %u\n", cqueue
->idx
);
207 return &cqueue
->qhead
[cqueue
->idx
* CPT_INST_SIZE
];
210 static __rte_always_inline
void
211 fill_cpt_inst(struct cpt_instance
*instance
, void *req
)
213 struct command_queue
*cqueue
;
214 cpt_inst_s_t
*cpt_ist_p
;
215 struct cpt_vf
*cptvf
= (struct cpt_vf
*)instance
;
216 struct cpt_request_info
*user_req
= (struct cpt_request_info
*)req
;
217 cqueue
= &cptvf
->cqueue
;
218 cpt_ist_p
= get_cpt_inst(cqueue
);
219 rte_prefetch_non_temporal(cpt_ist_p
);
221 /* EI0, EI1, EI2, EI3 are already prepared */
225 cpt_ist_p
->s8x
.res_addr
= user_req
->comp_baddr
;
229 cpt_ist_p
->s8x
.wq_ptr
= 0;
232 cpt_ist_p
->s8x
.ei0
= user_req
->ist
.ei0
;
234 cpt_ist_p
->s8x
.ei1
= user_req
->ist
.ei1
;
236 cpt_ist_p
->s8x
.ei2
= user_req
->ist
.ei2
;
238 cpt_ist_p
->s8x
.ei3
= user_req
->ist
.ei3
;
241 static __rte_always_inline
void
242 mark_cpt_inst(struct cpt_instance
*instance
)
244 struct cpt_vf
*cptvf
= (struct cpt_vf
*)instance
;
245 struct command_queue
*queue
= &cptvf
->cqueue
;
246 if (unlikely(++queue
->idx
>= DEFAULT_CMD_QCHUNK_SIZE
)) {
247 uint32_t cchunk
= queue
->cchunk
;
248 MOD_INC(cchunk
, DEFAULT_CMD_QCHUNKS
);
249 queue
->qhead
= queue
->chead
[cchunk
].head
;
251 queue
->cchunk
= cchunk
;
255 static __rte_always_inline
uint8_t
256 check_nb_command_id(struct cpt_request_info
*user_req
,
257 struct cpt_instance
*instance
)
259 uint8_t ret
= ERR_REQ_PENDING
;
260 struct cpt_vf
*cptvf
= (struct cpt_vf
*)instance
;
261 volatile cpt_res_s_t
*cptres
;
263 cptres
= (volatile cpt_res_s_t
*)user_req
->completion_addr
;
265 if (unlikely(cptres
->s8x
.compcode
== CPT_8X_COMP_E_NOTDONE
)) {
267 * Wait for some time for this command to get completed
270 if (rte_get_timer_cycles() < user_req
->time_out
)
273 * TODO: See if alternate caddr can be used to not loop
274 * longer than needed.
276 if ((cptres
->s8x
.compcode
== CPT_8X_COMP_E_NOTDONE
) &&
277 (user_req
->extra_time
< TIME_IN_RESET_COUNT
)) {
278 user_req
->extra_time
++;
282 if (cptres
->s8x
.compcode
!= CPT_8X_COMP_E_NOTDONE
)
285 ret
= ERR_REQ_TIMEOUT
;
286 CPT_LOG_DP_ERR("Request %p timedout", user_req
);
287 otx_cpt_poll_misc(cptvf
);
292 if (likely(cptres
->s8x
.compcode
== CPT_8X_COMP_E_GOOD
)) {
293 ret
= 0; /* success */
294 if (unlikely((uint8_t)*user_req
->alternate_caddr
)) {
295 ret
= (uint8_t)*user_req
->alternate_caddr
;
296 CPT_LOG_DP_ERR("Request %p : failed with microcode"
297 " error, MC completion code : 0x%x", user_req
,
300 CPT_LOG_DP_DEBUG("MC status %.8x\n",
301 *((volatile uint32_t *)user_req
->alternate_caddr
));
302 CPT_LOG_DP_DEBUG("HW status %.8x\n",
303 *((volatile uint32_t *)user_req
->completion_addr
));
304 } else if ((cptres
->s8x
.compcode
== CPT_8X_COMP_E_SWERR
) ||
305 (cptres
->s8x
.compcode
== CPT_8X_COMP_E_FAULT
)) {
306 ret
= (uint8_t)*user_req
->alternate_caddr
;
308 ret
= ERR_BAD_ALT_CCODE
;
309 CPT_LOG_DP_DEBUG("Request %p : failed with %s : err code :%x",
311 (cptres
->s8x
.compcode
== CPT_8X_COMP_E_FAULT
) ?
312 "DMA Fault" : "Software error", ret
);
314 CPT_LOG_DP_ERR("Request %p : unexpected completion code %d",
315 user_req
, cptres
->s8x
.compcode
);
316 ret
= (uint8_t)*user_req
->alternate_caddr
;
323 #endif /* _OTX_CRYPTODEV_HW_ACCESS_H_ */