]>
Commit | Line | Data |
---|---|---|
11fdf7f2 | 1 | /* SPDX-License-Identifier: BSD-3-Clause |
f67539c2 | 2 | * Copyright(c) 2015-2019 Intel Corporation |
11fdf7f2 TL |
3 | */ |
4 | ||
5 | #include <openssl/evp.h> | |
6 | ||
7 | #include <rte_mempool.h> | |
8 | #include <rte_mbuf.h> | |
9 | #include <rte_crypto_sym.h> | |
10 | #include <rte_bus_pci.h> | |
11 | #include <rte_byteorder.h> | |
12 | ||
13 | #include "qat_sym.h" | |
14 | ||
f67539c2 | 15 | |
11fdf7f2 TL |
16 | /** Decrypt a single partial block |
17 | * Depends on openssl libcrypto | |
18 | * Uses ECB+XOR to do CFB encryption, same result, more performant | |
19 | */ | |
20 | static inline int | |
21 | bpi_cipher_decrypt(uint8_t *src, uint8_t *dst, | |
22 | uint8_t *iv, int ivlen, int srclen, | |
23 | void *bpi_ctx) | |
24 | { | |
25 | EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; | |
26 | int encrypted_ivlen; | |
27 | uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; | |
28 | uint8_t *encr = encrypted_iv; | |
29 | ||
30 | /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */ | |
31 | if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) | |
32 | <= 0) | |
33 | goto cipher_decrypt_err; | |
34 | ||
35 | for (; srclen != 0; --srclen, ++dst, ++src, ++encr) | |
36 | *dst = *src ^ *encr; | |
37 | ||
38 | return 0; | |
39 | ||
40 | cipher_decrypt_err: | |
41 | QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed"); | |
42 | return -EINVAL; | |
43 | } | |
44 | ||
45 | ||
46 | static inline uint32_t | |
47 | qat_bpicipher_preprocess(struct qat_sym_session *ctx, | |
48 | struct rte_crypto_op *op) | |
49 | { | |
50 | int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); | |
51 | struct rte_crypto_sym_op *sym_op = op->sym; | |
52 | uint8_t last_block_len = block_len > 0 ? | |
53 | sym_op->cipher.data.length % block_len : 0; | |
54 | ||
55 | if (last_block_len && | |
56 | ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) { | |
57 | ||
58 | /* Decrypt last block */ | |
59 | uint8_t *last_block, *dst, *iv; | |
60 | uint32_t last_block_offset = sym_op->cipher.data.offset + | |
61 | sym_op->cipher.data.length - last_block_len; | |
62 | last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, | |
63 | uint8_t *, last_block_offset); | |
64 | ||
f67539c2 TL |
65 | if (unlikely((sym_op->m_dst != NULL) |
66 | && (sym_op->m_dst != sym_op->m_src))) | |
11fdf7f2 TL |
67 | /* out-of-place operation (OOP) */ |
68 | dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, | |
69 | uint8_t *, last_block_offset); | |
70 | else | |
71 | dst = last_block; | |
72 | ||
73 | if (last_block_len < sym_op->cipher.data.length) | |
74 | /* use previous block ciphertext as IV */ | |
75 | iv = last_block - block_len; | |
76 | else | |
77 | /* runt block, i.e. less than one full block */ | |
78 | iv = rte_crypto_op_ctod_offset(op, uint8_t *, | |
79 | ctx->cipher_iv.offset); | |
80 | ||
81 | #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG | |
82 | QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:", | |
83 | last_block, last_block_len); | |
84 | if (sym_op->m_dst != NULL) | |
85 | QAT_DP_HEXDUMP_LOG(DEBUG, "BPI:dst before pre-process:", | |
86 | dst, last_block_len); | |
87 | #endif | |
88 | bpi_cipher_decrypt(last_block, dst, iv, block_len, | |
89 | last_block_len, ctx->bpi_ctx); | |
90 | #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG | |
91 | QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:", | |
92 | last_block, last_block_len); | |
93 | if (sym_op->m_dst != NULL) | |
94 | QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:", | |
95 | dst, last_block_len); | |
96 | #endif | |
97 | } | |
98 | ||
99 | return sym_op->cipher.data.length - last_block_len; | |
100 | } | |
101 | ||
102 | static inline void | |
103 | set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, | |
104 | struct icp_qat_fw_la_cipher_req_params *cipher_param, | |
105 | struct rte_crypto_op *op, | |
106 | struct icp_qat_fw_la_bulk_req *qat_req) | |
107 | { | |
108 | /* copy IV into request if it fits */ | |
109 | if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) { | |
110 | rte_memcpy(cipher_param->u.cipher_IV_array, | |
111 | rte_crypto_op_ctod_offset(op, uint8_t *, | |
112 | iv_offset), | |
113 | iv_length); | |
114 | } else { | |
115 | ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( | |
116 | qat_req->comn_hdr.serv_specif_flags, | |
117 | ICP_QAT_FW_CIPH_IV_64BIT_PTR); | |
118 | cipher_param->u.s.cipher_IV_ptr = | |
119 | rte_crypto_op_ctophys_offset(op, | |
120 | iv_offset); | |
121 | } | |
122 | } | |
123 | ||
124 | /** Set IV for CCM is special case, 0th byte is set to q-1 | |
125 | * where q is padding of nonce in 16 byte block | |
126 | */ | |
127 | static inline void | |
128 | set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset, | |
129 | struct icp_qat_fw_la_cipher_req_params *cipher_param, | |
130 | struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz) | |
131 | { | |
132 | rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) + | |
133 | ICP_QAT_HW_CCM_NONCE_OFFSET, | |
134 | rte_crypto_op_ctod_offset(op, uint8_t *, | |
135 | iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, | |
136 | iv_length); | |
137 | *(uint8_t *)&cipher_param->u.cipher_IV_array[0] = | |
138 | q - ICP_QAT_HW_CCM_NONCE_OFFSET; | |
139 | ||
140 | if (aad_len_field_sz) | |
141 | rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET], | |
142 | rte_crypto_op_ctod_offset(op, uint8_t *, | |
143 | iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, | |
144 | iv_length); | |
145 | } | |
146 | ||
147 | int | |
148 | qat_sym_build_request(void *in_op, uint8_t *out_msg, | |
149 | void *op_cookie, enum qat_device_gen qat_dev_gen) | |
150 | { | |
151 | int ret = 0; | |
152 | struct qat_sym_session *ctx; | |
153 | struct icp_qat_fw_la_cipher_req_params *cipher_param; | |
154 | struct icp_qat_fw_la_auth_req_params *auth_param; | |
155 | register struct icp_qat_fw_la_bulk_req *qat_req; | |
156 | uint8_t do_auth = 0, do_cipher = 0, do_aead = 0; | |
157 | uint32_t cipher_len = 0, cipher_ofs = 0; | |
158 | uint32_t auth_len = 0, auth_ofs = 0; | |
159 | uint32_t min_ofs = 0; | |
160 | uint64_t src_buf_start = 0, dst_buf_start = 0; | |
f67539c2 | 161 | uint64_t auth_data_end = 0; |
11fdf7f2 | 162 | uint8_t do_sgl = 0; |
f67539c2 TL |
163 | uint8_t in_place = 1; |
164 | int alignment_adjustment = 0; | |
11fdf7f2 TL |
165 | struct rte_crypto_op *op = (struct rte_crypto_op *)in_op; |
166 | struct qat_sym_op_cookie *cookie = | |
167 | (struct qat_sym_op_cookie *)op_cookie; | |
168 | ||
169 | if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) { | |
170 | QAT_DP_LOG(ERR, "QAT PMD only supports symmetric crypto " | |
171 | "operation requests, op (%p) is not a " | |
172 | "symmetric operation.", op); | |
173 | return -EINVAL; | |
174 | } | |
175 | ||
176 | if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { | |
177 | QAT_DP_LOG(ERR, "QAT PMD only supports session oriented" | |
178 | " requests, op (%p) is sessionless.", op); | |
179 | return -EINVAL; | |
180 | } | |
181 | ||
182 | ctx = (struct qat_sym_session *)get_sym_session_private_data( | |
183 | op->sym->session, cryptodev_qat_driver_id); | |
184 | ||
185 | if (unlikely(ctx == NULL)) { | |
186 | QAT_DP_LOG(ERR, "Session was not created for this device"); | |
187 | return -EINVAL; | |
188 | } | |
189 | ||
190 | if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) { | |
191 | QAT_DP_LOG(ERR, "Session alg not supported on this device gen"); | |
192 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; | |
193 | return -EINVAL; | |
194 | } | |
195 | ||
196 | qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; | |
197 | rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); | |
198 | qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; | |
199 | cipher_param = (void *)&qat_req->serv_specif_rqpars; | |
f67539c2 TL |
200 | auth_param = (void *)((uint8_t *)cipher_param + |
201 | ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); | |
11fdf7f2 TL |
202 | |
203 | if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || | |
204 | ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { | |
205 | /* AES-GCM or AES-CCM */ | |
206 | if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || | |
207 | ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || | |
208 | (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128 | |
209 | && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE | |
210 | && ctx->qat_hash_alg == | |
211 | ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) { | |
212 | do_aead = 1; | |
213 | } else { | |
214 | do_auth = 1; | |
215 | do_cipher = 1; | |
216 | } | |
217 | } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { | |
218 | do_auth = 1; | |
219 | do_cipher = 0; | |
220 | } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { | |
221 | do_auth = 0; | |
222 | do_cipher = 1; | |
223 | } | |
224 | ||
225 | if (do_cipher) { | |
226 | ||
227 | if (ctx->qat_cipher_alg == | |
228 | ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || | |
229 | ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI || | |
230 | ctx->qat_cipher_alg == | |
231 | ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { | |
232 | ||
233 | if (unlikely( | |
234 | (op->sym->cipher.data.length % BYTE_LENGTH != 0) || | |
235 | (op->sym->cipher.data.offset % BYTE_LENGTH != 0))) { | |
236 | QAT_DP_LOG(ERR, | |
237 | "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values"); | |
238 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
239 | return -EINVAL; | |
240 | } | |
241 | cipher_len = op->sym->cipher.data.length >> 3; | |
242 | cipher_ofs = op->sym->cipher.data.offset >> 3; | |
243 | ||
244 | } else if (ctx->bpi_ctx) { | |
245 | /* DOCSIS - only send complete blocks to device | |
246 | * Process any partial block using CFB mode. | |
247 | * Even if 0 complete blocks, still send this to device | |
248 | * to get into rx queue for post-process and dequeuing | |
249 | */ | |
250 | cipher_len = qat_bpicipher_preprocess(ctx, op); | |
251 | cipher_ofs = op->sym->cipher.data.offset; | |
252 | } else { | |
253 | cipher_len = op->sym->cipher.data.length; | |
254 | cipher_ofs = op->sym->cipher.data.offset; | |
255 | } | |
256 | ||
257 | set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, | |
258 | cipher_param, op, qat_req); | |
259 | min_ofs = cipher_ofs; | |
260 | } | |
261 | ||
262 | if (do_auth) { | |
263 | ||
264 | if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 || | |
265 | ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 || | |
266 | ctx->qat_hash_alg == | |
267 | ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) { | |
268 | if (unlikely( | |
269 | (op->sym->auth.data.offset % BYTE_LENGTH != 0) || | |
270 | (op->sym->auth.data.length % BYTE_LENGTH != 0))) { | |
271 | QAT_DP_LOG(ERR, | |
272 | "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values"); | |
273 | op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; | |
274 | return -EINVAL; | |
275 | } | |
276 | auth_ofs = op->sym->auth.data.offset >> 3; | |
277 | auth_len = op->sym->auth.data.length >> 3; | |
278 | ||
279 | auth_param->u1.aad_adr = | |
280 | rte_crypto_op_ctophys_offset(op, | |
281 | ctx->auth_iv.offset); | |
282 | ||
283 | } else if (ctx->qat_hash_alg == | |
284 | ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || | |
285 | ctx->qat_hash_alg == | |
286 | ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { | |
287 | /* AES-GMAC */ | |
288 | set_cipher_iv(ctx->auth_iv.length, | |
289 | ctx->auth_iv.offset, | |
290 | cipher_param, op, qat_req); | |
291 | auth_ofs = op->sym->auth.data.offset; | |
292 | auth_len = op->sym->auth.data.length; | |
293 | ||
294 | auth_param->u1.aad_adr = 0; | |
295 | auth_param->u2.aad_sz = 0; | |
296 | ||
297 | /* | |
298 | * If len(iv)==12B fw computes J0 | |
299 | */ | |
300 | if (ctx->auth_iv.length == 12) { | |
301 | ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( | |
302 | qat_req->comn_hdr.serv_specif_flags, | |
303 | ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); | |
304 | ||
305 | } | |
306 | } else { | |
307 | auth_ofs = op->sym->auth.data.offset; | |
308 | auth_len = op->sym->auth.data.length; | |
309 | ||
310 | } | |
311 | min_ofs = auth_ofs; | |
312 | ||
f67539c2 TL |
313 | auth_param->auth_res_addr = |
314 | op->sym->auth.digest.phys_addr; | |
11fdf7f2 TL |
315 | |
316 | } | |
317 | ||
318 | if (do_aead) { | |
319 | /* | |
320 | * This address may used for setting AAD physical pointer | |
321 | * into IV offset from op | |
322 | */ | |
323 | rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr; | |
324 | if (ctx->qat_hash_alg == | |
325 | ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || | |
326 | ctx->qat_hash_alg == | |
327 | ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { | |
328 | /* | |
329 | * If len(iv)==12B fw computes J0 | |
330 | */ | |
331 | if (ctx->cipher_iv.length == 12) { | |
332 | ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( | |
333 | qat_req->comn_hdr.serv_specif_flags, | |
334 | ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); | |
335 | } | |
336 | set_cipher_iv(ctx->cipher_iv.length, | |
337 | ctx->cipher_iv.offset, | |
338 | cipher_param, op, qat_req); | |
339 | ||
340 | } else if (ctx->qat_hash_alg == | |
341 | ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) { | |
342 | ||
343 | /* In case of AES-CCM this may point to user selected | |
344 | * memory or iv offset in cypto_op | |
345 | */ | |
346 | uint8_t *aad_data = op->sym->aead.aad.data; | |
347 | /* This is true AAD length, it not includes 18 bytes of | |
348 | * preceding data | |
349 | */ | |
350 | uint8_t aad_ccm_real_len = 0; | |
351 | uint8_t aad_len_field_sz = 0; | |
352 | uint32_t msg_len_be = | |
353 | rte_bswap32(op->sym->aead.data.length); | |
354 | ||
355 | if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) { | |
356 | aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO; | |
357 | aad_ccm_real_len = ctx->aad_len - | |
358 | ICP_QAT_HW_CCM_AAD_B0_LEN - | |
359 | ICP_QAT_HW_CCM_AAD_LEN_INFO; | |
360 | } else { | |
361 | /* | |
362 | * aad_len not greater than 18, so no actual aad | |
363 | * data, then use IV after op for B0 block | |
364 | */ | |
365 | aad_data = rte_crypto_op_ctod_offset(op, | |
366 | uint8_t *, | |
367 | ctx->cipher_iv.offset); | |
368 | aad_phys_addr_aead = | |
369 | rte_crypto_op_ctophys_offset(op, | |
370 | ctx->cipher_iv.offset); | |
371 | } | |
372 | ||
373 | uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - | |
374 | ctx->cipher_iv.length; | |
375 | ||
376 | aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS( | |
377 | aad_len_field_sz, | |
378 | ctx->digest_length, q); | |
379 | ||
380 | if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) { | |
381 | memcpy(aad_data + ctx->cipher_iv.length + | |
382 | ICP_QAT_HW_CCM_NONCE_OFFSET + | |
383 | (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE), | |
384 | (uint8_t *)&msg_len_be, | |
385 | ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE); | |
386 | } else { | |
387 | memcpy(aad_data + ctx->cipher_iv.length + | |
388 | ICP_QAT_HW_CCM_NONCE_OFFSET, | |
389 | (uint8_t *)&msg_len_be | |
390 | + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE | |
391 | - q), q); | |
392 | } | |
393 | ||
394 | if (aad_len_field_sz > 0) { | |
395 | *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] | |
396 | = rte_bswap16(aad_ccm_real_len); | |
397 | ||
398 | if ((aad_ccm_real_len + aad_len_field_sz) | |
399 | % ICP_QAT_HW_CCM_AAD_B0_LEN) { | |
400 | uint8_t pad_len = 0; | |
401 | uint8_t pad_idx = 0; | |
402 | ||
403 | pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN - | |
404 | ((aad_ccm_real_len + aad_len_field_sz) % | |
405 | ICP_QAT_HW_CCM_AAD_B0_LEN); | |
406 | pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN + | |
407 | aad_ccm_real_len + aad_len_field_sz; | |
408 | memset(&aad_data[pad_idx], | |
409 | 0, pad_len); | |
410 | } | |
411 | ||
412 | } | |
413 | ||
414 | set_cipher_iv_ccm(ctx->cipher_iv.length, | |
415 | ctx->cipher_iv.offset, | |
416 | cipher_param, op, q, | |
417 | aad_len_field_sz); | |
418 | ||
419 | } | |
420 | ||
421 | cipher_len = op->sym->aead.data.length; | |
422 | cipher_ofs = op->sym->aead.data.offset; | |
423 | auth_len = op->sym->aead.data.length; | |
424 | auth_ofs = op->sym->aead.data.offset; | |
425 | ||
426 | auth_param->u1.aad_adr = aad_phys_addr_aead; | |
427 | auth_param->auth_res_addr = op->sym->aead.digest.phys_addr; | |
428 | min_ofs = op->sym->aead.data.offset; | |
429 | } | |
430 | ||
f67539c2 TL |
431 | if (op->sym->m_src->nb_segs > 1 || |
432 | (op->sym->m_dst && op->sym->m_dst->nb_segs > 1)) | |
11fdf7f2 TL |
433 | do_sgl = 1; |
434 | ||
435 | /* adjust for chain case */ | |
436 | if (do_cipher && do_auth) | |
437 | min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs; | |
438 | ||
439 | if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl)) | |
440 | min_ofs = 0; | |
441 | ||
f67539c2 TL |
442 | if (unlikely((op->sym->m_dst != NULL) && |
443 | (op->sym->m_dst != op->sym->m_src))) { | |
11fdf7f2 TL |
444 | /* Out-of-place operation (OOP) |
445 | * Don't align DMA start. DMA the minimum data-set | |
446 | * so as not to overwrite data in dest buffer | |
447 | */ | |
9f95a23c | 448 | in_place = 0; |
11fdf7f2 TL |
449 | src_buf_start = |
450 | rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); | |
451 | dst_buf_start = | |
452 | rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs); | |
453 | ||
454 | } else { | |
455 | /* In-place operation | |
456 | * Start DMA at nearest aligned address below min_ofs | |
457 | */ | |
458 | src_buf_start = | |
459 | rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs) | |
460 | & QAT_64_BTYE_ALIGN_MASK; | |
461 | ||
462 | if (unlikely((rte_pktmbuf_iova(op->sym->m_src) - | |
463 | rte_pktmbuf_headroom(op->sym->m_src)) | |
464 | > src_buf_start)) { | |
465 | /* alignment has pushed addr ahead of start of mbuf | |
466 | * so revert and take the performance hit | |
467 | */ | |
468 | src_buf_start = | |
469 | rte_pktmbuf_iova_offset(op->sym->m_src, | |
470 | min_ofs); | |
471 | } | |
472 | dst_buf_start = src_buf_start; | |
f67539c2 TL |
473 | |
474 | /* remember any adjustment for later, note, can be +/- */ | |
475 | alignment_adjustment = src_buf_start - | |
476 | rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); | |
11fdf7f2 TL |
477 | } |
478 | ||
479 | if (do_cipher || do_aead) { | |
480 | cipher_param->cipher_offset = | |
481 | (uint32_t)rte_pktmbuf_iova_offset( | |
482 | op->sym->m_src, cipher_ofs) - src_buf_start; | |
483 | cipher_param->cipher_length = cipher_len; | |
484 | } else { | |
485 | cipher_param->cipher_offset = 0; | |
486 | cipher_param->cipher_length = 0; | |
487 | } | |
488 | ||
489 | if (do_auth || do_aead) { | |
490 | auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset( | |
491 | op->sym->m_src, auth_ofs) - src_buf_start; | |
492 | auth_param->auth_len = auth_len; | |
493 | } else { | |
494 | auth_param->auth_off = 0; | |
495 | auth_param->auth_len = 0; | |
496 | } | |
497 | ||
498 | qat_req->comn_mid.dst_length = | |
499 | qat_req->comn_mid.src_length = | |
500 | (cipher_param->cipher_offset + cipher_param->cipher_length) | |
501 | > (auth_param->auth_off + auth_param->auth_len) ? | |
502 | (cipher_param->cipher_offset + cipher_param->cipher_length) | |
503 | : (auth_param->auth_off + auth_param->auth_len); | |
504 | ||
f67539c2 TL |
505 | if (do_auth && do_cipher) { |
506 | /* Handle digest-encrypted cases, i.e. | |
507 | * auth-gen-then-cipher-encrypt and | |
508 | * cipher-decrypt-then-auth-verify | |
509 | */ | |
510 | /* First find the end of the data */ | |
511 | if (do_sgl) { | |
512 | uint32_t remaining_off = auth_param->auth_off + | |
513 | auth_param->auth_len + alignment_adjustment; | |
514 | struct rte_mbuf *sgl_buf = | |
515 | (in_place ? | |
516 | op->sym->m_src : op->sym->m_dst); | |
517 | ||
518 | while (remaining_off >= rte_pktmbuf_data_len(sgl_buf) | |
519 | && sgl_buf->next != NULL) { | |
520 | remaining_off -= rte_pktmbuf_data_len(sgl_buf); | |
521 | sgl_buf = sgl_buf->next; | |
522 | } | |
523 | ||
524 | auth_data_end = (uint64_t)rte_pktmbuf_iova_offset( | |
525 | sgl_buf, remaining_off); | |
526 | } else { | |
527 | auth_data_end = (in_place ? | |
528 | src_buf_start : dst_buf_start) + | |
529 | auth_param->auth_off + auth_param->auth_len; | |
530 | } | |
531 | /* Then check if digest-encrypted conditions are met */ | |
532 | if ((auth_param->auth_off + auth_param->auth_len < | |
533 | cipher_param->cipher_offset + | |
534 | cipher_param->cipher_length) && | |
535 | (op->sym->auth.digest.phys_addr == | |
536 | auth_data_end)) { | |
537 | /* Handle partial digest encryption */ | |
538 | if (cipher_param->cipher_offset + | |
539 | cipher_param->cipher_length < | |
540 | auth_param->auth_off + | |
541 | auth_param->auth_len + | |
542 | ctx->digest_length) | |
543 | qat_req->comn_mid.dst_length = | |
544 | qat_req->comn_mid.src_length = | |
545 | auth_param->auth_off + | |
546 | auth_param->auth_len + | |
547 | ctx->digest_length; | |
548 | struct icp_qat_fw_comn_req_hdr *header = | |
549 | &qat_req->comn_hdr; | |
550 | ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( | |
551 | header->serv_specif_flags, | |
552 | ICP_QAT_FW_LA_DIGEST_IN_BUFFER); | |
553 | } | |
554 | } | |
555 | ||
11fdf7f2 TL |
556 | if (do_sgl) { |
557 | ||
558 | ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags, | |
559 | QAT_COMN_PTR_TYPE_SGL); | |
560 | ret = qat_sgl_fill_array(op->sym->m_src, | |
561 | (int64_t)(src_buf_start - rte_pktmbuf_iova(op->sym->m_src)), | |
562 | &cookie->qat_sgl_src, | |
563 | qat_req->comn_mid.src_length, | |
564 | QAT_SYM_SGL_MAX_NUMBER); | |
565 | ||
566 | if (unlikely(ret)) { | |
567 | QAT_DP_LOG(ERR, "QAT PMD Cannot fill sgl array"); | |
568 | return ret; | |
569 | } | |
570 | ||
f67539c2 | 571 | if (in_place) |
11fdf7f2 TL |
572 | qat_req->comn_mid.dest_data_addr = |
573 | qat_req->comn_mid.src_data_addr = | |
574 | cookie->qat_sgl_src_phys_addr; | |
575 | else { | |
576 | ret = qat_sgl_fill_array(op->sym->m_dst, | |
577 | (int64_t)(dst_buf_start - | |
578 | rte_pktmbuf_iova(op->sym->m_dst)), | |
579 | &cookie->qat_sgl_dst, | |
580 | qat_req->comn_mid.dst_length, | |
581 | QAT_SYM_SGL_MAX_NUMBER); | |
582 | ||
583 | if (unlikely(ret)) { | |
584 | QAT_DP_LOG(ERR, "QAT PMD can't fill sgl array"); | |
585 | return ret; | |
586 | } | |
587 | ||
588 | qat_req->comn_mid.src_data_addr = | |
589 | cookie->qat_sgl_src_phys_addr; | |
590 | qat_req->comn_mid.dest_data_addr = | |
591 | cookie->qat_sgl_dst_phys_addr; | |
592 | } | |
f67539c2 TL |
593 | qat_req->comn_mid.src_length = 0; |
594 | qat_req->comn_mid.dst_length = 0; | |
11fdf7f2 TL |
595 | } else { |
596 | qat_req->comn_mid.src_data_addr = src_buf_start; | |
597 | qat_req->comn_mid.dest_data_addr = dst_buf_start; | |
f67539c2 TL |
598 | } |
599 | ||
600 | /* Handle Single-Pass GCM */ | |
601 | if (ctx->is_single_pass) { | |
602 | cipher_param->spc_aad_addr = op->sym->aead.aad.phys_addr; | |
603 | cipher_param->spc_auth_res_addr = | |
604 | op->sym->aead.digest.phys_addr; | |
11fdf7f2 TL |
605 | } |
606 | ||
607 | #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG | |
608 | QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req, | |
609 | sizeof(struct icp_qat_fw_la_bulk_req)); | |
610 | QAT_DP_HEXDUMP_LOG(DEBUG, "src_data:", | |
611 | rte_pktmbuf_mtod(op->sym->m_src, uint8_t*), | |
612 | rte_pktmbuf_data_len(op->sym->m_src)); | |
613 | if (do_cipher) { | |
614 | uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op, | |
615 | uint8_t *, | |
616 | ctx->cipher_iv.offset); | |
617 | QAT_DP_HEXDUMP_LOG(DEBUG, "cipher iv:", cipher_iv_ptr, | |
618 | ctx->cipher_iv.length); | |
619 | } | |
620 | ||
621 | if (do_auth) { | |
622 | if (ctx->auth_iv.length) { | |
623 | uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op, | |
624 | uint8_t *, | |
625 | ctx->auth_iv.offset); | |
626 | QAT_DP_HEXDUMP_LOG(DEBUG, "auth iv:", auth_iv_ptr, | |
627 | ctx->auth_iv.length); | |
628 | } | |
629 | QAT_DP_HEXDUMP_LOG(DEBUG, "digest:", op->sym->auth.digest.data, | |
630 | ctx->digest_length); | |
631 | } | |
632 | ||
633 | if (do_aead) { | |
634 | QAT_DP_HEXDUMP_LOG(DEBUG, "digest:", op->sym->aead.digest.data, | |
635 | ctx->digest_length); | |
636 | QAT_DP_HEXDUMP_LOG(DEBUG, "aad:", op->sym->aead.aad.data, | |
637 | ctx->aad_len); | |
638 | } | |
639 | #endif | |
640 | return 0; | |
641 | } |