]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/crypto/qat/qat_common/qat_algs.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
ath10k: fix compilation warnings in wmi phyerr pull function
[mirror_ubuntu-bionic-kernel.git] / drivers / crypto / qat / qat_common / qat_algs.c
1 /*
2 This file is provided under a dual BSD/GPLv2 license. When using or
3 redistributing this file, you may do so under either license.
4
5 GPL LICENSE SUMMARY
6 Copyright(c) 2014 Intel Corporation.
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of version 2 of the GNU General Public License as
9 published by the Free Software Foundation.
10
11 This program is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Contact Information:
17 qat-linux@intel.com
18
19 BSD LICENSE
20 Copyright(c) 2014 Intel Corporation.
21 Redistribution and use in source and binary forms, with or without
22 modification, are permitted provided that the following conditions
23 are met:
24
25 * Redistributions of source code must retain the above copyright
26 notice, this list of conditions and the following disclaimer.
27 * Redistributions in binary form must reproduce the above copyright
28 notice, this list of conditions and the following disclaimer in
29 the documentation and/or other materials provided with the
30 distribution.
31 * Neither the name of Intel Corporation nor the names of its
32 contributors may be used to endorse or promote products derived
33 from this software without specific prior written permission.
34
35 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
36 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
37 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
38 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
39 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
41 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
42 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
43 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
44 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
45 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
46 */
47 #include <linux/module.h>
48 #include <linux/slab.h>
49 #include <linux/crypto.h>
50 #include <crypto/internal/aead.h>
51 #include <crypto/aes.h>
52 #include <crypto/sha.h>
53 #include <crypto/hash.h>
54 #include <crypto/algapi.h>
55 #include <crypto/authenc.h>
56 #include <crypto/rng.h>
57 #include <linux/dma-mapping.h>
58 #include "adf_accel_devices.h"
59 #include "adf_transport.h"
60 #include "adf_common_drv.h"
61 #include "qat_crypto.h"
62 #include "icp_qat_hw.h"
63 #include "icp_qat_fw.h"
64 #include "icp_qat_fw_la.h"
65
66 #define QAT_AES_HW_CONFIG_CBC_ENC(alg) \
67 ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
68 ICP_QAT_HW_CIPHER_NO_CONVERT, \
69 ICP_QAT_HW_CIPHER_ENCRYPT)
70
71 #define QAT_AES_HW_CONFIG_CBC_DEC(alg) \
72 ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
73 ICP_QAT_HW_CIPHER_KEY_CONVERT, \
74 ICP_QAT_HW_CIPHER_DECRYPT)
75
76 static atomic_t active_dev;
77
78 struct qat_alg_buf {
79 uint32_t len;
80 uint32_t resrvd;
81 uint64_t addr;
82 } __packed;
83
84 struct qat_alg_buf_list {
85 uint64_t resrvd;
86 uint32_t num_bufs;
87 uint32_t num_mapped_bufs;
88 struct qat_alg_buf bufers[];
89 } __packed __aligned(64);
90
91 /* Common content descriptor */
92 struct qat_alg_cd {
93 union {
94 struct qat_enc { /* Encrypt content desc */
95 struct icp_qat_hw_cipher_algo_blk cipher;
96 struct icp_qat_hw_auth_algo_blk hash;
97 } qat_enc_cd;
98 struct qat_dec { /* Decrytp content desc */
99 struct icp_qat_hw_auth_algo_blk hash;
100 struct icp_qat_hw_cipher_algo_blk cipher;
101 } qat_dec_cd;
102 };
103 } __aligned(64);
104
105 struct qat_alg_aead_ctx {
106 struct qat_alg_cd *enc_cd;
107 struct qat_alg_cd *dec_cd;
108 dma_addr_t enc_cd_paddr;
109 dma_addr_t dec_cd_paddr;
110 struct icp_qat_fw_la_bulk_req enc_fw_req;
111 struct icp_qat_fw_la_bulk_req dec_fw_req;
112 struct crypto_shash *hash_tfm;
113 enum icp_qat_hw_auth_algo qat_hash_alg;
114 struct qat_crypto_instance *inst;
115 struct crypto_tfm *tfm;
116 uint8_t salt[AES_BLOCK_SIZE];
117 spinlock_t lock; /* protects qat_alg_aead_ctx struct */
118 };
119
120 struct qat_alg_ablkcipher_ctx {
121 struct icp_qat_hw_cipher_algo_blk *enc_cd;
122 struct icp_qat_hw_cipher_algo_blk *dec_cd;
123 dma_addr_t enc_cd_paddr;
124 dma_addr_t dec_cd_paddr;
125 struct icp_qat_fw_la_bulk_req enc_fw_req;
126 struct icp_qat_fw_la_bulk_req dec_fw_req;
127 struct qat_crypto_instance *inst;
128 struct crypto_tfm *tfm;
129 spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */
130 };
131
132 static int get_current_node(void)
133 {
134 return cpu_data(current_thread_info()->cpu).phys_proc_id;
135 }
136
137 static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
138 {
139 switch (qat_hash_alg) {
140 case ICP_QAT_HW_AUTH_ALGO_SHA1:
141 return ICP_QAT_HW_SHA1_STATE1_SZ;
142 case ICP_QAT_HW_AUTH_ALGO_SHA256:
143 return ICP_QAT_HW_SHA256_STATE1_SZ;
144 case ICP_QAT_HW_AUTH_ALGO_SHA512:
145 return ICP_QAT_HW_SHA512_STATE1_SZ;
146 default:
147 return -EFAULT;
148 };
149 return -EFAULT;
150 }
151
152 static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
153 struct qat_alg_aead_ctx *ctx,
154 const uint8_t *auth_key,
155 unsigned int auth_keylen)
156 {
157 SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
158 struct sha1_state sha1;
159 struct sha256_state sha256;
160 struct sha512_state sha512;
161 int block_size = crypto_shash_blocksize(ctx->hash_tfm);
162 int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
163 char ipad[block_size];
164 char opad[block_size];
165 __be32 *hash_state_out;
166 __be64 *hash512_state_out;
167 int i, offset;
168
169 memset(ipad, 0, block_size);
170 memset(opad, 0, block_size);
171 shash->tfm = ctx->hash_tfm;
172 shash->flags = 0x0;
173
174 if (auth_keylen > block_size) {
175 int ret = crypto_shash_digest(shash, auth_key,
176 auth_keylen, ipad);
177 if (ret)
178 return ret;
179
180 memcpy(opad, ipad, digest_size);
181 } else {
182 memcpy(ipad, auth_key, auth_keylen);
183 memcpy(opad, auth_key, auth_keylen);
184 }
185
186 for (i = 0; i < block_size; i++) {
187 char *ipad_ptr = ipad + i;
188 char *opad_ptr = opad + i;
189 *ipad_ptr ^= 0x36;
190 *opad_ptr ^= 0x5C;
191 }
192
193 if (crypto_shash_init(shash))
194 return -EFAULT;
195
196 if (crypto_shash_update(shash, ipad, block_size))
197 return -EFAULT;
198
199 hash_state_out = (__be32 *)hash->sha.state1;
200 hash512_state_out = (__be64 *)hash_state_out;
201
202 switch (ctx->qat_hash_alg) {
203 case ICP_QAT_HW_AUTH_ALGO_SHA1:
204 if (crypto_shash_export(shash, &sha1))
205 return -EFAULT;
206 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
207 *hash_state_out = cpu_to_be32(*(sha1.state + i));
208 break;
209 case ICP_QAT_HW_AUTH_ALGO_SHA256:
210 if (crypto_shash_export(shash, &sha256))
211 return -EFAULT;
212 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
213 *hash_state_out = cpu_to_be32(*(sha256.state + i));
214 break;
215 case ICP_QAT_HW_AUTH_ALGO_SHA512:
216 if (crypto_shash_export(shash, &sha512))
217 return -EFAULT;
218 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
219 *hash512_state_out = cpu_to_be64(*(sha512.state + i));
220 break;
221 default:
222 return -EFAULT;
223 }
224
225 if (crypto_shash_init(shash))
226 return -EFAULT;
227
228 if (crypto_shash_update(shash, opad, block_size))
229 return -EFAULT;
230
231 offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
232 hash_state_out = (__be32 *)(hash->sha.state1 + offset);
233 hash512_state_out = (__be64 *)hash_state_out;
234
235 switch (ctx->qat_hash_alg) {
236 case ICP_QAT_HW_AUTH_ALGO_SHA1:
237 if (crypto_shash_export(shash, &sha1))
238 return -EFAULT;
239 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
240 *hash_state_out = cpu_to_be32(*(sha1.state + i));
241 break;
242 case ICP_QAT_HW_AUTH_ALGO_SHA256:
243 if (crypto_shash_export(shash, &sha256))
244 return -EFAULT;
245 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
246 *hash_state_out = cpu_to_be32(*(sha256.state + i));
247 break;
248 case ICP_QAT_HW_AUTH_ALGO_SHA512:
249 if (crypto_shash_export(shash, &sha512))
250 return -EFAULT;
251 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
252 *hash512_state_out = cpu_to_be64(*(sha512.state + i));
253 break;
254 default:
255 return -EFAULT;
256 }
257 memzero_explicit(ipad, block_size);
258 memzero_explicit(opad, block_size);
259 return 0;
260 }
261
262 static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
263 {
264 header->hdr_flags =
265 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
266 header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
267 header->comn_req_flags =
268 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
269 QAT_COMN_PTR_TYPE_SGL);
270 ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
271 ICP_QAT_FW_LA_PARTIAL_NONE);
272 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
273 ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
274 ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
275 ICP_QAT_FW_LA_NO_PROTO);
276 ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
277 ICP_QAT_FW_LA_NO_UPDATE_STATE);
278 }
279
280 static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx,
281 int alg,
282 struct crypto_authenc_keys *keys)
283 {
284 struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
285 unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
286 struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
287 struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
288 struct icp_qat_hw_auth_algo_blk *hash =
289 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
290 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
291 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
292 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
293 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
294 void *ptr = &req_tmpl->cd_ctrl;
295 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
296 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
297
298 /* CD setup */
299 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
300 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
301 hash->sha.inner_setup.auth_config.config =
302 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
303 ctx->qat_hash_alg, digestsize);
304 hash->sha.inner_setup.auth_counter.counter =
305 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
306
307 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
308 return -EFAULT;
309
310 /* Request setup */
311 qat_alg_init_common_hdr(header);
312 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
313 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
314 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
315 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
316 ICP_QAT_FW_LA_RET_AUTH_RES);
317 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
318 ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
319 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
320 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
321
322 /* Cipher CD config setup */
323 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
324 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
325 cipher_cd_ctrl->cipher_cfg_offset = 0;
326 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
327 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
328 /* Auth CD config setup */
329 hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
330 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
331 hash_cd_ctrl->inner_res_sz = digestsize;
332 hash_cd_ctrl->final_sz = digestsize;
333
334 switch (ctx->qat_hash_alg) {
335 case ICP_QAT_HW_AUTH_ALGO_SHA1:
336 hash_cd_ctrl->inner_state1_sz =
337 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
338 hash_cd_ctrl->inner_state2_sz =
339 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
340 break;
341 case ICP_QAT_HW_AUTH_ALGO_SHA256:
342 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
343 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
344 break;
345 case ICP_QAT_HW_AUTH_ALGO_SHA512:
346 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
347 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
348 break;
349 default:
350 break;
351 }
352 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
353 ((sizeof(struct icp_qat_hw_auth_setup) +
354 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
355 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
356 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
357 return 0;
358 }
359
360 static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx,
361 int alg,
362 struct crypto_authenc_keys *keys)
363 {
364 struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
365 unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
366 struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
367 struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
368 struct icp_qat_hw_cipher_algo_blk *cipher =
369 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
370 sizeof(struct icp_qat_hw_auth_setup) +
371 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
372 struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
373 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
374 struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
375 void *ptr = &req_tmpl->cd_ctrl;
376 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
377 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
378 struct icp_qat_fw_la_auth_req_params *auth_param =
379 (struct icp_qat_fw_la_auth_req_params *)
380 ((char *)&req_tmpl->serv_specif_rqpars +
381 sizeof(struct icp_qat_fw_la_cipher_req_params));
382
383 /* CD setup */
384 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
385 memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
386 hash->sha.inner_setup.auth_config.config =
387 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
388 ctx->qat_hash_alg,
389 digestsize);
390 hash->sha.inner_setup.auth_counter.counter =
391 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
392
393 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
394 return -EFAULT;
395
396 /* Request setup */
397 qat_alg_init_common_hdr(header);
398 header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
399 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
400 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
401 ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
402 ICP_QAT_FW_LA_NO_RET_AUTH_RES);
403 ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
404 ICP_QAT_FW_LA_CMP_AUTH_RES);
405 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
406 cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
407
408 /* Cipher CD config setup */
409 cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
410 cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
411 cipher_cd_ctrl->cipher_cfg_offset =
412 (sizeof(struct icp_qat_hw_auth_setup) +
413 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
414 ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
415 ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
416
417 /* Auth CD config setup */
418 hash_cd_ctrl->hash_cfg_offset = 0;
419 hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
420 hash_cd_ctrl->inner_res_sz = digestsize;
421 hash_cd_ctrl->final_sz = digestsize;
422
423 switch (ctx->qat_hash_alg) {
424 case ICP_QAT_HW_AUTH_ALGO_SHA1:
425 hash_cd_ctrl->inner_state1_sz =
426 round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
427 hash_cd_ctrl->inner_state2_sz =
428 round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
429 break;
430 case ICP_QAT_HW_AUTH_ALGO_SHA256:
431 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
432 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
433 break;
434 case ICP_QAT_HW_AUTH_ALGO_SHA512:
435 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
436 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
437 break;
438 default:
439 break;
440 }
441
442 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
443 ((sizeof(struct icp_qat_hw_auth_setup) +
444 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
445 auth_param->auth_res_sz = digestsize;
446 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
447 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
448 return 0;
449 }
450
451 static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,
452 struct icp_qat_fw_la_bulk_req *req,
453 struct icp_qat_hw_cipher_algo_blk *cd,
454 const uint8_t *key, unsigned int keylen)
455 {
456 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
457 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
458 struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
459
460 memcpy(cd->aes.key, key, keylen);
461 qat_alg_init_common_hdr(header);
462 header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
463 cd_pars->u.s.content_desc_params_sz =
464 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
465 /* Cipher CD config setup */
466 cd_ctrl->cipher_key_sz = keylen >> 3;
467 cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
468 cd_ctrl->cipher_cfg_offset = 0;
469 ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
470 ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
471 }
472
473 static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,
474 int alg, const uint8_t *key,
475 unsigned int keylen)
476 {
477 struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
478 struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
479 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
480
481 qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);
482 cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
483 enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
484 }
485
486 static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,
487 int alg, const uint8_t *key,
488 unsigned int keylen)
489 {
490 struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
491 struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
492 struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
493
494 qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);
495 cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
496 dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
497 }
498
499 static int qat_alg_validate_key(int key_len, int *alg)
500 {
501 switch (key_len) {
502 case AES_KEYSIZE_128:
503 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
504 break;
505 case AES_KEYSIZE_192:
506 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
507 break;
508 case AES_KEYSIZE_256:
509 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
510 break;
511 default:
512 return -EINVAL;
513 }
514 return 0;
515 }
516
517 static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx,
518 const uint8_t *key, unsigned int keylen)
519 {
520 struct crypto_authenc_keys keys;
521 int alg;
522
523 if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))
524 return -EFAULT;
525
526 if (crypto_authenc_extractkeys(&keys, key, keylen))
527 goto bad_key;
528
529 if (qat_alg_validate_key(keys.enckeylen, &alg))
530 goto bad_key;
531
532 if (qat_alg_aead_init_enc_session(ctx, alg, &keys))
533 goto error;
534
535 if (qat_alg_aead_init_dec_session(ctx, alg, &keys))
536 goto error;
537
538 return 0;
539 bad_key:
540 crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
541 return -EINVAL;
542 error:
543 return -EFAULT;
544 }
545
546 static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,
547 const uint8_t *key,
548 unsigned int keylen)
549 {
550 int alg;
551
552 if (qat_alg_validate_key(keylen, &alg))
553 goto bad_key;
554
555 qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen);
556 qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen);
557 return 0;
558 bad_key:
559 crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
560 return -EINVAL;
561 }
562
563 static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
564 unsigned int keylen)
565 {
566 struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
567 struct device *dev;
568
569 spin_lock(&ctx->lock);
570 if (ctx->enc_cd) {
571 /* rekeying */
572 dev = &GET_DEV(ctx->inst->accel_dev);
573 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
574 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
575 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
576 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
577 } else {
578 /* new key */
579 int node = get_current_node();
580 struct qat_crypto_instance *inst =
581 qat_crypto_get_instance_node(node);
582 if (!inst) {
583 spin_unlock(&ctx->lock);
584 return -EINVAL;
585 }
586
587 dev = &GET_DEV(inst->accel_dev);
588 ctx->inst = inst;
589 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
590 &ctx->enc_cd_paddr,
591 GFP_ATOMIC);
592 if (!ctx->enc_cd) {
593 spin_unlock(&ctx->lock);
594 return -ENOMEM;
595 }
596 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
597 &ctx->dec_cd_paddr,
598 GFP_ATOMIC);
599 if (!ctx->dec_cd) {
600 spin_unlock(&ctx->lock);
601 goto out_free_enc;
602 }
603 }
604 spin_unlock(&ctx->lock);
605 if (qat_alg_aead_init_sessions(ctx, key, keylen))
606 goto out_free_all;
607
608 return 0;
609
610 out_free_all:
611 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
612 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
613 ctx->dec_cd, ctx->dec_cd_paddr);
614 ctx->dec_cd = NULL;
615 out_free_enc:
616 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
617 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
618 ctx->enc_cd, ctx->enc_cd_paddr);
619 ctx->enc_cd = NULL;
620 return -ENOMEM;
621 }
622
623 static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
624 struct qat_crypto_request *qat_req)
625 {
626 struct device *dev = &GET_DEV(inst->accel_dev);
627 struct qat_alg_buf_list *bl = qat_req->buf.bl;
628 struct qat_alg_buf_list *blout = qat_req->buf.blout;
629 dma_addr_t blp = qat_req->buf.blp;
630 dma_addr_t blpout = qat_req->buf.bloutp;
631 size_t sz = qat_req->buf.sz;
632 size_t sz_out = qat_req->buf.sz_out;
633 int i;
634
635 for (i = 0; i < bl->num_bufs; i++)
636 dma_unmap_single(dev, bl->bufers[i].addr,
637 bl->bufers[i].len, DMA_BIDIRECTIONAL);
638
639 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
640 kfree(bl);
641 if (blp != blpout) {
642 /* If out of place operation dma unmap only data */
643 int bufless = blout->num_bufs - blout->num_mapped_bufs;
644
645 for (i = bufless; i < blout->num_bufs; i++) {
646 dma_unmap_single(dev, blout->bufers[i].addr,
647 blout->bufers[i].len,
648 DMA_BIDIRECTIONAL);
649 }
650 dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
651 kfree(blout);
652 }
653 }
654
655 static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
656 struct scatterlist *assoc, int assoclen,
657 struct scatterlist *sgl,
658 struct scatterlist *sglout, uint8_t *iv,
659 uint8_t ivlen,
660 struct qat_crypto_request *qat_req)
661 {
662 struct device *dev = &GET_DEV(inst->accel_dev);
663 int i, bufs = 0, sg_nctr = 0;
664 int n = sg_nents(sgl), assoc_n = sg_nents(assoc);
665 struct qat_alg_buf_list *bufl;
666 struct qat_alg_buf_list *buflout = NULL;
667 dma_addr_t blp;
668 dma_addr_t bloutp = 0;
669 struct scatterlist *sg;
670 size_t sz_out, sz = sizeof(struct qat_alg_buf_list) +
671 ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
672
673 if (unlikely(!n))
674 return -EINVAL;
675
676 bufl = kzalloc_node(sz, GFP_ATOMIC,
677 dev_to_node(&GET_DEV(inst->accel_dev)));
678 if (unlikely(!bufl))
679 return -ENOMEM;
680
681 blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE);
682 if (unlikely(dma_mapping_error(dev, blp)))
683 goto err;
684
685 for_each_sg(assoc, sg, assoc_n, i) {
686 if (!sg->length)
687 continue;
688
689 if (!(assoclen > 0))
690 break;
691
692 bufl->bufers[bufs].addr =
693 dma_map_single(dev, sg_virt(sg),
694 min_t(int, assoclen, sg->length),
695 DMA_BIDIRECTIONAL);
696 bufl->bufers[bufs].len = min_t(int, assoclen, sg->length);
697 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
698 goto err;
699 bufs++;
700 assoclen -= sg->length;
701 }
702
703 if (ivlen) {
704 bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen,
705 DMA_BIDIRECTIONAL);
706 bufl->bufers[bufs].len = ivlen;
707 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
708 goto err;
709 bufs++;
710 }
711
712 for_each_sg(sgl, sg, n, i) {
713 int y = sg_nctr + bufs;
714
715 if (!sg->length)
716 continue;
717
718 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
719 sg->length,
720 DMA_BIDIRECTIONAL);
721 bufl->bufers[y].len = sg->length;
722 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
723 goto err;
724 sg_nctr++;
725 }
726 bufl->num_bufs = sg_nctr + bufs;
727 qat_req->buf.bl = bufl;
728 qat_req->buf.blp = blp;
729 qat_req->buf.sz = sz;
730 /* Handle out of place operation */
731 if (sgl != sglout) {
732 struct qat_alg_buf *bufers;
733
734 n = sg_nents(sglout);
735 sz_out = sizeof(struct qat_alg_buf_list) +
736 ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
737 sg_nctr = 0;
738 buflout = kzalloc_node(sz_out, GFP_ATOMIC,
739 dev_to_node(&GET_DEV(inst->accel_dev)));
740 if (unlikely(!buflout))
741 goto err;
742 bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE);
743 if (unlikely(dma_mapping_error(dev, bloutp)))
744 goto err;
745 bufers = buflout->bufers;
746 /* For out of place operation dma map only data and
747 * reuse assoc mapping and iv */
748 for (i = 0; i < bufs; i++) {
749 bufers[i].len = bufl->bufers[i].len;
750 bufers[i].addr = bufl->bufers[i].addr;
751 }
752 for_each_sg(sglout, sg, n, i) {
753 int y = sg_nctr + bufs;
754
755 if (!sg->length)
756 continue;
757
758 bufers[y].addr = dma_map_single(dev, sg_virt(sg),
759 sg->length,
760 DMA_BIDIRECTIONAL);
761 if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
762 goto err;
763 bufers[y].len = sg->length;
764 sg_nctr++;
765 }
766 buflout->num_bufs = sg_nctr + bufs;
767 buflout->num_mapped_bufs = sg_nctr;
768 qat_req->buf.blout = buflout;
769 qat_req->buf.bloutp = bloutp;
770 qat_req->buf.sz_out = sz_out;
771 } else {
772 /* Otherwise set the src and dst to the same address */
773 qat_req->buf.bloutp = qat_req->buf.blp;
774 qat_req->buf.sz_out = 0;
775 }
776 return 0;
777 err:
778 dev_err(dev, "Failed to map buf for dma\n");
779 sg_nctr = 0;
780 for (i = 0; i < n + bufs; i++)
781 if (!dma_mapping_error(dev, bufl->bufers[i].addr))
782 dma_unmap_single(dev, bufl->bufers[i].addr,
783 bufl->bufers[i].len,
784 DMA_BIDIRECTIONAL);
785
786 if (!dma_mapping_error(dev, blp))
787 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
788 kfree(bufl);
789 if (sgl != sglout && buflout) {
790 n = sg_nents(sglout);
791 for (i = bufs; i < n + bufs; i++)
792 if (!dma_mapping_error(dev, buflout->bufers[i].addr))
793 dma_unmap_single(dev, buflout->bufers[i].addr,
794 buflout->bufers[i].len,
795 DMA_BIDIRECTIONAL);
796 if (!dma_mapping_error(dev, bloutp))
797 dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE);
798 kfree(buflout);
799 }
800 return -ENOMEM;
801 }
802
803 static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
804 struct qat_crypto_request *qat_req)
805 {
806 struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
807 struct qat_crypto_instance *inst = ctx->inst;
808 struct aead_request *areq = qat_req->aead_req;
809 uint8_t stat_filed = qat_resp->comn_resp.comn_status;
810 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
811
812 qat_alg_free_bufl(inst, qat_req);
813 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
814 res = -EBADMSG;
815 areq->base.complete(&areq->base, res);
816 }
817
818 static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
819 struct qat_crypto_request *qat_req)
820 {
821 struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx;
822 struct qat_crypto_instance *inst = ctx->inst;
823 struct ablkcipher_request *areq = qat_req->ablkcipher_req;
824 uint8_t stat_filed = qat_resp->comn_resp.comn_status;
825 int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
826
827 qat_alg_free_bufl(inst, qat_req);
828 if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
829 res = -EINVAL;
830 areq->base.complete(&areq->base, res);
831 }
832
833 void qat_alg_callback(void *resp)
834 {
835 struct icp_qat_fw_la_resp *qat_resp = resp;
836 struct qat_crypto_request *qat_req =
837 (void *)(__force long)qat_resp->opaque_data;
838
839 qat_req->cb(qat_resp, qat_req);
840 }
841
842 static int qat_alg_aead_dec(struct aead_request *areq)
843 {
844 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
845 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
846 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
847 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
848 struct icp_qat_fw_la_cipher_req_params *cipher_param;
849 struct icp_qat_fw_la_auth_req_params *auth_param;
850 struct icp_qat_fw_la_bulk_req *msg;
851 int digst_size = crypto_aead_crt(aead_tfm)->authsize;
852 int ret, ctr = 0;
853
854 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->assoclen,
855 areq->src, areq->dst, areq->iv,
856 AES_BLOCK_SIZE, qat_req);
857 if (unlikely(ret))
858 return ret;
859
860 msg = &qat_req->req;
861 *msg = ctx->dec_fw_req;
862 qat_req->aead_ctx = ctx;
863 qat_req->aead_req = areq;
864 qat_req->cb = qat_aead_alg_callback;
865 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
866 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
867 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
868 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
869 cipher_param->cipher_length = areq->cryptlen - digst_size;
870 cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE;
871 memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
872 auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
873 auth_param->auth_off = 0;
874 auth_param->auth_len = areq->assoclen +
875 cipher_param->cipher_length + AES_BLOCK_SIZE;
876 do {
877 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
878 } while (ret == -EAGAIN && ctr++ < 10);
879
880 if (ret == -EAGAIN) {
881 qat_alg_free_bufl(ctx->inst, qat_req);
882 return -EBUSY;
883 }
884 return -EINPROGRESS;
885 }
886
887 static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv,
888 int enc_iv)
889 {
890 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
891 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
892 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
893 struct qat_crypto_request *qat_req = aead_request_ctx(areq);
894 struct icp_qat_fw_la_cipher_req_params *cipher_param;
895 struct icp_qat_fw_la_auth_req_params *auth_param;
896 struct icp_qat_fw_la_bulk_req *msg;
897 int ret, ctr = 0;
898
899 ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->assoclen,
900 areq->src, areq->dst, iv, AES_BLOCK_SIZE,
901 qat_req);
902 if (unlikely(ret))
903 return ret;
904
905 msg = &qat_req->req;
906 *msg = ctx->enc_fw_req;
907 qat_req->aead_ctx = ctx;
908 qat_req->aead_req = areq;
909 qat_req->cb = qat_aead_alg_callback;
910 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
911 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
912 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
913 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
914 auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
915
916 if (enc_iv) {
917 cipher_param->cipher_length = areq->cryptlen + AES_BLOCK_SIZE;
918 cipher_param->cipher_offset = areq->assoclen;
919 } else {
920 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
921 cipher_param->cipher_length = areq->cryptlen;
922 cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE;
923 }
924 auth_param->auth_off = 0;
925 auth_param->auth_len = areq->assoclen + areq->cryptlen + AES_BLOCK_SIZE;
926
927 do {
928 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
929 } while (ret == -EAGAIN && ctr++ < 10);
930
931 if (ret == -EAGAIN) {
932 qat_alg_free_bufl(ctx->inst, qat_req);
933 return -EBUSY;
934 }
935 return -EINPROGRESS;
936 }
937
938 static int qat_alg_aead_enc(struct aead_request *areq)
939 {
940 return qat_alg_aead_enc_internal(areq, areq->iv, 0);
941 }
942
943 static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req)
944 {
945 struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq);
946 struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
947 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
948 __be64 seq;
949
950 memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE);
951 seq = cpu_to_be64(req->seq);
952 memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t),
953 &seq, sizeof(uint64_t));
954 return qat_alg_aead_enc_internal(&req->areq, req->giv, 1);
955 }
956
957 static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
958 const uint8_t *key,
959 unsigned int keylen)
960 {
961 struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
962 struct device *dev;
963
964 spin_lock(&ctx->lock);
965 if (ctx->enc_cd) {
966 /* rekeying */
967 dev = &GET_DEV(ctx->inst->accel_dev);
968 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
969 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
970 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
971 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
972 } else {
973 /* new key */
974 int node = get_current_node();
975 struct qat_crypto_instance *inst =
976 qat_crypto_get_instance_node(node);
977 if (!inst) {
978 spin_unlock(&ctx->lock);
979 return -EINVAL;
980 }
981
982 dev = &GET_DEV(inst->accel_dev);
983 ctx->inst = inst;
984 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
985 &ctx->enc_cd_paddr,
986 GFP_ATOMIC);
987 if (!ctx->enc_cd) {
988 spin_unlock(&ctx->lock);
989 return -ENOMEM;
990 }
991 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
992 &ctx->dec_cd_paddr,
993 GFP_ATOMIC);
994 if (!ctx->dec_cd) {
995 spin_unlock(&ctx->lock);
996 goto out_free_enc;
997 }
998 }
999 spin_unlock(&ctx->lock);
1000 if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen))
1001 goto out_free_all;
1002
1003 return 0;
1004
1005 out_free_all:
1006 memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd));
1007 dma_free_coherent(dev, sizeof(*ctx->enc_cd),
1008 ctx->dec_cd, ctx->dec_cd_paddr);
1009 ctx->dec_cd = NULL;
1010 out_free_enc:
1011 memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd));
1012 dma_free_coherent(dev, sizeof(*ctx->dec_cd),
1013 ctx->enc_cd, ctx->enc_cd_paddr);
1014 ctx->enc_cd = NULL;
1015 return -ENOMEM;
1016 }
1017
1018 static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
1019 {
1020 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
1021 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
1022 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1023 struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
1024 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1025 struct icp_qat_fw_la_bulk_req *msg;
1026 int ret, ctr = 0;
1027
1028 ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, 0, req->src, req->dst,
1029 NULL, 0, qat_req);
1030 if (unlikely(ret))
1031 return ret;
1032
1033 msg = &qat_req->req;
1034 *msg = ctx->enc_fw_req;
1035 qat_req->ablkcipher_ctx = ctx;
1036 qat_req->ablkcipher_req = req;
1037 qat_req->cb = qat_ablkcipher_alg_callback;
1038 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1039 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1040 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1041 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1042 cipher_param->cipher_length = req->nbytes;
1043 cipher_param->cipher_offset = 0;
1044 memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
1045 do {
1046 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1047 } while (ret == -EAGAIN && ctr++ < 10);
1048
1049 if (ret == -EAGAIN) {
1050 qat_alg_free_bufl(ctx->inst, qat_req);
1051 return -EBUSY;
1052 }
1053 return -EINPROGRESS;
1054 }
1055
1056 static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
1057 {
1058 struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
1059 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
1060 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1061 struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
1062 struct icp_qat_fw_la_cipher_req_params *cipher_param;
1063 struct icp_qat_fw_la_bulk_req *msg;
1064 int ret, ctr = 0;
1065
1066 ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, 0, req->src, req->dst,
1067 NULL, 0, qat_req);
1068 if (unlikely(ret))
1069 return ret;
1070
1071 msg = &qat_req->req;
1072 *msg = ctx->dec_fw_req;
1073 qat_req->ablkcipher_ctx = ctx;
1074 qat_req->ablkcipher_req = req;
1075 qat_req->cb = qat_ablkcipher_alg_callback;
1076 qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1077 qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1078 qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1079 cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1080 cipher_param->cipher_length = req->nbytes;
1081 cipher_param->cipher_offset = 0;
1082 memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
1083 do {
1084 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1085 } while (ret == -EAGAIN && ctr++ < 10);
1086
1087 if (ret == -EAGAIN) {
1088 qat_alg_free_bufl(ctx->inst, qat_req);
1089 return -EBUSY;
1090 }
1091 return -EINPROGRESS;
1092 }
1093
1094 static int qat_alg_aead_init(struct crypto_tfm *tfm,
1095 enum icp_qat_hw_auth_algo hash,
1096 const char *hash_name)
1097 {
1098 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
1099
1100 ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1101 if (IS_ERR(ctx->hash_tfm))
1102 return -EFAULT;
1103 spin_lock_init(&ctx->lock);
1104 ctx->qat_hash_alg = hash;
1105 crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
1106 sizeof(struct aead_request) +
1107 sizeof(struct qat_crypto_request));
1108 ctx->tfm = tfm;
1109 return 0;
1110 }
1111
1112 static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm)
1113 {
1114 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1115 }
1116
1117 static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm)
1118 {
1119 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1120 }
1121
1122 static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm)
1123 {
1124 return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1125 }
1126
1127 static void qat_alg_aead_exit(struct crypto_tfm *tfm)
1128 {
1129 struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
1130 struct qat_crypto_instance *inst = ctx->inst;
1131 struct device *dev;
1132
1133 if (!IS_ERR(ctx->hash_tfm))
1134 crypto_free_shash(ctx->hash_tfm);
1135
1136 if (!inst)
1137 return;
1138
1139 dev = &GET_DEV(inst->accel_dev);
1140 if (ctx->enc_cd) {
1141 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1142 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1143 ctx->enc_cd, ctx->enc_cd_paddr);
1144 }
1145 if (ctx->dec_cd) {
1146 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1147 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1148 ctx->dec_cd, ctx->dec_cd_paddr);
1149 }
1150 qat_crypto_put_instance(inst);
1151 }
1152
1153 static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm)
1154 {
1155 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1156
1157 spin_lock_init(&ctx->lock);
1158 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
1159 sizeof(struct qat_crypto_request);
1160 ctx->tfm = tfm;
1161 return 0;
1162 }
1163
1164 static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm)
1165 {
1166 struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1167 struct qat_crypto_instance *inst = ctx->inst;
1168 struct device *dev;
1169
1170 if (!inst)
1171 return;
1172
1173 dev = &GET_DEV(inst->accel_dev);
1174 if (ctx->enc_cd) {
1175 memset(ctx->enc_cd, 0,
1176 sizeof(struct icp_qat_hw_cipher_algo_blk));
1177 dma_free_coherent(dev,
1178 sizeof(struct icp_qat_hw_cipher_algo_blk),
1179 ctx->enc_cd, ctx->enc_cd_paddr);
1180 }
1181 if (ctx->dec_cd) {
1182 memset(ctx->dec_cd, 0,
1183 sizeof(struct icp_qat_hw_cipher_algo_blk));
1184 dma_free_coherent(dev,
1185 sizeof(struct icp_qat_hw_cipher_algo_blk),
1186 ctx->dec_cd, ctx->dec_cd_paddr);
1187 }
1188 qat_crypto_put_instance(inst);
1189 }
1190
1191 static struct crypto_alg qat_algs[] = { {
1192 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1193 .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1194 .cra_priority = 4001,
1195 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1196 .cra_blocksize = AES_BLOCK_SIZE,
1197 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1198 .cra_alignmask = 0,
1199 .cra_type = &crypto_aead_type,
1200 .cra_module = THIS_MODULE,
1201 .cra_init = qat_alg_aead_sha1_init,
1202 .cra_exit = qat_alg_aead_exit,
1203 .cra_u = {
1204 .aead = {
1205 .setkey = qat_alg_aead_setkey,
1206 .decrypt = qat_alg_aead_dec,
1207 .encrypt = qat_alg_aead_enc,
1208 .givencrypt = qat_alg_aead_genivenc,
1209 .ivsize = AES_BLOCK_SIZE,
1210 .maxauthsize = SHA1_DIGEST_SIZE,
1211 },
1212 },
1213 }, {
1214 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1215 .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1216 .cra_priority = 4001,
1217 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1218 .cra_blocksize = AES_BLOCK_SIZE,
1219 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1220 .cra_alignmask = 0,
1221 .cra_type = &crypto_aead_type,
1222 .cra_module = THIS_MODULE,
1223 .cra_init = qat_alg_aead_sha256_init,
1224 .cra_exit = qat_alg_aead_exit,
1225 .cra_u = {
1226 .aead = {
1227 .setkey = qat_alg_aead_setkey,
1228 .decrypt = qat_alg_aead_dec,
1229 .encrypt = qat_alg_aead_enc,
1230 .givencrypt = qat_alg_aead_genivenc,
1231 .ivsize = AES_BLOCK_SIZE,
1232 .maxauthsize = SHA256_DIGEST_SIZE,
1233 },
1234 },
1235 }, {
1236 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1237 .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1238 .cra_priority = 4001,
1239 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1240 .cra_blocksize = AES_BLOCK_SIZE,
1241 .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1242 .cra_alignmask = 0,
1243 .cra_type = &crypto_aead_type,
1244 .cra_module = THIS_MODULE,
1245 .cra_init = qat_alg_aead_sha512_init,
1246 .cra_exit = qat_alg_aead_exit,
1247 .cra_u = {
1248 .aead = {
1249 .setkey = qat_alg_aead_setkey,
1250 .decrypt = qat_alg_aead_dec,
1251 .encrypt = qat_alg_aead_enc,
1252 .givencrypt = qat_alg_aead_genivenc,
1253 .ivsize = AES_BLOCK_SIZE,
1254 .maxauthsize = SHA512_DIGEST_SIZE,
1255 },
1256 },
1257 }, {
1258 .cra_name = "cbc(aes)",
1259 .cra_driver_name = "qat_aes_cbc",
1260 .cra_priority = 4001,
1261 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1262 .cra_blocksize = AES_BLOCK_SIZE,
1263 .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
1264 .cra_alignmask = 0,
1265 .cra_type = &crypto_ablkcipher_type,
1266 .cra_module = THIS_MODULE,
1267 .cra_init = qat_alg_ablkcipher_init,
1268 .cra_exit = qat_alg_ablkcipher_exit,
1269 .cra_u = {
1270 .ablkcipher = {
1271 .setkey = qat_alg_ablkcipher_setkey,
1272 .decrypt = qat_alg_ablkcipher_decrypt,
1273 .encrypt = qat_alg_ablkcipher_encrypt,
1274 .min_keysize = AES_MIN_KEY_SIZE,
1275 .max_keysize = AES_MAX_KEY_SIZE,
1276 .ivsize = AES_BLOCK_SIZE,
1277 },
1278 },
1279 } };
1280
1281 int qat_algs_register(void)
1282 {
1283 if (atomic_add_return(1, &active_dev) == 1) {
1284 int i;
1285
1286 for (i = 0; i < ARRAY_SIZE(qat_algs); i++)
1287 qat_algs[i].cra_flags =
1288 (qat_algs[i].cra_type == &crypto_aead_type) ?
1289 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC :
1290 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
1291
1292 return crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs));
1293 }
1294 return 0;
1295 }
1296
1297 int qat_algs_unregister(void)
1298 {
1299 if (atomic_sub_return(1, &active_dev) == 0)
1300 return crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs));
1301 return 0;
1302 }
1303
1304 int qat_algs_init(void)
1305 {
1306 atomic_set(&active_dev, 0);
1307 crypto_get_default_rng();
1308 return 0;
1309 }
1310
1311 void qat_algs_exit(void)
1312 {
1313 crypto_put_default_rng();
1314 }
ubuntu-bionic-kernel mirror