]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - drivers/crypto/inside-secure/safexcel_hash.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
crypto: sha - split sha.h into sha1.h and sha2.h
[mirror_ubuntu-hirsute-kernel.git] / drivers / crypto / inside-secure / safexcel_hash.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2017 Marvell
4 *
5 * Antoine Tenart <antoine.tenart@free-electrons.com>
6 */
7
8 #include <crypto/aes.h>
9 #include <crypto/hmac.h>
10 #include <crypto/md5.h>
11 #include <crypto/sha1.h>
12 #include <crypto/sha2.h>
13 #include <crypto/sha3.h>
14 #include <crypto/skcipher.h>
15 #include <crypto/sm3.h>
16 #include <linux/device.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/dmapool.h>
19
20 #include "safexcel.h"
21
22 struct safexcel_ahash_ctx {
23 struct safexcel_context base;
24
25 u32 alg;
26 u8 key_sz;
27 bool cbcmac;
28 bool do_fallback;
29 bool fb_init_done;
30 bool fb_do_setkey;
31
32 struct crypto_cipher *kaes;
33 struct crypto_ahash *fback;
34 struct crypto_shash *shpre;
35 struct shash_desc *shdesc;
36 };
37
38 struct safexcel_ahash_req {
39 bool last_req;
40 bool finish;
41 bool hmac;
42 bool needs_inv;
43 bool hmac_zlen;
44 bool len_is_le;
45 bool not_first;
46 bool xcbcmac;
47
48 int nents;
49 dma_addr_t result_dma;
50
51 u32 digest;
52
53 u8 state_sz; /* expected state size, only set once */
54 u8 block_sz; /* block size, only set once */
55 u8 digest_sz; /* output digest size, only set once */
56 __le32 state[SHA3_512_BLOCK_SIZE /
57 sizeof(__le32)] __aligned(sizeof(__le32));
58
59 u64 len;
60 u64 processed;
61
62 u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
63 dma_addr_t cache_dma;
64 unsigned int cache_sz;
65
66 u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
67 };
68
69 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
70 {
71 return req->len - req->processed;
72 }
73
74 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
75 u32 input_length, u32 result_length,
76 bool cbcmac)
77 {
78 struct safexcel_token *token =
79 (struct safexcel_token *)cdesc->control_data.token;
80
81 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
82 token[0].packet_length = input_length;
83 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
84
85 input_length &= 15;
86 if (unlikely(cbcmac && input_length)) {
87 token[0].stat = 0;
88 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
89 token[1].packet_length = 16 - input_length;
90 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
91 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH;
92 } else {
93 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
94 eip197_noop_token(&token[1]);
95 }
96
97 token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
98 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
99 EIP197_TOKEN_STAT_LAST_PACKET;
100 token[2].packet_length = result_length;
101 token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
102 EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
103
104 eip197_noop_token(&token[3]);
105 }
106
107 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
108 struct safexcel_ahash_req *req,
109 struct safexcel_command_desc *cdesc)
110 {
111 struct safexcel_crypto_priv *priv = ctx->base.priv;
112 u64 count = 0;
113
114 cdesc->control_data.control0 = ctx->alg;
115 cdesc->control_data.control1 = 0;
116
117 /*
118 * Copy the input digest if needed, and setup the context
119 * fields. Do this now as we need it to setup the first command
120 * descriptor.
121 */
122 if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
123 if (req->xcbcmac)
124 memcpy(ctx->base.ctxr->data, &ctx->base.ipad, ctx->key_sz);
125 else
126 memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
127
128 if (!req->finish && req->xcbcmac)
129 cdesc->control_data.control0 |=
130 CONTEXT_CONTROL_DIGEST_XCM |
131 CONTEXT_CONTROL_TYPE_HASH_OUT |
132 CONTEXT_CONTROL_NO_FINISH_HASH |
133 CONTEXT_CONTROL_SIZE(req->state_sz /
134 sizeof(u32));
135 else
136 cdesc->control_data.control0 |=
137 CONTEXT_CONTROL_DIGEST_XCM |
138 CONTEXT_CONTROL_TYPE_HASH_OUT |
139 CONTEXT_CONTROL_SIZE(req->state_sz /
140 sizeof(u32));
141 return;
142 } else if (!req->processed) {
143 /* First - and possibly only - block of basic hash only */
144 if (req->finish)
145 cdesc->control_data.control0 |= req->digest |
146 CONTEXT_CONTROL_TYPE_HASH_OUT |
147 CONTEXT_CONTROL_RESTART_HASH |
148 /* ensure its not 0! */
149 CONTEXT_CONTROL_SIZE(1);
150 else
151 cdesc->control_data.control0 |= req->digest |
152 CONTEXT_CONTROL_TYPE_HASH_OUT |
153 CONTEXT_CONTROL_RESTART_HASH |
154 CONTEXT_CONTROL_NO_FINISH_HASH |
155 /* ensure its not 0! */
156 CONTEXT_CONTROL_SIZE(1);
157 return;
158 }
159
160 /* Hash continuation or HMAC, setup (inner) digest from state */
161 memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
162
163 if (req->finish) {
164 /* Compute digest count for hash/HMAC finish operations */
165 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
166 req->hmac_zlen || (req->processed != req->block_sz)) {
167 count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
168
169 /* This is a hardware limitation, as the
170 * counter must fit into an u32. This represents
171 * a fairly big amount of input data, so we
172 * shouldn't see this.
173 */
174 if (unlikely(count & 0xffffffff00000000ULL)) {
175 dev_warn(priv->dev,
176 "Input data is too big\n");
177 return;
178 }
179 }
180
181 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
182 /* Special case: zero length HMAC */
183 req->hmac_zlen ||
184 /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
185 (req->processed != req->block_sz)) {
186 /* Basic hash continue operation, need digest + cnt */
187 cdesc->control_data.control0 |=
188 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
189 CONTEXT_CONTROL_TYPE_HASH_OUT |
190 CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
191 /* For zero-len HMAC, don't finalize, already padded! */
192 if (req->hmac_zlen)
193 cdesc->control_data.control0 |=
194 CONTEXT_CONTROL_NO_FINISH_HASH;
195 cdesc->control_data.control1 |=
196 CONTEXT_CONTROL_DIGEST_CNT;
197 ctx->base.ctxr->data[req->state_sz >> 2] =
198 cpu_to_le32(count);
199 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
200
201 /* Clear zero-length HMAC flag for next operation! */
202 req->hmac_zlen = false;
203 } else { /* HMAC */
204 /* Need outer digest for HMAC finalization */
205 memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
206 &ctx->base.opad, req->state_sz);
207
208 /* Single pass HMAC - no digest count */
209 cdesc->control_data.control0 |=
210 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
211 CONTEXT_CONTROL_TYPE_HASH_OUT |
212 CONTEXT_CONTROL_DIGEST_HMAC;
213 }
214 } else { /* Hash continuation, do not finish yet */
215 cdesc->control_data.control0 |=
216 CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
217 CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
218 CONTEXT_CONTROL_TYPE_HASH_OUT |
219 CONTEXT_CONTROL_NO_FINISH_HASH;
220 }
221 }
222
223 static int safexcel_ahash_enqueue(struct ahash_request *areq);
224
225 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
226 int ring,
227 struct crypto_async_request *async,
228 bool *should_complete, int *ret)
229 {
230 struct safexcel_result_desc *rdesc;
231 struct ahash_request *areq = ahash_request_cast(async);
232 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
233 struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
234 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
235 u64 cache_len;
236
237 *ret = 0;
238
239 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
240 if (IS_ERR(rdesc)) {
241 dev_err(priv->dev,
242 "hash: result: could not retrieve the result descriptor\n");
243 *ret = PTR_ERR(rdesc);
244 } else {
245 *ret = safexcel_rdesc_check_errors(priv, rdesc);
246 }
247
248 safexcel_complete(priv, ring);
249
250 if (sreq->nents) {
251 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
252 sreq->nents = 0;
253 }
254
255 if (sreq->result_dma) {
256 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz,
257 DMA_FROM_DEVICE);
258 sreq->result_dma = 0;
259 }
260
261 if (sreq->cache_dma) {
262 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
263 DMA_TO_DEVICE);
264 sreq->cache_dma = 0;
265 sreq->cache_sz = 0;
266 }
267
268 if (sreq->finish) {
269 if (sreq->hmac &&
270 (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
271 /* Faking HMAC using hash - need to do outer hash */
272 memcpy(sreq->cache, sreq->state,
273 crypto_ahash_digestsize(ahash));
274
275 memcpy(sreq->state, &ctx->base.opad, sreq->digest_sz);
276
277 sreq->len = sreq->block_sz +
278 crypto_ahash_digestsize(ahash);
279 sreq->processed = sreq->block_sz;
280 sreq->hmac = 0;
281
282 if (priv->flags & EIP197_TRC_CACHE)
283 ctx->base.needs_inv = true;
284 areq->nbytes = 0;
285 safexcel_ahash_enqueue(areq);
286
287 *should_complete = false; /* Not done yet */
288 return 1;
289 }
290
291 if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM &&
292 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) {
293 /* Undo final XOR with 0xffffffff ...*/
294 *(__le32 *)areq->result = ~sreq->state[0];
295 } else {
296 memcpy(areq->result, sreq->state,
297 crypto_ahash_digestsize(ahash));
298 }
299 }
300
301 cache_len = safexcel_queued_len(sreq);
302 if (cache_len)
303 memcpy(sreq->cache, sreq->cache_next, cache_len);
304
305 *should_complete = true;
306
307 return 1;
308 }
309
310 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
311 int *commands, int *results)
312 {
313 struct ahash_request *areq = ahash_request_cast(async);
314 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
315 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
316 struct safexcel_crypto_priv *priv = ctx->base.priv;
317 struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
318 struct safexcel_result_desc *rdesc;
319 struct scatterlist *sg;
320 struct safexcel_token *dmmy;
321 int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0;
322 u64 queued, len;
323
324 queued = safexcel_queued_len(req);
325 if (queued <= HASH_CACHE_SIZE)
326 cache_len = queued;
327 else
328 cache_len = queued - areq->nbytes;
329
330 if (!req->finish && !req->last_req) {
331 /* If this is not the last request and the queued data does not
332 * fit into full cache blocks, cache it for the next send call.
333 */
334 extra = queued & (HASH_CACHE_SIZE - 1);
335
336 /* If this is not the last request and the queued data
337 * is a multiple of a block, cache the last one for now.
338 */
339 if (!extra)
340 extra = HASH_CACHE_SIZE;
341
342 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
343 req->cache_next, extra,
344 areq->nbytes - extra);
345
346 queued -= extra;
347
348 if (!queued) {
349 *commands = 0;
350 *results = 0;
351 return 0;
352 }
353
354 extra = 0;
355 }
356
357 if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {
358 if (unlikely(cache_len < AES_BLOCK_SIZE)) {
359 /*
360 * Cache contains less than 1 full block, complete.
361 */
362 extra = AES_BLOCK_SIZE - cache_len;
363 if (queued > cache_len) {
364 /* More data follows: borrow bytes */
365 u64 tmp = queued - cache_len;
366
367 skip = min_t(u64, tmp, extra);
368 sg_pcopy_to_buffer(areq->src,
369 sg_nents(areq->src),
370 req->cache + cache_len,
371 skip, 0);
372 }
373 extra -= skip;
374 memset(req->cache + cache_len + skip, 0, extra);
375 if (!ctx->cbcmac && extra) {
376 // 10- padding for XCBCMAC & CMAC
377 req->cache[cache_len + skip] = 0x80;
378 // HW will use K2 iso K3 - compensate!
379 for (i = 0; i < AES_BLOCK_SIZE / 4; i++) {
380 u32 *cache = (void *)req->cache;
381 u32 *ipad = ctx->base.ipad.word;
382 u32 x;
383
384 x = ipad[i] ^ ipad[i + 4];
385 cache[i] ^= swab(x);
386 }
387 }
388 cache_len = AES_BLOCK_SIZE;
389 queued = queued + extra;
390 }
391
392 /* XCBC continue: XOR previous result into 1st word */
393 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);
394 }
395
396 len = queued;
397 /* Add a command descriptor for the cached data, if any */
398 if (cache_len) {
399 req->cache_dma = dma_map_single(priv->dev, req->cache,
400 cache_len, DMA_TO_DEVICE);
401 if (dma_mapping_error(priv->dev, req->cache_dma))
402 return -EINVAL;
403
404 req->cache_sz = cache_len;
405 first_cdesc = safexcel_add_cdesc(priv, ring, 1,
406 (cache_len == len),
407 req->cache_dma, cache_len,
408 len, ctx->base.ctxr_dma,
409 &dmmy);
410 if (IS_ERR(first_cdesc)) {
411 ret = PTR_ERR(first_cdesc);
412 goto unmap_cache;
413 }
414 n_cdesc++;
415
416 queued -= cache_len;
417 if (!queued)
418 goto send_command;
419 }
420
421 /* Now handle the current ahash request buffer(s) */
422 req->nents = dma_map_sg(priv->dev, areq->src,
423 sg_nents_for_len(areq->src,
424 areq->nbytes),
425 DMA_TO_DEVICE);
426 if (!req->nents) {
427 ret = -ENOMEM;
428 goto cdesc_rollback;
429 }
430
431 for_each_sg(areq->src, sg, req->nents, i) {
432 int sglen = sg_dma_len(sg);
433
434 if (unlikely(sglen <= skip)) {
435 skip -= sglen;
436 continue;
437 }
438
439 /* Do not overflow the request */
440 if ((queued + skip) <= sglen)
441 sglen = queued;
442 else
443 sglen -= skip;
444
445 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
446 !(queued - sglen),
447 sg_dma_address(sg) + skip, sglen,
448 len, ctx->base.ctxr_dma, &dmmy);
449 if (IS_ERR(cdesc)) {
450 ret = PTR_ERR(cdesc);
451 goto unmap_sg;
452 }
453
454 if (!n_cdesc)
455 first_cdesc = cdesc;
456 n_cdesc++;
457
458 queued -= sglen;
459 if (!queued)
460 break;
461 skip = 0;
462 }
463
464 send_command:
465 /* Setup the context options */
466 safexcel_context_control(ctx, req, first_cdesc);
467
468 /* Add the token */
469 safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);
470
471 req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,
472 DMA_FROM_DEVICE);
473 if (dma_mapping_error(priv->dev, req->result_dma)) {
474 ret = -EINVAL;
475 goto unmap_sg;
476 }
477
478 /* Add a result descriptor */
479 rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
480 req->digest_sz);
481 if (IS_ERR(rdesc)) {
482 ret = PTR_ERR(rdesc);
483 goto unmap_result;
484 }
485
486 safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
487
488 req->processed += len - extra;
489
490 *commands = n_cdesc;
491 *results = 1;
492 return 0;
493
494 unmap_result:
495 dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,
496 DMA_FROM_DEVICE);
497 unmap_sg:
498 if (req->nents) {
499 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
500 req->nents = 0;
501 }
502 cdesc_rollback:
503 for (i = 0; i < n_cdesc; i++)
504 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
505 unmap_cache:
506 if (req->cache_dma) {
507 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
508 DMA_TO_DEVICE);
509 req->cache_dma = 0;
510 req->cache_sz = 0;
511 }
512
513 return ret;
514 }
515
516 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
517 int ring,
518 struct crypto_async_request *async,
519 bool *should_complete, int *ret)
520 {
521 struct safexcel_result_desc *rdesc;
522 struct ahash_request *areq = ahash_request_cast(async);
523 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
524 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
525 int enq_ret;
526
527 *ret = 0;
528
529 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
530 if (IS_ERR(rdesc)) {
531 dev_err(priv->dev,
532 "hash: invalidate: could not retrieve the result descriptor\n");
533 *ret = PTR_ERR(rdesc);
534 } else {
535 *ret = safexcel_rdesc_check_errors(priv, rdesc);
536 }
537
538 safexcel_complete(priv, ring);
539
540 if (ctx->base.exit_inv) {
541 dma_pool_free(priv->context_pool, ctx->base.ctxr,
542 ctx->base.ctxr_dma);
543
544 *should_complete = true;
545 return 1;
546 }
547
548 ring = safexcel_select_ring(priv);
549 ctx->base.ring = ring;
550
551 spin_lock_bh(&priv->ring[ring].queue_lock);
552 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
553 spin_unlock_bh(&priv->ring[ring].queue_lock);
554
555 if (enq_ret != -EINPROGRESS)
556 *ret = enq_ret;
557
558 queue_work(priv->ring[ring].workqueue,
559 &priv->ring[ring].work_data.work);
560
561 *should_complete = false;
562
563 return 1;
564 }
565
566 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
567 struct crypto_async_request *async,
568 bool *should_complete, int *ret)
569 {
570 struct ahash_request *areq = ahash_request_cast(async);
571 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
572 int err;
573
574 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
575
576 if (req->needs_inv) {
577 req->needs_inv = false;
578 err = safexcel_handle_inv_result(priv, ring, async,
579 should_complete, ret);
580 } else {
581 err = safexcel_handle_req_result(priv, ring, async,
582 should_complete, ret);
583 }
584
585 return err;
586 }
587
588 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
589 int ring, int *commands, int *results)
590 {
591 struct ahash_request *areq = ahash_request_cast(async);
592 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
593 int ret;
594
595 ret = safexcel_invalidate_cache(async, ctx->base.priv,
596 ctx->base.ctxr_dma, ring);
597 if (unlikely(ret))
598 return ret;
599
600 *commands = 1;
601 *results = 1;
602
603 return 0;
604 }
605
606 static int safexcel_ahash_send(struct crypto_async_request *async,
607 int ring, int *commands, int *results)
608 {
609 struct ahash_request *areq = ahash_request_cast(async);
610 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
611 int ret;
612
613 if (req->needs_inv)
614 ret = safexcel_ahash_send_inv(async, ring, commands, results);
615 else
616 ret = safexcel_ahash_send_req(async, ring, commands, results);
617
618 return ret;
619 }
620
621 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
622 {
623 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
624 struct safexcel_crypto_priv *priv = ctx->base.priv;
625 EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
626 struct safexcel_ahash_req *rctx = ahash_request_ctx(req);
627 struct safexcel_inv_result result = {};
628 int ring = ctx->base.ring;
629
630 memset(req, 0, EIP197_AHASH_REQ_SIZE);
631
632 /* create invalidation request */
633 init_completion(&result.completion);
634 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
635 safexcel_inv_complete, &result);
636
637 ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));
638 ctx = crypto_tfm_ctx(req->base.tfm);
639 ctx->base.exit_inv = true;
640 rctx->needs_inv = true;
641
642 spin_lock_bh(&priv->ring[ring].queue_lock);
643 crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
644 spin_unlock_bh(&priv->ring[ring].queue_lock);
645
646 queue_work(priv->ring[ring].workqueue,
647 &priv->ring[ring].work_data.work);
648
649 wait_for_completion(&result.completion);
650
651 if (result.error) {
652 dev_warn(priv->dev, "hash: completion error (%d)\n",
653 result.error);
654 return result.error;
655 }
656
657 return 0;
658 }
659
660 /* safexcel_ahash_cache: cache data until at least one request can be sent to
661 * the engine, aka. when there is at least 1 block size in the pipe.
662 */
663 static int safexcel_ahash_cache(struct ahash_request *areq)
664 {
665 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
666 u64 cache_len;
667
668 /* cache_len: everything accepted by the driver but not sent yet,
669 * tot sz handled by update() - last req sz - tot sz handled by send()
670 */
671 cache_len = safexcel_queued_len(req);
672
673 /*
674 * In case there isn't enough bytes to proceed (less than a
675 * block size), cache the data until we have enough.
676 */
677 if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
678 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
679 req->cache + cache_len,
680 areq->nbytes, 0);
681 return 0;
682 }
683
684 /* We couldn't cache all the data */
685 return -E2BIG;
686 }
687
688 static int safexcel_ahash_enqueue(struct ahash_request *areq)
689 {
690 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
691 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
692 struct safexcel_crypto_priv *priv = ctx->base.priv;
693 int ret, ring;
694
695 req->needs_inv = false;
696
697 if (ctx->base.ctxr) {
698 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
699 /* invalidate for *any* non-XCBC continuation */
700 ((req->not_first && !req->xcbcmac) ||
701 /* invalidate if (i)digest changed */
702 memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
703 /* invalidate for HMAC finish with odigest changed */
704 (req->finish && req->hmac &&
705 memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
706 &ctx->base.opad, req->state_sz))))
707 /*
708 * We're still setting needs_inv here, even though it is
709 * cleared right away, because the needs_inv flag can be
710 * set in other functions and we want to keep the same
711 * logic.
712 */
713 ctx->base.needs_inv = true;
714
715 if (ctx->base.needs_inv) {
716 ctx->base.needs_inv = false;
717 req->needs_inv = true;
718 }
719 } else {
720 ctx->base.ring = safexcel_select_ring(priv);
721 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
722 EIP197_GFP_FLAGS(areq->base),
723 &ctx->base.ctxr_dma);
724 if (!ctx->base.ctxr)
725 return -ENOMEM;
726 }
727 req->not_first = true;
728
729 ring = ctx->base.ring;
730
731 spin_lock_bh(&priv->ring[ring].queue_lock);
732 ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
733 spin_unlock_bh(&priv->ring[ring].queue_lock);
734
735 queue_work(priv->ring[ring].workqueue,
736 &priv->ring[ring].work_data.work);
737
738 return ret;
739 }
740
741 static int safexcel_ahash_update(struct ahash_request *areq)
742 {
743 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
744 int ret;
745
746 /* If the request is 0 length, do nothing */
747 if (!areq->nbytes)
748 return 0;
749
750 /* Add request to the cache if it fits */
751 ret = safexcel_ahash_cache(areq);
752
753 /* Update total request length */
754 req->len += areq->nbytes;
755
756 /* If not all data could fit into the cache, go process the excess.
757 * Also go process immediately for an HMAC IV precompute, which
758 * will never be finished at all, but needs to be processed anyway.
759 */
760 if ((ret && !req->finish) || req->last_req)
761 return safexcel_ahash_enqueue(areq);
762
763 return 0;
764 }
765
766 static int safexcel_ahash_final(struct ahash_request *areq)
767 {
768 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
769 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
770
771 req->finish = true;
772
773 if (unlikely(!req->len && !areq->nbytes)) {
774 /*
775 * If we have an overall 0 length *hash* request:
776 * The HW cannot do 0 length hash, so we provide the correct
777 * result directly here.
778 */
779 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
780 memcpy(areq->result, md5_zero_message_hash,
781 MD5_DIGEST_SIZE);
782 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
783 memcpy(areq->result, sha1_zero_message_hash,
784 SHA1_DIGEST_SIZE);
785 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
786 memcpy(areq->result, sha224_zero_message_hash,
787 SHA224_DIGEST_SIZE);
788 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
789 memcpy(areq->result, sha256_zero_message_hash,
790 SHA256_DIGEST_SIZE);
791 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
792 memcpy(areq->result, sha384_zero_message_hash,
793 SHA384_DIGEST_SIZE);
794 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
795 memcpy(areq->result, sha512_zero_message_hash,
796 SHA512_DIGEST_SIZE);
797 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) {
798 memcpy(areq->result,
799 EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE);
800 }
801
802 return 0;
803 } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&
804 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
805 req->len == sizeof(u32) && !areq->nbytes)) {
806 /* Zero length CRC32 */
807 memcpy(areq->result, &ctx->base.ipad, sizeof(u32));
808 return 0;
809 } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
810 !areq->nbytes)) {
811 /* Zero length CBC MAC */
812 memset(areq->result, 0, AES_BLOCK_SIZE);
813 return 0;
814 } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&
815 !areq->nbytes)) {
816 /* Zero length (X)CBC/CMAC */
817 int i;
818
819 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) {
820 u32 *result = (void *)areq->result;
821
822 /* K3 */
823 result[i] = swab(ctx->base.ipad.word[i + 4]);
824 }
825 areq->result[0] ^= 0x80; // 10- padding
826 crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result);
827 return 0;
828 } else if (unlikely(req->hmac &&
829 (req->len == req->block_sz) &&
830 !areq->nbytes)) {
831 /*
832 * If we have an overall 0 length *HMAC* request:
833 * For HMAC, we need to finalize the inner digest
834 * and then perform the outer hash.
835 */
836
837 /* generate pad block in the cache */
838 /* start with a hash block of all zeroes */
839 memset(req->cache, 0, req->block_sz);
840 /* set the first byte to 0x80 to 'append a 1 bit' */
841 req->cache[0] = 0x80;
842 /* add the length in bits in the last 2 bytes */
843 if (req->len_is_le) {
844 /* Little endian length word (e.g. MD5) */
845 req->cache[req->block_sz-8] = (req->block_sz << 3) &
846 255;
847 req->cache[req->block_sz-7] = (req->block_sz >> 5);
848 } else {
849 /* Big endian length word (e.g. any SHA) */
850 req->cache[req->block_sz-2] = (req->block_sz >> 5);
851 req->cache[req->block_sz-1] = (req->block_sz << 3) &
852 255;
853 }
854
855 req->len += req->block_sz; /* plus 1 hash block */
856
857 /* Set special zero-length HMAC flag */
858 req->hmac_zlen = true;
859
860 /* Finalize HMAC */
861 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
862 } else if (req->hmac) {
863 /* Finalize HMAC */
864 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
865 }
866
867 return safexcel_ahash_enqueue(areq);
868 }
869
870 static int safexcel_ahash_finup(struct ahash_request *areq)
871 {
872 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
873
874 req->finish = true;
875
876 safexcel_ahash_update(areq);
877 return safexcel_ahash_final(areq);
878 }
879
880 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
881 {
882 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
883 struct safexcel_ahash_export_state *export = out;
884
885 export->len = req->len;
886 export->processed = req->processed;
887
888 export->digest = req->digest;
889
890 memcpy(export->state, req->state, req->state_sz);
891 memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
892
893 return 0;
894 }
895
896 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
897 {
898 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
899 const struct safexcel_ahash_export_state *export = in;
900 int ret;
901
902 ret = crypto_ahash_init(areq);
903 if (ret)
904 return ret;
905
906 req->len = export->len;
907 req->processed = export->processed;
908
909 req->digest = export->digest;
910
911 memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
912 memcpy(req->state, export->state, req->state_sz);
913
914 return 0;
915 }
916
917 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
918 {
919 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
920 struct safexcel_alg_template *tmpl =
921 container_of(__crypto_ahash_alg(tfm->__crt_alg),
922 struct safexcel_alg_template, alg.ahash);
923
924 ctx->base.priv = tmpl->priv;
925 ctx->base.send = safexcel_ahash_send;
926 ctx->base.handle_result = safexcel_handle_result;
927 ctx->fb_do_setkey = false;
928
929 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
930 sizeof(struct safexcel_ahash_req));
931 return 0;
932 }
933
934 static int safexcel_sha1_init(struct ahash_request *areq)
935 {
936 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
937 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
938
939 memset(req, 0, sizeof(*req));
940
941 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
942 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
943 req->state_sz = SHA1_DIGEST_SIZE;
944 req->digest_sz = SHA1_DIGEST_SIZE;
945 req->block_sz = SHA1_BLOCK_SIZE;
946
947 return 0;
948 }
949
950 static int safexcel_sha1_digest(struct ahash_request *areq)
951 {
952 int ret = safexcel_sha1_init(areq);
953
954 if (ret)
955 return ret;
956
957 return safexcel_ahash_finup(areq);
958 }
959
960 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
961 {
962 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
963 struct safexcel_crypto_priv *priv = ctx->base.priv;
964 int ret;
965
966 /* context not allocated, skip invalidation */
967 if (!ctx->base.ctxr)
968 return;
969
970 if (priv->flags & EIP197_TRC_CACHE) {
971 ret = safexcel_ahash_exit_inv(tfm);
972 if (ret)
973 dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
974 } else {
975 dma_pool_free(priv->context_pool, ctx->base.ctxr,
976 ctx->base.ctxr_dma);
977 }
978 }
979
980 struct safexcel_alg_template safexcel_alg_sha1 = {
981 .type = SAFEXCEL_ALG_TYPE_AHASH,
982 .algo_mask = SAFEXCEL_ALG_SHA1,
983 .alg.ahash = {
984 .init = safexcel_sha1_init,
985 .update = safexcel_ahash_update,
986 .final = safexcel_ahash_final,
987 .finup = safexcel_ahash_finup,
988 .digest = safexcel_sha1_digest,
989 .export = safexcel_ahash_export,
990 .import = safexcel_ahash_import,
991 .halg = {
992 .digestsize = SHA1_DIGEST_SIZE,
993 .statesize = sizeof(struct safexcel_ahash_export_state),
994 .base = {
995 .cra_name = "sha1",
996 .cra_driver_name = "safexcel-sha1",
997 .cra_priority = SAFEXCEL_CRA_PRIORITY,
998 .cra_flags = CRYPTO_ALG_ASYNC |
999 CRYPTO_ALG_ALLOCATES_MEMORY |
1000 CRYPTO_ALG_KERN_DRIVER_ONLY,
1001 .cra_blocksize = SHA1_BLOCK_SIZE,
1002 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1003 .cra_init = safexcel_ahash_cra_init,
1004 .cra_exit = safexcel_ahash_cra_exit,
1005 .cra_module = THIS_MODULE,
1006 },
1007 },
1008 },
1009 };
1010
1011 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
1012 {
1013 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1014 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1015
1016 memset(req, 0, sizeof(*req));
1017
1018 /* Start from ipad precompute */
1019 memcpy(req->state, &ctx->base.ipad, SHA1_DIGEST_SIZE);
1020 /* Already processed the key^ipad part now! */
1021 req->len = SHA1_BLOCK_SIZE;
1022 req->processed = SHA1_BLOCK_SIZE;
1023
1024 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1025 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1026 req->state_sz = SHA1_DIGEST_SIZE;
1027 req->digest_sz = SHA1_DIGEST_SIZE;
1028 req->block_sz = SHA1_BLOCK_SIZE;
1029 req->hmac = true;
1030
1031 return 0;
1032 }
1033
1034 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
1035 {
1036 int ret = safexcel_hmac_sha1_init(areq);
1037
1038 if (ret)
1039 return ret;
1040
1041 return safexcel_ahash_finup(areq);
1042 }
1043
1044 struct safexcel_ahash_result {
1045 struct completion completion;
1046 int error;
1047 };
1048
1049 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
1050 {
1051 struct safexcel_ahash_result *result = req->data;
1052
1053 if (error == -EINPROGRESS)
1054 return;
1055
1056 result->error = error;
1057 complete(&result->completion);
1058 }
1059
1060 static int safexcel_hmac_init_pad(struct ahash_request *areq,
1061 unsigned int blocksize, const u8 *key,
1062 unsigned int keylen, u8 *ipad, u8 *opad)
1063 {
1064 struct safexcel_ahash_result result;
1065 struct scatterlist sg;
1066 int ret, i;
1067 u8 *keydup;
1068
1069 if (keylen <= blocksize) {
1070 memcpy(ipad, key, keylen);
1071 } else {
1072 keydup = kmemdup(key, keylen, GFP_KERNEL);
1073 if (!keydup)
1074 return -ENOMEM;
1075
1076 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1077 safexcel_ahash_complete, &result);
1078 sg_init_one(&sg, keydup, keylen);
1079 ahash_request_set_crypt(areq, &sg, ipad, keylen);
1080 init_completion(&result.completion);
1081
1082 ret = crypto_ahash_digest(areq);
1083 if (ret == -EINPROGRESS || ret == -EBUSY) {
1084 wait_for_completion_interruptible(&result.completion);
1085 ret = result.error;
1086 }
1087
1088 /* Avoid leaking */
1089 kfree_sensitive(keydup);
1090
1091 if (ret)
1092 return ret;
1093
1094 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1095 }
1096
1097 memset(ipad + keylen, 0, blocksize - keylen);
1098 memcpy(opad, ipad, blocksize);
1099
1100 for (i = 0; i < blocksize; i++) {
1101 ipad[i] ^= HMAC_IPAD_VALUE;
1102 opad[i] ^= HMAC_OPAD_VALUE;
1103 }
1104
1105 return 0;
1106 }
1107
1108 static int safexcel_hmac_init_iv(struct ahash_request *areq,
1109 unsigned int blocksize, u8 *pad, void *state)
1110 {
1111 struct safexcel_ahash_result result;
1112 struct safexcel_ahash_req *req;
1113 struct scatterlist sg;
1114 int ret;
1115
1116 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1117 safexcel_ahash_complete, &result);
1118 sg_init_one(&sg, pad, blocksize);
1119 ahash_request_set_crypt(areq, &sg, pad, blocksize);
1120 init_completion(&result.completion);
1121
1122 ret = crypto_ahash_init(areq);
1123 if (ret)
1124 return ret;
1125
1126 req = ahash_request_ctx(areq);
1127 req->hmac = true;
1128 req->last_req = true;
1129
1130 ret = crypto_ahash_update(areq);
1131 if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1132 return ret;
1133
1134 wait_for_completion_interruptible(&result.completion);
1135 if (result.error)
1136 return result.error;
1137
1138 return crypto_ahash_export(areq, state);
1139 }
1140
1141 static int __safexcel_hmac_setkey(const char *alg, const u8 *key,
1142 unsigned int keylen,
1143 void *istate, void *ostate)
1144 {
1145 struct ahash_request *areq;
1146 struct crypto_ahash *tfm;
1147 unsigned int blocksize;
1148 u8 *ipad, *opad;
1149 int ret;
1150
1151 tfm = crypto_alloc_ahash(alg, 0, 0);
1152 if (IS_ERR(tfm))
1153 return PTR_ERR(tfm);
1154
1155 areq = ahash_request_alloc(tfm, GFP_KERNEL);
1156 if (!areq) {
1157 ret = -ENOMEM;
1158 goto free_ahash;
1159 }
1160
1161 crypto_ahash_clear_flags(tfm, ~0);
1162 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1163
1164 ipad = kcalloc(2, blocksize, GFP_KERNEL);
1165 if (!ipad) {
1166 ret = -ENOMEM;
1167 goto free_request;
1168 }
1169
1170 opad = ipad + blocksize;
1171
1172 ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
1173 if (ret)
1174 goto free_ipad;
1175
1176 ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
1177 if (ret)
1178 goto free_ipad;
1179
1180 ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
1181
1182 free_ipad:
1183 kfree(ipad);
1184 free_request:
1185 ahash_request_free(areq);
1186 free_ahash:
1187 crypto_free_ahash(tfm);
1188
1189 return ret;
1190 }
1191
1192 int safexcel_hmac_setkey(struct safexcel_context *base, const u8 *key,
1193 unsigned int keylen, const char *alg,
1194 unsigned int state_sz)
1195 {
1196 struct safexcel_crypto_priv *priv = base->priv;
1197 struct safexcel_ahash_export_state istate, ostate;
1198 int ret;
1199
1200 ret = __safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
1201 if (ret)
1202 return ret;
1203
1204 if (priv->flags & EIP197_TRC_CACHE && base->ctxr &&
1205 (memcmp(&base->ipad, istate.state, state_sz) ||
1206 memcmp(&base->opad, ostate.state, state_sz)))
1207 base->needs_inv = true;
1208
1209 memcpy(&base->ipad, &istate.state, state_sz);
1210 memcpy(&base->opad, &ostate.state, state_sz);
1211
1212 return 0;
1213 }
1214
1215 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
1216 unsigned int keylen, const char *alg,
1217 unsigned int state_sz)
1218 {
1219 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
1220
1221 return safexcel_hmac_setkey(&ctx->base, key, keylen, alg, state_sz);
1222 }
1223
1224 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
1225 unsigned int keylen)
1226 {
1227 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
1228 SHA1_DIGEST_SIZE);
1229 }
1230
1231 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
1232 .type = SAFEXCEL_ALG_TYPE_AHASH,
1233 .algo_mask = SAFEXCEL_ALG_SHA1,
1234 .alg.ahash = {
1235 .init = safexcel_hmac_sha1_init,
1236 .update = safexcel_ahash_update,
1237 .final = safexcel_ahash_final,
1238 .finup = safexcel_ahash_finup,
1239 .digest = safexcel_hmac_sha1_digest,
1240 .setkey = safexcel_hmac_sha1_setkey,
1241 .export = safexcel_ahash_export,
1242 .import = safexcel_ahash_import,
1243 .halg = {
1244 .digestsize = SHA1_DIGEST_SIZE,
1245 .statesize = sizeof(struct safexcel_ahash_export_state),
1246 .base = {
1247 .cra_name = "hmac(sha1)",
1248 .cra_driver_name = "safexcel-hmac-sha1",
1249 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1250 .cra_flags = CRYPTO_ALG_ASYNC |
1251 CRYPTO_ALG_ALLOCATES_MEMORY |
1252 CRYPTO_ALG_KERN_DRIVER_ONLY,
1253 .cra_blocksize = SHA1_BLOCK_SIZE,
1254 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1255 .cra_init = safexcel_ahash_cra_init,
1256 .cra_exit = safexcel_ahash_cra_exit,
1257 .cra_module = THIS_MODULE,
1258 },
1259 },
1260 },
1261 };
1262
1263 static int safexcel_sha256_init(struct ahash_request *areq)
1264 {
1265 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1266 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1267
1268 memset(req, 0, sizeof(*req));
1269
1270 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1271 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1272 req->state_sz = SHA256_DIGEST_SIZE;
1273 req->digest_sz = SHA256_DIGEST_SIZE;
1274 req->block_sz = SHA256_BLOCK_SIZE;
1275
1276 return 0;
1277 }
1278
1279 static int safexcel_sha256_digest(struct ahash_request *areq)
1280 {
1281 int ret = safexcel_sha256_init(areq);
1282
1283 if (ret)
1284 return ret;
1285
1286 return safexcel_ahash_finup(areq);
1287 }
1288
1289 struct safexcel_alg_template safexcel_alg_sha256 = {
1290 .type = SAFEXCEL_ALG_TYPE_AHASH,
1291 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1292 .alg.ahash = {
1293 .init = safexcel_sha256_init,
1294 .update = safexcel_ahash_update,
1295 .final = safexcel_ahash_final,
1296 .finup = safexcel_ahash_finup,
1297 .digest = safexcel_sha256_digest,
1298 .export = safexcel_ahash_export,
1299 .import = safexcel_ahash_import,
1300 .halg = {
1301 .digestsize = SHA256_DIGEST_SIZE,
1302 .statesize = sizeof(struct safexcel_ahash_export_state),
1303 .base = {
1304 .cra_name = "sha256",
1305 .cra_driver_name = "safexcel-sha256",
1306 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1307 .cra_flags = CRYPTO_ALG_ASYNC |
1308 CRYPTO_ALG_ALLOCATES_MEMORY |
1309 CRYPTO_ALG_KERN_DRIVER_ONLY,
1310 .cra_blocksize = SHA256_BLOCK_SIZE,
1311 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1312 .cra_init = safexcel_ahash_cra_init,
1313 .cra_exit = safexcel_ahash_cra_exit,
1314 .cra_module = THIS_MODULE,
1315 },
1316 },
1317 },
1318 };
1319
1320 static int safexcel_sha224_init(struct ahash_request *areq)
1321 {
1322 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1323 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1324
1325 memset(req, 0, sizeof(*req));
1326
1327 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1328 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1329 req->state_sz = SHA256_DIGEST_SIZE;
1330 req->digest_sz = SHA256_DIGEST_SIZE;
1331 req->block_sz = SHA256_BLOCK_SIZE;
1332
1333 return 0;
1334 }
1335
1336 static int safexcel_sha224_digest(struct ahash_request *areq)
1337 {
1338 int ret = safexcel_sha224_init(areq);
1339
1340 if (ret)
1341 return ret;
1342
1343 return safexcel_ahash_finup(areq);
1344 }
1345
1346 struct safexcel_alg_template safexcel_alg_sha224 = {
1347 .type = SAFEXCEL_ALG_TYPE_AHASH,
1348 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1349 .alg.ahash = {
1350 .init = safexcel_sha224_init,
1351 .update = safexcel_ahash_update,
1352 .final = safexcel_ahash_final,
1353 .finup = safexcel_ahash_finup,
1354 .digest = safexcel_sha224_digest,
1355 .export = safexcel_ahash_export,
1356 .import = safexcel_ahash_import,
1357 .halg = {
1358 .digestsize = SHA224_DIGEST_SIZE,
1359 .statesize = sizeof(struct safexcel_ahash_export_state),
1360 .base = {
1361 .cra_name = "sha224",
1362 .cra_driver_name = "safexcel-sha224",
1363 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1364 .cra_flags = CRYPTO_ALG_ASYNC |
1365 CRYPTO_ALG_ALLOCATES_MEMORY |
1366 CRYPTO_ALG_KERN_DRIVER_ONLY,
1367 .cra_blocksize = SHA224_BLOCK_SIZE,
1368 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1369 .cra_init = safexcel_ahash_cra_init,
1370 .cra_exit = safexcel_ahash_cra_exit,
1371 .cra_module = THIS_MODULE,
1372 },
1373 },
1374 },
1375 };
1376
1377 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
1378 unsigned int keylen)
1379 {
1380 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
1381 SHA256_DIGEST_SIZE);
1382 }
1383
1384 static int safexcel_hmac_sha224_init(struct ahash_request *areq)
1385 {
1386 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1387 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1388
1389 memset(req, 0, sizeof(*req));
1390
1391 /* Start from ipad precompute */
1392 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1393 /* Already processed the key^ipad part now! */
1394 req->len = SHA256_BLOCK_SIZE;
1395 req->processed = SHA256_BLOCK_SIZE;
1396
1397 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1398 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1399 req->state_sz = SHA256_DIGEST_SIZE;
1400 req->digest_sz = SHA256_DIGEST_SIZE;
1401 req->block_sz = SHA256_BLOCK_SIZE;
1402 req->hmac = true;
1403
1404 return 0;
1405 }
1406
1407 static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
1408 {
1409 int ret = safexcel_hmac_sha224_init(areq);
1410
1411 if (ret)
1412 return ret;
1413
1414 return safexcel_ahash_finup(areq);
1415 }
1416
1417 struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
1418 .type = SAFEXCEL_ALG_TYPE_AHASH,
1419 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1420 .alg.ahash = {
1421 .init = safexcel_hmac_sha224_init,
1422 .update = safexcel_ahash_update,
1423 .final = safexcel_ahash_final,
1424 .finup = safexcel_ahash_finup,
1425 .digest = safexcel_hmac_sha224_digest,
1426 .setkey = safexcel_hmac_sha224_setkey,
1427 .export = safexcel_ahash_export,
1428 .import = safexcel_ahash_import,
1429 .halg = {
1430 .digestsize = SHA224_DIGEST_SIZE,
1431 .statesize = sizeof(struct safexcel_ahash_export_state),
1432 .base = {
1433 .cra_name = "hmac(sha224)",
1434 .cra_driver_name = "safexcel-hmac-sha224",
1435 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1436 .cra_flags = CRYPTO_ALG_ASYNC |
1437 CRYPTO_ALG_ALLOCATES_MEMORY |
1438 CRYPTO_ALG_KERN_DRIVER_ONLY,
1439 .cra_blocksize = SHA224_BLOCK_SIZE,
1440 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1441 .cra_init = safexcel_ahash_cra_init,
1442 .cra_exit = safexcel_ahash_cra_exit,
1443 .cra_module = THIS_MODULE,
1444 },
1445 },
1446 },
1447 };
1448
1449 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
1450 unsigned int keylen)
1451 {
1452 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
1453 SHA256_DIGEST_SIZE);
1454 }
1455
1456 static int safexcel_hmac_sha256_init(struct ahash_request *areq)
1457 {
1458 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1459 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1460
1461 memset(req, 0, sizeof(*req));
1462
1463 /* Start from ipad precompute */
1464 memcpy(req->state, &ctx->base.ipad, SHA256_DIGEST_SIZE);
1465 /* Already processed the key^ipad part now! */
1466 req->len = SHA256_BLOCK_SIZE;
1467 req->processed = SHA256_BLOCK_SIZE;
1468
1469 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1470 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1471 req->state_sz = SHA256_DIGEST_SIZE;
1472 req->digest_sz = SHA256_DIGEST_SIZE;
1473 req->block_sz = SHA256_BLOCK_SIZE;
1474 req->hmac = true;
1475
1476 return 0;
1477 }
1478
1479 static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
1480 {
1481 int ret = safexcel_hmac_sha256_init(areq);
1482
1483 if (ret)
1484 return ret;
1485
1486 return safexcel_ahash_finup(areq);
1487 }
1488
1489 struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
1490 .type = SAFEXCEL_ALG_TYPE_AHASH,
1491 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1492 .alg.ahash = {
1493 .init = safexcel_hmac_sha256_init,
1494 .update = safexcel_ahash_update,
1495 .final = safexcel_ahash_final,
1496 .finup = safexcel_ahash_finup,
1497 .digest = safexcel_hmac_sha256_digest,
1498 .setkey = safexcel_hmac_sha256_setkey,
1499 .export = safexcel_ahash_export,
1500 .import = safexcel_ahash_import,
1501 .halg = {
1502 .digestsize = SHA256_DIGEST_SIZE,
1503 .statesize = sizeof(struct safexcel_ahash_export_state),
1504 .base = {
1505 .cra_name = "hmac(sha256)",
1506 .cra_driver_name = "safexcel-hmac-sha256",
1507 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1508 .cra_flags = CRYPTO_ALG_ASYNC |
1509 CRYPTO_ALG_ALLOCATES_MEMORY |
1510 CRYPTO_ALG_KERN_DRIVER_ONLY,
1511 .cra_blocksize = SHA256_BLOCK_SIZE,
1512 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1513 .cra_init = safexcel_ahash_cra_init,
1514 .cra_exit = safexcel_ahash_cra_exit,
1515 .cra_module = THIS_MODULE,
1516 },
1517 },
1518 },
1519 };
1520
1521 static int safexcel_sha512_init(struct ahash_request *areq)
1522 {
1523 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1524 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1525
1526 memset(req, 0, sizeof(*req));
1527
1528 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1529 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1530 req->state_sz = SHA512_DIGEST_SIZE;
1531 req->digest_sz = SHA512_DIGEST_SIZE;
1532 req->block_sz = SHA512_BLOCK_SIZE;
1533
1534 return 0;
1535 }
1536
1537 static int safexcel_sha512_digest(struct ahash_request *areq)
1538 {
1539 int ret = safexcel_sha512_init(areq);
1540
1541 if (ret)
1542 return ret;
1543
1544 return safexcel_ahash_finup(areq);
1545 }
1546
1547 struct safexcel_alg_template safexcel_alg_sha512 = {
1548 .type = SAFEXCEL_ALG_TYPE_AHASH,
1549 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1550 .alg.ahash = {
1551 .init = safexcel_sha512_init,
1552 .update = safexcel_ahash_update,
1553 .final = safexcel_ahash_final,
1554 .finup = safexcel_ahash_finup,
1555 .digest = safexcel_sha512_digest,
1556 .export = safexcel_ahash_export,
1557 .import = safexcel_ahash_import,
1558 .halg = {
1559 .digestsize = SHA512_DIGEST_SIZE,
1560 .statesize = sizeof(struct safexcel_ahash_export_state),
1561 .base = {
1562 .cra_name = "sha512",
1563 .cra_driver_name = "safexcel-sha512",
1564 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1565 .cra_flags = CRYPTO_ALG_ASYNC |
1566 CRYPTO_ALG_ALLOCATES_MEMORY |
1567 CRYPTO_ALG_KERN_DRIVER_ONLY,
1568 .cra_blocksize = SHA512_BLOCK_SIZE,
1569 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1570 .cra_init = safexcel_ahash_cra_init,
1571 .cra_exit = safexcel_ahash_cra_exit,
1572 .cra_module = THIS_MODULE,
1573 },
1574 },
1575 },
1576 };
1577
1578 static int safexcel_sha384_init(struct ahash_request *areq)
1579 {
1580 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1581 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1582
1583 memset(req, 0, sizeof(*req));
1584
1585 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1586 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1587 req->state_sz = SHA512_DIGEST_SIZE;
1588 req->digest_sz = SHA512_DIGEST_SIZE;
1589 req->block_sz = SHA512_BLOCK_SIZE;
1590
1591 return 0;
1592 }
1593
1594 static int safexcel_sha384_digest(struct ahash_request *areq)
1595 {
1596 int ret = safexcel_sha384_init(areq);
1597
1598 if (ret)
1599 return ret;
1600
1601 return safexcel_ahash_finup(areq);
1602 }
1603
1604 struct safexcel_alg_template safexcel_alg_sha384 = {
1605 .type = SAFEXCEL_ALG_TYPE_AHASH,
1606 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1607 .alg.ahash = {
1608 .init = safexcel_sha384_init,
1609 .update = safexcel_ahash_update,
1610 .final = safexcel_ahash_final,
1611 .finup = safexcel_ahash_finup,
1612 .digest = safexcel_sha384_digest,
1613 .export = safexcel_ahash_export,
1614 .import = safexcel_ahash_import,
1615 .halg = {
1616 .digestsize = SHA384_DIGEST_SIZE,
1617 .statesize = sizeof(struct safexcel_ahash_export_state),
1618 .base = {
1619 .cra_name = "sha384",
1620 .cra_driver_name = "safexcel-sha384",
1621 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1622 .cra_flags = CRYPTO_ALG_ASYNC |
1623 CRYPTO_ALG_ALLOCATES_MEMORY |
1624 CRYPTO_ALG_KERN_DRIVER_ONLY,
1625 .cra_blocksize = SHA384_BLOCK_SIZE,
1626 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1627 .cra_init = safexcel_ahash_cra_init,
1628 .cra_exit = safexcel_ahash_cra_exit,
1629 .cra_module = THIS_MODULE,
1630 },
1631 },
1632 },
1633 };
1634
1635 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
1636 unsigned int keylen)
1637 {
1638 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
1639 SHA512_DIGEST_SIZE);
1640 }
1641
1642 static int safexcel_hmac_sha512_init(struct ahash_request *areq)
1643 {
1644 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1645 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1646
1647 memset(req, 0, sizeof(*req));
1648
1649 /* Start from ipad precompute */
1650 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1651 /* Already processed the key^ipad part now! */
1652 req->len = SHA512_BLOCK_SIZE;
1653 req->processed = SHA512_BLOCK_SIZE;
1654
1655 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1656 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1657 req->state_sz = SHA512_DIGEST_SIZE;
1658 req->digest_sz = SHA512_DIGEST_SIZE;
1659 req->block_sz = SHA512_BLOCK_SIZE;
1660 req->hmac = true;
1661
1662 return 0;
1663 }
1664
1665 static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
1666 {
1667 int ret = safexcel_hmac_sha512_init(areq);
1668
1669 if (ret)
1670 return ret;
1671
1672 return safexcel_ahash_finup(areq);
1673 }
1674
1675 struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
1676 .type = SAFEXCEL_ALG_TYPE_AHASH,
1677 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1678 .alg.ahash = {
1679 .init = safexcel_hmac_sha512_init,
1680 .update = safexcel_ahash_update,
1681 .final = safexcel_ahash_final,
1682 .finup = safexcel_ahash_finup,
1683 .digest = safexcel_hmac_sha512_digest,
1684 .setkey = safexcel_hmac_sha512_setkey,
1685 .export = safexcel_ahash_export,
1686 .import = safexcel_ahash_import,
1687 .halg = {
1688 .digestsize = SHA512_DIGEST_SIZE,
1689 .statesize = sizeof(struct safexcel_ahash_export_state),
1690 .base = {
1691 .cra_name = "hmac(sha512)",
1692 .cra_driver_name = "safexcel-hmac-sha512",
1693 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1694 .cra_flags = CRYPTO_ALG_ASYNC |
1695 CRYPTO_ALG_ALLOCATES_MEMORY |
1696 CRYPTO_ALG_KERN_DRIVER_ONLY,
1697 .cra_blocksize = SHA512_BLOCK_SIZE,
1698 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1699 .cra_init = safexcel_ahash_cra_init,
1700 .cra_exit = safexcel_ahash_cra_exit,
1701 .cra_module = THIS_MODULE,
1702 },
1703 },
1704 },
1705 };
1706
1707 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
1708 unsigned int keylen)
1709 {
1710 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
1711 SHA512_DIGEST_SIZE);
1712 }
1713
1714 static int safexcel_hmac_sha384_init(struct ahash_request *areq)
1715 {
1716 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1717 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1718
1719 memset(req, 0, sizeof(*req));
1720
1721 /* Start from ipad precompute */
1722 memcpy(req->state, &ctx->base.ipad, SHA512_DIGEST_SIZE);
1723 /* Already processed the key^ipad part now! */
1724 req->len = SHA512_BLOCK_SIZE;
1725 req->processed = SHA512_BLOCK_SIZE;
1726
1727 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1728 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1729 req->state_sz = SHA512_DIGEST_SIZE;
1730 req->digest_sz = SHA512_DIGEST_SIZE;
1731 req->block_sz = SHA512_BLOCK_SIZE;
1732 req->hmac = true;
1733
1734 return 0;
1735 }
1736
1737 static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
1738 {
1739 int ret = safexcel_hmac_sha384_init(areq);
1740
1741 if (ret)
1742 return ret;
1743
1744 return safexcel_ahash_finup(areq);
1745 }
1746
1747 struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
1748 .type = SAFEXCEL_ALG_TYPE_AHASH,
1749 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1750 .alg.ahash = {
1751 .init = safexcel_hmac_sha384_init,
1752 .update = safexcel_ahash_update,
1753 .final = safexcel_ahash_final,
1754 .finup = safexcel_ahash_finup,
1755 .digest = safexcel_hmac_sha384_digest,
1756 .setkey = safexcel_hmac_sha384_setkey,
1757 .export = safexcel_ahash_export,
1758 .import = safexcel_ahash_import,
1759 .halg = {
1760 .digestsize = SHA384_DIGEST_SIZE,
1761 .statesize = sizeof(struct safexcel_ahash_export_state),
1762 .base = {
1763 .cra_name = "hmac(sha384)",
1764 .cra_driver_name = "safexcel-hmac-sha384",
1765 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1766 .cra_flags = CRYPTO_ALG_ASYNC |
1767 CRYPTO_ALG_ALLOCATES_MEMORY |
1768 CRYPTO_ALG_KERN_DRIVER_ONLY,
1769 .cra_blocksize = SHA384_BLOCK_SIZE,
1770 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1771 .cra_init = safexcel_ahash_cra_init,
1772 .cra_exit = safexcel_ahash_cra_exit,
1773 .cra_module = THIS_MODULE,
1774 },
1775 },
1776 },
1777 };
1778
1779 static int safexcel_md5_init(struct ahash_request *areq)
1780 {
1781 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1782 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1783
1784 memset(req, 0, sizeof(*req));
1785
1786 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1787 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1788 req->state_sz = MD5_DIGEST_SIZE;
1789 req->digest_sz = MD5_DIGEST_SIZE;
1790 req->block_sz = MD5_HMAC_BLOCK_SIZE;
1791
1792 return 0;
1793 }
1794
1795 static int safexcel_md5_digest(struct ahash_request *areq)
1796 {
1797 int ret = safexcel_md5_init(areq);
1798
1799 if (ret)
1800 return ret;
1801
1802 return safexcel_ahash_finup(areq);
1803 }
1804
1805 struct safexcel_alg_template safexcel_alg_md5 = {
1806 .type = SAFEXCEL_ALG_TYPE_AHASH,
1807 .algo_mask = SAFEXCEL_ALG_MD5,
1808 .alg.ahash = {
1809 .init = safexcel_md5_init,
1810 .update = safexcel_ahash_update,
1811 .final = safexcel_ahash_final,
1812 .finup = safexcel_ahash_finup,
1813 .digest = safexcel_md5_digest,
1814 .export = safexcel_ahash_export,
1815 .import = safexcel_ahash_import,
1816 .halg = {
1817 .digestsize = MD5_DIGEST_SIZE,
1818 .statesize = sizeof(struct safexcel_ahash_export_state),
1819 .base = {
1820 .cra_name = "md5",
1821 .cra_driver_name = "safexcel-md5",
1822 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1823 .cra_flags = CRYPTO_ALG_ASYNC |
1824 CRYPTO_ALG_ALLOCATES_MEMORY |
1825 CRYPTO_ALG_KERN_DRIVER_ONLY,
1826 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1827 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1828 .cra_init = safexcel_ahash_cra_init,
1829 .cra_exit = safexcel_ahash_cra_exit,
1830 .cra_module = THIS_MODULE,
1831 },
1832 },
1833 },
1834 };
1835
1836 static int safexcel_hmac_md5_init(struct ahash_request *areq)
1837 {
1838 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1839 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1840
1841 memset(req, 0, sizeof(*req));
1842
1843 /* Start from ipad precompute */
1844 memcpy(req->state, &ctx->base.ipad, MD5_DIGEST_SIZE);
1845 /* Already processed the key^ipad part now! */
1846 req->len = MD5_HMAC_BLOCK_SIZE;
1847 req->processed = MD5_HMAC_BLOCK_SIZE;
1848
1849 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1850 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1851 req->state_sz = MD5_DIGEST_SIZE;
1852 req->digest_sz = MD5_DIGEST_SIZE;
1853 req->block_sz = MD5_HMAC_BLOCK_SIZE;
1854 req->len_is_le = true; /* MD5 is little endian! ... */
1855 req->hmac = true;
1856
1857 return 0;
1858 }
1859
1860 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
1861 unsigned int keylen)
1862 {
1863 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
1864 MD5_DIGEST_SIZE);
1865 }
1866
1867 static int safexcel_hmac_md5_digest(struct ahash_request *areq)
1868 {
1869 int ret = safexcel_hmac_md5_init(areq);
1870
1871 if (ret)
1872 return ret;
1873
1874 return safexcel_ahash_finup(areq);
1875 }
1876
1877 struct safexcel_alg_template safexcel_alg_hmac_md5 = {
1878 .type = SAFEXCEL_ALG_TYPE_AHASH,
1879 .algo_mask = SAFEXCEL_ALG_MD5,
1880 .alg.ahash = {
1881 .init = safexcel_hmac_md5_init,
1882 .update = safexcel_ahash_update,
1883 .final = safexcel_ahash_final,
1884 .finup = safexcel_ahash_finup,
1885 .digest = safexcel_hmac_md5_digest,
1886 .setkey = safexcel_hmac_md5_setkey,
1887 .export = safexcel_ahash_export,
1888 .import = safexcel_ahash_import,
1889 .halg = {
1890 .digestsize = MD5_DIGEST_SIZE,
1891 .statesize = sizeof(struct safexcel_ahash_export_state),
1892 .base = {
1893 .cra_name = "hmac(md5)",
1894 .cra_driver_name = "safexcel-hmac-md5",
1895 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1896 .cra_flags = CRYPTO_ALG_ASYNC |
1897 CRYPTO_ALG_ALLOCATES_MEMORY |
1898 CRYPTO_ALG_KERN_DRIVER_ONLY,
1899 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1900 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1901 .cra_init = safexcel_ahash_cra_init,
1902 .cra_exit = safexcel_ahash_cra_exit,
1903 .cra_module = THIS_MODULE,
1904 },
1905 },
1906 },
1907 };
1908
1909 static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
1910 {
1911 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
1912 int ret = safexcel_ahash_cra_init(tfm);
1913
1914 /* Default 'key' is all zeroes */
1915 memset(&ctx->base.ipad, 0, sizeof(u32));
1916 return ret;
1917 }
1918
1919 static int safexcel_crc32_init(struct ahash_request *areq)
1920 {
1921 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1922 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1923
1924 memset(req, 0, sizeof(*req));
1925
1926 /* Start from loaded key */
1927 req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]);
1928 /* Set processed to non-zero to enable invalidation detection */
1929 req->len = sizeof(u32);
1930 req->processed = sizeof(u32);
1931
1932 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32;
1933 req->digest = CONTEXT_CONTROL_DIGEST_XCM;
1934 req->state_sz = sizeof(u32);
1935 req->digest_sz = sizeof(u32);
1936 req->block_sz = sizeof(u32);
1937
1938 return 0;
1939 }
1940
1941 static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key,
1942 unsigned int keylen)
1943 {
1944 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1945
1946 if (keylen != sizeof(u32))
1947 return -EINVAL;
1948
1949 memcpy(&ctx->base.ipad, key, sizeof(u32));
1950 return 0;
1951 }
1952
1953 static int safexcel_crc32_digest(struct ahash_request *areq)
1954 {
1955 return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);
1956 }
1957
1958 struct safexcel_alg_template safexcel_alg_crc32 = {
1959 .type = SAFEXCEL_ALG_TYPE_AHASH,
1960 .algo_mask = 0,
1961 .alg.ahash = {
1962 .init = safexcel_crc32_init,
1963 .update = safexcel_ahash_update,
1964 .final = safexcel_ahash_final,
1965 .finup = safexcel_ahash_finup,
1966 .digest = safexcel_crc32_digest,
1967 .setkey = safexcel_crc32_setkey,
1968 .export = safexcel_ahash_export,
1969 .import = safexcel_ahash_import,
1970 .halg = {
1971 .digestsize = sizeof(u32),
1972 .statesize = sizeof(struct safexcel_ahash_export_state),
1973 .base = {
1974 .cra_name = "crc32",
1975 .cra_driver_name = "safexcel-crc32",
1976 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1977 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY |
1978 CRYPTO_ALG_ASYNC |
1979 CRYPTO_ALG_ALLOCATES_MEMORY |
1980 CRYPTO_ALG_KERN_DRIVER_ONLY,
1981 .cra_blocksize = 1,
1982 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1983 .cra_init = safexcel_crc32_cra_init,
1984 .cra_exit = safexcel_ahash_cra_exit,
1985 .cra_module = THIS_MODULE,
1986 },
1987 },
1988 },
1989 };
1990
1991 static int safexcel_cbcmac_init(struct ahash_request *areq)
1992 {
1993 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1994 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1995
1996 memset(req, 0, sizeof(*req));
1997
1998 /* Start from loaded keys */
1999 memcpy(req->state, &ctx->base.ipad, ctx->key_sz);
2000 /* Set processed to non-zero to enable invalidation detection */
2001 req->len = AES_BLOCK_SIZE;
2002 req->processed = AES_BLOCK_SIZE;
2003
2004 req->digest = CONTEXT_CONTROL_DIGEST_XCM;
2005 req->state_sz = ctx->key_sz;
2006 req->digest_sz = AES_BLOCK_SIZE;
2007 req->block_sz = AES_BLOCK_SIZE;
2008 req->xcbcmac = true;
2009
2010 return 0;
2011 }
2012
2013 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2014 unsigned int len)
2015 {
2016 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2017 struct crypto_aes_ctx aes;
2018 int ret, i;
2019
2020 ret = aes_expandkey(&aes, key, len);
2021 if (ret)
2022 return ret;
2023
2024 memset(&ctx->base.ipad, 0, 2 * AES_BLOCK_SIZE);
2025 for (i = 0; i < len / sizeof(u32); i++)
2026 ctx->base.ipad.be[i + 8] = cpu_to_be32(aes.key_enc[i]);
2027
2028 if (len == AES_KEYSIZE_192) {
2029 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2030 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2031 } else if (len == AES_KEYSIZE_256) {
2032 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2033 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2034 } else {
2035 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2036 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2037 }
2038 ctx->cbcmac = true;
2039
2040 memzero_explicit(&aes, sizeof(aes));
2041 return 0;
2042 }
2043
2044 static int safexcel_cbcmac_digest(struct ahash_request *areq)
2045 {
2046 return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);
2047 }
2048
2049 struct safexcel_alg_template safexcel_alg_cbcmac = {
2050 .type = SAFEXCEL_ALG_TYPE_AHASH,
2051 .algo_mask = 0,
2052 .alg.ahash = {
2053 .init = safexcel_cbcmac_init,
2054 .update = safexcel_ahash_update,
2055 .final = safexcel_ahash_final,
2056 .finup = safexcel_ahash_finup,
2057 .digest = safexcel_cbcmac_digest,
2058 .setkey = safexcel_cbcmac_setkey,
2059 .export = safexcel_ahash_export,
2060 .import = safexcel_ahash_import,
2061 .halg = {
2062 .digestsize = AES_BLOCK_SIZE,
2063 .statesize = sizeof(struct safexcel_ahash_export_state),
2064 .base = {
2065 .cra_name = "cbcmac(aes)",
2066 .cra_driver_name = "safexcel-cbcmac-aes",
2067 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2068 .cra_flags = CRYPTO_ALG_ASYNC |
2069 CRYPTO_ALG_ALLOCATES_MEMORY |
2070 CRYPTO_ALG_KERN_DRIVER_ONLY,
2071 .cra_blocksize = 1,
2072 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2073 .cra_init = safexcel_ahash_cra_init,
2074 .cra_exit = safexcel_ahash_cra_exit,
2075 .cra_module = THIS_MODULE,
2076 },
2077 },
2078 },
2079 };
2080
2081 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2082 unsigned int len)
2083 {
2084 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2085 struct crypto_aes_ctx aes;
2086 u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)];
2087 int ret, i;
2088
2089 ret = aes_expandkey(&aes, key, len);
2090 if (ret)
2091 return ret;
2092
2093 /* precompute the XCBC key material */
2094 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2095 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2096 CRYPTO_TFM_REQ_MASK);
2097 ret = crypto_cipher_setkey(ctx->kaes, key, len);
2098 if (ret)
2099 return ret;
2100
2101 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2102 "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1");
2103 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp,
2104 "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2");
2105 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE,
2106 "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
2107 for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
2108 ctx->base.ipad.word[i] = swab(key_tmp[i]);
2109
2110 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2111 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2112 CRYPTO_TFM_REQ_MASK);
2113 ret = crypto_cipher_setkey(ctx->kaes,
2114 (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2115 AES_MIN_KEY_SIZE);
2116 if (ret)
2117 return ret;
2118
2119 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2120 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2121 ctx->cbcmac = false;
2122
2123 memzero_explicit(&aes, sizeof(aes));
2124 return 0;
2125 }
2126
2127 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm)
2128 {
2129 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2130
2131 safexcel_ahash_cra_init(tfm);
2132 ctx->kaes = crypto_alloc_cipher("aes", 0, 0);
2133 return PTR_ERR_OR_ZERO(ctx->kaes);
2134 }
2135
2136 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm)
2137 {
2138 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2139
2140 crypto_free_cipher(ctx->kaes);
2141 safexcel_ahash_cra_exit(tfm);
2142 }
2143
2144 struct safexcel_alg_template safexcel_alg_xcbcmac = {
2145 .type = SAFEXCEL_ALG_TYPE_AHASH,
2146 .algo_mask = 0,
2147 .alg.ahash = {
2148 .init = safexcel_cbcmac_init,
2149 .update = safexcel_ahash_update,
2150 .final = safexcel_ahash_final,
2151 .finup = safexcel_ahash_finup,
2152 .digest = safexcel_cbcmac_digest,
2153 .setkey = safexcel_xcbcmac_setkey,
2154 .export = safexcel_ahash_export,
2155 .import = safexcel_ahash_import,
2156 .halg = {
2157 .digestsize = AES_BLOCK_SIZE,
2158 .statesize = sizeof(struct safexcel_ahash_export_state),
2159 .base = {
2160 .cra_name = "xcbc(aes)",
2161 .cra_driver_name = "safexcel-xcbc-aes",
2162 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2163 .cra_flags = CRYPTO_ALG_ASYNC |
2164 CRYPTO_ALG_ALLOCATES_MEMORY |
2165 CRYPTO_ALG_KERN_DRIVER_ONLY,
2166 .cra_blocksize = AES_BLOCK_SIZE,
2167 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2168 .cra_init = safexcel_xcbcmac_cra_init,
2169 .cra_exit = safexcel_xcbcmac_cra_exit,
2170 .cra_module = THIS_MODULE,
2171 },
2172 },
2173 },
2174 };
2175
2176 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2177 unsigned int len)
2178 {
2179 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2180 struct crypto_aes_ctx aes;
2181 __be64 consts[4];
2182 u64 _const[2];
2183 u8 msb_mask, gfmask;
2184 int ret, i;
2185
2186 ret = aes_expandkey(&aes, key, len);
2187 if (ret)
2188 return ret;
2189
2190 for (i = 0; i < len / sizeof(u32); i++)
2191 ctx->base.ipad.word[i + 8] = swab(aes.key_enc[i]);
2192
2193 /* precompute the CMAC key material */
2194 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2195 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2196 CRYPTO_TFM_REQ_MASK);
2197 ret = crypto_cipher_setkey(ctx->kaes, key, len);
2198 if (ret)
2199 return ret;
2200
2201 /* code below borrowed from crypto/cmac.c */
2202 /* encrypt the zero block */
2203 memset(consts, 0, AES_BLOCK_SIZE);
2204 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)consts, (u8 *)consts);
2205
2206 gfmask = 0x87;
2207 _const[0] = be64_to_cpu(consts[1]);
2208 _const[1] = be64_to_cpu(consts[0]);
2209
2210 /* gf(2^128) multiply zero-ciphertext with u and u^2 */
2211 for (i = 0; i < 4; i += 2) {
2212 msb_mask = ((s64)_const[1] >> 63) & gfmask;
2213 _const[1] = (_const[1] << 1) | (_const[0] >> 63);
2214 _const[0] = (_const[0] << 1) ^ msb_mask;
2215
2216 consts[i + 0] = cpu_to_be64(_const[1]);
2217 consts[i + 1] = cpu_to_be64(_const[0]);
2218 }
2219 /* end of code borrowed from crypto/cmac.c */
2220
2221 for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
2222 ctx->base.ipad.be[i] = cpu_to_be32(((u32 *)consts)[i]);
2223
2224 if (len == AES_KEYSIZE_192) {
2225 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2226 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2227 } else if (len == AES_KEYSIZE_256) {
2228 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2229 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2230 } else {
2231 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2232 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2233 }
2234 ctx->cbcmac = false;
2235
2236 memzero_explicit(&aes, sizeof(aes));
2237 return 0;
2238 }
2239
2240 struct safexcel_alg_template safexcel_alg_cmac = {
2241 .type = SAFEXCEL_ALG_TYPE_AHASH,
2242 .algo_mask = 0,
2243 .alg.ahash = {
2244 .init = safexcel_cbcmac_init,
2245 .update = safexcel_ahash_update,
2246 .final = safexcel_ahash_final,
2247 .finup = safexcel_ahash_finup,
2248 .digest = safexcel_cbcmac_digest,
2249 .setkey = safexcel_cmac_setkey,
2250 .export = safexcel_ahash_export,
2251 .import = safexcel_ahash_import,
2252 .halg = {
2253 .digestsize = AES_BLOCK_SIZE,
2254 .statesize = sizeof(struct safexcel_ahash_export_state),
2255 .base = {
2256 .cra_name = "cmac(aes)",
2257 .cra_driver_name = "safexcel-cmac-aes",
2258 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2259 .cra_flags = CRYPTO_ALG_ASYNC |
2260 CRYPTO_ALG_ALLOCATES_MEMORY |
2261 CRYPTO_ALG_KERN_DRIVER_ONLY,
2262 .cra_blocksize = AES_BLOCK_SIZE,
2263 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2264 .cra_init = safexcel_xcbcmac_cra_init,
2265 .cra_exit = safexcel_xcbcmac_cra_exit,
2266 .cra_module = THIS_MODULE,
2267 },
2268 },
2269 },
2270 };
2271
2272 static int safexcel_sm3_init(struct ahash_request *areq)
2273 {
2274 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2275 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2276
2277 memset(req, 0, sizeof(*req));
2278
2279 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2280 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2281 req->state_sz = SM3_DIGEST_SIZE;
2282 req->digest_sz = SM3_DIGEST_SIZE;
2283 req->block_sz = SM3_BLOCK_SIZE;
2284
2285 return 0;
2286 }
2287
2288 static int safexcel_sm3_digest(struct ahash_request *areq)
2289 {
2290 int ret = safexcel_sm3_init(areq);
2291
2292 if (ret)
2293 return ret;
2294
2295 return safexcel_ahash_finup(areq);
2296 }
2297
2298 struct safexcel_alg_template safexcel_alg_sm3 = {
2299 .type = SAFEXCEL_ALG_TYPE_AHASH,
2300 .algo_mask = SAFEXCEL_ALG_SM3,
2301 .alg.ahash = {
2302 .init = safexcel_sm3_init,
2303 .update = safexcel_ahash_update,
2304 .final = safexcel_ahash_final,
2305 .finup = safexcel_ahash_finup,
2306 .digest = safexcel_sm3_digest,
2307 .export = safexcel_ahash_export,
2308 .import = safexcel_ahash_import,
2309 .halg = {
2310 .digestsize = SM3_DIGEST_SIZE,
2311 .statesize = sizeof(struct safexcel_ahash_export_state),
2312 .base = {
2313 .cra_name = "sm3",
2314 .cra_driver_name = "safexcel-sm3",
2315 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2316 .cra_flags = CRYPTO_ALG_ASYNC |
2317 CRYPTO_ALG_ALLOCATES_MEMORY |
2318 CRYPTO_ALG_KERN_DRIVER_ONLY,
2319 .cra_blocksize = SM3_BLOCK_SIZE,
2320 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2321 .cra_init = safexcel_ahash_cra_init,
2322 .cra_exit = safexcel_ahash_cra_exit,
2323 .cra_module = THIS_MODULE,
2324 },
2325 },
2326 },
2327 };
2328
2329 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
2330 unsigned int keylen)
2331 {
2332 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3",
2333 SM3_DIGEST_SIZE);
2334 }
2335
2336 static int safexcel_hmac_sm3_init(struct ahash_request *areq)
2337 {
2338 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2339 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2340
2341 memset(req, 0, sizeof(*req));
2342
2343 /* Start from ipad precompute */
2344 memcpy(req->state, &ctx->base.ipad, SM3_DIGEST_SIZE);
2345 /* Already processed the key^ipad part now! */
2346 req->len = SM3_BLOCK_SIZE;
2347 req->processed = SM3_BLOCK_SIZE;
2348
2349 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2350 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2351 req->state_sz = SM3_DIGEST_SIZE;
2352 req->digest_sz = SM3_DIGEST_SIZE;
2353 req->block_sz = SM3_BLOCK_SIZE;
2354 req->hmac = true;
2355
2356 return 0;
2357 }
2358
2359 static int safexcel_hmac_sm3_digest(struct ahash_request *areq)
2360 {
2361 int ret = safexcel_hmac_sm3_init(areq);
2362
2363 if (ret)
2364 return ret;
2365
2366 return safexcel_ahash_finup(areq);
2367 }
2368
2369 struct safexcel_alg_template safexcel_alg_hmac_sm3 = {
2370 .type = SAFEXCEL_ALG_TYPE_AHASH,
2371 .algo_mask = SAFEXCEL_ALG_SM3,
2372 .alg.ahash = {
2373 .init = safexcel_hmac_sm3_init,
2374 .update = safexcel_ahash_update,
2375 .final = safexcel_ahash_final,
2376 .finup = safexcel_ahash_finup,
2377 .digest = safexcel_hmac_sm3_digest,
2378 .setkey = safexcel_hmac_sm3_setkey,
2379 .export = safexcel_ahash_export,
2380 .import = safexcel_ahash_import,
2381 .halg = {
2382 .digestsize = SM3_DIGEST_SIZE,
2383 .statesize = sizeof(struct safexcel_ahash_export_state),
2384 .base = {
2385 .cra_name = "hmac(sm3)",
2386 .cra_driver_name = "safexcel-hmac-sm3",
2387 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2388 .cra_flags = CRYPTO_ALG_ASYNC |
2389 CRYPTO_ALG_ALLOCATES_MEMORY |
2390 CRYPTO_ALG_KERN_DRIVER_ONLY,
2391 .cra_blocksize = SM3_BLOCK_SIZE,
2392 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2393 .cra_init = safexcel_ahash_cra_init,
2394 .cra_exit = safexcel_ahash_cra_exit,
2395 .cra_module = THIS_MODULE,
2396 },
2397 },
2398 },
2399 };
2400
2401 static int safexcel_sha3_224_init(struct ahash_request *areq)
2402 {
2403 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2404 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2405 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2406
2407 memset(req, 0, sizeof(*req));
2408
2409 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2410 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2411 req->state_sz = SHA3_224_DIGEST_SIZE;
2412 req->digest_sz = SHA3_224_DIGEST_SIZE;
2413 req->block_sz = SHA3_224_BLOCK_SIZE;
2414 ctx->do_fallback = false;
2415 ctx->fb_init_done = false;
2416 return 0;
2417 }
2418
2419 static int safexcel_sha3_fbcheck(struct ahash_request *req)
2420 {
2421 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2422 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2423 struct ahash_request *subreq = ahash_request_ctx(req);
2424 int ret = 0;
2425
2426 if (ctx->do_fallback) {
2427 ahash_request_set_tfm(subreq, ctx->fback);
2428 ahash_request_set_callback(subreq, req->base.flags,
2429 req->base.complete, req->base.data);
2430 ahash_request_set_crypt(subreq, req->src, req->result,
2431 req->nbytes);
2432 if (!ctx->fb_init_done) {
2433 if (ctx->fb_do_setkey) {
2434 /* Set fallback cipher HMAC key */
2435 u8 key[SHA3_224_BLOCK_SIZE];
2436
2437 memcpy(key, &ctx->base.ipad,
2438 crypto_ahash_blocksize(ctx->fback) / 2);
2439 memcpy(key +
2440 crypto_ahash_blocksize(ctx->fback) / 2,
2441 &ctx->base.opad,
2442 crypto_ahash_blocksize(ctx->fback) / 2);
2443 ret = crypto_ahash_setkey(ctx->fback, key,
2444 crypto_ahash_blocksize(ctx->fback));
2445 memzero_explicit(key,
2446 crypto_ahash_blocksize(ctx->fback));
2447 ctx->fb_do_setkey = false;
2448 }
2449 ret = ret ?: crypto_ahash_init(subreq);
2450 ctx->fb_init_done = true;
2451 }
2452 }
2453 return ret;
2454 }
2455
2456 static int safexcel_sha3_update(struct ahash_request *req)
2457 {
2458 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2459 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2460 struct ahash_request *subreq = ahash_request_ctx(req);
2461
2462 ctx->do_fallback = true;
2463 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
2464 }
2465
2466 static int safexcel_sha3_final(struct ahash_request *req)
2467 {
2468 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2469 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2470 struct ahash_request *subreq = ahash_request_ctx(req);
2471
2472 ctx->do_fallback = true;
2473 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
2474 }
2475
2476 static int safexcel_sha3_finup(struct ahash_request *req)
2477 {
2478 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2479 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2480 struct ahash_request *subreq = ahash_request_ctx(req);
2481
2482 ctx->do_fallback |= !req->nbytes;
2483 if (ctx->do_fallback)
2484 /* Update or ex/import happened or len 0, cannot use the HW */
2485 return safexcel_sha3_fbcheck(req) ?:
2486 crypto_ahash_finup(subreq);
2487 else
2488 return safexcel_ahash_finup(req);
2489 }
2490
2491 static int safexcel_sha3_digest_fallback(struct ahash_request *req)
2492 {
2493 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2494 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2495 struct ahash_request *subreq = ahash_request_ctx(req);
2496
2497 ctx->do_fallback = true;
2498 ctx->fb_init_done = false;
2499 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);
2500 }
2501
2502 static int safexcel_sha3_224_digest(struct ahash_request *req)
2503 {
2504 if (req->nbytes)
2505 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);
2506
2507 /* HW cannot do zero length hash, use fallback instead */
2508 return safexcel_sha3_digest_fallback(req);
2509 }
2510
2511 static int safexcel_sha3_export(struct ahash_request *req, void *out)
2512 {
2513 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2514 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2515 struct ahash_request *subreq = ahash_request_ctx(req);
2516
2517 ctx->do_fallback = true;
2518 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
2519 }
2520
2521 static int safexcel_sha3_import(struct ahash_request *req, const void *in)
2522 {
2523 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2524 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2525 struct ahash_request *subreq = ahash_request_ctx(req);
2526
2527 ctx->do_fallback = true;
2528 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
2529 // return safexcel_ahash_import(req, in);
2530 }
2531
2532 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
2533 {
2534 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
2535 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2536
2537 safexcel_ahash_cra_init(tfm);
2538
2539 /* Allocate fallback implementation */
2540 ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
2541 CRYPTO_ALG_ASYNC |
2542 CRYPTO_ALG_NEED_FALLBACK);
2543 if (IS_ERR(ctx->fback))
2544 return PTR_ERR(ctx->fback);
2545
2546 /* Update statesize from fallback algorithm! */
2547 crypto_hash_alg_common(ahash)->statesize =
2548 crypto_ahash_statesize(ctx->fback);
2549 crypto_ahash_set_reqsize(ahash, max(sizeof(struct safexcel_ahash_req),
2550 sizeof(struct ahash_request) +
2551 crypto_ahash_reqsize(ctx->fback)));
2552 return 0;
2553 }
2554
2555 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm)
2556 {
2557 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2558
2559 crypto_free_ahash(ctx->fback);
2560 safexcel_ahash_cra_exit(tfm);
2561 }
2562
2563 struct safexcel_alg_template safexcel_alg_sha3_224 = {
2564 .type = SAFEXCEL_ALG_TYPE_AHASH,
2565 .algo_mask = SAFEXCEL_ALG_SHA3,
2566 .alg.ahash = {
2567 .init = safexcel_sha3_224_init,
2568 .update = safexcel_sha3_update,
2569 .final = safexcel_sha3_final,
2570 .finup = safexcel_sha3_finup,
2571 .digest = safexcel_sha3_224_digest,
2572 .export = safexcel_sha3_export,
2573 .import = safexcel_sha3_import,
2574 .halg = {
2575 .digestsize = SHA3_224_DIGEST_SIZE,
2576 .statesize = sizeof(struct safexcel_ahash_export_state),
2577 .base = {
2578 .cra_name = "sha3-224",
2579 .cra_driver_name = "safexcel-sha3-224",
2580 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2581 .cra_flags = CRYPTO_ALG_ASYNC |
2582 CRYPTO_ALG_KERN_DRIVER_ONLY |
2583 CRYPTO_ALG_NEED_FALLBACK,
2584 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2585 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2586 .cra_init = safexcel_sha3_cra_init,
2587 .cra_exit = safexcel_sha3_cra_exit,
2588 .cra_module = THIS_MODULE,
2589 },
2590 },
2591 },
2592 };
2593
2594 static int safexcel_sha3_256_init(struct ahash_request *areq)
2595 {
2596 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2597 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2598 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2599
2600 memset(req, 0, sizeof(*req));
2601
2602 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2603 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2604 req->state_sz = SHA3_256_DIGEST_SIZE;
2605 req->digest_sz = SHA3_256_DIGEST_SIZE;
2606 req->block_sz = SHA3_256_BLOCK_SIZE;
2607 ctx->do_fallback = false;
2608 ctx->fb_init_done = false;
2609 return 0;
2610 }
2611
2612 static int safexcel_sha3_256_digest(struct ahash_request *req)
2613 {
2614 if (req->nbytes)
2615 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);
2616
2617 /* HW cannot do zero length hash, use fallback instead */
2618 return safexcel_sha3_digest_fallback(req);
2619 }
2620
2621 struct safexcel_alg_template safexcel_alg_sha3_256 = {
2622 .type = SAFEXCEL_ALG_TYPE_AHASH,
2623 .algo_mask = SAFEXCEL_ALG_SHA3,
2624 .alg.ahash = {
2625 .init = safexcel_sha3_256_init,
2626 .update = safexcel_sha3_update,
2627 .final = safexcel_sha3_final,
2628 .finup = safexcel_sha3_finup,
2629 .digest = safexcel_sha3_256_digest,
2630 .export = safexcel_sha3_export,
2631 .import = safexcel_sha3_import,
2632 .halg = {
2633 .digestsize = SHA3_256_DIGEST_SIZE,
2634 .statesize = sizeof(struct safexcel_ahash_export_state),
2635 .base = {
2636 .cra_name = "sha3-256",
2637 .cra_driver_name = "safexcel-sha3-256",
2638 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2639 .cra_flags = CRYPTO_ALG_ASYNC |
2640 CRYPTO_ALG_KERN_DRIVER_ONLY |
2641 CRYPTO_ALG_NEED_FALLBACK,
2642 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2643 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2644 .cra_init = safexcel_sha3_cra_init,
2645 .cra_exit = safexcel_sha3_cra_exit,
2646 .cra_module = THIS_MODULE,
2647 },
2648 },
2649 },
2650 };
2651
2652 static int safexcel_sha3_384_init(struct ahash_request *areq)
2653 {
2654 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2655 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2656 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2657
2658 memset(req, 0, sizeof(*req));
2659
2660 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2661 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2662 req->state_sz = SHA3_384_DIGEST_SIZE;
2663 req->digest_sz = SHA3_384_DIGEST_SIZE;
2664 req->block_sz = SHA3_384_BLOCK_SIZE;
2665 ctx->do_fallback = false;
2666 ctx->fb_init_done = false;
2667 return 0;
2668 }
2669
2670 static int safexcel_sha3_384_digest(struct ahash_request *req)
2671 {
2672 if (req->nbytes)
2673 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);
2674
2675 /* HW cannot do zero length hash, use fallback instead */
2676 return safexcel_sha3_digest_fallback(req);
2677 }
2678
2679 struct safexcel_alg_template safexcel_alg_sha3_384 = {
2680 .type = SAFEXCEL_ALG_TYPE_AHASH,
2681 .algo_mask = SAFEXCEL_ALG_SHA3,
2682 .alg.ahash = {
2683 .init = safexcel_sha3_384_init,
2684 .update = safexcel_sha3_update,
2685 .final = safexcel_sha3_final,
2686 .finup = safexcel_sha3_finup,
2687 .digest = safexcel_sha3_384_digest,
2688 .export = safexcel_sha3_export,
2689 .import = safexcel_sha3_import,
2690 .halg = {
2691 .digestsize = SHA3_384_DIGEST_SIZE,
2692 .statesize = sizeof(struct safexcel_ahash_export_state),
2693 .base = {
2694 .cra_name = "sha3-384",
2695 .cra_driver_name = "safexcel-sha3-384",
2696 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2697 .cra_flags = CRYPTO_ALG_ASYNC |
2698 CRYPTO_ALG_KERN_DRIVER_ONLY |
2699 CRYPTO_ALG_NEED_FALLBACK,
2700 .cra_blocksize = SHA3_384_BLOCK_SIZE,
2701 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2702 .cra_init = safexcel_sha3_cra_init,
2703 .cra_exit = safexcel_sha3_cra_exit,
2704 .cra_module = THIS_MODULE,
2705 },
2706 },
2707 },
2708 };
2709
2710 static int safexcel_sha3_512_init(struct ahash_request *areq)
2711 {
2712 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2713 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2714 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2715
2716 memset(req, 0, sizeof(*req));
2717
2718 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2719 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2720 req->state_sz = SHA3_512_DIGEST_SIZE;
2721 req->digest_sz = SHA3_512_DIGEST_SIZE;
2722 req->block_sz = SHA3_512_BLOCK_SIZE;
2723 ctx->do_fallback = false;
2724 ctx->fb_init_done = false;
2725 return 0;
2726 }
2727
2728 static int safexcel_sha3_512_digest(struct ahash_request *req)
2729 {
2730 if (req->nbytes)
2731 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);
2732
2733 /* HW cannot do zero length hash, use fallback instead */
2734 return safexcel_sha3_digest_fallback(req);
2735 }
2736
2737 struct safexcel_alg_template safexcel_alg_sha3_512 = {
2738 .type = SAFEXCEL_ALG_TYPE_AHASH,
2739 .algo_mask = SAFEXCEL_ALG_SHA3,
2740 .alg.ahash = {
2741 .init = safexcel_sha3_512_init,
2742 .update = safexcel_sha3_update,
2743 .final = safexcel_sha3_final,
2744 .finup = safexcel_sha3_finup,
2745 .digest = safexcel_sha3_512_digest,
2746 .export = safexcel_sha3_export,
2747 .import = safexcel_sha3_import,
2748 .halg = {
2749 .digestsize = SHA3_512_DIGEST_SIZE,
2750 .statesize = sizeof(struct safexcel_ahash_export_state),
2751 .base = {
2752 .cra_name = "sha3-512",
2753 .cra_driver_name = "safexcel-sha3-512",
2754 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2755 .cra_flags = CRYPTO_ALG_ASYNC |
2756 CRYPTO_ALG_KERN_DRIVER_ONLY |
2757 CRYPTO_ALG_NEED_FALLBACK,
2758 .cra_blocksize = SHA3_512_BLOCK_SIZE,
2759 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2760 .cra_init = safexcel_sha3_cra_init,
2761 .cra_exit = safexcel_sha3_cra_exit,
2762 .cra_module = THIS_MODULE,
2763 },
2764 },
2765 },
2766 };
2767
2768 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg)
2769 {
2770 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2771 int ret;
2772
2773 ret = safexcel_sha3_cra_init(tfm);
2774 if (ret)
2775 return ret;
2776
2777 /* Allocate precalc basic digest implementation */
2778 ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
2779 if (IS_ERR(ctx->shpre))
2780 return PTR_ERR(ctx->shpre);
2781
2782 ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) +
2783 crypto_shash_descsize(ctx->shpre), GFP_KERNEL);
2784 if (!ctx->shdesc) {
2785 crypto_free_shash(ctx->shpre);
2786 return -ENOMEM;
2787 }
2788 ctx->shdesc->tfm = ctx->shpre;
2789 return 0;
2790 }
2791
2792 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm)
2793 {
2794 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2795
2796 crypto_free_ahash(ctx->fback);
2797 crypto_free_shash(ctx->shpre);
2798 kfree(ctx->shdesc);
2799 safexcel_ahash_cra_exit(tfm);
2800 }
2801
2802 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key,
2803 unsigned int keylen)
2804 {
2805 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2806 int ret = 0;
2807
2808 if (keylen > crypto_ahash_blocksize(tfm)) {
2809 /*
2810 * If the key is larger than the blocksize, then hash it
2811 * first using our fallback cipher
2812 */
2813 ret = crypto_shash_digest(ctx->shdesc, key, keylen,
2814 ctx->base.ipad.byte);
2815 keylen = crypto_shash_digestsize(ctx->shpre);
2816
2817 /*
2818 * If the digest is larger than half the blocksize, we need to
2819 * move the rest to opad due to the way our HMAC infra works.
2820 */
2821 if (keylen > crypto_ahash_blocksize(tfm) / 2)
2822 /* Buffers overlap, need to use memmove iso memcpy! */
2823 memmove(&ctx->base.opad,
2824 ctx->base.ipad.byte +
2825 crypto_ahash_blocksize(tfm) / 2,
2826 keylen - crypto_ahash_blocksize(tfm) / 2);
2827 } else {
2828 /*
2829 * Copy the key to our ipad & opad buffers
2830 * Note that ipad and opad each contain one half of the key,
2831 * to match the existing HMAC driver infrastructure.
2832 */
2833 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2834 memcpy(&ctx->base.ipad, key, keylen);
2835 } else {
2836 memcpy(&ctx->base.ipad, key,
2837 crypto_ahash_blocksize(tfm) / 2);
2838 memcpy(&ctx->base.opad,
2839 key + crypto_ahash_blocksize(tfm) / 2,
2840 keylen - crypto_ahash_blocksize(tfm) / 2);
2841 }
2842 }
2843
2844 /* Pad key with zeroes */
2845 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2846 memset(ctx->base.ipad.byte + keylen, 0,
2847 crypto_ahash_blocksize(tfm) / 2 - keylen);
2848 memset(&ctx->base.opad, 0, crypto_ahash_blocksize(tfm) / 2);
2849 } else {
2850 memset(ctx->base.opad.byte + keylen -
2851 crypto_ahash_blocksize(tfm) / 2, 0,
2852 crypto_ahash_blocksize(tfm) - keylen);
2853 }
2854
2855 /* If doing fallback, still need to set the new key! */
2856 ctx->fb_do_setkey = true;
2857 return ret;
2858 }
2859
2860 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
2861 {
2862 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2863 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2864 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2865
2866 memset(req, 0, sizeof(*req));
2867
2868 /* Copy (half of) the key */
2869 memcpy(req->state, &ctx->base.ipad, SHA3_224_BLOCK_SIZE / 2);
2870 /* Start of HMAC should have len == processed == blocksize */
2871 req->len = SHA3_224_BLOCK_SIZE;
2872 req->processed = SHA3_224_BLOCK_SIZE;
2873 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2874 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2875 req->state_sz = SHA3_224_BLOCK_SIZE / 2;
2876 req->digest_sz = SHA3_224_DIGEST_SIZE;
2877 req->block_sz = SHA3_224_BLOCK_SIZE;
2878 req->hmac = true;
2879 ctx->do_fallback = false;
2880 ctx->fb_init_done = false;
2881 return 0;
2882 }
2883
2884 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)
2885 {
2886 if (req->nbytes)
2887 return safexcel_hmac_sha3_224_init(req) ?:
2888 safexcel_ahash_finup(req);
2889
2890 /* HW cannot do zero length HMAC, use fallback instead */
2891 return safexcel_sha3_digest_fallback(req);
2892 }
2893
2894 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm)
2895 {
2896 return safexcel_hmac_sha3_cra_init(tfm, "sha3-224");
2897 }
2898
2899 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = {
2900 .type = SAFEXCEL_ALG_TYPE_AHASH,
2901 .algo_mask = SAFEXCEL_ALG_SHA3,
2902 .alg.ahash = {
2903 .init = safexcel_hmac_sha3_224_init,
2904 .update = safexcel_sha3_update,
2905 .final = safexcel_sha3_final,
2906 .finup = safexcel_sha3_finup,
2907 .digest = safexcel_hmac_sha3_224_digest,
2908 .setkey = safexcel_hmac_sha3_setkey,
2909 .export = safexcel_sha3_export,
2910 .import = safexcel_sha3_import,
2911 .halg = {
2912 .digestsize = SHA3_224_DIGEST_SIZE,
2913 .statesize = sizeof(struct safexcel_ahash_export_state),
2914 .base = {
2915 .cra_name = "hmac(sha3-224)",
2916 .cra_driver_name = "safexcel-hmac-sha3-224",
2917 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2918 .cra_flags = CRYPTO_ALG_ASYNC |
2919 CRYPTO_ALG_KERN_DRIVER_ONLY |
2920 CRYPTO_ALG_NEED_FALLBACK,
2921 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2922 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2923 .cra_init = safexcel_hmac_sha3_224_cra_init,
2924 .cra_exit = safexcel_hmac_sha3_cra_exit,
2925 .cra_module = THIS_MODULE,
2926 },
2927 },
2928 },
2929 };
2930
2931 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
2932 {
2933 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2934 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2935 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2936
2937 memset(req, 0, sizeof(*req));
2938
2939 /* Copy (half of) the key */
2940 memcpy(req->state, &ctx->base.ipad, SHA3_256_BLOCK_SIZE / 2);
2941 /* Start of HMAC should have len == processed == blocksize */
2942 req->len = SHA3_256_BLOCK_SIZE;
2943 req->processed = SHA3_256_BLOCK_SIZE;
2944 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2945 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2946 req->state_sz = SHA3_256_BLOCK_SIZE / 2;
2947 req->digest_sz = SHA3_256_DIGEST_SIZE;
2948 req->block_sz = SHA3_256_BLOCK_SIZE;
2949 req->hmac = true;
2950 ctx->do_fallback = false;
2951 ctx->fb_init_done = false;
2952 return 0;
2953 }
2954
2955 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)
2956 {
2957 if (req->nbytes)
2958 return safexcel_hmac_sha3_256_init(req) ?:
2959 safexcel_ahash_finup(req);
2960
2961 /* HW cannot do zero length HMAC, use fallback instead */
2962 return safexcel_sha3_digest_fallback(req);
2963 }
2964
2965 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm)
2966 {
2967 return safexcel_hmac_sha3_cra_init(tfm, "sha3-256");
2968 }
2969
2970 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = {
2971 .type = SAFEXCEL_ALG_TYPE_AHASH,
2972 .algo_mask = SAFEXCEL_ALG_SHA3,
2973 .alg.ahash = {
2974 .init = safexcel_hmac_sha3_256_init,
2975 .update = safexcel_sha3_update,
2976 .final = safexcel_sha3_final,
2977 .finup = safexcel_sha3_finup,
2978 .digest = safexcel_hmac_sha3_256_digest,
2979 .setkey = safexcel_hmac_sha3_setkey,
2980 .export = safexcel_sha3_export,
2981 .import = safexcel_sha3_import,
2982 .halg = {
2983 .digestsize = SHA3_256_DIGEST_SIZE,
2984 .statesize = sizeof(struct safexcel_ahash_export_state),
2985 .base = {
2986 .cra_name = "hmac(sha3-256)",
2987 .cra_driver_name = "safexcel-hmac-sha3-256",
2988 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2989 .cra_flags = CRYPTO_ALG_ASYNC |
2990 CRYPTO_ALG_KERN_DRIVER_ONLY |
2991 CRYPTO_ALG_NEED_FALLBACK,
2992 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2993 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2994 .cra_init = safexcel_hmac_sha3_256_cra_init,
2995 .cra_exit = safexcel_hmac_sha3_cra_exit,
2996 .cra_module = THIS_MODULE,
2997 },
2998 },
2999 },
3000 };
3001
3002 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
3003 {
3004 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3005 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3006 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3007
3008 memset(req, 0, sizeof(*req));
3009
3010 /* Copy (half of) the key */
3011 memcpy(req->state, &ctx->base.ipad, SHA3_384_BLOCK_SIZE / 2);
3012 /* Start of HMAC should have len == processed == blocksize */
3013 req->len = SHA3_384_BLOCK_SIZE;
3014 req->processed = SHA3_384_BLOCK_SIZE;
3015 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
3016 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3017 req->state_sz = SHA3_384_BLOCK_SIZE / 2;
3018 req->digest_sz = SHA3_384_DIGEST_SIZE;
3019 req->block_sz = SHA3_384_BLOCK_SIZE;
3020 req->hmac = true;
3021 ctx->do_fallback = false;
3022 ctx->fb_init_done = false;
3023 return 0;
3024 }
3025
3026 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)
3027 {
3028 if (req->nbytes)
3029 return safexcel_hmac_sha3_384_init(req) ?:
3030 safexcel_ahash_finup(req);
3031
3032 /* HW cannot do zero length HMAC, use fallback instead */
3033 return safexcel_sha3_digest_fallback(req);
3034 }
3035
3036 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm)
3037 {
3038 return safexcel_hmac_sha3_cra_init(tfm, "sha3-384");
3039 }
3040
3041 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = {
3042 .type = SAFEXCEL_ALG_TYPE_AHASH,
3043 .algo_mask = SAFEXCEL_ALG_SHA3,
3044 .alg.ahash = {
3045 .init = safexcel_hmac_sha3_384_init,
3046 .update = safexcel_sha3_update,
3047 .final = safexcel_sha3_final,
3048 .finup = safexcel_sha3_finup,
3049 .digest = safexcel_hmac_sha3_384_digest,
3050 .setkey = safexcel_hmac_sha3_setkey,
3051 .export = safexcel_sha3_export,
3052 .import = safexcel_sha3_import,
3053 .halg = {
3054 .digestsize = SHA3_384_DIGEST_SIZE,
3055 .statesize = sizeof(struct safexcel_ahash_export_state),
3056 .base = {
3057 .cra_name = "hmac(sha3-384)",
3058 .cra_driver_name = "safexcel-hmac-sha3-384",
3059 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3060 .cra_flags = CRYPTO_ALG_ASYNC |
3061 CRYPTO_ALG_KERN_DRIVER_ONLY |
3062 CRYPTO_ALG_NEED_FALLBACK,
3063 .cra_blocksize = SHA3_384_BLOCK_SIZE,
3064 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3065 .cra_init = safexcel_hmac_sha3_384_cra_init,
3066 .cra_exit = safexcel_hmac_sha3_cra_exit,
3067 .cra_module = THIS_MODULE,
3068 },
3069 },
3070 },
3071 };
3072
3073 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
3074 {
3075 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3076 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3077 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3078
3079 memset(req, 0, sizeof(*req));
3080
3081 /* Copy (half of) the key */
3082 memcpy(req->state, &ctx->base.ipad, SHA3_512_BLOCK_SIZE / 2);
3083 /* Start of HMAC should have len == processed == blocksize */
3084 req->len = SHA3_512_BLOCK_SIZE;
3085 req->processed = SHA3_512_BLOCK_SIZE;
3086 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
3087 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3088 req->state_sz = SHA3_512_BLOCK_SIZE / 2;
3089 req->digest_sz = SHA3_512_DIGEST_SIZE;
3090 req->block_sz = SHA3_512_BLOCK_SIZE;
3091 req->hmac = true;
3092 ctx->do_fallback = false;
3093 ctx->fb_init_done = false;
3094 return 0;
3095 }
3096
3097 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)
3098 {
3099 if (req->nbytes)
3100 return safexcel_hmac_sha3_512_init(req) ?:
3101 safexcel_ahash_finup(req);
3102
3103 /* HW cannot do zero length HMAC, use fallback instead */
3104 return safexcel_sha3_digest_fallback(req);
3105 }
3106
3107 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm)
3108 {
3109 return safexcel_hmac_sha3_cra_init(tfm, "sha3-512");
3110 }
3111 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = {
3112 .type = SAFEXCEL_ALG_TYPE_AHASH,
3113 .algo_mask = SAFEXCEL_ALG_SHA3,
3114 .alg.ahash = {
3115 .init = safexcel_hmac_sha3_512_init,
3116 .update = safexcel_sha3_update,
3117 .final = safexcel_sha3_final,
3118 .finup = safexcel_sha3_finup,
3119 .digest = safexcel_hmac_sha3_512_digest,
3120 .setkey = safexcel_hmac_sha3_setkey,
3121 .export = safexcel_sha3_export,
3122 .import = safexcel_sha3_import,
3123 .halg = {
3124 .digestsize = SHA3_512_DIGEST_SIZE,
3125 .statesize = sizeof(struct safexcel_ahash_export_state),
3126 .base = {
3127 .cra_name = "hmac(sha3-512)",
3128 .cra_driver_name = "safexcel-hmac-sha3-512",
3129 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3130 .cra_flags = CRYPTO_ALG_ASYNC |
3131 CRYPTO_ALG_KERN_DRIVER_ONLY |
3132 CRYPTO_ALG_NEED_FALLBACK,
3133 .cra_blocksize = SHA3_512_BLOCK_SIZE,
3134 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3135 .cra_init = safexcel_hmac_sha3_512_cra_init,
3136 .cra_exit = safexcel_hmac_sha3_cra_exit,
3137 .cra_module = THIS_MODULE,
3138 },
3139 },
3140 },
3141 };
Ubuntu Hirsute Linux kernel mirror