4 * Support for OMAP SHA1/MD5 HW acceleration.
6 * Copyright (c) 2010 Nokia Corporation
7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as published
11 * by the Free Software Foundation.
13 * Some ideas are from old omap-sha1-md5.c driver.
16 #define pr_fmt(fmt) "%s: " fmt, __func__
18 #include <linux/err.h>
19 #include <linux/device.h>
20 #include <linux/module.h>
21 #include <linux/init.h>
22 #include <linux/errno.h>
23 #include <linux/interrupt.h>
24 #include <linux/kernel.h>
25 #include <linux/clk.h>
26 #include <linux/irq.h>
28 #include <linux/platform_device.h>
29 #include <linux/scatterlist.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/delay.h>
32 #include <linux/crypto.h>
33 #include <linux/cryptohash.h>
34 #include <crypto/scatterwalk.h>
35 #include <crypto/algapi.h>
36 #include <crypto/sha.h>
37 #include <crypto/hash.h>
38 #include <crypto/internal/hash.h>
42 #include <mach/irqs.h>
44 #define SHA_REG_DIGEST(x) (0x00 + ((x) * 0x04))
45 #define SHA_REG_DIN(x) (0x1C + ((x) * 0x04))
47 #define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
48 #define MD5_DIGEST_SIZE 16
50 #define SHA_REG_DIGCNT 0x14
52 #define SHA_REG_CTRL 0x18
53 #define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
54 #define SHA_REG_CTRL_CLOSE_HASH (1 << 4)
55 #define SHA_REG_CTRL_ALGO_CONST (1 << 3)
56 #define SHA_REG_CTRL_ALGO (1 << 2)
57 #define SHA_REG_CTRL_INPUT_READY (1 << 1)
58 #define SHA_REG_CTRL_OUTPUT_READY (1 << 0)
60 #define SHA_REG_REV 0x5C
61 #define SHA_REG_REV_MAJOR 0xF0
62 #define SHA_REG_REV_MINOR 0x0F
64 #define SHA_REG_MASK 0x60
65 #define SHA_REG_MASK_DMA_EN (1 << 3)
66 #define SHA_REG_MASK_IT_EN (1 << 2)
67 #define SHA_REG_MASK_SOFTRESET (1 << 1)
68 #define SHA_REG_AUTOIDLE (1 << 0)
70 #define SHA_REG_SYSSTATUS 0x64
71 #define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
73 #define DEFAULT_TIMEOUT_INTERVAL HZ
75 #define FLAGS_FINUP 0x0002
76 #define FLAGS_FINAL 0x0004
77 #define FLAGS_SG 0x0008
78 #define FLAGS_SHA1 0x0010
79 #define FLAGS_DMA_ACTIVE 0x0020
80 #define FLAGS_OUTPUT_READY 0x0040
81 #define FLAGS_INIT 0x0100
82 #define FLAGS_CPU 0x0200
83 #define FLAGS_HMAC 0x0400
84 #define FLAGS_ERROR 0x0800
85 #define FLAGS_BUSY 0x1000
90 #define OMAP_ALIGN_MASK (sizeof(u32)-1)
91 #define OMAP_ALIGNED __attribute__((aligned(sizeof(u32))))
93 #define BUFLEN PAGE_SIZE
97 struct omap_sham_reqctx
{
98 struct omap_sham_dev
*dd
;
102 u8 digest
[SHA1_DIGEST_SIZE
] OMAP_ALIGNED
;
109 struct scatterlist
*sg
;
110 unsigned int offset
; /* offset in current sg */
111 unsigned int total
; /* total request */
113 u8 buffer
[0] OMAP_ALIGNED
;
116 struct omap_sham_hmac_ctx
{
117 struct crypto_shash
*shash
;
118 u8 ipad
[SHA1_MD5_BLOCK_SIZE
];
119 u8 opad
[SHA1_MD5_BLOCK_SIZE
];
122 struct omap_sham_ctx
{
123 struct omap_sham_dev
*dd
;
128 struct crypto_shash
*fallback
;
130 struct omap_sham_hmac_ctx base
[0];
133 #define OMAP_SHAM_QUEUE_LENGTH 1
135 struct omap_sham_dev
{
136 struct list_head list
;
137 unsigned long phys_base
;
139 void __iomem
*io_base
;
146 struct tasklet_struct done_task
;
147 struct tasklet_struct queue_task
;
150 struct crypto_queue queue
;
151 struct ahash_request
*req
;
154 struct omap_sham_drv
{
155 struct list_head dev_list
;
160 static struct omap_sham_drv sham
= {
161 .dev_list
= LIST_HEAD_INIT(sham
.dev_list
),
162 .lock
= __SPIN_LOCK_UNLOCKED(sham
.lock
),
165 static inline u32
omap_sham_read(struct omap_sham_dev
*dd
, u32 offset
)
167 return __raw_readl(dd
->io_base
+ offset
);
170 static inline void omap_sham_write(struct omap_sham_dev
*dd
,
171 u32 offset
, u32 value
)
173 __raw_writel(value
, dd
->io_base
+ offset
);
176 static inline void omap_sham_write_mask(struct omap_sham_dev
*dd
, u32 address
,
181 val
= omap_sham_read(dd
, address
);
184 omap_sham_write(dd
, address
, val
);
187 static inline int omap_sham_wait(struct omap_sham_dev
*dd
, u32 offset
, u32 bit
)
189 unsigned long timeout
= jiffies
+ DEFAULT_TIMEOUT_INTERVAL
;
191 while (!(omap_sham_read(dd
, offset
) & bit
)) {
192 if (time_is_before_jiffies(timeout
))
199 static void omap_sham_copy_hash(struct ahash_request
*req
, int out
)
201 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
202 u32
*hash
= (u32
*)ctx
->digest
;
205 /* MD5 is almost unused. So copy sha1 size to reduce code */
206 for (i
= 0; i
< SHA1_DIGEST_SIZE
/ sizeof(u32
); i
++) {
208 hash
[i
] = omap_sham_read(ctx
->dd
,
211 omap_sham_write(ctx
->dd
,
212 SHA_REG_DIGEST(i
), hash
[i
]);
216 static void omap_sham_copy_ready_hash(struct ahash_request
*req
)
218 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
219 u32
*in
= (u32
*)ctx
->digest
;
220 u32
*hash
= (u32
*)req
->result
;
226 if (likely(ctx
->flags
& FLAGS_SHA1
)) {
227 /* SHA1 results are in big endian */
228 for (i
= 0; i
< SHA1_DIGEST_SIZE
/ sizeof(u32
); i
++)
229 hash
[i
] = be32_to_cpu(in
[i
]);
231 /* MD5 results are in little endian */
232 for (i
= 0; i
< MD5_DIGEST_SIZE
/ sizeof(u32
); i
++)
233 hash
[i
] = le32_to_cpu(in
[i
]);
237 static int omap_sham_hw_init(struct omap_sham_dev
*dd
)
239 clk_enable(dd
->iclk
);
241 if (!(dd
->flags
& FLAGS_INIT
)) {
242 omap_sham_write_mask(dd
, SHA_REG_MASK
,
243 SHA_REG_MASK_SOFTRESET
, SHA_REG_MASK_SOFTRESET
);
245 if (omap_sham_wait(dd
, SHA_REG_SYSSTATUS
,
246 SHA_REG_SYSSTATUS_RESETDONE
))
249 dd
->flags
|= FLAGS_INIT
;
256 static void omap_sham_write_ctrl(struct omap_sham_dev
*dd
, size_t length
,
259 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
260 u32 val
= length
<< 5, mask
;
262 if (likely(ctx
->digcnt
))
263 omap_sham_write(dd
, SHA_REG_DIGCNT
, ctx
->digcnt
);
265 omap_sham_write_mask(dd
, SHA_REG_MASK
,
266 SHA_REG_MASK_IT_EN
| (dma
? SHA_REG_MASK_DMA_EN
: 0),
267 SHA_REG_MASK_IT_EN
| SHA_REG_MASK_DMA_EN
);
269 * Setting ALGO_CONST only for the first iteration
270 * and CLOSE_HASH only for the last one.
272 if (ctx
->flags
& FLAGS_SHA1
)
273 val
|= SHA_REG_CTRL_ALGO
;
275 val
|= SHA_REG_CTRL_ALGO_CONST
;
277 val
|= SHA_REG_CTRL_CLOSE_HASH
;
279 mask
= SHA_REG_CTRL_ALGO_CONST
| SHA_REG_CTRL_CLOSE_HASH
|
280 SHA_REG_CTRL_ALGO
| SHA_REG_CTRL_LENGTH
;
282 omap_sham_write_mask(dd
, SHA_REG_CTRL
, val
, mask
);
285 static int omap_sham_xmit_cpu(struct omap_sham_dev
*dd
, const u8
*buf
,
286 size_t length
, int final
)
288 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
290 const u32
*buffer
= (const u32
*)buf
;
292 dev_dbg(dd
->dev
, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
293 ctx
->digcnt
, length
, final
);
295 omap_sham_write_ctrl(dd
, length
, final
, 0);
297 /* should be non-zero before next lines to disable clocks later */
298 ctx
->digcnt
+= length
;
300 if (omap_sham_wait(dd
, SHA_REG_CTRL
, SHA_REG_CTRL_INPUT_READY
))
304 ctx
->flags
|= FLAGS_FINAL
; /* catch last interrupt */
306 len32
= DIV_ROUND_UP(length
, sizeof(u32
));
308 for (count
= 0; count
< len32
; count
++)
309 omap_sham_write(dd
, SHA_REG_DIN(count
), buffer
[count
]);
314 static int omap_sham_xmit_dma(struct omap_sham_dev
*dd
, dma_addr_t dma_addr
,
315 size_t length
, int final
)
317 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
320 dev_dbg(dd
->dev
, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
321 ctx
->digcnt
, length
, final
);
323 len32
= DIV_ROUND_UP(length
, sizeof(u32
));
325 omap_set_dma_transfer_params(dd
->dma_lch
, OMAP_DMA_DATA_TYPE_S32
, len32
,
326 1, OMAP_DMA_SYNC_PACKET
, dd
->dma
,
327 OMAP_DMA_DST_SYNC_PREFETCH
);
329 omap_set_dma_src_params(dd
->dma_lch
, 0, OMAP_DMA_AMODE_POST_INC
,
332 omap_sham_write_ctrl(dd
, length
, final
, 1);
334 ctx
->digcnt
+= length
;
337 ctx
->flags
|= FLAGS_FINAL
; /* catch last interrupt */
339 dd
->flags
|= FLAGS_DMA_ACTIVE
;
341 omap_start_dma(dd
->dma_lch
);
346 static size_t omap_sham_append_buffer(struct omap_sham_reqctx
*ctx
,
347 const u8
*data
, size_t length
)
349 size_t count
= min(length
, ctx
->buflen
- ctx
->bufcnt
);
351 count
= min(count
, ctx
->total
);
354 memcpy(ctx
->buffer
+ ctx
->bufcnt
, data
, count
);
355 ctx
->bufcnt
+= count
;
360 static size_t omap_sham_append_sg(struct omap_sham_reqctx
*ctx
)
365 count
= omap_sham_append_buffer(ctx
,
366 sg_virt(ctx
->sg
) + ctx
->offset
,
367 ctx
->sg
->length
- ctx
->offset
);
370 ctx
->offset
+= count
;
372 if (ctx
->offset
== ctx
->sg
->length
) {
373 ctx
->sg
= sg_next(ctx
->sg
);
384 static int omap_sham_xmit_dma_map(struct omap_sham_dev
*dd
,
385 struct omap_sham_reqctx
*ctx
,
386 size_t length
, int final
)
388 ctx
->dma_addr
= dma_map_single(dd
->dev
, ctx
->buffer
, ctx
->buflen
,
390 if (dma_mapping_error(dd
->dev
, ctx
->dma_addr
)) {
391 dev_err(dd
->dev
, "dma %u bytes error\n", ctx
->buflen
);
395 ctx
->flags
&= ~FLAGS_SG
;
397 /* next call does not fail... so no unmap in the case of error */
398 return omap_sham_xmit_dma(dd
, ctx
->dma_addr
, length
, final
);
401 static int omap_sham_update_dma_slow(struct omap_sham_dev
*dd
)
403 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
407 omap_sham_append_sg(ctx
);
409 final
= (ctx
->flags
& FLAGS_FINUP
) && !ctx
->total
;
411 dev_dbg(dd
->dev
, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
412 ctx
->bufcnt
, ctx
->digcnt
, final
);
414 if (final
|| (ctx
->bufcnt
== ctx
->buflen
&& ctx
->total
)) {
417 return omap_sham_xmit_dma_map(dd
, ctx
, count
, final
);
423 /* Start address alignment */
424 #define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
425 /* SHA1 block size alignment */
426 #define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
428 static int omap_sham_update_dma_start(struct omap_sham_dev
*dd
)
430 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
431 unsigned int length
, final
, tail
;
432 struct scatterlist
*sg
;
437 if (ctx
->bufcnt
|| ctx
->offset
)
438 return omap_sham_update_dma_slow(dd
);
440 dev_dbg(dd
->dev
, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
441 ctx
->digcnt
, ctx
->bufcnt
, ctx
->total
);
446 return omap_sham_update_dma_slow(dd
);
448 if (!sg_is_last(sg
) && !SG_SA(sg
))
449 /* size is not SHA1_BLOCK_SIZE aligned */
450 return omap_sham_update_dma_slow(dd
);
452 length
= min(ctx
->total
, sg
->length
);
454 if (sg_is_last(sg
)) {
455 if (!(ctx
->flags
& FLAGS_FINUP
)) {
456 /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
457 tail
= length
& (SHA1_MD5_BLOCK_SIZE
- 1);
458 /* without finup() we need one block to close hash */
460 tail
= SHA1_MD5_BLOCK_SIZE
;
465 if (!dma_map_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
)) {
466 dev_err(dd
->dev
, "dma_map_sg error\n");
470 ctx
->flags
|= FLAGS_SG
;
472 ctx
->total
-= length
;
473 ctx
->offset
= length
; /* offset where to start slow */
475 final
= (ctx
->flags
& FLAGS_FINUP
) && !ctx
->total
;
477 /* next call does not fail... so no unmap in the case of error */
478 return omap_sham_xmit_dma(dd
, sg_dma_address(ctx
->sg
), length
, final
);
481 static int omap_sham_update_cpu(struct omap_sham_dev
*dd
)
483 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
486 omap_sham_append_sg(ctx
);
487 bufcnt
= ctx
->bufcnt
;
490 return omap_sham_xmit_cpu(dd
, ctx
->buffer
, bufcnt
, 1);
493 static int omap_sham_update_dma_stop(struct omap_sham_dev
*dd
)
495 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
497 omap_stop_dma(dd
->dma_lch
);
498 if (ctx
->flags
& FLAGS_SG
) {
499 dma_unmap_sg(dd
->dev
, ctx
->sg
, 1, DMA_TO_DEVICE
);
500 if (ctx
->sg
->length
== ctx
->offset
) {
501 ctx
->sg
= sg_next(ctx
->sg
);
506 dma_unmap_single(dd
->dev
, ctx
->dma_addr
, ctx
->buflen
,
513 static int omap_sham_init(struct ahash_request
*req
)
515 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
516 struct omap_sham_ctx
*tctx
= crypto_ahash_ctx(tfm
);
517 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
518 struct omap_sham_dev
*dd
= NULL
, *tmp
;
520 spin_lock_bh(&sham
.lock
);
522 list_for_each_entry(tmp
, &sham
.dev_list
, list
) {
530 spin_unlock_bh(&sham
.lock
);
536 dev_dbg(dd
->dev
, "init: digest size: %d\n",
537 crypto_ahash_digestsize(tfm
));
539 if (crypto_ahash_digestsize(tfm
) == SHA1_DIGEST_SIZE
)
540 ctx
->flags
|= FLAGS_SHA1
;
544 ctx
->buflen
= BUFLEN
;
546 if (tctx
->flags
& FLAGS_HMAC
) {
547 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
549 memcpy(ctx
->buffer
, bctx
->ipad
, SHA1_MD5_BLOCK_SIZE
);
550 ctx
->bufcnt
= SHA1_MD5_BLOCK_SIZE
;
551 ctx
->flags
|= FLAGS_HMAC
;
558 static int omap_sham_update_req(struct omap_sham_dev
*dd
)
560 struct ahash_request
*req
= dd
->req
;
561 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
564 dev_dbg(dd
->dev
, "update_req: total: %u, digcnt: %d, finup: %d\n",
565 ctx
->total
, ctx
->digcnt
, (ctx
->flags
& FLAGS_FINUP
) != 0);
567 if (ctx
->flags
& FLAGS_CPU
)
568 err
= omap_sham_update_cpu(dd
);
570 err
= omap_sham_update_dma_start(dd
);
572 /* wait for dma completion before can take more data */
573 dev_dbg(dd
->dev
, "update: err: %d, digcnt: %d\n", err
, ctx
->digcnt
);
578 static int omap_sham_final_req(struct omap_sham_dev
*dd
)
580 struct ahash_request
*req
= dd
->req
;
581 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
582 int err
= 0, use_dma
= 1;
584 if (ctx
->bufcnt
<= 64)
585 /* faster to handle last block with cpu */
589 err
= omap_sham_xmit_dma_map(dd
, ctx
, ctx
->bufcnt
, 1);
591 err
= omap_sham_xmit_cpu(dd
, ctx
->buffer
, ctx
->bufcnt
, 1);
595 dev_dbg(dd
->dev
, "final_req: err: %d\n", err
);
600 static int omap_sham_finish_hmac(struct ahash_request
*req
)
602 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
603 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
604 int bs
= crypto_shash_blocksize(bctx
->shash
);
605 int ds
= crypto_shash_digestsize(bctx
->shash
);
607 struct shash_desc shash
;
608 char ctx
[crypto_shash_descsize(bctx
->shash
)];
611 desc
.shash
.tfm
= bctx
->shash
;
612 desc
.shash
.flags
= 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
614 return crypto_shash_init(&desc
.shash
) ?:
615 crypto_shash_update(&desc
.shash
, bctx
->opad
, bs
) ?:
616 crypto_shash_finup(&desc
.shash
, req
->result
, ds
, req
->result
);
619 static int omap_sham_finish(struct ahash_request
*req
)
621 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
622 struct omap_sham_dev
*dd
= ctx
->dd
;
626 omap_sham_copy_ready_hash(req
);
627 if (ctx
->flags
& FLAGS_HMAC
)
628 err
= omap_sham_finish_hmac(req
);
631 dev_dbg(dd
->dev
, "digcnt: %d, bufcnt: %d\n", ctx
->digcnt
, ctx
->bufcnt
);
636 static void omap_sham_finish_req(struct ahash_request
*req
, int err
)
638 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
639 struct omap_sham_dev
*dd
= ctx
->dd
;
642 omap_sham_copy_hash(req
, 1);
643 if (ctx
->flags
& FLAGS_FINAL
)
644 err
= omap_sham_finish(req
);
646 ctx
->flags
|= FLAGS_ERROR
;
649 clk_disable(dd
->iclk
);
650 dd
->flags
&= ~FLAGS_BUSY
;
652 if (req
->base
.complete
)
653 req
->base
.complete(&req
->base
, err
);
656 static int omap_sham_handle_queue(struct omap_sham_dev
*dd
,
657 struct ahash_request
*req
)
659 struct crypto_async_request
*async_req
, *backlog
;
660 struct omap_sham_reqctx
*ctx
;
662 int err
= 0, ret
= 0;
664 spin_lock_irqsave(&dd
->lock
, flags
);
666 ret
= ahash_enqueue_request(&dd
->queue
, req
);
667 if (dd
->flags
& FLAGS_BUSY
) {
668 spin_unlock_irqrestore(&dd
->lock
, flags
);
671 backlog
= crypto_get_backlog(&dd
->queue
);
672 async_req
= crypto_dequeue_request(&dd
->queue
);
674 dd
->flags
|= FLAGS_BUSY
;
675 spin_unlock_irqrestore(&dd
->lock
, flags
);
681 backlog
->complete(backlog
, -EINPROGRESS
);
683 req
= ahash_request_cast(async_req
);
685 ctx
= ahash_request_ctx(req
);
687 dev_dbg(dd
->dev
, "handling new req, op: %lu, nbytes: %d\n",
688 ctx
->op
, req
->nbytes
);
690 err
= omap_sham_hw_init(dd
);
694 omap_set_dma_dest_params(dd
->dma_lch
, 0,
695 OMAP_DMA_AMODE_CONSTANT
,
696 dd
->phys_base
+ SHA_REG_DIN(0), 0, 16);
698 omap_set_dma_dest_burst_mode(dd
->dma_lch
,
699 OMAP_DMA_DATA_BURST_16
);
701 omap_set_dma_src_burst_mode(dd
->dma_lch
,
702 OMAP_DMA_DATA_BURST_4
);
705 /* request has changed - restore hash */
706 omap_sham_copy_hash(req
, 0);
708 if (ctx
->op
== OP_UPDATE
) {
709 err
= omap_sham_update_req(dd
);
710 if (err
!= -EINPROGRESS
&& (ctx
->flags
& FLAGS_FINUP
))
711 /* no final() after finup() */
712 err
= omap_sham_final_req(dd
);
713 } else if (ctx
->op
== OP_FINAL
) {
714 err
= omap_sham_final_req(dd
);
717 if (err
!= -EINPROGRESS
) {
718 /* done_task will not finish it, so do it here */
719 omap_sham_finish_req(req
, err
);
720 tasklet_schedule(&dd
->queue_task
);
723 dev_dbg(dd
->dev
, "exit, err: %d\n", err
);
728 static int omap_sham_enqueue(struct ahash_request
*req
, unsigned int op
)
730 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
731 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
732 struct omap_sham_dev
*dd
= tctx
->dd
;
736 return omap_sham_handle_queue(dd
, req
);
739 static int omap_sham_update(struct ahash_request
*req
)
741 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
746 ctx
->total
= req
->nbytes
;
750 if (ctx
->flags
& FLAGS_FINUP
) {
751 if ((ctx
->digcnt
+ ctx
->bufcnt
+ ctx
->total
) < 9) {
753 * OMAP HW accel works only with buffers >= 9
754 * will switch to bypass in final()
755 * final has the same request and data
757 omap_sham_append_sg(ctx
);
759 } else if (ctx
->bufcnt
+ ctx
->total
<= SHA1_MD5_BLOCK_SIZE
) {
761 * faster to use CPU for short transfers
763 ctx
->flags
|= FLAGS_CPU
;
765 } else if (ctx
->bufcnt
+ ctx
->total
< ctx
->buflen
) {
766 omap_sham_append_sg(ctx
);
770 return omap_sham_enqueue(req
, OP_UPDATE
);
773 static int omap_sham_shash_digest(struct crypto_shash
*shash
, u32 flags
,
774 const u8
*data
, unsigned int len
, u8
*out
)
777 struct shash_desc shash
;
778 char ctx
[crypto_shash_descsize(shash
)];
781 desc
.shash
.tfm
= shash
;
782 desc
.shash
.flags
= flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
784 return crypto_shash_digest(&desc
.shash
, data
, len
, out
);
787 static int omap_sham_final_shash(struct ahash_request
*req
)
789 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(req
->base
.tfm
);
790 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
792 return omap_sham_shash_digest(tctx
->fallback
, req
->base
.flags
,
793 ctx
->buffer
, ctx
->bufcnt
, req
->result
);
796 static int omap_sham_final(struct ahash_request
*req
)
798 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
800 ctx
->flags
|= FLAGS_FINUP
;
802 if (ctx
->flags
& FLAGS_ERROR
)
803 return 0; /* uncompleted hash is not needed */
805 /* OMAP HW accel works only with buffers >= 9 */
806 /* HMAC is always >= 9 because ipad == block size */
807 if ((ctx
->digcnt
+ ctx
->bufcnt
) < 9)
808 return omap_sham_final_shash(req
);
809 else if (ctx
->bufcnt
)
810 return omap_sham_enqueue(req
, OP_FINAL
);
812 /* copy ready hash (+ finalize hmac) */
813 return omap_sham_finish(req
);
816 static int omap_sham_finup(struct ahash_request
*req
)
818 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
821 ctx
->flags
|= FLAGS_FINUP
;
823 err1
= omap_sham_update(req
);
824 if (err1
== -EINPROGRESS
|| err1
== -EBUSY
)
827 * final() has to be always called to cleanup resources
828 * even if udpate() failed, except EINPROGRESS
830 err2
= omap_sham_final(req
);
835 static int omap_sham_digest(struct ahash_request
*req
)
837 return omap_sham_init(req
) ?: omap_sham_finup(req
);
840 static int omap_sham_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
843 struct omap_sham_ctx
*tctx
= crypto_ahash_ctx(tfm
);
844 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
845 int bs
= crypto_shash_blocksize(bctx
->shash
);
846 int ds
= crypto_shash_digestsize(bctx
->shash
);
848 err
= crypto_shash_setkey(tctx
->fallback
, key
, keylen
);
853 err
= omap_sham_shash_digest(bctx
->shash
,
854 crypto_shash_get_flags(bctx
->shash
),
855 key
, keylen
, bctx
->ipad
);
860 memcpy(bctx
->ipad
, key
, keylen
);
863 memset(bctx
->ipad
+ keylen
, 0, bs
- keylen
);
864 memcpy(bctx
->opad
, bctx
->ipad
, bs
);
866 for (i
= 0; i
< bs
; i
++) {
867 bctx
->ipad
[i
] ^= 0x36;
868 bctx
->opad
[i
] ^= 0x5c;
874 static int omap_sham_cra_init_alg(struct crypto_tfm
*tfm
, const char *alg_base
)
876 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(tfm
);
877 const char *alg_name
= crypto_tfm_alg_name(tfm
);
879 /* Allocate a fallback and abort if it failed. */
880 tctx
->fallback
= crypto_alloc_shash(alg_name
, 0,
881 CRYPTO_ALG_NEED_FALLBACK
);
882 if (IS_ERR(tctx
->fallback
)) {
883 pr_err("omap-sham: fallback driver '%s' "
884 "could not be loaded.\n", alg_name
);
885 return PTR_ERR(tctx
->fallback
);
888 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
889 sizeof(struct omap_sham_reqctx
) + BUFLEN
);
892 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
893 tctx
->flags
|= FLAGS_HMAC
;
894 bctx
->shash
= crypto_alloc_shash(alg_base
, 0,
895 CRYPTO_ALG_NEED_FALLBACK
);
896 if (IS_ERR(bctx
->shash
)) {
897 pr_err("omap-sham: base driver '%s' "
898 "could not be loaded.\n", alg_base
);
899 crypto_free_shash(tctx
->fallback
);
900 return PTR_ERR(bctx
->shash
);
908 static int omap_sham_cra_init(struct crypto_tfm
*tfm
)
910 return omap_sham_cra_init_alg(tfm
, NULL
);
913 static int omap_sham_cra_sha1_init(struct crypto_tfm
*tfm
)
915 return omap_sham_cra_init_alg(tfm
, "sha1");
918 static int omap_sham_cra_md5_init(struct crypto_tfm
*tfm
)
920 return omap_sham_cra_init_alg(tfm
, "md5");
923 static void omap_sham_cra_exit(struct crypto_tfm
*tfm
)
925 struct omap_sham_ctx
*tctx
= crypto_tfm_ctx(tfm
);
927 crypto_free_shash(tctx
->fallback
);
928 tctx
->fallback
= NULL
;
930 if (tctx
->flags
& FLAGS_HMAC
) {
931 struct omap_sham_hmac_ctx
*bctx
= tctx
->base
;
932 crypto_free_shash(bctx
->shash
);
936 static struct ahash_alg algs
[] = {
938 .init
= omap_sham_init
,
939 .update
= omap_sham_update
,
940 .final
= omap_sham_final
,
941 .finup
= omap_sham_finup
,
942 .digest
= omap_sham_digest
,
943 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
946 .cra_driver_name
= "omap-sha1",
948 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
950 CRYPTO_ALG_NEED_FALLBACK
,
951 .cra_blocksize
= SHA1_BLOCK_SIZE
,
952 .cra_ctxsize
= sizeof(struct omap_sham_ctx
),
954 .cra_module
= THIS_MODULE
,
955 .cra_init
= omap_sham_cra_init
,
956 .cra_exit
= omap_sham_cra_exit
,
960 .init
= omap_sham_init
,
961 .update
= omap_sham_update
,
962 .final
= omap_sham_final
,
963 .finup
= omap_sham_finup
,
964 .digest
= omap_sham_digest
,
965 .halg
.digestsize
= MD5_DIGEST_SIZE
,
968 .cra_driver_name
= "omap-md5",
970 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
972 CRYPTO_ALG_NEED_FALLBACK
,
973 .cra_blocksize
= SHA1_BLOCK_SIZE
,
974 .cra_ctxsize
= sizeof(struct omap_sham_ctx
),
975 .cra_alignmask
= OMAP_ALIGN_MASK
,
976 .cra_module
= THIS_MODULE
,
977 .cra_init
= omap_sham_cra_init
,
978 .cra_exit
= omap_sham_cra_exit
,
982 .init
= omap_sham_init
,
983 .update
= omap_sham_update
,
984 .final
= omap_sham_final
,
985 .finup
= omap_sham_finup
,
986 .digest
= omap_sham_digest
,
987 .setkey
= omap_sham_setkey
,
988 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
990 .cra_name
= "hmac(sha1)",
991 .cra_driver_name
= "omap-hmac-sha1",
993 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
995 CRYPTO_ALG_NEED_FALLBACK
,
996 .cra_blocksize
= SHA1_BLOCK_SIZE
,
997 .cra_ctxsize
= sizeof(struct omap_sham_ctx
) +
998 sizeof(struct omap_sham_hmac_ctx
),
999 .cra_alignmask
= OMAP_ALIGN_MASK
,
1000 .cra_module
= THIS_MODULE
,
1001 .cra_init
= omap_sham_cra_sha1_init
,
1002 .cra_exit
= omap_sham_cra_exit
,
1006 .init
= omap_sham_init
,
1007 .update
= omap_sham_update
,
1008 .final
= omap_sham_final
,
1009 .finup
= omap_sham_finup
,
1010 .digest
= omap_sham_digest
,
1011 .setkey
= omap_sham_setkey
,
1012 .halg
.digestsize
= MD5_DIGEST_SIZE
,
1014 .cra_name
= "hmac(md5)",
1015 .cra_driver_name
= "omap-hmac-md5",
1016 .cra_priority
= 100,
1017 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
1019 CRYPTO_ALG_NEED_FALLBACK
,
1020 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1021 .cra_ctxsize
= sizeof(struct omap_sham_ctx
) +
1022 sizeof(struct omap_sham_hmac_ctx
),
1023 .cra_alignmask
= OMAP_ALIGN_MASK
,
1024 .cra_module
= THIS_MODULE
,
1025 .cra_init
= omap_sham_cra_md5_init
,
1026 .cra_exit
= omap_sham_cra_exit
,
1031 static void omap_sham_done_task(unsigned long data
)
1033 struct omap_sham_dev
*dd
= (struct omap_sham_dev
*)data
;
1034 struct ahash_request
*req
= dd
->req
;
1035 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(req
);
1036 int ready
= 0, err
= 0;
1038 if (ctx
->flags
& FLAGS_OUTPUT_READY
) {
1039 ctx
->flags
&= ~FLAGS_OUTPUT_READY
;
1043 if (dd
->flags
& FLAGS_DMA_ACTIVE
) {
1044 dd
->flags
&= ~FLAGS_DMA_ACTIVE
;
1045 omap_sham_update_dma_stop(dd
);
1047 err
= omap_sham_update_dma_start(dd
);
1050 err
= dd
->err
? : err
;
1052 if (err
!= -EINPROGRESS
&& (ready
|| err
)) {
1053 dev_dbg(dd
->dev
, "update done: err: %d\n", err
);
1054 /* finish curent request */
1055 omap_sham_finish_req(req
, err
);
1056 /* start new request */
1057 omap_sham_handle_queue(dd
, NULL
);
1061 static void omap_sham_queue_task(unsigned long data
)
1063 struct omap_sham_dev
*dd
= (struct omap_sham_dev
*)data
;
1065 omap_sham_handle_queue(dd
, NULL
);
1068 static irqreturn_t
omap_sham_irq(int irq
, void *dev_id
)
1070 struct omap_sham_dev
*dd
= dev_id
;
1071 struct omap_sham_reqctx
*ctx
= ahash_request_ctx(dd
->req
);
1074 dev_err(dd
->dev
, "unknown interrupt.\n");
1078 if (unlikely(ctx
->flags
& FLAGS_FINAL
))
1079 /* final -> allow device to go to power-saving mode */
1080 omap_sham_write_mask(dd
, SHA_REG_CTRL
, 0, SHA_REG_CTRL_LENGTH
);
1082 omap_sham_write_mask(dd
, SHA_REG_CTRL
, SHA_REG_CTRL_OUTPUT_READY
,
1083 SHA_REG_CTRL_OUTPUT_READY
);
1084 omap_sham_read(dd
, SHA_REG_CTRL
);
1086 ctx
->flags
|= FLAGS_OUTPUT_READY
;
1088 tasklet_schedule(&dd
->done_task
);
1093 static void omap_sham_dma_callback(int lch
, u16 ch_status
, void *data
)
1095 struct omap_sham_dev
*dd
= data
;
1097 if (ch_status
!= OMAP_DMA_BLOCK_IRQ
) {
1098 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status
);
1100 dd
->flags
&= ~FLAGS_INIT
; /* request to re-initialize */
1103 tasklet_schedule(&dd
->done_task
);
1106 static int omap_sham_dma_init(struct omap_sham_dev
*dd
)
1112 err
= omap_request_dma(dd
->dma
, dev_name(dd
->dev
),
1113 omap_sham_dma_callback
, dd
, &dd
->dma_lch
);
1115 dev_err(dd
->dev
, "Unable to request DMA channel\n");
1122 static void omap_sham_dma_cleanup(struct omap_sham_dev
*dd
)
1124 if (dd
->dma_lch
>= 0) {
1125 omap_free_dma(dd
->dma_lch
);
1130 static int __devinit
omap_sham_probe(struct platform_device
*pdev
)
1132 struct omap_sham_dev
*dd
;
1133 struct device
*dev
= &pdev
->dev
;
1134 struct resource
*res
;
1137 dd
= kzalloc(sizeof(struct omap_sham_dev
), GFP_KERNEL
);
1139 dev_err(dev
, "unable to alloc data struct.\n");
1144 platform_set_drvdata(pdev
, dd
);
1146 INIT_LIST_HEAD(&dd
->list
);
1147 spin_lock_init(&dd
->lock
);
1148 tasklet_init(&dd
->done_task
, omap_sham_done_task
, (unsigned long)dd
);
1149 tasklet_init(&dd
->queue_task
, omap_sham_queue_task
, (unsigned long)dd
);
1150 crypto_init_queue(&dd
->queue
, OMAP_SHAM_QUEUE_LENGTH
);
1154 /* Get the base address */
1155 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1157 dev_err(dev
, "no MEM resource info\n");
1161 dd
->phys_base
= res
->start
;
1164 res
= platform_get_resource(pdev
, IORESOURCE_DMA
, 0);
1166 dev_err(dev
, "no DMA resource info\n");
1170 dd
->dma
= res
->start
;
1173 dd
->irq
= platform_get_irq(pdev
, 0);
1175 dev_err(dev
, "no IRQ resource info\n");
1180 err
= request_irq(dd
->irq
, omap_sham_irq
,
1181 IRQF_TRIGGER_LOW
, dev_name(dev
), dd
);
1183 dev_err(dev
, "unable to request irq.\n");
1187 err
= omap_sham_dma_init(dd
);
1191 /* Initializing the clock */
1192 dd
->iclk
= clk_get(dev
, "ick");
1193 if (IS_ERR(dd
->iclk
)) {
1194 dev_err(dev
, "clock intialization failed.\n");
1195 err
= PTR_ERR(dd
->iclk
);
1199 dd
->io_base
= ioremap(dd
->phys_base
, SZ_4K
);
1201 dev_err(dev
, "can't ioremap\n");
1206 clk_enable(dd
->iclk
);
1207 dev_info(dev
, "hw accel on OMAP rev %u.%u\n",
1208 (omap_sham_read(dd
, SHA_REG_REV
) & SHA_REG_REV_MAJOR
) >> 4,
1209 omap_sham_read(dd
, SHA_REG_REV
) & SHA_REG_REV_MINOR
);
1210 clk_disable(dd
->iclk
);
1212 spin_lock(&sham
.lock
);
1213 list_add_tail(&dd
->list
, &sham
.dev_list
);
1214 spin_unlock(&sham
.lock
);
1216 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++) {
1217 err
= crypto_register_ahash(&algs
[i
]);
1225 for (j
= 0; j
< i
; j
++)
1226 crypto_unregister_ahash(&algs
[j
]);
1227 iounmap(dd
->io_base
);
1231 omap_sham_dma_cleanup(dd
);
1234 free_irq(dd
->irq
, dd
);
1239 dev_err(dev
, "initialization failed.\n");
1244 static int __devexit
omap_sham_remove(struct platform_device
*pdev
)
1246 static struct omap_sham_dev
*dd
;
1249 dd
= platform_get_drvdata(pdev
);
1252 spin_lock(&sham
.lock
);
1253 list_del(&dd
->list
);
1254 spin_unlock(&sham
.lock
);
1255 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++)
1256 crypto_unregister_ahash(&algs
[i
]);
1257 tasklet_kill(&dd
->done_task
);
1258 tasklet_kill(&dd
->queue_task
);
1259 iounmap(dd
->io_base
);
1261 omap_sham_dma_cleanup(dd
);
1263 free_irq(dd
->irq
, dd
);
1270 static struct platform_driver omap_sham_driver
= {
1271 .probe
= omap_sham_probe
,
1272 .remove
= omap_sham_remove
,
1274 .name
= "omap-sham",
1275 .owner
= THIS_MODULE
,
1279 static int __init
omap_sham_mod_init(void)
1281 pr_info("loading %s driver\n", "omap-sham");
1283 if (!cpu_class_is_omap2() ||
1284 (omap_type() != OMAP2_DEVICE_TYPE_SEC
&&
1285 omap_type() != OMAP2_DEVICE_TYPE_EMU
)) {
1286 pr_err("Unsupported cpu\n");
1290 return platform_driver_register(&omap_sham_driver
);
1293 static void __exit
omap_sham_mod_exit(void)
1295 platform_driver_unregister(&omap_sham_driver
);
1298 module_init(omap_sham_mod_init
);
1299 module_exit(omap_sham_mod_exit
);
1301 MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
1302 MODULE_LICENSE("GPL v2");
1303 MODULE_AUTHOR("Dmitry Kasatkin");