1 // SPDX-License-Identifier: GPL-2.0-only
5 * Support for SAHARA cryptographic accelerator.
7 * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
8 * Copyright (c) 2013 Vista Silicon S.L.
9 * Author: Javier Martin <javier.martin@vista-silicon.com>
11 * Based on omap-aes.c and tegra-aes.c
14 #include <crypto/aes.h>
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/skcipher.h>
17 #include <crypto/scatterwalk.h>
18 #include <crypto/sha.h>
20 #include <linux/clk.h>
21 #include <linux/crypto.h>
22 #include <linux/interrupt.h>
24 #include <linux/irq.h>
25 #include <linux/kernel.h>
26 #include <linux/kthread.h>
27 #include <linux/module.h>
28 #include <linux/mutex.h>
30 #include <linux/of_device.h>
31 #include <linux/platform_device.h>
33 #define SHA_BUFFER_LEN PAGE_SIZE
34 #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE
36 #define SAHARA_NAME "sahara"
37 #define SAHARA_VERSION_3 3
38 #define SAHARA_VERSION_4 4
39 #define SAHARA_TIMEOUT_MS 1000
40 #define SAHARA_MAX_HW_DESC 2
41 #define SAHARA_MAX_HW_LINK 20
43 #define FLAGS_MODE_MASK 0x000f
44 #define FLAGS_ENCRYPT BIT(0)
45 #define FLAGS_CBC BIT(1)
46 #define FLAGS_NEW_KEY BIT(3)
48 #define SAHARA_HDR_BASE 0x00800000
49 #define SAHARA_HDR_SKHA_ALG_AES 0
50 #define SAHARA_HDR_SKHA_OP_ENC (1 << 2)
51 #define SAHARA_HDR_SKHA_MODE_ECB (0 << 3)
52 #define SAHARA_HDR_SKHA_MODE_CBC (1 << 3)
53 #define SAHARA_HDR_FORM_DATA (5 << 16)
54 #define SAHARA_HDR_FORM_KEY (8 << 16)
55 #define SAHARA_HDR_LLO (1 << 24)
56 #define SAHARA_HDR_CHA_SKHA (1 << 28)
57 #define SAHARA_HDR_CHA_MDHA (2 << 28)
58 #define SAHARA_HDR_PARITY_BIT (1 << 31)
60 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
61 #define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000
62 #define SAHARA_HDR_MDHA_HASH 0xA0850000
63 #define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000
64 #define SAHARA_HDR_MDHA_ALG_SHA1 0
65 #define SAHARA_HDR_MDHA_ALG_MD5 1
66 #define SAHARA_HDR_MDHA_ALG_SHA256 2
67 #define SAHARA_HDR_MDHA_ALG_SHA224 3
68 #define SAHARA_HDR_MDHA_PDATA (1 << 2)
69 #define SAHARA_HDR_MDHA_HMAC (1 << 3)
70 #define SAHARA_HDR_MDHA_INIT (1 << 5)
71 #define SAHARA_HDR_MDHA_IPAD (1 << 6)
72 #define SAHARA_HDR_MDHA_OPAD (1 << 7)
73 #define SAHARA_HDR_MDHA_SWAP (1 << 8)
74 #define SAHARA_HDR_MDHA_MAC_FULL (1 << 9)
75 #define SAHARA_HDR_MDHA_SSL (1 << 10)
77 /* SAHARA can only process one request at a time */
78 #define SAHARA_QUEUE_LENGTH 1
80 #define SAHARA_REG_VERSION 0x00
81 #define SAHARA_REG_DAR 0x04
82 #define SAHARA_REG_CONTROL 0x08
83 #define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24)
84 #define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16)
85 #define SAHARA_CONTROL_RNG_AUTORSD (1 << 7)
86 #define SAHARA_CONTROL_ENABLE_INT (1 << 4)
87 #define SAHARA_REG_CMD 0x0C
88 #define SAHARA_CMD_RESET (1 << 0)
89 #define SAHARA_CMD_CLEAR_INT (1 << 8)
90 #define SAHARA_CMD_CLEAR_ERR (1 << 9)
91 #define SAHARA_CMD_SINGLE_STEP (1 << 10)
92 #define SAHARA_CMD_MODE_BATCH (1 << 16)
93 #define SAHARA_CMD_MODE_DEBUG (1 << 18)
94 #define SAHARA_REG_STATUS 0x10
95 #define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7)
96 #define SAHARA_STATE_IDLE 0
97 #define SAHARA_STATE_BUSY 1
98 #define SAHARA_STATE_ERR 2
99 #define SAHARA_STATE_FAULT 3
100 #define SAHARA_STATE_COMPLETE 4
101 #define SAHARA_STATE_COMP_FLAG (1 << 2)
102 #define SAHARA_STATUS_DAR_FULL (1 << 3)
103 #define SAHARA_STATUS_ERROR (1 << 4)
104 #define SAHARA_STATUS_SECURE (1 << 5)
105 #define SAHARA_STATUS_FAIL (1 << 6)
106 #define SAHARA_STATUS_INIT (1 << 7)
107 #define SAHARA_STATUS_RNG_RESEED (1 << 8)
108 #define SAHARA_STATUS_ACTIVE_RNG (1 << 9)
109 #define SAHARA_STATUS_ACTIVE_MDHA (1 << 10)
110 #define SAHARA_STATUS_ACTIVE_SKHA (1 << 11)
111 #define SAHARA_STATUS_MODE_BATCH (1 << 16)
112 #define SAHARA_STATUS_MODE_DEDICATED (1 << 17)
113 #define SAHARA_STATUS_MODE_DEBUG (1 << 18)
114 #define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff)
115 #define SAHARA_REG_ERRSTATUS 0x14
116 #define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf)
117 #define SAHARA_ERRSOURCE_CHA 14
118 #define SAHARA_ERRSOURCE_DMA 15
119 #define SAHARA_ERRSTATUS_DMA_DIR (1 << 8)
120 #define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
121 #define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
122 #define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff)
123 #define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3)
124 #define SAHARA_REG_FADDR 0x18
125 #define SAHARA_REG_CDAR 0x1C
126 #define SAHARA_REG_IDAR 0x20
128 struct sahara_hw_desc
{
137 struct sahara_hw_link
{
146 /* AES-specific context */
148 u8 key
[AES_KEYSIZE_128
];
149 struct crypto_sync_skcipher
*fallback
;
152 struct sahara_aes_reqctx
{
157 * struct sahara_sha_reqctx - private data per request
158 * @buf: holds data for requests smaller than block_size
159 * @rembuf: used to prepare one block_size-aligned request
160 * @context: hw-specific context for request. Digest is extracted from this
161 * @mode: specifies what type of hw-descriptor needs to be built
162 * @digest_size: length of digest for this request
163 * @context_size: length of hw-context for this request.
164 * Always digest_size + 4
165 * @buf_cnt: number of bytes saved in buf
166 * @sg_in_idx: number of hw links
167 * @in_sg: scatterlist for input data
168 * @in_sg_chain: scatterlists for chained input data
169 * @total: total number of bytes for transfer
170 * @last: is this the last block
171 * @first: is this the first block
172 * @active: inside a transfer
174 struct sahara_sha_reqctx
{
175 u8 buf
[SAHARA_MAX_SHA_BLOCK_SIZE
];
176 u8 rembuf
[SAHARA_MAX_SHA_BLOCK_SIZE
];
177 u8 context
[SHA256_DIGEST_SIZE
+ 4];
179 unsigned int digest_size
;
180 unsigned int context_size
;
181 unsigned int buf_cnt
;
182 unsigned int sg_in_idx
;
183 struct scatterlist
*in_sg
;
184 struct scatterlist in_sg_chain
[2];
192 struct device
*device
;
193 unsigned int version
;
194 void __iomem
*regs_base
;
197 struct mutex queue_mutex
;
198 struct task_struct
*kthread
;
199 struct completion dma_completion
;
201 struct sahara_ctx
*ctx
;
202 struct crypto_queue queue
;
205 struct sahara_hw_desc
*hw_desc
[SAHARA_MAX_HW_DESC
];
206 dma_addr_t hw_phys_desc
[SAHARA_MAX_HW_DESC
];
209 dma_addr_t key_phys_base
;
212 dma_addr_t iv_phys_base
;
215 dma_addr_t context_phys_base
;
217 struct sahara_hw_link
*hw_link
[SAHARA_MAX_HW_LINK
];
218 dma_addr_t hw_phys_link
[SAHARA_MAX_HW_LINK
];
221 struct scatterlist
*in_sg
;
223 struct scatterlist
*out_sg
;
229 static struct sahara_dev
*dev_ptr
;
231 static inline void sahara_write(struct sahara_dev
*dev
, u32 data
, u32 reg
)
233 writel(data
, dev
->regs_base
+ reg
);
236 static inline unsigned int sahara_read(struct sahara_dev
*dev
, u32 reg
)
238 return readl(dev
->regs_base
+ reg
);
241 static u32
sahara_aes_key_hdr(struct sahara_dev
*dev
)
243 u32 hdr
= SAHARA_HDR_BASE
| SAHARA_HDR_SKHA_ALG_AES
|
244 SAHARA_HDR_FORM_KEY
| SAHARA_HDR_LLO
|
245 SAHARA_HDR_CHA_SKHA
| SAHARA_HDR_PARITY_BIT
;
247 if (dev
->flags
& FLAGS_CBC
) {
248 hdr
|= SAHARA_HDR_SKHA_MODE_CBC
;
249 hdr
^= SAHARA_HDR_PARITY_BIT
;
252 if (dev
->flags
& FLAGS_ENCRYPT
) {
253 hdr
|= SAHARA_HDR_SKHA_OP_ENC
;
254 hdr
^= SAHARA_HDR_PARITY_BIT
;
260 static u32
sahara_aes_data_link_hdr(struct sahara_dev
*dev
)
262 return SAHARA_HDR_BASE
| SAHARA_HDR_FORM_DATA
|
263 SAHARA_HDR_CHA_SKHA
| SAHARA_HDR_PARITY_BIT
;
266 static const char *sahara_err_src
[16] = {
269 "Descriptor length error",
270 "Descriptor length or pointer error",
272 "Link pointer error",
273 "Input buffer error",
274 "Output buffer error",
275 "Output buffer starvation",
276 "Internal state fault",
277 "General descriptor problem",
279 "Descriptor address error",
280 "Link address error",
285 static const char *sahara_err_dmasize
[4] = {
287 "Half-word transfer",
292 static const char *sahara_err_dmasrc
[8] = {
295 "Internal IP bus error",
297 "DMA crosses 256 byte boundary",
303 static const char *sahara_cha_errsrc
[12] = {
304 "Input buffer non-empty",
309 "Write during processing",
310 "CTX read during processing",
312 "Input buffer disabled/underflow",
313 "Output buffer disabled/overflow",
314 "DES key parity error",
318 static const char *sahara_cha_err
[4] = { "No error", "SKHA", "MDHA", "RNG" };
320 static void sahara_decode_error(struct sahara_dev
*dev
, unsigned int error
)
322 u8 source
= SAHARA_ERRSTATUS_GET_SOURCE(error
);
323 u16 chasrc
= ffs(SAHARA_ERRSTATUS_GET_CHASRC(error
));
325 dev_err(dev
->device
, "%s: Error Register = 0x%08x\n", __func__
, error
);
327 dev_err(dev
->device
, " - %s.\n", sahara_err_src
[source
]);
329 if (source
== SAHARA_ERRSOURCE_DMA
) {
330 if (error
& SAHARA_ERRSTATUS_DMA_DIR
)
331 dev_err(dev
->device
, " * DMA read.\n");
333 dev_err(dev
->device
, " * DMA write.\n");
335 dev_err(dev
->device
, " * %s.\n",
336 sahara_err_dmasize
[SAHARA_ERRSTATUS_GET_DMASZ(error
)]);
337 dev_err(dev
->device
, " * %s.\n",
338 sahara_err_dmasrc
[SAHARA_ERRSTATUS_GET_DMASRC(error
)]);
339 } else if (source
== SAHARA_ERRSOURCE_CHA
) {
340 dev_err(dev
->device
, " * %s.\n",
341 sahara_cha_errsrc
[chasrc
]);
342 dev_err(dev
->device
, " * %s.\n",
343 sahara_cha_err
[SAHARA_ERRSTATUS_GET_CHAERR(error
)]);
345 dev_err(dev
->device
, "\n");
348 static const char *sahara_state
[4] = { "Idle", "Busy", "Error", "HW Fault" };
350 static void sahara_decode_status(struct sahara_dev
*dev
, unsigned int status
)
354 if (!__is_defined(DEBUG
))
357 state
= SAHARA_STATUS_GET_STATE(status
);
359 dev_dbg(dev
->device
, "%s: Status Register = 0x%08x\n",
362 dev_dbg(dev
->device
, " - State = %d:\n", state
);
363 if (state
& SAHARA_STATE_COMP_FLAG
)
364 dev_dbg(dev
->device
, " * Descriptor completed. IRQ pending.\n");
366 dev_dbg(dev
->device
, " * %s.\n",
367 sahara_state
[state
& ~SAHARA_STATE_COMP_FLAG
]);
369 if (status
& SAHARA_STATUS_DAR_FULL
)
370 dev_dbg(dev
->device
, " - DAR Full.\n");
371 if (status
& SAHARA_STATUS_ERROR
)
372 dev_dbg(dev
->device
, " - Error.\n");
373 if (status
& SAHARA_STATUS_SECURE
)
374 dev_dbg(dev
->device
, " - Secure.\n");
375 if (status
& SAHARA_STATUS_FAIL
)
376 dev_dbg(dev
->device
, " - Fail.\n");
377 if (status
& SAHARA_STATUS_RNG_RESEED
)
378 dev_dbg(dev
->device
, " - RNG Reseed Request.\n");
379 if (status
& SAHARA_STATUS_ACTIVE_RNG
)
380 dev_dbg(dev
->device
, " - RNG Active.\n");
381 if (status
& SAHARA_STATUS_ACTIVE_MDHA
)
382 dev_dbg(dev
->device
, " - MDHA Active.\n");
383 if (status
& SAHARA_STATUS_ACTIVE_SKHA
)
384 dev_dbg(dev
->device
, " - SKHA Active.\n");
386 if (status
& SAHARA_STATUS_MODE_BATCH
)
387 dev_dbg(dev
->device
, " - Batch Mode.\n");
388 else if (status
& SAHARA_STATUS_MODE_DEDICATED
)
389 dev_dbg(dev
->device
, " - Dedicated Mode.\n");
390 else if (status
& SAHARA_STATUS_MODE_DEBUG
)
391 dev_dbg(dev
->device
, " - Debug Mode.\n");
393 dev_dbg(dev
->device
, " - Internal state = 0x%02x\n",
394 SAHARA_STATUS_GET_ISTATE(status
));
396 dev_dbg(dev
->device
, "Current DAR: 0x%08x\n",
397 sahara_read(dev
, SAHARA_REG_CDAR
));
398 dev_dbg(dev
->device
, "Initial DAR: 0x%08x\n\n",
399 sahara_read(dev
, SAHARA_REG_IDAR
));
402 static void sahara_dump_descriptors(struct sahara_dev
*dev
)
406 if (!__is_defined(DEBUG
))
409 for (i
= 0; i
< SAHARA_MAX_HW_DESC
; i
++) {
410 dev_dbg(dev
->device
, "Descriptor (%d) (%pad):\n",
411 i
, &dev
->hw_phys_desc
[i
]);
412 dev_dbg(dev
->device
, "\thdr = 0x%08x\n", dev
->hw_desc
[i
]->hdr
);
413 dev_dbg(dev
->device
, "\tlen1 = %u\n", dev
->hw_desc
[i
]->len1
);
414 dev_dbg(dev
->device
, "\tp1 = 0x%08x\n", dev
->hw_desc
[i
]->p1
);
415 dev_dbg(dev
->device
, "\tlen2 = %u\n", dev
->hw_desc
[i
]->len2
);
416 dev_dbg(dev
->device
, "\tp2 = 0x%08x\n", dev
->hw_desc
[i
]->p2
);
417 dev_dbg(dev
->device
, "\tnext = 0x%08x\n",
418 dev
->hw_desc
[i
]->next
);
420 dev_dbg(dev
->device
, "\n");
423 static void sahara_dump_links(struct sahara_dev
*dev
)
427 if (!__is_defined(DEBUG
))
430 for (i
= 0; i
< SAHARA_MAX_HW_LINK
; i
++) {
431 dev_dbg(dev
->device
, "Link (%d) (%pad):\n",
432 i
, &dev
->hw_phys_link
[i
]);
433 dev_dbg(dev
->device
, "\tlen = %u\n", dev
->hw_link
[i
]->len
);
434 dev_dbg(dev
->device
, "\tp = 0x%08x\n", dev
->hw_link
[i
]->p
);
435 dev_dbg(dev
->device
, "\tnext = 0x%08x\n",
436 dev
->hw_link
[i
]->next
);
438 dev_dbg(dev
->device
, "\n");
441 static int sahara_hw_descriptor_create(struct sahara_dev
*dev
)
443 struct sahara_ctx
*ctx
= dev
->ctx
;
444 struct scatterlist
*sg
;
449 /* Copy new key if necessary */
450 if (ctx
->flags
& FLAGS_NEW_KEY
) {
451 memcpy(dev
->key_base
, ctx
->key
, ctx
->keylen
);
452 ctx
->flags
&= ~FLAGS_NEW_KEY
;
454 if (dev
->flags
& FLAGS_CBC
) {
455 dev
->hw_desc
[idx
]->len1
= AES_BLOCK_SIZE
;
456 dev
->hw_desc
[idx
]->p1
= dev
->iv_phys_base
;
458 dev
->hw_desc
[idx
]->len1
= 0;
459 dev
->hw_desc
[idx
]->p1
= 0;
461 dev
->hw_desc
[idx
]->len2
= ctx
->keylen
;
462 dev
->hw_desc
[idx
]->p2
= dev
->key_phys_base
;
463 dev
->hw_desc
[idx
]->next
= dev
->hw_phys_desc
[1];
465 dev
->hw_desc
[idx
]->hdr
= sahara_aes_key_hdr(dev
);
470 dev
->nb_in_sg
= sg_nents_for_len(dev
->in_sg
, dev
->total
);
471 if (dev
->nb_in_sg
< 0) {
472 dev_err(dev
->device
, "Invalid numbers of src SG.\n");
473 return dev
->nb_in_sg
;
475 dev
->nb_out_sg
= sg_nents_for_len(dev
->out_sg
, dev
->total
);
476 if (dev
->nb_out_sg
< 0) {
477 dev_err(dev
->device
, "Invalid numbers of dst SG.\n");
478 return dev
->nb_out_sg
;
480 if ((dev
->nb_in_sg
+ dev
->nb_out_sg
) > SAHARA_MAX_HW_LINK
) {
481 dev_err(dev
->device
, "not enough hw links (%d)\n",
482 dev
->nb_in_sg
+ dev
->nb_out_sg
);
486 ret
= dma_map_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
,
488 if (ret
!= dev
->nb_in_sg
) {
489 dev_err(dev
->device
, "couldn't map in sg\n");
492 ret
= dma_map_sg(dev
->device
, dev
->out_sg
, dev
->nb_out_sg
,
494 if (ret
!= dev
->nb_out_sg
) {
495 dev_err(dev
->device
, "couldn't map out sg\n");
499 /* Create input links */
500 dev
->hw_desc
[idx
]->p1
= dev
->hw_phys_link
[0];
502 for (i
= 0; i
< dev
->nb_in_sg
; i
++) {
503 dev
->hw_link
[i
]->len
= sg
->length
;
504 dev
->hw_link
[i
]->p
= sg
->dma_address
;
505 if (i
== (dev
->nb_in_sg
- 1)) {
506 dev
->hw_link
[i
]->next
= 0;
508 dev
->hw_link
[i
]->next
= dev
->hw_phys_link
[i
+ 1];
513 /* Create output links */
514 dev
->hw_desc
[idx
]->p2
= dev
->hw_phys_link
[i
];
516 for (j
= i
; j
< dev
->nb_out_sg
+ i
; j
++) {
517 dev
->hw_link
[j
]->len
= sg
->length
;
518 dev
->hw_link
[j
]->p
= sg
->dma_address
;
519 if (j
== (dev
->nb_out_sg
+ i
- 1)) {
520 dev
->hw_link
[j
]->next
= 0;
522 dev
->hw_link
[j
]->next
= dev
->hw_phys_link
[j
+ 1];
527 /* Fill remaining fields of hw_desc[1] */
528 dev
->hw_desc
[idx
]->hdr
= sahara_aes_data_link_hdr(dev
);
529 dev
->hw_desc
[idx
]->len1
= dev
->total
;
530 dev
->hw_desc
[idx
]->len2
= dev
->total
;
531 dev
->hw_desc
[idx
]->next
= 0;
533 sahara_dump_descriptors(dev
);
534 sahara_dump_links(dev
);
536 sahara_write(dev
, dev
->hw_phys_desc
[0], SAHARA_REG_DAR
);
541 dma_unmap_sg(dev
->device
, dev
->out_sg
, dev
->nb_out_sg
,
544 dma_unmap_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
,
550 static int sahara_aes_process(struct skcipher_request
*req
)
552 struct sahara_dev
*dev
= dev_ptr
;
553 struct sahara_ctx
*ctx
;
554 struct sahara_aes_reqctx
*rctx
;
556 unsigned long timeout
;
558 /* Request is ready to be dispatched by the device */
560 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
561 req
->cryptlen
, req
->src
, req
->dst
);
563 /* assign new request to device */
564 dev
->total
= req
->cryptlen
;
565 dev
->in_sg
= req
->src
;
566 dev
->out_sg
= req
->dst
;
568 rctx
= skcipher_request_ctx(req
);
569 ctx
= crypto_skcipher_ctx(crypto_skcipher_reqtfm(req
));
570 rctx
->mode
&= FLAGS_MODE_MASK
;
571 dev
->flags
= (dev
->flags
& ~FLAGS_MODE_MASK
) | rctx
->mode
;
573 if ((dev
->flags
& FLAGS_CBC
) && req
->iv
)
574 memcpy(dev
->iv_base
, req
->iv
, AES_KEYSIZE_128
);
576 /* assign new context to device */
579 reinit_completion(&dev
->dma_completion
);
581 ret
= sahara_hw_descriptor_create(dev
);
585 timeout
= wait_for_completion_timeout(&dev
->dma_completion
,
586 msecs_to_jiffies(SAHARA_TIMEOUT_MS
));
588 dev_err(dev
->device
, "AES timeout\n");
592 dma_unmap_sg(dev
->device
, dev
->out_sg
, dev
->nb_out_sg
,
594 dma_unmap_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
,
600 static int sahara_aes_setkey(struct crypto_skcipher
*tfm
, const u8
*key
,
603 struct sahara_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
606 ctx
->keylen
= keylen
;
608 /* SAHARA only supports 128bit keys */
609 if (keylen
== AES_KEYSIZE_128
) {
610 memcpy(ctx
->key
, key
, keylen
);
611 ctx
->flags
|= FLAGS_NEW_KEY
;
615 if (keylen
!= AES_KEYSIZE_192
&& keylen
!= AES_KEYSIZE_256
)
619 * The requested key size is not supported by HW, do a fallback.
621 crypto_sync_skcipher_clear_flags(ctx
->fallback
, CRYPTO_TFM_REQ_MASK
);
622 crypto_sync_skcipher_set_flags(ctx
->fallback
, tfm
->base
.crt_flags
&
623 CRYPTO_TFM_REQ_MASK
);
625 ret
= crypto_sync_skcipher_setkey(ctx
->fallback
, key
, keylen
);
627 tfm
->base
.crt_flags
&= ~CRYPTO_TFM_RES_MASK
;
628 tfm
->base
.crt_flags
|= crypto_sync_skcipher_get_flags(ctx
->fallback
) &
633 static int sahara_aes_crypt(struct skcipher_request
*req
, unsigned long mode
)
635 struct sahara_aes_reqctx
*rctx
= skcipher_request_ctx(req
);
636 struct sahara_dev
*dev
= dev_ptr
;
639 dev_dbg(dev
->device
, "nbytes: %d, enc: %d, cbc: %d\n",
640 req
->cryptlen
, !!(mode
& FLAGS_ENCRYPT
), !!(mode
& FLAGS_CBC
));
642 if (!IS_ALIGNED(req
->cryptlen
, AES_BLOCK_SIZE
)) {
644 "request size is not exact amount of AES blocks\n");
650 mutex_lock(&dev
->queue_mutex
);
651 err
= crypto_enqueue_request(&dev
->queue
, &req
->base
);
652 mutex_unlock(&dev
->queue_mutex
);
654 wake_up_process(dev
->kthread
);
659 static int sahara_aes_ecb_encrypt(struct skcipher_request
*req
)
661 struct sahara_ctx
*ctx
= crypto_skcipher_ctx(
662 crypto_skcipher_reqtfm(req
));
665 if (unlikely(ctx
->keylen
!= AES_KEYSIZE_128
)) {
666 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq
, ctx
->fallback
);
668 skcipher_request_set_sync_tfm(subreq
, ctx
->fallback
);
669 skcipher_request_set_callback(subreq
, req
->base
.flags
,
671 skcipher_request_set_crypt(subreq
, req
->src
, req
->dst
,
672 req
->cryptlen
, req
->iv
);
673 err
= crypto_skcipher_encrypt(subreq
);
674 skcipher_request_zero(subreq
);
678 return sahara_aes_crypt(req
, FLAGS_ENCRYPT
);
681 static int sahara_aes_ecb_decrypt(struct skcipher_request
*req
)
683 struct sahara_ctx
*ctx
= crypto_skcipher_ctx(
684 crypto_skcipher_reqtfm(req
));
687 if (unlikely(ctx
->keylen
!= AES_KEYSIZE_128
)) {
688 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq
, ctx
->fallback
);
690 skcipher_request_set_sync_tfm(subreq
, ctx
->fallback
);
691 skcipher_request_set_callback(subreq
, req
->base
.flags
,
693 skcipher_request_set_crypt(subreq
, req
->src
, req
->dst
,
694 req
->cryptlen
, req
->iv
);
695 err
= crypto_skcipher_decrypt(subreq
);
696 skcipher_request_zero(subreq
);
700 return sahara_aes_crypt(req
, 0);
703 static int sahara_aes_cbc_encrypt(struct skcipher_request
*req
)
705 struct sahara_ctx
*ctx
= crypto_skcipher_ctx(
706 crypto_skcipher_reqtfm(req
));
709 if (unlikely(ctx
->keylen
!= AES_KEYSIZE_128
)) {
710 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq
, ctx
->fallback
);
712 skcipher_request_set_sync_tfm(subreq
, ctx
->fallback
);
713 skcipher_request_set_callback(subreq
, req
->base
.flags
,
715 skcipher_request_set_crypt(subreq
, req
->src
, req
->dst
,
716 req
->cryptlen
, req
->iv
);
717 err
= crypto_skcipher_encrypt(subreq
);
718 skcipher_request_zero(subreq
);
722 return sahara_aes_crypt(req
, FLAGS_ENCRYPT
| FLAGS_CBC
);
725 static int sahara_aes_cbc_decrypt(struct skcipher_request
*req
)
727 struct sahara_ctx
*ctx
= crypto_skcipher_ctx(
728 crypto_skcipher_reqtfm(req
));
731 if (unlikely(ctx
->keylen
!= AES_KEYSIZE_128
)) {
732 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq
, ctx
->fallback
);
734 skcipher_request_set_sync_tfm(subreq
, ctx
->fallback
);
735 skcipher_request_set_callback(subreq
, req
->base
.flags
,
737 skcipher_request_set_crypt(subreq
, req
->src
, req
->dst
,
738 req
->cryptlen
, req
->iv
);
739 err
= crypto_skcipher_decrypt(subreq
);
740 skcipher_request_zero(subreq
);
744 return sahara_aes_crypt(req
, FLAGS_CBC
);
747 static int sahara_aes_init_tfm(struct crypto_skcipher
*tfm
)
749 const char *name
= crypto_tfm_alg_name(&tfm
->base
);
750 struct sahara_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
752 ctx
->fallback
= crypto_alloc_sync_skcipher(name
, 0,
753 CRYPTO_ALG_NEED_FALLBACK
);
754 if (IS_ERR(ctx
->fallback
)) {
755 pr_err("Error allocating fallback algo %s\n", name
);
756 return PTR_ERR(ctx
->fallback
);
759 crypto_skcipher_set_reqsize(tfm
, sizeof(struct sahara_aes_reqctx
));
764 static void sahara_aes_exit_tfm(struct crypto_skcipher
*tfm
)
766 struct sahara_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
768 crypto_free_sync_skcipher(ctx
->fallback
);
771 static u32
sahara_sha_init_hdr(struct sahara_dev
*dev
,
772 struct sahara_sha_reqctx
*rctx
)
779 hdr
|= SAHARA_HDR_MDHA_SET_MODE_HASH
;
780 hdr
|= SAHARA_HDR_MDHA_INIT
;
782 hdr
|= SAHARA_HDR_MDHA_SET_MODE_MD_KEY
;
786 hdr
|= SAHARA_HDR_MDHA_PDATA
;
788 if (hweight_long(hdr
) % 2 == 0)
789 hdr
|= SAHARA_HDR_PARITY_BIT
;
794 static int sahara_sha_hw_links_create(struct sahara_dev
*dev
,
795 struct sahara_sha_reqctx
*rctx
,
798 struct scatterlist
*sg
;
802 dev
->in_sg
= rctx
->in_sg
;
804 dev
->nb_in_sg
= sg_nents_for_len(dev
->in_sg
, rctx
->total
);
805 if (dev
->nb_in_sg
< 0) {
806 dev_err(dev
->device
, "Invalid numbers of src SG.\n");
807 return dev
->nb_in_sg
;
809 if ((dev
->nb_in_sg
) > SAHARA_MAX_HW_LINK
) {
810 dev_err(dev
->device
, "not enough hw links (%d)\n",
811 dev
->nb_in_sg
+ dev
->nb_out_sg
);
816 ret
= dma_map_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
, DMA_TO_DEVICE
);
820 for (i
= start
; i
< dev
->nb_in_sg
+ start
; i
++) {
821 dev
->hw_link
[i
]->len
= sg
->length
;
822 dev
->hw_link
[i
]->p
= sg
->dma_address
;
823 if (i
== (dev
->nb_in_sg
+ start
- 1)) {
824 dev
->hw_link
[i
]->next
= 0;
826 dev
->hw_link
[i
]->next
= dev
->hw_phys_link
[i
+ 1];
834 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev
*dev
,
835 struct sahara_sha_reqctx
*rctx
,
836 struct ahash_request
*req
,
843 /* Create initial descriptor: #8*/
844 dev
->hw_desc
[index
]->hdr
= sahara_sha_init_hdr(dev
, rctx
);
846 /* Create hash descriptor: #10. Must follow #6. */
847 dev
->hw_desc
[index
]->hdr
= SAHARA_HDR_MDHA_HASH
;
849 dev
->hw_desc
[index
]->len1
= rctx
->total
;
850 if (dev
->hw_desc
[index
]->len1
== 0) {
851 /* if len1 is 0, p1 must be 0, too */
852 dev
->hw_desc
[index
]->p1
= 0;
855 /* Create input links */
856 dev
->hw_desc
[index
]->p1
= dev
->hw_phys_link
[index
];
857 i
= sahara_sha_hw_links_create(dev
, rctx
, index
);
859 rctx
->sg_in_idx
= index
;
864 dev
->hw_desc
[index
]->p2
= dev
->hw_phys_link
[i
];
866 /* Save the context for the next operation */
867 result_len
= rctx
->context_size
;
868 dev
->hw_link
[i
]->p
= dev
->context_phys_base
;
870 dev
->hw_link
[i
]->len
= result_len
;
871 dev
->hw_desc
[index
]->len2
= result_len
;
873 dev
->hw_link
[i
]->next
= 0;
879 * Load descriptor aka #6
881 * To load a previously saved context back to the MDHA unit
887 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev
*dev
,
888 struct sahara_sha_reqctx
*rctx
,
889 struct ahash_request
*req
,
892 dev
->hw_desc
[index
]->hdr
= sahara_sha_init_hdr(dev
, rctx
);
894 dev
->hw_desc
[index
]->len1
= rctx
->context_size
;
895 dev
->hw_desc
[index
]->p1
= dev
->hw_phys_link
[index
];
896 dev
->hw_desc
[index
]->len2
= 0;
897 dev
->hw_desc
[index
]->p2
= 0;
899 dev
->hw_link
[index
]->len
= rctx
->context_size
;
900 dev
->hw_link
[index
]->p
= dev
->context_phys_base
;
901 dev
->hw_link
[index
]->next
= 0;
906 static int sahara_walk_and_recalc(struct scatterlist
*sg
, unsigned int nbytes
)
908 if (!sg
|| !sg
->length
)
911 while (nbytes
&& sg
) {
912 if (nbytes
<= sg
->length
) {
917 nbytes
-= sg
->length
;
924 static int sahara_sha_prepare_request(struct ahash_request
*req
)
926 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
927 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
928 unsigned int hash_later
;
929 unsigned int block_size
;
932 block_size
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
934 /* append bytes from previous operation */
935 len
= rctx
->buf_cnt
+ req
->nbytes
;
937 /* only the last transfer can be padded in hardware */
938 if (!rctx
->last
&& (len
< block_size
)) {
939 /* to few data, save for next operation */
940 scatterwalk_map_and_copy(rctx
->buf
+ rctx
->buf_cnt
, req
->src
,
942 rctx
->buf_cnt
+= req
->nbytes
;
947 /* add data from previous operation first */
949 memcpy(rctx
->rembuf
, rctx
->buf
, rctx
->buf_cnt
);
951 /* data must always be a multiple of block_size */
952 hash_later
= rctx
->last
? 0 : len
& (block_size
- 1);
954 unsigned int offset
= req
->nbytes
- hash_later
;
955 /* Save remaining bytes for later use */
956 scatterwalk_map_and_copy(rctx
->buf
, req
->src
, offset
,
960 /* nbytes should now be multiple of blocksize */
961 req
->nbytes
= req
->nbytes
- hash_later
;
963 sahara_walk_and_recalc(req
->src
, req
->nbytes
);
965 /* have data from previous operation and current */
966 if (rctx
->buf_cnt
&& req
->nbytes
) {
967 sg_init_table(rctx
->in_sg_chain
, 2);
968 sg_set_buf(rctx
->in_sg_chain
, rctx
->rembuf
, rctx
->buf_cnt
);
970 sg_chain(rctx
->in_sg_chain
, 2, req
->src
);
972 rctx
->total
= req
->nbytes
+ rctx
->buf_cnt
;
973 rctx
->in_sg
= rctx
->in_sg_chain
;
975 req
->src
= rctx
->in_sg_chain
;
976 /* only data from previous operation */
977 } else if (rctx
->buf_cnt
) {
979 rctx
->in_sg
= req
->src
;
981 rctx
->in_sg
= rctx
->in_sg_chain
;
982 /* buf was copied into rembuf above */
983 sg_init_one(rctx
->in_sg
, rctx
->rembuf
, rctx
->buf_cnt
);
984 rctx
->total
= rctx
->buf_cnt
;
985 /* no data from previous operation */
987 rctx
->in_sg
= req
->src
;
988 rctx
->total
= req
->nbytes
;
989 req
->src
= rctx
->in_sg
;
992 /* on next call, we only have the remaining data in the buffer */
993 rctx
->buf_cnt
= hash_later
;
998 static int sahara_sha_process(struct ahash_request
*req
)
1000 struct sahara_dev
*dev
= dev_ptr
;
1001 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1003 unsigned long timeout
;
1005 ret
= sahara_sha_prepare_request(req
);
1010 sahara_sha_hw_data_descriptor_create(dev
, rctx
, req
, 0);
1011 dev
->hw_desc
[0]->next
= 0;
1014 memcpy(dev
->context_base
, rctx
->context
, rctx
->context_size
);
1016 sahara_sha_hw_context_descriptor_create(dev
, rctx
, req
, 0);
1017 dev
->hw_desc
[0]->next
= dev
->hw_phys_desc
[1];
1018 sahara_sha_hw_data_descriptor_create(dev
, rctx
, req
, 1);
1019 dev
->hw_desc
[1]->next
= 0;
1022 sahara_dump_descriptors(dev
);
1023 sahara_dump_links(dev
);
1025 reinit_completion(&dev
->dma_completion
);
1027 sahara_write(dev
, dev
->hw_phys_desc
[0], SAHARA_REG_DAR
);
1029 timeout
= wait_for_completion_timeout(&dev
->dma_completion
,
1030 msecs_to_jiffies(SAHARA_TIMEOUT_MS
));
1032 dev_err(dev
->device
, "SHA timeout\n");
1036 if (rctx
->sg_in_idx
)
1037 dma_unmap_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
,
1040 memcpy(rctx
->context
, dev
->context_base
, rctx
->context_size
);
1043 memcpy(req
->result
, rctx
->context
, rctx
->digest_size
);
1048 static int sahara_queue_manage(void *data
)
1050 struct sahara_dev
*dev
= (struct sahara_dev
*)data
;
1051 struct crypto_async_request
*async_req
;
1052 struct crypto_async_request
*backlog
;
1056 __set_current_state(TASK_INTERRUPTIBLE
);
1058 mutex_lock(&dev
->queue_mutex
);
1059 backlog
= crypto_get_backlog(&dev
->queue
);
1060 async_req
= crypto_dequeue_request(&dev
->queue
);
1061 mutex_unlock(&dev
->queue_mutex
);
1064 backlog
->complete(backlog
, -EINPROGRESS
);
1067 if (crypto_tfm_alg_type(async_req
->tfm
) ==
1068 CRYPTO_ALG_TYPE_AHASH
) {
1069 struct ahash_request
*req
=
1070 ahash_request_cast(async_req
);
1072 ret
= sahara_sha_process(req
);
1074 struct skcipher_request
*req
=
1075 skcipher_request_cast(async_req
);
1077 ret
= sahara_aes_process(req
);
1080 async_req
->complete(async_req
, ret
);
1086 } while (!kthread_should_stop());
1091 static int sahara_sha_enqueue(struct ahash_request
*req
, int last
)
1093 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1094 struct sahara_dev
*dev
= dev_ptr
;
1097 if (!req
->nbytes
&& !last
)
1102 if (!rctx
->active
) {
1107 mutex_lock(&dev
->queue_mutex
);
1108 ret
= crypto_enqueue_request(&dev
->queue
, &req
->base
);
1109 mutex_unlock(&dev
->queue_mutex
);
1111 wake_up_process(dev
->kthread
);
1116 static int sahara_sha_init(struct ahash_request
*req
)
1118 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
1119 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1121 memset(rctx
, 0, sizeof(*rctx
));
1123 switch (crypto_ahash_digestsize(tfm
)) {
1124 case SHA1_DIGEST_SIZE
:
1125 rctx
->mode
|= SAHARA_HDR_MDHA_ALG_SHA1
;
1126 rctx
->digest_size
= SHA1_DIGEST_SIZE
;
1128 case SHA256_DIGEST_SIZE
:
1129 rctx
->mode
|= SAHARA_HDR_MDHA_ALG_SHA256
;
1130 rctx
->digest_size
= SHA256_DIGEST_SIZE
;
1136 rctx
->context_size
= rctx
->digest_size
+ 4;
1142 static int sahara_sha_update(struct ahash_request
*req
)
1144 return sahara_sha_enqueue(req
, 0);
1147 static int sahara_sha_final(struct ahash_request
*req
)
1150 return sahara_sha_enqueue(req
, 1);
1153 static int sahara_sha_finup(struct ahash_request
*req
)
1155 return sahara_sha_enqueue(req
, 1);
1158 static int sahara_sha_digest(struct ahash_request
*req
)
1160 sahara_sha_init(req
);
1162 return sahara_sha_finup(req
);
1165 static int sahara_sha_export(struct ahash_request
*req
, void *out
)
1167 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1169 memcpy(out
, rctx
, sizeof(struct sahara_sha_reqctx
));
1174 static int sahara_sha_import(struct ahash_request
*req
, const void *in
)
1176 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1178 memcpy(rctx
, in
, sizeof(struct sahara_sha_reqctx
));
1183 static int sahara_sha_cra_init(struct crypto_tfm
*tfm
)
1185 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
1186 sizeof(struct sahara_sha_reqctx
) +
1187 SHA_BUFFER_LEN
+ SHA256_BLOCK_SIZE
);
1192 static struct skcipher_alg aes_algs
[] = {
1194 .base
.cra_name
= "ecb(aes)",
1195 .base
.cra_driver_name
= "sahara-ecb-aes",
1196 .base
.cra_priority
= 300,
1197 .base
.cra_flags
= CRYPTO_ALG_ASYNC
| CRYPTO_ALG_NEED_FALLBACK
,
1198 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
1199 .base
.cra_ctxsize
= sizeof(struct sahara_ctx
),
1200 .base
.cra_alignmask
= 0x0,
1201 .base
.cra_module
= THIS_MODULE
,
1203 .init
= sahara_aes_init_tfm
,
1204 .exit
= sahara_aes_exit_tfm
,
1205 .min_keysize
= AES_MIN_KEY_SIZE
,
1206 .max_keysize
= AES_MAX_KEY_SIZE
,
1207 .setkey
= sahara_aes_setkey
,
1208 .encrypt
= sahara_aes_ecb_encrypt
,
1209 .decrypt
= sahara_aes_ecb_decrypt
,
1211 .base
.cra_name
= "cbc(aes)",
1212 .base
.cra_driver_name
= "sahara-cbc-aes",
1213 .base
.cra_priority
= 300,
1214 .base
.cra_flags
= CRYPTO_ALG_ASYNC
| CRYPTO_ALG_NEED_FALLBACK
,
1215 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
1216 .base
.cra_ctxsize
= sizeof(struct sahara_ctx
),
1217 .base
.cra_alignmask
= 0x0,
1218 .base
.cra_module
= THIS_MODULE
,
1220 .init
= sahara_aes_init_tfm
,
1221 .exit
= sahara_aes_exit_tfm
,
1222 .min_keysize
= AES_MIN_KEY_SIZE
,
1223 .max_keysize
= AES_MAX_KEY_SIZE
,
1224 .ivsize
= AES_BLOCK_SIZE
,
1225 .setkey
= sahara_aes_setkey
,
1226 .encrypt
= sahara_aes_cbc_encrypt
,
1227 .decrypt
= sahara_aes_cbc_decrypt
,
1231 static struct ahash_alg sha_v3_algs
[] = {
1233 .init
= sahara_sha_init
,
1234 .update
= sahara_sha_update
,
1235 .final
= sahara_sha_final
,
1236 .finup
= sahara_sha_finup
,
1237 .digest
= sahara_sha_digest
,
1238 .export
= sahara_sha_export
,
1239 .import
= sahara_sha_import
,
1240 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
1241 .halg
.statesize
= sizeof(struct sahara_sha_reqctx
),
1244 .cra_driver_name
= "sahara-sha1",
1245 .cra_priority
= 300,
1246 .cra_flags
= CRYPTO_ALG_ASYNC
|
1247 CRYPTO_ALG_NEED_FALLBACK
,
1248 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1249 .cra_ctxsize
= sizeof(struct sahara_ctx
),
1251 .cra_module
= THIS_MODULE
,
1252 .cra_init
= sahara_sha_cra_init
,
1257 static struct ahash_alg sha_v4_algs
[] = {
1259 .init
= sahara_sha_init
,
1260 .update
= sahara_sha_update
,
1261 .final
= sahara_sha_final
,
1262 .finup
= sahara_sha_finup
,
1263 .digest
= sahara_sha_digest
,
1264 .export
= sahara_sha_export
,
1265 .import
= sahara_sha_import
,
1266 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
1267 .halg
.statesize
= sizeof(struct sahara_sha_reqctx
),
1269 .cra_name
= "sha256",
1270 .cra_driver_name
= "sahara-sha256",
1271 .cra_priority
= 300,
1272 .cra_flags
= CRYPTO_ALG_ASYNC
|
1273 CRYPTO_ALG_NEED_FALLBACK
,
1274 .cra_blocksize
= SHA256_BLOCK_SIZE
,
1275 .cra_ctxsize
= sizeof(struct sahara_ctx
),
1277 .cra_module
= THIS_MODULE
,
1278 .cra_init
= sahara_sha_cra_init
,
1283 static irqreturn_t
sahara_irq_handler(int irq
, void *data
)
1285 struct sahara_dev
*dev
= (struct sahara_dev
*)data
;
1286 unsigned int stat
= sahara_read(dev
, SAHARA_REG_STATUS
);
1287 unsigned int err
= sahara_read(dev
, SAHARA_REG_ERRSTATUS
);
1289 sahara_write(dev
, SAHARA_CMD_CLEAR_INT
| SAHARA_CMD_CLEAR_ERR
,
1292 sahara_decode_status(dev
, stat
);
1294 if (SAHARA_STATUS_GET_STATE(stat
) == SAHARA_STATE_BUSY
) {
1296 } else if (SAHARA_STATUS_GET_STATE(stat
) == SAHARA_STATE_COMPLETE
) {
1299 sahara_decode_error(dev
, err
);
1300 dev
->error
= -EINVAL
;
1303 complete(&dev
->dma_completion
);
1309 static int sahara_register_algs(struct sahara_dev
*dev
)
1312 unsigned int i
, j
, k
, l
;
1314 for (i
= 0; i
< ARRAY_SIZE(aes_algs
); i
++) {
1315 err
= crypto_register_skcipher(&aes_algs
[i
]);
1320 for (k
= 0; k
< ARRAY_SIZE(sha_v3_algs
); k
++) {
1321 err
= crypto_register_ahash(&sha_v3_algs
[k
]);
1323 goto err_sha_v3_algs
;
1326 if (dev
->version
> SAHARA_VERSION_3
)
1327 for (l
= 0; l
< ARRAY_SIZE(sha_v4_algs
); l
++) {
1328 err
= crypto_register_ahash(&sha_v4_algs
[l
]);
1330 goto err_sha_v4_algs
;
1336 for (j
= 0; j
< l
; j
++)
1337 crypto_unregister_ahash(&sha_v4_algs
[j
]);
1340 for (j
= 0; j
< k
; j
++)
1341 crypto_unregister_ahash(&sha_v3_algs
[j
]);
1344 for (j
= 0; j
< i
; j
++)
1345 crypto_unregister_skcipher(&aes_algs
[j
]);
1350 static void sahara_unregister_algs(struct sahara_dev
*dev
)
1354 for (i
= 0; i
< ARRAY_SIZE(aes_algs
); i
++)
1355 crypto_unregister_skcipher(&aes_algs
[i
]);
1357 for (i
= 0; i
< ARRAY_SIZE(sha_v3_algs
); i
++)
1358 crypto_unregister_ahash(&sha_v3_algs
[i
]);
1360 if (dev
->version
> SAHARA_VERSION_3
)
1361 for (i
= 0; i
< ARRAY_SIZE(sha_v4_algs
); i
++)
1362 crypto_unregister_ahash(&sha_v4_algs
[i
]);
1365 static const struct platform_device_id sahara_platform_ids
[] = {
1366 { .name
= "sahara-imx27" },
1369 MODULE_DEVICE_TABLE(platform
, sahara_platform_ids
);
1371 static const struct of_device_id sahara_dt_ids
[] = {
1372 { .compatible
= "fsl,imx53-sahara" },
1373 { .compatible
= "fsl,imx27-sahara" },
1376 MODULE_DEVICE_TABLE(of
, sahara_dt_ids
);
1378 static int sahara_probe(struct platform_device
*pdev
)
1380 struct sahara_dev
*dev
;
1386 dev
= devm_kzalloc(&pdev
->dev
, sizeof(*dev
), GFP_KERNEL
);
1390 dev
->device
= &pdev
->dev
;
1391 platform_set_drvdata(pdev
, dev
);
1393 /* Get the base address */
1394 dev
->regs_base
= devm_platform_ioremap_resource(pdev
, 0);
1395 if (IS_ERR(dev
->regs_base
))
1396 return PTR_ERR(dev
->regs_base
);
1399 irq
= platform_get_irq(pdev
, 0);
1403 err
= devm_request_irq(&pdev
->dev
, irq
, sahara_irq_handler
,
1404 0, dev_name(&pdev
->dev
), dev
);
1406 dev_err(&pdev
->dev
, "failed to request irq\n");
1411 dev
->clk_ipg
= devm_clk_get(&pdev
->dev
, "ipg");
1412 if (IS_ERR(dev
->clk_ipg
)) {
1413 dev_err(&pdev
->dev
, "Could not get ipg clock\n");
1414 return PTR_ERR(dev
->clk_ipg
);
1417 dev
->clk_ahb
= devm_clk_get(&pdev
->dev
, "ahb");
1418 if (IS_ERR(dev
->clk_ahb
)) {
1419 dev_err(&pdev
->dev
, "Could not get ahb clock\n");
1420 return PTR_ERR(dev
->clk_ahb
);
1423 /* Allocate HW descriptors */
1424 dev
->hw_desc
[0] = dmam_alloc_coherent(&pdev
->dev
,
1425 SAHARA_MAX_HW_DESC
* sizeof(struct sahara_hw_desc
),
1426 &dev
->hw_phys_desc
[0], GFP_KERNEL
);
1427 if (!dev
->hw_desc
[0]) {
1428 dev_err(&pdev
->dev
, "Could not allocate hw descriptors\n");
1431 dev
->hw_desc
[1] = dev
->hw_desc
[0] + 1;
1432 dev
->hw_phys_desc
[1] = dev
->hw_phys_desc
[0] +
1433 sizeof(struct sahara_hw_desc
);
1435 /* Allocate space for iv and key */
1436 dev
->key_base
= dmam_alloc_coherent(&pdev
->dev
, 2 * AES_KEYSIZE_128
,
1437 &dev
->key_phys_base
, GFP_KERNEL
);
1438 if (!dev
->key_base
) {
1439 dev_err(&pdev
->dev
, "Could not allocate memory for key\n");
1442 dev
->iv_base
= dev
->key_base
+ AES_KEYSIZE_128
;
1443 dev
->iv_phys_base
= dev
->key_phys_base
+ AES_KEYSIZE_128
;
1445 /* Allocate space for context: largest digest + message length field */
1446 dev
->context_base
= dmam_alloc_coherent(&pdev
->dev
,
1447 SHA256_DIGEST_SIZE
+ 4,
1448 &dev
->context_phys_base
, GFP_KERNEL
);
1449 if (!dev
->context_base
) {
1450 dev_err(&pdev
->dev
, "Could not allocate memory for MDHA context\n");
1454 /* Allocate space for HW links */
1455 dev
->hw_link
[0] = dmam_alloc_coherent(&pdev
->dev
,
1456 SAHARA_MAX_HW_LINK
* sizeof(struct sahara_hw_link
),
1457 &dev
->hw_phys_link
[0], GFP_KERNEL
);
1458 if (!dev
->hw_link
[0]) {
1459 dev_err(&pdev
->dev
, "Could not allocate hw links\n");
1462 for (i
= 1; i
< SAHARA_MAX_HW_LINK
; i
++) {
1463 dev
->hw_phys_link
[i
] = dev
->hw_phys_link
[i
- 1] +
1464 sizeof(struct sahara_hw_link
);
1465 dev
->hw_link
[i
] = dev
->hw_link
[i
- 1] + 1;
1468 crypto_init_queue(&dev
->queue
, SAHARA_QUEUE_LENGTH
);
1470 mutex_init(&dev
->queue_mutex
);
1474 dev
->kthread
= kthread_run(sahara_queue_manage
, dev
, "sahara_crypto");
1475 if (IS_ERR(dev
->kthread
)) {
1476 return PTR_ERR(dev
->kthread
);
1479 init_completion(&dev
->dma_completion
);
1481 err
= clk_prepare_enable(dev
->clk_ipg
);
1484 err
= clk_prepare_enable(dev
->clk_ahb
);
1486 goto clk_ipg_disable
;
1488 version
= sahara_read(dev
, SAHARA_REG_VERSION
);
1489 if (of_device_is_compatible(pdev
->dev
.of_node
, "fsl,imx27-sahara")) {
1490 if (version
!= SAHARA_VERSION_3
)
1492 } else if (of_device_is_compatible(pdev
->dev
.of_node
,
1493 "fsl,imx53-sahara")) {
1494 if (((version
>> 8) & 0xff) != SAHARA_VERSION_4
)
1496 version
= (version
>> 8) & 0xff;
1498 if (err
== -ENODEV
) {
1499 dev_err(&pdev
->dev
, "SAHARA version %d not supported\n",
1504 dev
->version
= version
;
1506 sahara_write(dev
, SAHARA_CMD_RESET
| SAHARA_CMD_MODE_BATCH
,
1508 sahara_write(dev
, SAHARA_CONTROL_SET_THROTTLE(0) |
1509 SAHARA_CONTROL_SET_MAXBURST(8) |
1510 SAHARA_CONTROL_RNG_AUTORSD
|
1511 SAHARA_CONTROL_ENABLE_INT
,
1512 SAHARA_REG_CONTROL
);
1514 err
= sahara_register_algs(dev
);
1518 dev_info(&pdev
->dev
, "SAHARA version %d initialized\n", version
);
1523 kthread_stop(dev
->kthread
);
1525 clk_disable_unprepare(dev
->clk_ahb
);
1527 clk_disable_unprepare(dev
->clk_ipg
);
1532 static int sahara_remove(struct platform_device
*pdev
)
1534 struct sahara_dev
*dev
= platform_get_drvdata(pdev
);
1536 kthread_stop(dev
->kthread
);
1538 sahara_unregister_algs(dev
);
1540 clk_disable_unprepare(dev
->clk_ipg
);
1541 clk_disable_unprepare(dev
->clk_ahb
);
1548 static struct platform_driver sahara_driver
= {
1549 .probe
= sahara_probe
,
1550 .remove
= sahara_remove
,
1552 .name
= SAHARA_NAME
,
1553 .of_match_table
= sahara_dt_ids
,
1555 .id_table
= sahara_platform_ids
,
1558 module_platform_driver(sahara_driver
);
1560 MODULE_LICENSE("GPL");
1561 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1562 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1563 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");