]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/crypto/sahara.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branches 'for-4.4/upstream-fixes', 'for-4.5/async-suspend', 'for-4.5/container...
[mirror_ubuntu-bionic-kernel.git] / drivers / crypto / sahara.c
1 /*
2 * Cryptographic API.
3 *
4 * Support for SAHARA cryptographic accelerator.
5 *
6 * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
7 * Copyright (c) 2013 Vista Silicon S.L.
8 * Author: Javier Martin <javier.martin@vista-silicon.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as published
12 * by the Free Software Foundation.
13 *
14 * Based on omap-aes.c and tegra-aes.c
15 */
16
17 #include <crypto/algapi.h>
18 #include <crypto/aes.h>
19 #include <crypto/hash.h>
20 #include <crypto/internal/hash.h>
21 #include <crypto/scatterwalk.h>
22 #include <crypto/sha.h>
23
24 #include <linux/clk.h>
25 #include <linux/crypto.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/kernel.h>
30 #include <linux/kthread.h>
31 #include <linux/module.h>
32 #include <linux/mutex.h>
33 #include <linux/of.h>
34 #include <linux/of_device.h>
35 #include <linux/platform_device.h>
36
37 #define SHA_BUFFER_LEN PAGE_SIZE
38 #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE
39
40 #define SAHARA_NAME "sahara"
41 #define SAHARA_VERSION_3 3
42 #define SAHARA_VERSION_4 4
43 #define SAHARA_TIMEOUT_MS 1000
44 #define SAHARA_MAX_HW_DESC 2
45 #define SAHARA_MAX_HW_LINK 20
46
47 #define FLAGS_MODE_MASK 0x000f
48 #define FLAGS_ENCRYPT BIT(0)
49 #define FLAGS_CBC BIT(1)
50 #define FLAGS_NEW_KEY BIT(3)
51
52 #define SAHARA_HDR_BASE 0x00800000
53 #define SAHARA_HDR_SKHA_ALG_AES 0
54 #define SAHARA_HDR_SKHA_OP_ENC (1 << 2)
55 #define SAHARA_HDR_SKHA_MODE_ECB (0 << 3)
56 #define SAHARA_HDR_SKHA_MODE_CBC (1 << 3)
57 #define SAHARA_HDR_FORM_DATA (5 << 16)
58 #define SAHARA_HDR_FORM_KEY (8 << 16)
59 #define SAHARA_HDR_LLO (1 << 24)
60 #define SAHARA_HDR_CHA_SKHA (1 << 28)
61 #define SAHARA_HDR_CHA_MDHA (2 << 28)
62 #define SAHARA_HDR_PARITY_BIT (1 << 31)
63
64 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
65 #define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000
66 #define SAHARA_HDR_MDHA_HASH 0xA0850000
67 #define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000
68 #define SAHARA_HDR_MDHA_ALG_SHA1 0
69 #define SAHARA_HDR_MDHA_ALG_MD5 1
70 #define SAHARA_HDR_MDHA_ALG_SHA256 2
71 #define SAHARA_HDR_MDHA_ALG_SHA224 3
72 #define SAHARA_HDR_MDHA_PDATA (1 << 2)
73 #define SAHARA_HDR_MDHA_HMAC (1 << 3)
74 #define SAHARA_HDR_MDHA_INIT (1 << 5)
75 #define SAHARA_HDR_MDHA_IPAD (1 << 6)
76 #define SAHARA_HDR_MDHA_OPAD (1 << 7)
77 #define SAHARA_HDR_MDHA_SWAP (1 << 8)
78 #define SAHARA_HDR_MDHA_MAC_FULL (1 << 9)
79 #define SAHARA_HDR_MDHA_SSL (1 << 10)
80
81 /* SAHARA can only process one request at a time */
82 #define SAHARA_QUEUE_LENGTH 1
83
84 #define SAHARA_REG_VERSION 0x00
85 #define SAHARA_REG_DAR 0x04
86 #define SAHARA_REG_CONTROL 0x08
87 #define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24)
88 #define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16)
89 #define SAHARA_CONTROL_RNG_AUTORSD (1 << 7)
90 #define SAHARA_CONTROL_ENABLE_INT (1 << 4)
91 #define SAHARA_REG_CMD 0x0C
92 #define SAHARA_CMD_RESET (1 << 0)
93 #define SAHARA_CMD_CLEAR_INT (1 << 8)
94 #define SAHARA_CMD_CLEAR_ERR (1 << 9)
95 #define SAHARA_CMD_SINGLE_STEP (1 << 10)
96 #define SAHARA_CMD_MODE_BATCH (1 << 16)
97 #define SAHARA_CMD_MODE_DEBUG (1 << 18)
98 #define SAHARA_REG_STATUS 0x10
99 #define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7)
100 #define SAHARA_STATE_IDLE 0
101 #define SAHARA_STATE_BUSY 1
102 #define SAHARA_STATE_ERR 2
103 #define SAHARA_STATE_FAULT 3
104 #define SAHARA_STATE_COMPLETE 4
105 #define SAHARA_STATE_COMP_FLAG (1 << 2)
106 #define SAHARA_STATUS_DAR_FULL (1 << 3)
107 #define SAHARA_STATUS_ERROR (1 << 4)
108 #define SAHARA_STATUS_SECURE (1 << 5)
109 #define SAHARA_STATUS_FAIL (1 << 6)
110 #define SAHARA_STATUS_INIT (1 << 7)
111 #define SAHARA_STATUS_RNG_RESEED (1 << 8)
112 #define SAHARA_STATUS_ACTIVE_RNG (1 << 9)
113 #define SAHARA_STATUS_ACTIVE_MDHA (1 << 10)
114 #define SAHARA_STATUS_ACTIVE_SKHA (1 << 11)
115 #define SAHARA_STATUS_MODE_BATCH (1 << 16)
116 #define SAHARA_STATUS_MODE_DEDICATED (1 << 17)
117 #define SAHARA_STATUS_MODE_DEBUG (1 << 18)
118 #define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff)
119 #define SAHARA_REG_ERRSTATUS 0x14
120 #define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf)
121 #define SAHARA_ERRSOURCE_CHA 14
122 #define SAHARA_ERRSOURCE_DMA 15
123 #define SAHARA_ERRSTATUS_DMA_DIR (1 << 8)
124 #define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
125 #define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
126 #define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff)
127 #define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3)
128 #define SAHARA_REG_FADDR 0x18
129 #define SAHARA_REG_CDAR 0x1C
130 #define SAHARA_REG_IDAR 0x20
131
132 struct sahara_hw_desc {
133 u32 hdr;
134 u32 len1;
135 dma_addr_t p1;
136 u32 len2;
137 dma_addr_t p2;
138 dma_addr_t next;
139 };
140
141 struct sahara_hw_link {
142 u32 len;
143 dma_addr_t p;
144 dma_addr_t next;
145 };
146
147 struct sahara_ctx {
148 unsigned long flags;
149
150 /* AES-specific context */
151 int keylen;
152 u8 key[AES_KEYSIZE_128];
153 struct crypto_ablkcipher *fallback;
154
155 /* SHA-specific context */
156 struct crypto_shash *shash_fallback;
157 };
158
159 struct sahara_aes_reqctx {
160 unsigned long mode;
161 };
162
163 /*
164 * struct sahara_sha_reqctx - private data per request
165 * @buf: holds data for requests smaller than block_size
166 * @rembuf: used to prepare one block_size-aligned request
167 * @context: hw-specific context for request. Digest is extracted from this
168 * @mode: specifies what type of hw-descriptor needs to be built
169 * @digest_size: length of digest for this request
170 * @context_size: length of hw-context for this request.
171 * Always digest_size + 4
172 * @buf_cnt: number of bytes saved in buf
173 * @sg_in_idx: number of hw links
174 * @in_sg: scatterlist for input data
175 * @in_sg_chain: scatterlists for chained input data
176 * @total: total number of bytes for transfer
177 * @last: is this the last block
178 * @first: is this the first block
179 * @active: inside a transfer
180 */
181 struct sahara_sha_reqctx {
182 u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE];
183 u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE];
184 u8 context[SHA256_DIGEST_SIZE + 4];
185 struct mutex mutex;
186 unsigned int mode;
187 unsigned int digest_size;
188 unsigned int context_size;
189 unsigned int buf_cnt;
190 unsigned int sg_in_idx;
191 struct scatterlist *in_sg;
192 struct scatterlist in_sg_chain[2];
193 size_t total;
194 unsigned int last;
195 unsigned int first;
196 unsigned int active;
197 };
198
199 struct sahara_dev {
200 struct device *device;
201 unsigned int version;
202 void __iomem *regs_base;
203 struct clk *clk_ipg;
204 struct clk *clk_ahb;
205 struct mutex queue_mutex;
206 struct task_struct *kthread;
207 struct completion dma_completion;
208
209 struct sahara_ctx *ctx;
210 spinlock_t lock;
211 struct crypto_queue queue;
212 unsigned long flags;
213
214 struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC];
215 dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC];
216
217 u8 *key_base;
218 dma_addr_t key_phys_base;
219
220 u8 *iv_base;
221 dma_addr_t iv_phys_base;
222
223 u8 *context_base;
224 dma_addr_t context_phys_base;
225
226 struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK];
227 dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK];
228
229 size_t total;
230 struct scatterlist *in_sg;
231 unsigned int nb_in_sg;
232 struct scatterlist *out_sg;
233 unsigned int nb_out_sg;
234
235 u32 error;
236 };
237
238 static struct sahara_dev *dev_ptr;
239
240 static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
241 {
242 writel(data, dev->regs_base + reg);
243 }
244
245 static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
246 {
247 return readl(dev->regs_base + reg);
248 }
249
250 static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
251 {
252 u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
253 SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
254 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
255
256 if (dev->flags & FLAGS_CBC) {
257 hdr |= SAHARA_HDR_SKHA_MODE_CBC;
258 hdr ^= SAHARA_HDR_PARITY_BIT;
259 }
260
261 if (dev->flags & FLAGS_ENCRYPT) {
262 hdr |= SAHARA_HDR_SKHA_OP_ENC;
263 hdr ^= SAHARA_HDR_PARITY_BIT;
264 }
265
266 return hdr;
267 }
268
269 static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
270 {
271 return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
272 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
273 }
274
275 static const char *sahara_err_src[16] = {
276 "No error",
277 "Header error",
278 "Descriptor length error",
279 "Descriptor length or pointer error",
280 "Link length error",
281 "Link pointer error",
282 "Input buffer error",
283 "Output buffer error",
284 "Output buffer starvation",
285 "Internal state fault",
286 "General descriptor problem",
287 "Reserved",
288 "Descriptor address error",
289 "Link address error",
290 "CHA error",
291 "DMA error"
292 };
293
294 static const char *sahara_err_dmasize[4] = {
295 "Byte transfer",
296 "Half-word transfer",
297 "Word transfer",
298 "Reserved"
299 };
300
301 static const char *sahara_err_dmasrc[8] = {
302 "No error",
303 "AHB bus error",
304 "Internal IP bus error",
305 "Parity error",
306 "DMA crosses 256 byte boundary",
307 "DMA is busy",
308 "Reserved",
309 "DMA HW error"
310 };
311
312 static const char *sahara_cha_errsrc[12] = {
313 "Input buffer non-empty",
314 "Illegal address",
315 "Illegal mode",
316 "Illegal data size",
317 "Illegal key size",
318 "Write during processing",
319 "CTX read during processing",
320 "HW error",
321 "Input buffer disabled/underflow",
322 "Output buffer disabled/overflow",
323 "DES key parity error",
324 "Reserved"
325 };
326
327 static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
328
329 static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
330 {
331 u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
332 u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
333
334 dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
335
336 dev_err(dev->device, " - %s.\n", sahara_err_src[source]);
337
338 if (source == SAHARA_ERRSOURCE_DMA) {
339 if (error & SAHARA_ERRSTATUS_DMA_DIR)
340 dev_err(dev->device, " * DMA read.\n");
341 else
342 dev_err(dev->device, " * DMA write.\n");
343
344 dev_err(dev->device, " * %s.\n",
345 sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
346 dev_err(dev->device, " * %s.\n",
347 sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
348 } else if (source == SAHARA_ERRSOURCE_CHA) {
349 dev_err(dev->device, " * %s.\n",
350 sahara_cha_errsrc[chasrc]);
351 dev_err(dev->device, " * %s.\n",
352 sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
353 }
354 dev_err(dev->device, "\n");
355 }
356
357 static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
358
359 static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
360 {
361 u8 state;
362
363 if (!IS_ENABLED(DEBUG))
364 return;
365
366 state = SAHARA_STATUS_GET_STATE(status);
367
368 dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
369 __func__, status);
370
371 dev_dbg(dev->device, " - State = %d:\n", state);
372 if (state & SAHARA_STATE_COMP_FLAG)
373 dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n");
374
375 dev_dbg(dev->device, " * %s.\n",
376 sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
377
378 if (status & SAHARA_STATUS_DAR_FULL)
379 dev_dbg(dev->device, " - DAR Full.\n");
380 if (status & SAHARA_STATUS_ERROR)
381 dev_dbg(dev->device, " - Error.\n");
382 if (status & SAHARA_STATUS_SECURE)
383 dev_dbg(dev->device, " - Secure.\n");
384 if (status & SAHARA_STATUS_FAIL)
385 dev_dbg(dev->device, " - Fail.\n");
386 if (status & SAHARA_STATUS_RNG_RESEED)
387 dev_dbg(dev->device, " - RNG Reseed Request.\n");
388 if (status & SAHARA_STATUS_ACTIVE_RNG)
389 dev_dbg(dev->device, " - RNG Active.\n");
390 if (status & SAHARA_STATUS_ACTIVE_MDHA)
391 dev_dbg(dev->device, " - MDHA Active.\n");
392 if (status & SAHARA_STATUS_ACTIVE_SKHA)
393 dev_dbg(dev->device, " - SKHA Active.\n");
394
395 if (status & SAHARA_STATUS_MODE_BATCH)
396 dev_dbg(dev->device, " - Batch Mode.\n");
397 else if (status & SAHARA_STATUS_MODE_DEDICATED)
398 dev_dbg(dev->device, " - Decidated Mode.\n");
399 else if (status & SAHARA_STATUS_MODE_DEBUG)
400 dev_dbg(dev->device, " - Debug Mode.\n");
401
402 dev_dbg(dev->device, " - Internal state = 0x%02x\n",
403 SAHARA_STATUS_GET_ISTATE(status));
404
405 dev_dbg(dev->device, "Current DAR: 0x%08x\n",
406 sahara_read(dev, SAHARA_REG_CDAR));
407 dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
408 sahara_read(dev, SAHARA_REG_IDAR));
409 }
410
411 static void sahara_dump_descriptors(struct sahara_dev *dev)
412 {
413 int i;
414
415 if (!IS_ENABLED(DEBUG))
416 return;
417
418 for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
419 dev_dbg(dev->device, "Descriptor (%d) (0x%08x):\n",
420 i, dev->hw_phys_desc[i]);
421 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
422 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
423 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
424 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
425 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
426 dev_dbg(dev->device, "\tnext = 0x%08x\n",
427 dev->hw_desc[i]->next);
428 }
429 dev_dbg(dev->device, "\n");
430 }
431
432 static void sahara_dump_links(struct sahara_dev *dev)
433 {
434 int i;
435
436 if (!IS_ENABLED(DEBUG))
437 return;
438
439 for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
440 dev_dbg(dev->device, "Link (%d) (0x%08x):\n",
441 i, dev->hw_phys_link[i]);
442 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
443 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
444 dev_dbg(dev->device, "\tnext = 0x%08x\n",
445 dev->hw_link[i]->next);
446 }
447 dev_dbg(dev->device, "\n");
448 }
449
450 static int sahara_hw_descriptor_create(struct sahara_dev *dev)
451 {
452 struct sahara_ctx *ctx = dev->ctx;
453 struct scatterlist *sg;
454 int ret;
455 int i, j;
456 int idx = 0;
457
458 /* Copy new key if necessary */
459 if (ctx->flags & FLAGS_NEW_KEY) {
460 memcpy(dev->key_base, ctx->key, ctx->keylen);
461 ctx->flags &= ~FLAGS_NEW_KEY;
462
463 if (dev->flags & FLAGS_CBC) {
464 dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;
465 dev->hw_desc[idx]->p1 = dev->iv_phys_base;
466 } else {
467 dev->hw_desc[idx]->len1 = 0;
468 dev->hw_desc[idx]->p1 = 0;
469 }
470 dev->hw_desc[idx]->len2 = ctx->keylen;
471 dev->hw_desc[idx]->p2 = dev->key_phys_base;
472 dev->hw_desc[idx]->next = dev->hw_phys_desc[1];
473
474 dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);
475
476 idx++;
477 }
478
479 dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total);
480 dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total);
481 if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
482 dev_err(dev->device, "not enough hw links (%d)\n",
483 dev->nb_in_sg + dev->nb_out_sg);
484 return -EINVAL;
485 }
486
487 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
488 DMA_TO_DEVICE);
489 if (ret != dev->nb_in_sg) {
490 dev_err(dev->device, "couldn't map in sg\n");
491 goto unmap_in;
492 }
493 ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
494 DMA_FROM_DEVICE);
495 if (ret != dev->nb_out_sg) {
496 dev_err(dev->device, "couldn't map out sg\n");
497 goto unmap_out;
498 }
499
500 /* Create input links */
501 dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];
502 sg = dev->in_sg;
503 for (i = 0; i < dev->nb_in_sg; i++) {
504 dev->hw_link[i]->len = sg->length;
505 dev->hw_link[i]->p = sg->dma_address;
506 if (i == (dev->nb_in_sg - 1)) {
507 dev->hw_link[i]->next = 0;
508 } else {
509 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
510 sg = sg_next(sg);
511 }
512 }
513
514 /* Create output links */
515 dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];
516 sg = dev->out_sg;
517 for (j = i; j < dev->nb_out_sg + i; j++) {
518 dev->hw_link[j]->len = sg->length;
519 dev->hw_link[j]->p = sg->dma_address;
520 if (j == (dev->nb_out_sg + i - 1)) {
521 dev->hw_link[j]->next = 0;
522 } else {
523 dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
524 sg = sg_next(sg);
525 }
526 }
527
528 /* Fill remaining fields of hw_desc[1] */
529 dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);
530 dev->hw_desc[idx]->len1 = dev->total;
531 dev->hw_desc[idx]->len2 = dev->total;
532 dev->hw_desc[idx]->next = 0;
533
534 sahara_dump_descriptors(dev);
535 sahara_dump_links(dev);
536
537 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
538
539 return 0;
540
541 unmap_out:
542 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
543 DMA_TO_DEVICE);
544 unmap_in:
545 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
546 DMA_FROM_DEVICE);
547
548 return -EINVAL;
549 }
550
551 static int sahara_aes_process(struct ablkcipher_request *req)
552 {
553 struct sahara_dev *dev = dev_ptr;
554 struct sahara_ctx *ctx;
555 struct sahara_aes_reqctx *rctx;
556 int ret;
557 unsigned long timeout;
558
559 /* Request is ready to be dispatched by the device */
560 dev_dbg(dev->device,
561 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
562 req->nbytes, req->src, req->dst);
563
564 /* assign new request to device */
565 dev->total = req->nbytes;
566 dev->in_sg = req->src;
567 dev->out_sg = req->dst;
568
569 rctx = ablkcipher_request_ctx(req);
570 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
571 rctx->mode &= FLAGS_MODE_MASK;
572 dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
573
574 if ((dev->flags & FLAGS_CBC) && req->info)
575 memcpy(dev->iv_base, req->info, AES_KEYSIZE_128);
576
577 /* assign new context to device */
578 dev->ctx = ctx;
579
580 reinit_completion(&dev->dma_completion);
581
582 ret = sahara_hw_descriptor_create(dev);
583 if (ret)
584 return -EINVAL;
585
586 timeout = wait_for_completion_timeout(&dev->dma_completion,
587 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
588 if (!timeout) {
589 dev_err(dev->device, "AES timeout\n");
590 return -ETIMEDOUT;
591 }
592
593 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
594 DMA_TO_DEVICE);
595 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
596 DMA_FROM_DEVICE);
597
598 return 0;
599 }
600
601 static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
602 unsigned int keylen)
603 {
604 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
605 int ret;
606
607 ctx->keylen = keylen;
608
609 /* SAHARA only supports 128bit keys */
610 if (keylen == AES_KEYSIZE_128) {
611 memcpy(ctx->key, key, keylen);
612 ctx->flags |= FLAGS_NEW_KEY;
613 return 0;
614 }
615
616 if (keylen != AES_KEYSIZE_128 &&
617 keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
618 return -EINVAL;
619
620 /*
621 * The requested key size is not supported by HW, do a fallback.
622 */
623 ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
624 ctx->fallback->base.crt_flags |=
625 (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
626
627 ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen);
628 if (ret) {
629 struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm);
630
631 tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK;
632 tfm_aux->crt_flags |=
633 (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK);
634 }
635 return ret;
636 }
637
638 static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
639 {
640 struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req);
641 struct sahara_dev *dev = dev_ptr;
642 int err = 0;
643
644 dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
645 req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
646
647 if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
648 dev_err(dev->device,
649 "request size is not exact amount of AES blocks\n");
650 return -EINVAL;
651 }
652
653 rctx->mode = mode;
654
655 mutex_lock(&dev->queue_mutex);
656 err = ablkcipher_enqueue_request(&dev->queue, req);
657 mutex_unlock(&dev->queue_mutex);
658
659 wake_up_process(dev->kthread);
660
661 return err;
662 }
663
664 static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
665 {
666 struct crypto_tfm *tfm =
667 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
668 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
669 crypto_ablkcipher_reqtfm(req));
670 int err;
671
672 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
673 ablkcipher_request_set_tfm(req, ctx->fallback);
674 err = crypto_ablkcipher_encrypt(req);
675 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
676 return err;
677 }
678
679 return sahara_aes_crypt(req, FLAGS_ENCRYPT);
680 }
681
682 static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
683 {
684 struct crypto_tfm *tfm =
685 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
686 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
687 crypto_ablkcipher_reqtfm(req));
688 int err;
689
690 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
691 ablkcipher_request_set_tfm(req, ctx->fallback);
692 err = crypto_ablkcipher_decrypt(req);
693 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
694 return err;
695 }
696
697 return sahara_aes_crypt(req, 0);
698 }
699
700 static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
701 {
702 struct crypto_tfm *tfm =
703 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
704 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
705 crypto_ablkcipher_reqtfm(req));
706 int err;
707
708 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
709 ablkcipher_request_set_tfm(req, ctx->fallback);
710 err = crypto_ablkcipher_encrypt(req);
711 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
712 return err;
713 }
714
715 return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
716 }
717
718 static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
719 {
720 struct crypto_tfm *tfm =
721 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
722 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
723 crypto_ablkcipher_reqtfm(req));
724 int err;
725
726 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
727 ablkcipher_request_set_tfm(req, ctx->fallback);
728 err = crypto_ablkcipher_decrypt(req);
729 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
730 return err;
731 }
732
733 return sahara_aes_crypt(req, FLAGS_CBC);
734 }
735
736 static int sahara_aes_cra_init(struct crypto_tfm *tfm)
737 {
738 const char *name = crypto_tfm_alg_name(tfm);
739 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
740
741 ctx->fallback = crypto_alloc_ablkcipher(name, 0,
742 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
743 if (IS_ERR(ctx->fallback)) {
744 pr_err("Error allocating fallback algo %s\n", name);
745 return PTR_ERR(ctx->fallback);
746 }
747
748 tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx);
749
750 return 0;
751 }
752
753 static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
754 {
755 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
756
757 if (ctx->fallback)
758 crypto_free_ablkcipher(ctx->fallback);
759 ctx->fallback = NULL;
760 }
761
762 static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
763 struct sahara_sha_reqctx *rctx)
764 {
765 u32 hdr = 0;
766
767 hdr = rctx->mode;
768
769 if (rctx->first) {
770 hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH;
771 hdr |= SAHARA_HDR_MDHA_INIT;
772 } else {
773 hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY;
774 }
775
776 if (rctx->last)
777 hdr |= SAHARA_HDR_MDHA_PDATA;
778
779 if (hweight_long(hdr) % 2 == 0)
780 hdr |= SAHARA_HDR_PARITY_BIT;
781
782 return hdr;
783 }
784
785 static int sahara_sha_hw_links_create(struct sahara_dev *dev,
786 struct sahara_sha_reqctx *rctx,
787 int start)
788 {
789 struct scatterlist *sg;
790 unsigned int i;
791 int ret;
792
793 dev->in_sg = rctx->in_sg;
794
795 dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total);
796 if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {
797 dev_err(dev->device, "not enough hw links (%d)\n",
798 dev->nb_in_sg + dev->nb_out_sg);
799 return -EINVAL;
800 }
801
802 sg = dev->in_sg;
803 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE);
804 if (!ret)
805 return -EFAULT;
806
807 for (i = start; i < dev->nb_in_sg + start; i++) {
808 dev->hw_link[i]->len = sg->length;
809 dev->hw_link[i]->p = sg->dma_address;
810 if (i == (dev->nb_in_sg + start - 1)) {
811 dev->hw_link[i]->next = 0;
812 } else {
813 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
814 sg = sg_next(sg);
815 }
816 }
817
818 return i;
819 }
820
821 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev,
822 struct sahara_sha_reqctx *rctx,
823 struct ahash_request *req,
824 int index)
825 {
826 unsigned result_len;
827 int i = index;
828
829 if (rctx->first)
830 /* Create initial descriptor: #8*/
831 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
832 else
833 /* Create hash descriptor: #10. Must follow #6. */
834 dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;
835
836 dev->hw_desc[index]->len1 = rctx->total;
837 if (dev->hw_desc[index]->len1 == 0) {
838 /* if len1 is 0, p1 must be 0, too */
839 dev->hw_desc[index]->p1 = 0;
840 rctx->sg_in_idx = 0;
841 } else {
842 /* Create input links */
843 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
844 i = sahara_sha_hw_links_create(dev, rctx, index);
845
846 rctx->sg_in_idx = index;
847 if (i < 0)
848 return i;
849 }
850
851 dev->hw_desc[index]->p2 = dev->hw_phys_link[i];
852
853 /* Save the context for the next operation */
854 result_len = rctx->context_size;
855 dev->hw_link[i]->p = dev->context_phys_base;
856
857 dev->hw_link[i]->len = result_len;
858 dev->hw_desc[index]->len2 = result_len;
859
860 dev->hw_link[i]->next = 0;
861
862 return 0;
863 }
864
865 /*
866 * Load descriptor aka #6
867 *
868 * To load a previously saved context back to the MDHA unit
869 *
870 * p1: Saved Context
871 * p2: NULL
872 *
873 */
874 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev,
875 struct sahara_sha_reqctx *rctx,
876 struct ahash_request *req,
877 int index)
878 {
879 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
880
881 dev->hw_desc[index]->len1 = rctx->context_size;
882 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
883 dev->hw_desc[index]->len2 = 0;
884 dev->hw_desc[index]->p2 = 0;
885
886 dev->hw_link[index]->len = rctx->context_size;
887 dev->hw_link[index]->p = dev->context_phys_base;
888 dev->hw_link[index]->next = 0;
889
890 return 0;
891 }
892
893 static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes)
894 {
895 if (!sg || !sg->length)
896 return nbytes;
897
898 while (nbytes && sg) {
899 if (nbytes <= sg->length) {
900 sg->length = nbytes;
901 sg_mark_end(sg);
902 break;
903 }
904 nbytes -= sg->length;
905 sg = sg_next(sg);
906 }
907
908 return nbytes;
909 }
910
911 static int sahara_sha_prepare_request(struct ahash_request *req)
912 {
913 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
914 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
915 unsigned int hash_later;
916 unsigned int block_size;
917 unsigned int len;
918
919 block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
920
921 /* append bytes from previous operation */
922 len = rctx->buf_cnt + req->nbytes;
923
924 /* only the last transfer can be padded in hardware */
925 if (!rctx->last && (len < block_size)) {
926 /* to few data, save for next operation */
927 scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,
928 0, req->nbytes, 0);
929 rctx->buf_cnt += req->nbytes;
930
931 return 0;
932 }
933
934 /* add data from previous operation first */
935 if (rctx->buf_cnt)
936 memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);
937
938 /* data must always be a multiple of block_size */
939 hash_later = rctx->last ? 0 : len & (block_size - 1);
940 if (hash_later) {
941 unsigned int offset = req->nbytes - hash_later;
942 /* Save remaining bytes for later use */
943 scatterwalk_map_and_copy(rctx->buf, req->src, offset,
944 hash_later, 0);
945 }
946
947 /* nbytes should now be multiple of blocksize */
948 req->nbytes = req->nbytes - hash_later;
949
950 sahara_walk_and_recalc(req->src, req->nbytes);
951
952 /* have data from previous operation and current */
953 if (rctx->buf_cnt && req->nbytes) {
954 sg_init_table(rctx->in_sg_chain, 2);
955 sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);
956
957 sg_chain(rctx->in_sg_chain, 2, req->src);
958
959 rctx->total = req->nbytes + rctx->buf_cnt;
960 rctx->in_sg = rctx->in_sg_chain;
961
962 req->src = rctx->in_sg_chain;
963 /* only data from previous operation */
964 } else if (rctx->buf_cnt) {
965 if (req->src)
966 rctx->in_sg = req->src;
967 else
968 rctx->in_sg = rctx->in_sg_chain;
969 /* buf was copied into rembuf above */
970 sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);
971 rctx->total = rctx->buf_cnt;
972 /* no data from previous operation */
973 } else {
974 rctx->in_sg = req->src;
975 rctx->total = req->nbytes;
976 req->src = rctx->in_sg;
977 }
978
979 /* on next call, we only have the remaining data in the buffer */
980 rctx->buf_cnt = hash_later;
981
982 return -EINPROGRESS;
983 }
984
985 static int sahara_sha_process(struct ahash_request *req)
986 {
987 struct sahara_dev *dev = dev_ptr;
988 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
989 int ret;
990 unsigned long timeout;
991
992 ret = sahara_sha_prepare_request(req);
993 if (!ret)
994 return ret;
995
996 if (rctx->first) {
997 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0);
998 dev->hw_desc[0]->next = 0;
999 rctx->first = 0;
1000 } else {
1001 memcpy(dev->context_base, rctx->context, rctx->context_size);
1002
1003 sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0);
1004 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
1005 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1);
1006 dev->hw_desc[1]->next = 0;
1007 }
1008
1009 sahara_dump_descriptors(dev);
1010 sahara_dump_links(dev);
1011
1012 reinit_completion(&dev->dma_completion);
1013
1014 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
1015
1016 timeout = wait_for_completion_timeout(&dev->dma_completion,
1017 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
1018 if (!timeout) {
1019 dev_err(dev->device, "SHA timeout\n");
1020 return -ETIMEDOUT;
1021 }
1022
1023 if (rctx->sg_in_idx)
1024 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
1025 DMA_TO_DEVICE);
1026
1027 memcpy(rctx->context, dev->context_base, rctx->context_size);
1028
1029 if (req->result)
1030 memcpy(req->result, rctx->context, rctx->digest_size);
1031
1032 return 0;
1033 }
1034
1035 static int sahara_queue_manage(void *data)
1036 {
1037 struct sahara_dev *dev = (struct sahara_dev *)data;
1038 struct crypto_async_request *async_req;
1039 struct crypto_async_request *backlog;
1040 int ret = 0;
1041
1042 do {
1043 __set_current_state(TASK_INTERRUPTIBLE);
1044
1045 mutex_lock(&dev->queue_mutex);
1046 backlog = crypto_get_backlog(&dev->queue);
1047 async_req = crypto_dequeue_request(&dev->queue);
1048 mutex_unlock(&dev->queue_mutex);
1049
1050 if (backlog)
1051 backlog->complete(backlog, -EINPROGRESS);
1052
1053 if (async_req) {
1054 if (crypto_tfm_alg_type(async_req->tfm) ==
1055 CRYPTO_ALG_TYPE_AHASH) {
1056 struct ahash_request *req =
1057 ahash_request_cast(async_req);
1058
1059 ret = sahara_sha_process(req);
1060 } else {
1061 struct ablkcipher_request *req =
1062 ablkcipher_request_cast(async_req);
1063
1064 ret = sahara_aes_process(req);
1065 }
1066
1067 async_req->complete(async_req, ret);
1068
1069 continue;
1070 }
1071
1072 schedule();
1073 } while (!kthread_should_stop());
1074
1075 return 0;
1076 }
1077
1078 static int sahara_sha_enqueue(struct ahash_request *req, int last)
1079 {
1080 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1081 struct sahara_dev *dev = dev_ptr;
1082 int ret;
1083
1084 if (!req->nbytes && !last)
1085 return 0;
1086
1087 mutex_lock(&rctx->mutex);
1088 rctx->last = last;
1089
1090 if (!rctx->active) {
1091 rctx->active = 1;
1092 rctx->first = 1;
1093 }
1094
1095 mutex_lock(&dev->queue_mutex);
1096 ret = crypto_enqueue_request(&dev->queue, &req->base);
1097 mutex_unlock(&dev->queue_mutex);
1098
1099 wake_up_process(dev->kthread);
1100 mutex_unlock(&rctx->mutex);
1101
1102 return ret;
1103 }
1104
1105 static int sahara_sha_init(struct ahash_request *req)
1106 {
1107 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1108 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1109
1110 memset(rctx, 0, sizeof(*rctx));
1111
1112 switch (crypto_ahash_digestsize(tfm)) {
1113 case SHA1_DIGEST_SIZE:
1114 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;
1115 rctx->digest_size = SHA1_DIGEST_SIZE;
1116 break;
1117 case SHA256_DIGEST_SIZE:
1118 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;
1119 rctx->digest_size = SHA256_DIGEST_SIZE;
1120 break;
1121 default:
1122 return -EINVAL;
1123 }
1124
1125 rctx->context_size = rctx->digest_size + 4;
1126 rctx->active = 0;
1127
1128 mutex_init(&rctx->mutex);
1129
1130 return 0;
1131 }
1132
1133 static int sahara_sha_update(struct ahash_request *req)
1134 {
1135 return sahara_sha_enqueue(req, 0);
1136 }
1137
1138 static int sahara_sha_final(struct ahash_request *req)
1139 {
1140 req->nbytes = 0;
1141 return sahara_sha_enqueue(req, 1);
1142 }
1143
1144 static int sahara_sha_finup(struct ahash_request *req)
1145 {
1146 return sahara_sha_enqueue(req, 1);
1147 }
1148
1149 static int sahara_sha_digest(struct ahash_request *req)
1150 {
1151 sahara_sha_init(req);
1152
1153 return sahara_sha_finup(req);
1154 }
1155
1156 static int sahara_sha_export(struct ahash_request *req, void *out)
1157 {
1158 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1159 struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1160 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1161
1162 memcpy(out, ctx, sizeof(struct sahara_ctx));
1163 memcpy(out + sizeof(struct sahara_sha_reqctx), rctx,
1164 sizeof(struct sahara_sha_reqctx));
1165
1166 return 0;
1167 }
1168
1169 static int sahara_sha_import(struct ahash_request *req, const void *in)
1170 {
1171 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1172 struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1173 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1174
1175 memcpy(ctx, in, sizeof(struct sahara_ctx));
1176 memcpy(rctx, in + sizeof(struct sahara_sha_reqctx),
1177 sizeof(struct sahara_sha_reqctx));
1178
1179 return 0;
1180 }
1181
1182 static int sahara_sha_cra_init(struct crypto_tfm *tfm)
1183 {
1184 const char *name = crypto_tfm_alg_name(tfm);
1185 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1186
1187 ctx->shash_fallback = crypto_alloc_shash(name, 0,
1188 CRYPTO_ALG_NEED_FALLBACK);
1189 if (IS_ERR(ctx->shash_fallback)) {
1190 pr_err("Error allocating fallback algo %s\n", name);
1191 return PTR_ERR(ctx->shash_fallback);
1192 }
1193 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1194 sizeof(struct sahara_sha_reqctx) +
1195 SHA_BUFFER_LEN + SHA256_BLOCK_SIZE);
1196
1197 return 0;
1198 }
1199
1200 static void sahara_sha_cra_exit(struct crypto_tfm *tfm)
1201 {
1202 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1203
1204 crypto_free_shash(ctx->shash_fallback);
1205 ctx->shash_fallback = NULL;
1206 }
1207
1208 static struct crypto_alg aes_algs[] = {
1209 {
1210 .cra_name = "ecb(aes)",
1211 .cra_driver_name = "sahara-ecb-aes",
1212 .cra_priority = 300,
1213 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1214 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1215 .cra_blocksize = AES_BLOCK_SIZE,
1216 .cra_ctxsize = sizeof(struct sahara_ctx),
1217 .cra_alignmask = 0x0,
1218 .cra_type = &crypto_ablkcipher_type,
1219 .cra_module = THIS_MODULE,
1220 .cra_init = sahara_aes_cra_init,
1221 .cra_exit = sahara_aes_cra_exit,
1222 .cra_u.ablkcipher = {
1223 .min_keysize = AES_MIN_KEY_SIZE ,
1224 .max_keysize = AES_MAX_KEY_SIZE,
1225 .setkey = sahara_aes_setkey,
1226 .encrypt = sahara_aes_ecb_encrypt,
1227 .decrypt = sahara_aes_ecb_decrypt,
1228 }
1229 }, {
1230 .cra_name = "cbc(aes)",
1231 .cra_driver_name = "sahara-cbc-aes",
1232 .cra_priority = 300,
1233 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1234 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1235 .cra_blocksize = AES_BLOCK_SIZE,
1236 .cra_ctxsize = sizeof(struct sahara_ctx),
1237 .cra_alignmask = 0x0,
1238 .cra_type = &crypto_ablkcipher_type,
1239 .cra_module = THIS_MODULE,
1240 .cra_init = sahara_aes_cra_init,
1241 .cra_exit = sahara_aes_cra_exit,
1242 .cra_u.ablkcipher = {
1243 .min_keysize = AES_MIN_KEY_SIZE ,
1244 .max_keysize = AES_MAX_KEY_SIZE,
1245 .ivsize = AES_BLOCK_SIZE,
1246 .setkey = sahara_aes_setkey,
1247 .encrypt = sahara_aes_cbc_encrypt,
1248 .decrypt = sahara_aes_cbc_decrypt,
1249 }
1250 }
1251 };
1252
1253 static struct ahash_alg sha_v3_algs[] = {
1254 {
1255 .init = sahara_sha_init,
1256 .update = sahara_sha_update,
1257 .final = sahara_sha_final,
1258 .finup = sahara_sha_finup,
1259 .digest = sahara_sha_digest,
1260 .export = sahara_sha_export,
1261 .import = sahara_sha_import,
1262 .halg.digestsize = SHA1_DIGEST_SIZE,
1263 .halg.base = {
1264 .cra_name = "sha1",
1265 .cra_driver_name = "sahara-sha1",
1266 .cra_priority = 300,
1267 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1268 CRYPTO_ALG_ASYNC |
1269 CRYPTO_ALG_NEED_FALLBACK,
1270 .cra_blocksize = SHA1_BLOCK_SIZE,
1271 .cra_ctxsize = sizeof(struct sahara_ctx),
1272 .cra_alignmask = 0,
1273 .cra_module = THIS_MODULE,
1274 .cra_init = sahara_sha_cra_init,
1275 .cra_exit = sahara_sha_cra_exit,
1276 }
1277 },
1278 };
1279
1280 static struct ahash_alg sha_v4_algs[] = {
1281 {
1282 .init = sahara_sha_init,
1283 .update = sahara_sha_update,
1284 .final = sahara_sha_final,
1285 .finup = sahara_sha_finup,
1286 .digest = sahara_sha_digest,
1287 .export = sahara_sha_export,
1288 .import = sahara_sha_import,
1289 .halg.digestsize = SHA256_DIGEST_SIZE,
1290 .halg.base = {
1291 .cra_name = "sha256",
1292 .cra_driver_name = "sahara-sha256",
1293 .cra_priority = 300,
1294 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1295 CRYPTO_ALG_ASYNC |
1296 CRYPTO_ALG_NEED_FALLBACK,
1297 .cra_blocksize = SHA256_BLOCK_SIZE,
1298 .cra_ctxsize = sizeof(struct sahara_ctx),
1299 .cra_alignmask = 0,
1300 .cra_module = THIS_MODULE,
1301 .cra_init = sahara_sha_cra_init,
1302 .cra_exit = sahara_sha_cra_exit,
1303 }
1304 },
1305 };
1306
1307 static irqreturn_t sahara_irq_handler(int irq, void *data)
1308 {
1309 struct sahara_dev *dev = (struct sahara_dev *)data;
1310 unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
1311 unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
1312
1313 sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
1314 SAHARA_REG_CMD);
1315
1316 sahara_decode_status(dev, stat);
1317
1318 if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
1319 return IRQ_NONE;
1320 } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
1321 dev->error = 0;
1322 } else {
1323 sahara_decode_error(dev, err);
1324 dev->error = -EINVAL;
1325 }
1326
1327 complete(&dev->dma_completion);
1328
1329 return IRQ_HANDLED;
1330 }
1331
1332
1333 static int sahara_register_algs(struct sahara_dev *dev)
1334 {
1335 int err;
1336 unsigned int i, j, k, l;
1337
1338 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1339 INIT_LIST_HEAD(&aes_algs[i].cra_list);
1340 err = crypto_register_alg(&aes_algs[i]);
1341 if (err)
1342 goto err_aes_algs;
1343 }
1344
1345 for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) {
1346 err = crypto_register_ahash(&sha_v3_algs[k]);
1347 if (err)
1348 goto err_sha_v3_algs;
1349 }
1350
1351 if (dev->version > SAHARA_VERSION_3)
1352 for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) {
1353 err = crypto_register_ahash(&sha_v4_algs[l]);
1354 if (err)
1355 goto err_sha_v4_algs;
1356 }
1357
1358 return 0;
1359
1360 err_sha_v4_algs:
1361 for (j = 0; j < l; j++)
1362 crypto_unregister_ahash(&sha_v4_algs[j]);
1363
1364 err_sha_v3_algs:
1365 for (j = 0; j < k; j++)
1366 crypto_unregister_ahash(&sha_v4_algs[j]);
1367
1368 err_aes_algs:
1369 for (j = 0; j < i; j++)
1370 crypto_unregister_alg(&aes_algs[j]);
1371
1372 return err;
1373 }
1374
1375 static void sahara_unregister_algs(struct sahara_dev *dev)
1376 {
1377 unsigned int i;
1378
1379 for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1380 crypto_unregister_alg(&aes_algs[i]);
1381
1382 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1383 crypto_unregister_ahash(&sha_v3_algs[i]);
1384
1385 if (dev->version > SAHARA_VERSION_3)
1386 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1387 crypto_unregister_ahash(&sha_v4_algs[i]);
1388 }
1389
1390 static struct platform_device_id sahara_platform_ids[] = {
1391 { .name = "sahara-imx27" },
1392 { /* sentinel */ }
1393 };
1394 MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
1395
1396 static struct of_device_id sahara_dt_ids[] = {
1397 { .compatible = "fsl,imx53-sahara" },
1398 { .compatible = "fsl,imx27-sahara" },
1399 { /* sentinel */ }
1400 };
1401 MODULE_DEVICE_TABLE(of, sahara_dt_ids);
1402
1403 static int sahara_probe(struct platform_device *pdev)
1404 {
1405 struct sahara_dev *dev;
1406 struct resource *res;
1407 u32 version;
1408 int irq;
1409 int err;
1410 int i;
1411
1412 dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL);
1413 if (dev == NULL) {
1414 dev_err(&pdev->dev, "unable to alloc data struct.\n");
1415 return -ENOMEM;
1416 }
1417
1418 dev->device = &pdev->dev;
1419 platform_set_drvdata(pdev, dev);
1420
1421 /* Get the base address */
1422 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1423 dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
1424 if (IS_ERR(dev->regs_base))
1425 return PTR_ERR(dev->regs_base);
1426
1427 /* Get the IRQ */
1428 irq = platform_get_irq(pdev, 0);
1429 if (irq < 0) {
1430 dev_err(&pdev->dev, "failed to get irq resource\n");
1431 return irq;
1432 }
1433
1434 err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
1435 0, dev_name(&pdev->dev), dev);
1436 if (err) {
1437 dev_err(&pdev->dev, "failed to request irq\n");
1438 return err;
1439 }
1440
1441 /* clocks */
1442 dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1443 if (IS_ERR(dev->clk_ipg)) {
1444 dev_err(&pdev->dev, "Could not get ipg clock\n");
1445 return PTR_ERR(dev->clk_ipg);
1446 }
1447
1448 dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
1449 if (IS_ERR(dev->clk_ahb)) {
1450 dev_err(&pdev->dev, "Could not get ahb clock\n");
1451 return PTR_ERR(dev->clk_ahb);
1452 }
1453
1454 /* Allocate HW descriptors */
1455 dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev,
1456 SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1457 &dev->hw_phys_desc[0], GFP_KERNEL);
1458 if (!dev->hw_desc[0]) {
1459 dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
1460 return -ENOMEM;
1461 }
1462 dev->hw_desc[1] = dev->hw_desc[0] + 1;
1463 dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
1464 sizeof(struct sahara_hw_desc);
1465
1466 /* Allocate space for iv and key */
1467 dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
1468 &dev->key_phys_base, GFP_KERNEL);
1469 if (!dev->key_base) {
1470 dev_err(&pdev->dev, "Could not allocate memory for key\n");
1471 return -ENOMEM;
1472 }
1473 dev->iv_base = dev->key_base + AES_KEYSIZE_128;
1474 dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
1475
1476 /* Allocate space for context: largest digest + message length field */
1477 dev->context_base = dmam_alloc_coherent(&pdev->dev,
1478 SHA256_DIGEST_SIZE + 4,
1479 &dev->context_phys_base, GFP_KERNEL);
1480 if (!dev->context_base) {
1481 dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n");
1482 return -ENOMEM;
1483 }
1484
1485 /* Allocate space for HW links */
1486 dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev,
1487 SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1488 &dev->hw_phys_link[0], GFP_KERNEL);
1489 if (!dev->hw_link[0]) {
1490 dev_err(&pdev->dev, "Could not allocate hw links\n");
1491 return -ENOMEM;
1492 }
1493 for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
1494 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
1495 sizeof(struct sahara_hw_link);
1496 dev->hw_link[i] = dev->hw_link[i - 1] + 1;
1497 }
1498
1499 crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1500
1501 spin_lock_init(&dev->lock);
1502 mutex_init(&dev->queue_mutex);
1503
1504 dev_ptr = dev;
1505
1506 dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto");
1507 if (IS_ERR(dev->kthread)) {
1508 return PTR_ERR(dev->kthread);
1509 }
1510
1511 init_completion(&dev->dma_completion);
1512
1513 err = clk_prepare_enable(dev->clk_ipg);
1514 if (err)
1515 return err;
1516 err = clk_prepare_enable(dev->clk_ahb);
1517 if (err)
1518 goto clk_ipg_disable;
1519
1520 version = sahara_read(dev, SAHARA_REG_VERSION);
1521 if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {
1522 if (version != SAHARA_VERSION_3)
1523 err = -ENODEV;
1524 } else if (of_device_is_compatible(pdev->dev.of_node,
1525 "fsl,imx53-sahara")) {
1526 if (((version >> 8) & 0xff) != SAHARA_VERSION_4)
1527 err = -ENODEV;
1528 version = (version >> 8) & 0xff;
1529 }
1530 if (err == -ENODEV) {
1531 dev_err(&pdev->dev, "SAHARA version %d not supported\n",
1532 version);
1533 goto err_algs;
1534 }
1535
1536 dev->version = version;
1537
1538 sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
1539 SAHARA_REG_CMD);
1540 sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
1541 SAHARA_CONTROL_SET_MAXBURST(8) |
1542 SAHARA_CONTROL_RNG_AUTORSD |
1543 SAHARA_CONTROL_ENABLE_INT,
1544 SAHARA_REG_CONTROL);
1545
1546 err = sahara_register_algs(dev);
1547 if (err)
1548 goto err_algs;
1549
1550 dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1551
1552 return 0;
1553
1554 err_algs:
1555 kthread_stop(dev->kthread);
1556 dev_ptr = NULL;
1557 clk_disable_unprepare(dev->clk_ahb);
1558 clk_ipg_disable:
1559 clk_disable_unprepare(dev->clk_ipg);
1560
1561 return err;
1562 }
1563
1564 static int sahara_remove(struct platform_device *pdev)
1565 {
1566 struct sahara_dev *dev = platform_get_drvdata(pdev);
1567
1568 kthread_stop(dev->kthread);
1569
1570 sahara_unregister_algs(dev);
1571
1572 clk_disable_unprepare(dev->clk_ipg);
1573 clk_disable_unprepare(dev->clk_ahb);
1574
1575 dev_ptr = NULL;
1576
1577 return 0;
1578 }
1579
1580 static struct platform_driver sahara_driver = {
1581 .probe = sahara_probe,
1582 .remove = sahara_remove,
1583 .driver = {
1584 .name = SAHARA_NAME,
1585 .of_match_table = sahara_dt_ids,
1586 },
1587 .id_table = sahara_platform_ids,
1588 };
1589
1590 module_platform_driver(sahara_driver);
1591
1592 MODULE_LICENSE("GPL");
1593 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1594 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1595 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
ubuntu-bionic-kernel mirror