]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/crypto/sahara.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: sahara - remove unneeded mutex in the exported state
[mirror_ubuntu-artful-kernel.git] / drivers / crypto / sahara.c
CommitLineData
5de88752
JM		 1/*
2 * Cryptographic API.
3 *
4 * Support for SAHARA cryptographic accelerator.
5 *
5a2bb93f 6 * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
5de88752
JM		 7 * Copyright (c) 2013 Vista Silicon S.L.
8 * Author: Javier Martin <javier.martin@vista-silicon.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as published
12 * by the Free Software Foundation.
13 *
14 * Based on omap-aes.c and tegra-aes.c
15 */
16
17#include <crypto/algapi.h>
18#include <crypto/aes.h>
5a2bb93f
ST		 19#include <crypto/hash.h>
20#include <crypto/internal/hash.h>
21#include <crypto/scatterwalk.h>
22#include <crypto/sha.h>
5de88752
JM		 23
24#include <linux/clk.h>
25#include <linux/crypto.h>
26#include <linux/interrupt.h>
27#include <linux/io.h>
28#include <linux/irq.h>
29#include <linux/kernel.h>
c0c3c89a 30#include <linux/kthread.h>
5de88752 31#include <linux/module.h>
c0c3c89a 32#include <linux/mutex.h>
5de88752 33#include <linux/of.h>
5ed903b3 34#include <linux/of_device.h>
5de88752
JM		 35#include <linux/platform_device.h>
36
5a2bb93f
ST		 37#define SHA_BUFFER_LEN PAGE_SIZE
38#define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE
39
5de88752
JM		 40#define SAHARA_NAME "sahara"
41#define SAHARA_VERSION_3 3
5ed903b3 42#define SAHARA_VERSION_4 4
5de88752
JM		 43#define SAHARA_TIMEOUT_MS 1000
44#define SAHARA_MAX_HW_DESC 2
45#define SAHARA_MAX_HW_LINK 20
46
47#define FLAGS_MODE_MASK 0x000f
48#define FLAGS_ENCRYPT BIT(0)
49#define FLAGS_CBC BIT(1)
50#define FLAGS_NEW_KEY BIT(3)
5de88752
JM		 51
52#define SAHARA_HDR_BASE 0x00800000
53#define SAHARA_HDR_SKHA_ALG_AES 0
54#define SAHARA_HDR_SKHA_OP_ENC (1 << 2)
55#define SAHARA_HDR_SKHA_MODE_ECB (0 << 3)
56#define SAHARA_HDR_SKHA_MODE_CBC (1 << 3)
57#define SAHARA_HDR_FORM_DATA (5 << 16)
58#define SAHARA_HDR_FORM_KEY (8 << 16)
59#define SAHARA_HDR_LLO (1 << 24)
60#define SAHARA_HDR_CHA_SKHA (1 << 28)
61#define SAHARA_HDR_CHA_MDHA (2 << 28)
62#define SAHARA_HDR_PARITY_BIT (1 << 31)
63
5a2bb93f
ST		 64#define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
65#define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000
66#define SAHARA_HDR_MDHA_HASH 0xA0850000
67#define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000
68#define SAHARA_HDR_MDHA_ALG_SHA1 0
69#define SAHARA_HDR_MDHA_ALG_MD5 1
70#define SAHARA_HDR_MDHA_ALG_SHA256 2
71#define SAHARA_HDR_MDHA_ALG_SHA224 3
72#define SAHARA_HDR_MDHA_PDATA (1 << 2)
73#define SAHARA_HDR_MDHA_HMAC (1 << 3)
74#define SAHARA_HDR_MDHA_INIT (1 << 5)
75#define SAHARA_HDR_MDHA_IPAD (1 << 6)
76#define SAHARA_HDR_MDHA_OPAD (1 << 7)
77#define SAHARA_HDR_MDHA_SWAP (1 << 8)
78#define SAHARA_HDR_MDHA_MAC_FULL (1 << 9)
79#define SAHARA_HDR_MDHA_SSL (1 << 10)
80
5de88752
JM		 81/* SAHARA can only process one request at a time */
82#define SAHARA_QUEUE_LENGTH 1
83
84#define SAHARA_REG_VERSION 0x00
85#define SAHARA_REG_DAR 0x04
86#define SAHARA_REG_CONTROL 0x08
87#define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24)
88#define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16)
89#define SAHARA_CONTROL_RNG_AUTORSD (1 << 7)
90#define SAHARA_CONTROL_ENABLE_INT (1 << 4)
91#define SAHARA_REG_CMD 0x0C
92#define SAHARA_CMD_RESET (1 << 0)
93#define SAHARA_CMD_CLEAR_INT (1 << 8)
94#define SAHARA_CMD_CLEAR_ERR (1 << 9)
95#define SAHARA_CMD_SINGLE_STEP (1 << 10)
96#define SAHARA_CMD_MODE_BATCH (1 << 16)
97#define SAHARA_CMD_MODE_DEBUG (1 << 18)
98#define SAHARA_REG_STATUS 0x10
99#define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7)
100#define SAHARA_STATE_IDLE 0
101#define SAHARA_STATE_BUSY 1
102#define SAHARA_STATE_ERR 2
103#define SAHARA_STATE_FAULT 3
104#define SAHARA_STATE_COMPLETE 4
105#define SAHARA_STATE_COMP_FLAG (1 << 2)
106#define SAHARA_STATUS_DAR_FULL (1 << 3)
107#define SAHARA_STATUS_ERROR (1 << 4)
108#define SAHARA_STATUS_SECURE (1 << 5)
109#define SAHARA_STATUS_FAIL (1 << 6)
110#define SAHARA_STATUS_INIT (1 << 7)
111#define SAHARA_STATUS_RNG_RESEED (1 << 8)
112#define SAHARA_STATUS_ACTIVE_RNG (1 << 9)
113#define SAHARA_STATUS_ACTIVE_MDHA (1 << 10)
114#define SAHARA_STATUS_ACTIVE_SKHA (1 << 11)
115#define SAHARA_STATUS_MODE_BATCH (1 << 16)
116#define SAHARA_STATUS_MODE_DEDICATED (1 << 17)
117#define SAHARA_STATUS_MODE_DEBUG (1 << 18)
118#define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff)
119#define SAHARA_REG_ERRSTATUS 0x14
120#define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf)
121#define SAHARA_ERRSOURCE_CHA 14
122#define SAHARA_ERRSOURCE_DMA 15
123#define SAHARA_ERRSTATUS_DMA_DIR (1 << 8)
124#define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
125#define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
126#define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff)
127#define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3)
128#define SAHARA_REG_FADDR 0x18
129#define SAHARA_REG_CDAR 0x1C
130#define SAHARA_REG_IDAR 0x20
131
132struct sahara_hw_desc {
75d3f811
AB		 133 u32 hdr;
134 u32 len1;
135 u32 p1;
136 u32 len2;
137 u32 p2;
138 u32 next;
5de88752
JM		 139};
140
141struct sahara_hw_link {
75d3f811
AB		 142 u32 len;
143 u32 p;
144 u32 next;
5de88752
JM		 145};
146
147struct sahara_ctx {
5de88752 148 unsigned long flags;
5a2bb93f
ST		 149
150 /* AES-specific context */
5de88752
JM		 151 int keylen;
152 u8 key[AES_KEYSIZE_128];
153 struct crypto_ablkcipher *fallback;
5a2bb93f
ST		 154
155 /* SHA-specific context */
156 struct crypto_shash *shash_fallback;
5de88752
JM		 157};
158
159struct sahara_aes_reqctx {
160 unsigned long mode;
161};
162
5a2bb93f
ST		 163/*
164 * struct sahara_sha_reqctx - private data per request
165 * @buf: holds data for requests smaller than block_size
166 * @rembuf: used to prepare one block_size-aligned request
167 * @context: hw-specific context for request. Digest is extracted from this
168 * @mode: specifies what type of hw-descriptor needs to be built
169 * @digest_size: length of digest for this request
170 * @context_size: length of hw-context for this request.
171 * Always digest_size + 4
172 * @buf_cnt: number of bytes saved in buf
173 * @sg_in_idx: number of hw links
174 * @in_sg: scatterlist for input data
175 * @in_sg_chain: scatterlists for chained input data
5a2bb93f
ST		 176 * @total: total number of bytes for transfer
177 * @last: is this the last block
178 * @first: is this the first block
179 * @active: inside a transfer
180 */
181struct sahara_sha_reqctx {
182 u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE];
183 u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE];
184 u8 context[SHA256_DIGEST_SIZE + 4];
5a2bb93f
ST		 185 unsigned int mode;
186 unsigned int digest_size;
187 unsigned int context_size;
188 unsigned int buf_cnt;
189 unsigned int sg_in_idx;
190 struct scatterlist *in_sg;
191 struct scatterlist in_sg_chain[2];
5a2bb93f
ST		 192 size_t total;
193 unsigned int last;
194 unsigned int first;
195 unsigned int active;
196};
197
5de88752
JM		 198struct sahara_dev {
199 struct device *device;
5ed903b3 200 unsigned int version;
5de88752
JM		 201 void __iomem *regs_base;
202 struct clk *clk_ipg;
203 struct clk *clk_ahb;
c0c3c89a
ST		 204 struct mutex queue_mutex;
205 struct task_struct *kthread;
206 struct completion dma_completion;
5de88752
JM		 207
208 struct sahara_ctx *ctx;
209 spinlock_t lock;
210 struct crypto_queue queue;
211 unsigned long flags;
212
5de88752
JM		 213 struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC];
214 dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC];
215
216 u8 *key_base;
217 dma_addr_t key_phys_base;
218
219 u8 *iv_base;
220 dma_addr_t iv_phys_base;
221
5a2bb93f
ST		 222 u8 *context_base;
223 dma_addr_t context_phys_base;
224
5de88752
JM		 225 struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK];
226 dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK];
227
5de88752
JM		 228 size_t total;
229 struct scatterlist *in_sg;
f8e28a0d 230 int nb_in_sg;
5de88752 231 struct scatterlist *out_sg;
f8e28a0d 232 int nb_out_sg;
5de88752
JM		 233
234 u32 error;
5de88752
JM		 235};
236
237static struct sahara_dev *dev_ptr;
238
239static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
240{
241 writel(data, dev->regs_base + reg);
242}
243
244static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
245{
246 return readl(dev->regs_base + reg);
247}
248
249static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
250{
251 u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
252 SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
253 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
254
255 if (dev->flags & FLAGS_CBC) {
256 hdr |= SAHARA_HDR_SKHA_MODE_CBC;
257 hdr ^= SAHARA_HDR_PARITY_BIT;
258 }
259
260 if (dev->flags & FLAGS_ENCRYPT) {
261 hdr |= SAHARA_HDR_SKHA_OP_ENC;
262 hdr ^= SAHARA_HDR_PARITY_BIT;
263 }
264
265 return hdr;
266}
267
268static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
269{
270 return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
271 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
272}
273
cac367bf 274static const char *sahara_err_src[16] = {
5de88752
JM		 275 "No error",
276 "Header error",
277 "Descriptor length error",
278 "Descriptor length or pointer error",
279 "Link length error",
280 "Link pointer error",
281 "Input buffer error",
282 "Output buffer error",
283 "Output buffer starvation",
284 "Internal state fault",
285 "General descriptor problem",
286 "Reserved",
287 "Descriptor address error",
288 "Link address error",
289 "CHA error",
290 "DMA error"
291};
292
cac367bf 293static const char *sahara_err_dmasize[4] = {
5de88752
JM		 294 "Byte transfer",
295 "Half-word transfer",
296 "Word transfer",
297 "Reserved"
298};
299
cac367bf 300static const char *sahara_err_dmasrc[8] = {
5de88752
JM		 301 "No error",
302 "AHB bus error",
303 "Internal IP bus error",
304 "Parity error",
305 "DMA crosses 256 byte boundary",
306 "DMA is busy",
307 "Reserved",
308 "DMA HW error"
309};
310
cac367bf 311static const char *sahara_cha_errsrc[12] = {
5de88752
JM		 312 "Input buffer non-empty",
313 "Illegal address",
314 "Illegal mode",
315 "Illegal data size",
316 "Illegal key size",
317 "Write during processing",
318 "CTX read during processing",
319 "HW error",
320 "Input buffer disabled/underflow",
321 "Output buffer disabled/overflow",
322 "DES key parity error",
323 "Reserved"
324};
325
cac367bf 326static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
5de88752
JM		 327
328static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
329{
330 u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
331 u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
332
333 dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
334
335 dev_err(dev->device, " - %s.\n", sahara_err_src[source]);
336
337 if (source == SAHARA_ERRSOURCE_DMA) {
338 if (error & SAHARA_ERRSTATUS_DMA_DIR)
339 dev_err(dev->device, " * DMA read.\n");
340 else
341 dev_err(dev->device, " * DMA write.\n");
342
343 dev_err(dev->device, " * %s.\n",
344 sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
345 dev_err(dev->device, " * %s.\n",
346 sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
347 } else if (source == SAHARA_ERRSOURCE_CHA) {
348 dev_err(dev->device, " * %s.\n",
349 sahara_cha_errsrc[chasrc]);
350 dev_err(dev->device, " * %s.\n",
351 sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
352 }
353 dev_err(dev->device, "\n");
354}
355
cac367bf 356static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
5de88752
JM		 357
358static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
359{
360 u8 state;
361
362 if (!IS_ENABLED(DEBUG))
363 return;
364
365 state = SAHARA_STATUS_GET_STATE(status);
366
367 dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
368 __func__, status);
369
370 dev_dbg(dev->device, " - State = %d:\n", state);
371 if (state & SAHARA_STATE_COMP_FLAG)
372 dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n");
373
374 dev_dbg(dev->device, " * %s.\n",
375 sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
376
377 if (status & SAHARA_STATUS_DAR_FULL)
378 dev_dbg(dev->device, " - DAR Full.\n");
379 if (status & SAHARA_STATUS_ERROR)
380 dev_dbg(dev->device, " - Error.\n");
381 if (status & SAHARA_STATUS_SECURE)
382 dev_dbg(dev->device, " - Secure.\n");
383 if (status & SAHARA_STATUS_FAIL)
384 dev_dbg(dev->device, " - Fail.\n");
385 if (status & SAHARA_STATUS_RNG_RESEED)
386 dev_dbg(dev->device, " - RNG Reseed Request.\n");
387 if (status & SAHARA_STATUS_ACTIVE_RNG)
388 dev_dbg(dev->device, " - RNG Active.\n");
389 if (status & SAHARA_STATUS_ACTIVE_MDHA)
390 dev_dbg(dev->device, " - MDHA Active.\n");
391 if (status & SAHARA_STATUS_ACTIVE_SKHA)
392 dev_dbg(dev->device, " - SKHA Active.\n");
393
394 if (status & SAHARA_STATUS_MODE_BATCH)
395 dev_dbg(dev->device, " - Batch Mode.\n");
396 else if (status & SAHARA_STATUS_MODE_DEDICATED)
397 dev_dbg(dev->device, " - Decidated Mode.\n");
398 else if (status & SAHARA_STATUS_MODE_DEBUG)
399 dev_dbg(dev->device, " - Debug Mode.\n");
400
401 dev_dbg(dev->device, " - Internal state = 0x%02x\n",
402 SAHARA_STATUS_GET_ISTATE(status));
403
404 dev_dbg(dev->device, "Current DAR: 0x%08x\n",
405 sahara_read(dev, SAHARA_REG_CDAR));
406 dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
407 sahara_read(dev, SAHARA_REG_IDAR));
408}
409
410static void sahara_dump_descriptors(struct sahara_dev *dev)
411{
412 int i;
413
414 if (!IS_ENABLED(DEBUG))
415 return;
416
417 for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
d4b98f20
AB		 418 dev_dbg(dev->device, "Descriptor (%d) (%pad):\n",
419 i, &dev->hw_phys_desc[i]);
5de88752
JM		 420 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
421 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
422 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
423 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
424 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
425 dev_dbg(dev->device, "\tnext = 0x%08x\n",
426 dev->hw_desc[i]->next);
427 }
428 dev_dbg(dev->device, "\n");
429}
430
431static void sahara_dump_links(struct sahara_dev *dev)
432{
433 int i;
434
435 if (!IS_ENABLED(DEBUG))
436 return;
437
438 for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
d4b98f20
AB		 439 dev_dbg(dev->device, "Link (%d) (%pad):\n",
440 i, &dev->hw_phys_link[i]);
5de88752
JM		 441 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
442 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
443 dev_dbg(dev->device, "\tnext = 0x%08x\n",
444 dev->hw_link[i]->next);
445 }
446 dev_dbg(dev->device, "\n");
447}
448
5de88752
JM		 449static int sahara_hw_descriptor_create(struct sahara_dev *dev)
450{
451 struct sahara_ctx *ctx = dev->ctx;
452 struct scatterlist *sg;
453 int ret;
454 int i, j;
1711045f 455 int idx = 0;
5de88752
JM		 456
457 /* Copy new key if necessary */
458 if (ctx->flags & FLAGS_NEW_KEY) {
459 memcpy(dev->key_base, ctx->key, ctx->keylen);
460 ctx->flags &= ~FLAGS_NEW_KEY;
461
462 if (dev->flags & FLAGS_CBC) {
1711045f
ST		 463 dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;
464 dev->hw_desc[idx]->p1 = dev->iv_phys_base;
5de88752 465 } else {
1711045f
ST		 466 dev->hw_desc[idx]->len1 = 0;
467 dev->hw_desc[idx]->p1 = 0;
5de88752 468 }
1711045f
ST		 469 dev->hw_desc[idx]->len2 = ctx->keylen;
470 dev->hw_desc[idx]->p2 = dev->key_phys_base;
471 dev->hw_desc[idx]->next = dev->hw_phys_desc[1];
472
473 dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);
474
475 idx++;
5de88752 476 }
5de88752 477
d23afa1a 478 dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total);
6c2b74d4
LC		 479 if (dev->nb_in_sg < 0) {
480 dev_err(dev->device, "Invalid numbers of src SG.\n");
481 return dev->nb_in_sg;
482 }
d23afa1a 483 dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total);
6c2b74d4
LC		 484 if (dev->nb_out_sg < 0) {
485 dev_err(dev->device, "Invalid numbers of dst SG.\n");
486 return dev->nb_out_sg;
487 }
5de88752
JM		 488 if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
489 dev_err(dev->device, "not enough hw links (%d)\n",
490 dev->nb_in_sg + dev->nb_out_sg);
491 return -EINVAL;
492 }
493
494 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
495 DMA_TO_DEVICE);
496 if (ret != dev->nb_in_sg) {
497 dev_err(dev->device, "couldn't map in sg\n");
498 goto unmap_in;
499 }
500 ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
501 DMA_FROM_DEVICE);
502 if (ret != dev->nb_out_sg) {
503 dev_err(dev->device, "couldn't map out sg\n");
504 goto unmap_out;
505 }
506
507 /* Create input links */
1711045f 508 dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];
5de88752
JM		 509 sg = dev->in_sg;
510 for (i = 0; i < dev->nb_in_sg; i++) {
511 dev->hw_link[i]->len = sg->length;
512 dev->hw_link[i]->p = sg->dma_address;
513 if (i == (dev->nb_in_sg - 1)) {
514 dev->hw_link[i]->next = 0;
515 } else {
516 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
517 sg = sg_next(sg);
518 }
519 }
520
521 /* Create output links */
1711045f 522 dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];
5de88752
JM		 523 sg = dev->out_sg;
524 for (j = i; j < dev->nb_out_sg + i; j++) {
525 dev->hw_link[j]->len = sg->length;
526 dev->hw_link[j]->p = sg->dma_address;
527 if (j == (dev->nb_out_sg + i - 1)) {
528 dev->hw_link[j]->next = 0;
529 } else {
530 dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
531 sg = sg_next(sg);
532 }
533 }
534
535 /* Fill remaining fields of hw_desc[1] */
1711045f
ST		 536 dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);
537 dev->hw_desc[idx]->len1 = dev->total;
538 dev->hw_desc[idx]->len2 = dev->total;
539 dev->hw_desc[idx]->next = 0;
5de88752
JM		 540
541 sahara_dump_descriptors(dev);
542 sahara_dump_links(dev);
543
5de88752
JM		 544 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
545
546 return 0;
547
548unmap_out:
549 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
550 DMA_TO_DEVICE);
551unmap_in:
552 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
553 DMA_FROM_DEVICE);
554
555 return -EINVAL;
556}
557
c0c3c89a 558static int sahara_aes_process(struct ablkcipher_request *req)
5de88752 559{
c0c3c89a 560 struct sahara_dev *dev = dev_ptr;
5de88752
JM		 561 struct sahara_ctx *ctx;
562 struct sahara_aes_reqctx *rctx;
5de88752 563 int ret;
58ed798b 564 unsigned long timeout;
5de88752 565
5de88752
JM		 566 /* Request is ready to be dispatched by the device */
567 dev_dbg(dev->device,
568 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
569 req->nbytes, req->src, req->dst);
570
571 /* assign new request to device */
5de88752
JM		 572 dev->total = req->nbytes;
573 dev->in_sg = req->src;
574 dev->out_sg = req->dst;
575
576 rctx = ablkcipher_request_ctx(req);
577 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
578 rctx->mode &= FLAGS_MODE_MASK;
579 dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
580
581 if ((dev->flags & FLAGS_CBC) && req->info)
582 memcpy(dev->iv_base, req->info, AES_KEYSIZE_128);
583
584 /* assign new context to device */
5de88752
JM		 585 dev->ctx = ctx;
586
c0c3c89a
ST		 587 reinit_completion(&dev->dma_completion);
588
5de88752 589 ret = sahara_hw_descriptor_create(dev);
6cf02fca
NMG		 590 if (ret)
591 return -EINVAL;
c0c3c89a 592
58ed798b 593 timeout = wait_for_completion_timeout(&dev->dma_completion,
c0c3c89a 594 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
58ed798b 595 if (!timeout) {
c0c3c89a
ST		 596 dev_err(dev->device, "AES timeout\n");
597 return -ETIMEDOUT;
5de88752 598 }
c0c3c89a
ST		 599
600 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
601 DMA_TO_DEVICE);
602 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
603 DMA_FROM_DEVICE);
604
605 return 0;
5de88752
JM		 606}
607
608static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
609 unsigned int keylen)
610{
611 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
612 int ret;
613
614 ctx->keylen = keylen;
615
616 /* SAHARA only supports 128bit keys */
617 if (keylen == AES_KEYSIZE_128) {
618 memcpy(ctx->key, key, keylen);
619 ctx->flags |= FLAGS_NEW_KEY;
620 return 0;
621 }
622
623 if (keylen != AES_KEYSIZE_128 &&
624 keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
625 return -EINVAL;
626
627 /*
628 * The requested key size is not supported by HW, do a fallback.
629 */
630 ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
631 ctx->fallback->base.crt_flags |=
632 (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
633
634 ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen);
635 if (ret) {
636 struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm);
637
638 tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK;
639 tfm_aux->crt_flags |=
640 (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK);
641 }
642 return ret;
643}
644
645static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
646{
5de88752
JM		 647 struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req);
648 struct sahara_dev *dev = dev_ptr;
649 int err = 0;
5de88752
JM		 650
651 dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
652 req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
653
654 if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
655 dev_err(dev->device,
656 "request size is not exact amount of AES blocks\n");
657 return -EINVAL;
658 }
659
5de88752 660 rctx->mode = mode;
c0c3c89a
ST		 661
662 mutex_lock(&dev->queue_mutex);
5de88752 663 err = ablkcipher_enqueue_request(&dev->queue, req);
c0c3c89a 664 mutex_unlock(&dev->queue_mutex);
5de88752 665
c0c3c89a 666 wake_up_process(dev->kthread);
5de88752
JM		 667
668 return err;
669}
670
671static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
672{
673 struct crypto_tfm *tfm =
674 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
675 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
676 crypto_ablkcipher_reqtfm(req));
677 int err;
678
679 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
680 ablkcipher_request_set_tfm(req, ctx->fallback);
681 err = crypto_ablkcipher_encrypt(req);
682 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
683 return err;
684 }
685
686 return sahara_aes_crypt(req, FLAGS_ENCRYPT);
687}
688
689static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
690{
691 struct crypto_tfm *tfm =
692 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
693 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
694 crypto_ablkcipher_reqtfm(req));
695 int err;
696
697 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
698 ablkcipher_request_set_tfm(req, ctx->fallback);
699 err = crypto_ablkcipher_decrypt(req);
700 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
701 return err;
702 }
703
704 return sahara_aes_crypt(req, 0);
705}
706
707static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
708{
709 struct crypto_tfm *tfm =
710 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
711 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
712 crypto_ablkcipher_reqtfm(req));
713 int err;
714
715 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
716 ablkcipher_request_set_tfm(req, ctx->fallback);
717 err = crypto_ablkcipher_encrypt(req);
718 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
719 return err;
720 }
721
722 return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
723}
724
725static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
726{
727 struct crypto_tfm *tfm =
728 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
729 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
730 crypto_ablkcipher_reqtfm(req));
731 int err;
732
733 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
734 ablkcipher_request_set_tfm(req, ctx->fallback);
735 err = crypto_ablkcipher_decrypt(req);
736 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
737 return err;
738 }
739
740 return sahara_aes_crypt(req, FLAGS_CBC);
741}
742
743static int sahara_aes_cra_init(struct crypto_tfm *tfm)
744{
efa59e2e 745 const char *name = crypto_tfm_alg_name(tfm);
5de88752
JM		 746 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
747
748 ctx->fallback = crypto_alloc_ablkcipher(name, 0,
749 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
750 if (IS_ERR(ctx->fallback)) {
751 pr_err("Error allocating fallback algo %s\n", name);
752 return PTR_ERR(ctx->fallback);
753 }
754
755 tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx);
756
757 return 0;
758}
759
760static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
761{
762 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
763
764 if (ctx->fallback)
765 crypto_free_ablkcipher(ctx->fallback);
766 ctx->fallback = NULL;
767}
768
5a2bb93f
ST		 769static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
770 struct sahara_sha_reqctx *rctx)
771{
772 u32 hdr = 0;
773
774 hdr = rctx->mode;
775
776 if (rctx->first) {
777 hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH;
778 hdr |= SAHARA_HDR_MDHA_INIT;
779 } else {
780 hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY;
781 }
782
783 if (rctx->last)
784 hdr |= SAHARA_HDR_MDHA_PDATA;
785
786 if (hweight_long(hdr) % 2 == 0)
787 hdr |= SAHARA_HDR_PARITY_BIT;
788
789 return hdr;
790}
791
792static int sahara_sha_hw_links_create(struct sahara_dev *dev,
793 struct sahara_sha_reqctx *rctx,
794 int start)
795{
796 struct scatterlist *sg;
797 unsigned int i;
798 int ret;
799
800 dev->in_sg = rctx->in_sg;
801
d23afa1a 802 dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total);
6c2b74d4
LC		 803 if (dev->nb_in_sg < 0) {
804 dev_err(dev->device, "Invalid numbers of src SG.\n");
805 return dev->nb_in_sg;
806 }
5a2bb93f
ST		 807 if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {
808 dev_err(dev->device, "not enough hw links (%d)\n",
809 dev->nb_in_sg + dev->nb_out_sg);
810 return -EINVAL;
811 }
812
640eec52
LC		 813 sg = dev->in_sg;
814 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE);
815 if (!ret)
816 return -EFAULT;
817
818 for (i = start; i < dev->nb_in_sg + start; i++) {
819 dev->hw_link[i]->len = sg->length;
820 dev->hw_link[i]->p = sg->dma_address;
821 if (i == (dev->nb_in_sg + start - 1)) {
822 dev->hw_link[i]->next = 0;
823 } else {
5a2bb93f
ST		 824 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
825 sg = sg_next(sg);
5a2bb93f
ST		 826 }
827 }
828
829 return i;
830}
831
832static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev,
833 struct sahara_sha_reqctx *rctx,
834 struct ahash_request *req,
835 int index)
836{
837 unsigned result_len;
838 int i = index;
839
840 if (rctx->first)
841 /* Create initial descriptor: #8*/
842 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
843 else
844 /* Create hash descriptor: #10. Must follow #6. */
845 dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;
846
847 dev->hw_desc[index]->len1 = rctx->total;
848 if (dev->hw_desc[index]->len1 == 0) {
849 /* if len1 is 0, p1 must be 0, too */
850 dev->hw_desc[index]->p1 = 0;
851 rctx->sg_in_idx = 0;
852 } else {
853 /* Create input links */
854 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
855 i = sahara_sha_hw_links_create(dev, rctx, index);
856
857 rctx->sg_in_idx = index;
858 if (i < 0)
859 return i;
860 }
861
862 dev->hw_desc[index]->p2 = dev->hw_phys_link[i];
863
864 /* Save the context for the next operation */
865 result_len = rctx->context_size;
866 dev->hw_link[i]->p = dev->context_phys_base;
867
868 dev->hw_link[i]->len = result_len;
869 dev->hw_desc[index]->len2 = result_len;
870
871 dev->hw_link[i]->next = 0;
872
873 return 0;
874}
875
876/*
877 * Load descriptor aka #6
878 *
879 * To load a previously saved context back to the MDHA unit
880 *
881 * p1: Saved Context
882 * p2: NULL
883 *
884 */
885static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev,
886 struct sahara_sha_reqctx *rctx,
887 struct ahash_request *req,
888 int index)
889{
890 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
891
892 dev->hw_desc[index]->len1 = rctx->context_size;
893 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
894 dev->hw_desc[index]->len2 = 0;
895 dev->hw_desc[index]->p2 = 0;
896
897 dev->hw_link[index]->len = rctx->context_size;
898 dev->hw_link[index]->p = dev->context_phys_base;
899 dev->hw_link[index]->next = 0;
900
901 return 0;
902}
903
904static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes)
905{
906 if (!sg || !sg->length)
907 return nbytes;
908
909 while (nbytes && sg) {
910 if (nbytes <= sg->length) {
911 sg->length = nbytes;
912 sg_mark_end(sg);
913 break;
914 }
915 nbytes -= sg->length;
5be4d4c9 916 sg = sg_next(sg);
5a2bb93f
ST		 917 }
918
919 return nbytes;
920}
921
922static int sahara_sha_prepare_request(struct ahash_request *req)
923{
924 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
925 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
926 unsigned int hash_later;
927 unsigned int block_size;
928 unsigned int len;
929
930 block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
931
932 /* append bytes from previous operation */
933 len = rctx->buf_cnt + req->nbytes;
934
935 /* only the last transfer can be padded in hardware */
936 if (!rctx->last && (len < block_size)) {
937 /* to few data, save for next operation */
938 scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,
939 0, req->nbytes, 0);
940 rctx->buf_cnt += req->nbytes;
941
942 return 0;
943 }
944
945 /* add data from previous operation first */
946 if (rctx->buf_cnt)
947 memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);
948
949 /* data must always be a multiple of block_size */
950 hash_later = rctx->last ? 0 : len & (block_size - 1);
951 if (hash_later) {
952 unsigned int offset = req->nbytes - hash_later;
953 /* Save remaining bytes for later use */
954 scatterwalk_map_and_copy(rctx->buf, req->src, offset,
955 hash_later, 0);
956 }
957
958 /* nbytes should now be multiple of blocksize */
959 req->nbytes = req->nbytes - hash_later;
960
961 sahara_walk_and_recalc(req->src, req->nbytes);
962
963 /* have data from previous operation and current */
964 if (rctx->buf_cnt && req->nbytes) {
965 sg_init_table(rctx->in_sg_chain, 2);
966 sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);
967
c56f6d12 968 sg_chain(rctx->in_sg_chain, 2, req->src);
5a2bb93f
ST		 969
970 rctx->total = req->nbytes + rctx->buf_cnt;
971 rctx->in_sg = rctx->in_sg_chain;
972
5a2bb93f
ST		 973 req->src = rctx->in_sg_chain;
974 /* only data from previous operation */
975 } else if (rctx->buf_cnt) {
976 if (req->src)
977 rctx->in_sg = req->src;
978 else
979 rctx->in_sg = rctx->in_sg_chain;
980 /* buf was copied into rembuf above */
981 sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);
982 rctx->total = rctx->buf_cnt;
5a2bb93f
ST		 983 /* no data from previous operation */
984 } else {
985 rctx->in_sg = req->src;
986 rctx->total = req->nbytes;
987 req->src = rctx->in_sg;
5a2bb93f
ST		 988 }
989
990 /* on next call, we only have the remaining data in the buffer */
991 rctx->buf_cnt = hash_later;
992
993 return -EINPROGRESS;
994}
995
5a2bb93f
ST		 996static int sahara_sha_process(struct ahash_request *req)
997{
998 struct sahara_dev *dev = dev_ptr;
999 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
df586cbb 1000 int ret;
58ed798b 1001 unsigned long timeout;
5a2bb93f
ST		 1002
1003 ret = sahara_sha_prepare_request(req);
1004 if (!ret)
1005 return ret;
1006
1007 if (rctx->first) {
1008 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0);
1009 dev->hw_desc[0]->next = 0;
1010 rctx->first = 0;
1011 } else {
1012 memcpy(dev->context_base, rctx->context, rctx->context_size);
1013
1014 sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0);
1015 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
1016 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1);
1017 dev->hw_desc[1]->next = 0;
1018 }
1019
1020 sahara_dump_descriptors(dev);
1021 sahara_dump_links(dev);
1022
1023 reinit_completion(&dev->dma_completion);
1024
1025 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
1026
58ed798b 1027 timeout = wait_for_completion_timeout(&dev->dma_completion,
5a2bb93f 1028 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
58ed798b 1029 if (!timeout) {
5a2bb93f
ST		 1030 dev_err(dev->device, "SHA timeout\n");
1031 return -ETIMEDOUT;
1032 }
1033
1034 if (rctx->sg_in_idx)
640eec52
LC		 1035 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
1036 DMA_TO_DEVICE);
5a2bb93f
ST		 1037
1038 memcpy(rctx->context, dev->context_base, rctx->context_size);
1039
1040 if (req->result)
1041 memcpy(req->result, rctx->context, rctx->digest_size);
1042
1043 return 0;
1044}
1045
c0c3c89a
ST		 1046static int sahara_queue_manage(void *data)
1047{
1048 struct sahara_dev *dev = (struct sahara_dev *)data;
1049 struct crypto_async_request *async_req;
ddacc621 1050 struct crypto_async_request *backlog;
c0c3c89a
ST		 1051 int ret = 0;
1052
1053 do {
1054 __set_current_state(TASK_INTERRUPTIBLE);
1055
1056 mutex_lock(&dev->queue_mutex);
ddacc621 1057 backlog = crypto_get_backlog(&dev->queue);
c0c3c89a
ST		 1058 async_req = crypto_dequeue_request(&dev->queue);
1059 mutex_unlock(&dev->queue_mutex);
1060
ddacc621
ST		 1061 if (backlog)
1062 backlog->complete(backlog, -EINPROGRESS);
1063
c0c3c89a 1064 if (async_req) {
5a2bb93f
ST		 1065 if (crypto_tfm_alg_type(async_req->tfm) ==
1066 CRYPTO_ALG_TYPE_AHASH) {
1067 struct ahash_request *req =
1068 ahash_request_cast(async_req);
1069
1070 ret = sahara_sha_process(req);
1071 } else {
1072 struct ablkcipher_request *req =
1073 ablkcipher_request_cast(async_req);
c0c3c89a 1074
5a2bb93f
ST		 1075 ret = sahara_aes_process(req);
1076 }
c0c3c89a
ST		 1077
1078 async_req->complete(async_req, ret);
1079
1080 continue;
1081 }
1082
1083 schedule();
1084 } while (!kthread_should_stop());
1085
1086 return 0;
1087}
1088
5a2bb93f
ST		 1089static int sahara_sha_enqueue(struct ahash_request *req, int last)
1090{
1091 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1092 struct sahara_dev *dev = dev_ptr;
1093 int ret;
1094
1095 if (!req->nbytes && !last)
1096 return 0;
1097
5a2bb93f
ST		 1098 rctx->last = last;
1099
1100 if (!rctx->active) {
1101 rctx->active = 1;
1102 rctx->first = 1;
1103 }
1104
1105 mutex_lock(&dev->queue_mutex);
1106 ret = crypto_enqueue_request(&dev->queue, &req->base);
1107 mutex_unlock(&dev->queue_mutex);
1108
1109 wake_up_process(dev->kthread);
5a2bb93f
ST		 1110
1111 return ret;
1112}
1113
1114static int sahara_sha_init(struct ahash_request *req)
1115{
1116 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1117 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1118
1119 memset(rctx, 0, sizeof(*rctx));
1120
1121 switch (crypto_ahash_digestsize(tfm)) {
1122 case SHA1_DIGEST_SIZE:
1123 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;
1124 rctx->digest_size = SHA1_DIGEST_SIZE;
1125 break;
1126 case SHA256_DIGEST_SIZE:
1127 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;
1128 rctx->digest_size = SHA256_DIGEST_SIZE;
1129 break;
1130 default:
1131 return -EINVAL;
1132 }
1133
1134 rctx->context_size = rctx->digest_size + 4;
1135 rctx->active = 0;
1136
5a2bb93f
ST		 1137 return 0;
1138}
1139
1140static int sahara_sha_update(struct ahash_request *req)
1141{
1142 return sahara_sha_enqueue(req, 0);
1143}
1144
1145static int sahara_sha_final(struct ahash_request *req)
1146{
1147 req->nbytes = 0;
1148 return sahara_sha_enqueue(req, 1);
1149}
1150
1151static int sahara_sha_finup(struct ahash_request *req)
1152{
1153 return sahara_sha_enqueue(req, 1);
1154}
1155
1156static int sahara_sha_digest(struct ahash_request *req)
1157{
1158 sahara_sha_init(req);
1159
1160 return sahara_sha_finup(req);
1161}
1162
1163static int sahara_sha_export(struct ahash_request *req, void *out)
1164{
1165 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1166 struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1167 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1168
1169 memcpy(out, ctx, sizeof(struct sahara_ctx));
1170 memcpy(out + sizeof(struct sahara_sha_reqctx), rctx,
1171 sizeof(struct sahara_sha_reqctx));
1172
1173 return 0;
1174}
1175
1176static int sahara_sha_import(struct ahash_request *req, const void *in)
1177{
1178 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1179 struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1180 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1181
1182 memcpy(ctx, in, sizeof(struct sahara_ctx));
1183 memcpy(rctx, in + sizeof(struct sahara_sha_reqctx),
1184 sizeof(struct sahara_sha_reqctx));
1185
1186 return 0;
1187}
1188
1189static int sahara_sha_cra_init(struct crypto_tfm *tfm)
1190{
1191 const char *name = crypto_tfm_alg_name(tfm);
1192 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1193
1194 ctx->shash_fallback = crypto_alloc_shash(name, 0,
1195 CRYPTO_ALG_NEED_FALLBACK);
1196 if (IS_ERR(ctx->shash_fallback)) {
1197 pr_err("Error allocating fallback algo %s\n", name);
1198 return PTR_ERR(ctx->shash_fallback);
1199 }
1200 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1201 sizeof(struct sahara_sha_reqctx) +
1202 SHA_BUFFER_LEN + SHA256_BLOCK_SIZE);
1203
1204 return 0;
1205}
1206
1207static void sahara_sha_cra_exit(struct crypto_tfm *tfm)
1208{
1209 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1210
1211 crypto_free_shash(ctx->shash_fallback);
1212 ctx->shash_fallback = NULL;
1213}
1214
5de88752
JM		 1215static struct crypto_alg aes_algs[] = {
1216{
1217 .cra_name = "ecb(aes)",
1218 .cra_driver_name = "sahara-ecb-aes",
1219 .cra_priority = 300,
1220 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1221 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1222 .cra_blocksize = AES_BLOCK_SIZE,
1223 .cra_ctxsize = sizeof(struct sahara_ctx),
1224 .cra_alignmask = 0x0,
1225 .cra_type = &crypto_ablkcipher_type,
1226 .cra_module = THIS_MODULE,
1227 .cra_init = sahara_aes_cra_init,
1228 .cra_exit = sahara_aes_cra_exit,
1229 .cra_u.ablkcipher = {
1230 .min_keysize = AES_MIN_KEY_SIZE ,
1231 .max_keysize = AES_MAX_KEY_SIZE,
1232 .setkey = sahara_aes_setkey,
1233 .encrypt = sahara_aes_ecb_encrypt,
1234 .decrypt = sahara_aes_ecb_decrypt,
1235 }
1236}, {
1237 .cra_name = "cbc(aes)",
1238 .cra_driver_name = "sahara-cbc-aes",
1239 .cra_priority = 300,
1240 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1241 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1242 .cra_blocksize = AES_BLOCK_SIZE,
1243 .cra_ctxsize = sizeof(struct sahara_ctx),
1244 .cra_alignmask = 0x0,
1245 .cra_type = &crypto_ablkcipher_type,
1246 .cra_module = THIS_MODULE,
1247 .cra_init = sahara_aes_cra_init,
1248 .cra_exit = sahara_aes_cra_exit,
1249 .cra_u.ablkcipher = {
1250 .min_keysize = AES_MIN_KEY_SIZE ,
1251 .max_keysize = AES_MAX_KEY_SIZE,
1252 .ivsize = AES_BLOCK_SIZE,
1253 .setkey = sahara_aes_setkey,
1254 .encrypt = sahara_aes_cbc_encrypt,
1255 .decrypt = sahara_aes_cbc_decrypt,
1256 }
1257}
1258};
1259
5a2bb93f
ST		 1260static struct ahash_alg sha_v3_algs[] = {
1261{
1262 .init = sahara_sha_init,
1263 .update = sahara_sha_update,
1264 .final = sahara_sha_final,
1265 .finup = sahara_sha_finup,
1266 .digest = sahara_sha_digest,
1267 .export = sahara_sha_export,
1268 .import = sahara_sha_import,
1269 .halg.digestsize = SHA1_DIGEST_SIZE,
1270 .halg.base = {
1271 .cra_name = "sha1",
1272 .cra_driver_name = "sahara-sha1",
1273 .cra_priority = 300,
1274 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1275 CRYPTO_ALG_ASYNC |
1276 CRYPTO_ALG_NEED_FALLBACK,
1277 .cra_blocksize = SHA1_BLOCK_SIZE,
1278 .cra_ctxsize = sizeof(struct sahara_ctx),
1279 .cra_alignmask = 0,
1280 .cra_module = THIS_MODULE,
1281 .cra_init = sahara_sha_cra_init,
1282 .cra_exit = sahara_sha_cra_exit,
1283 }
1284},
1285};
1286
1287static struct ahash_alg sha_v4_algs[] = {
1288{
1289 .init = sahara_sha_init,
1290 .update = sahara_sha_update,
1291 .final = sahara_sha_final,
1292 .finup = sahara_sha_finup,
1293 .digest = sahara_sha_digest,
1294 .export = sahara_sha_export,
1295 .import = sahara_sha_import,
1296 .halg.digestsize = SHA256_DIGEST_SIZE,
1297 .halg.base = {
1298 .cra_name = "sha256",
1299 .cra_driver_name = "sahara-sha256",
1300 .cra_priority = 300,
1301 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1302 CRYPTO_ALG_ASYNC |
1303 CRYPTO_ALG_NEED_FALLBACK,
1304 .cra_blocksize = SHA256_BLOCK_SIZE,
1305 .cra_ctxsize = sizeof(struct sahara_ctx),
1306 .cra_alignmask = 0,
1307 .cra_module = THIS_MODULE,
1308 .cra_init = sahara_sha_cra_init,
1309 .cra_exit = sahara_sha_cra_exit,
1310 }
1311},
1312};
1313
5de88752
JM		 1314static irqreturn_t sahara_irq_handler(int irq, void *data)
1315{
1316 struct sahara_dev *dev = (struct sahara_dev *)data;
1317 unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
1318 unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
1319
5de88752
JM		 1320 sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
1321 SAHARA_REG_CMD);
1322
1323 sahara_decode_status(dev, stat);
1324
1325 if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
1326 return IRQ_NONE;
1327 } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
1328 dev->error = 0;
1329 } else {
1330 sahara_decode_error(dev, err);
1331 dev->error = -EINVAL;
1332 }
1333
c0c3c89a 1334 complete(&dev->dma_completion);
5de88752
JM		 1335
1336 return IRQ_HANDLED;
1337}
1338
1339
1340static int sahara_register_algs(struct sahara_dev *dev)
1341{
5a2bb93f
ST		 1342 int err;
1343 unsigned int i, j, k, l;
5de88752
JM		 1344
1345 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1346 INIT_LIST_HEAD(&aes_algs[i].cra_list);
1347 err = crypto_register_alg(&aes_algs[i]);
1348 if (err)
1349 goto err_aes_algs;
1350 }
1351
5a2bb93f
ST		 1352 for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) {
1353 err = crypto_register_ahash(&sha_v3_algs[k]);
1354 if (err)
1355 goto err_sha_v3_algs;
1356 }
1357
1358 if (dev->version > SAHARA_VERSION_3)
1359 for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) {
1360 err = crypto_register_ahash(&sha_v4_algs[l]);
1361 if (err)
1362 goto err_sha_v4_algs;
1363 }
1364
5de88752
JM		 1365 return 0;
1366
5a2bb93f
ST		 1367err_sha_v4_algs:
1368 for (j = 0; j < l; j++)
1369 crypto_unregister_ahash(&sha_v4_algs[j]);
1370
1371err_sha_v3_algs:
1372 for (j = 0; j < k; j++)
1373 crypto_unregister_ahash(&sha_v4_algs[j]);
1374
5de88752
JM		 1375err_aes_algs:
1376 for (j = 0; j < i; j++)
1377 crypto_unregister_alg(&aes_algs[j]);
1378
1379 return err;
1380}
1381
1382static void sahara_unregister_algs(struct sahara_dev *dev)
1383{
5a2bb93f 1384 unsigned int i;
5de88752
JM		 1385
1386 for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1387 crypto_unregister_alg(&aes_algs[i]);
5a2bb93f
ST		 1388
1389 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1390 crypto_unregister_ahash(&sha_v3_algs[i]);
1391
1392 if (dev->version > SAHARA_VERSION_3)
1393 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1394 crypto_unregister_ahash(&sha_v4_algs[i]);
5de88752
JM		 1395}
1396
1397static struct platform_device_id sahara_platform_ids[] = {
1398 { .name = "sahara-imx27" },
1399 { /* sentinel */ }
1400};
1401MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
1402
1403static struct of_device_id sahara_dt_ids[] = {
5ed903b3 1404 { .compatible = "fsl,imx53-sahara" },
5de88752
JM		 1405 { .compatible = "fsl,imx27-sahara" },
1406 { /* sentinel */ }
1407};
68be0b1a 1408MODULE_DEVICE_TABLE(of, sahara_dt_ids);
5de88752
JM		 1409
1410static int sahara_probe(struct platform_device *pdev)
1411{
1412 struct sahara_dev *dev;
1413 struct resource *res;
1414 u32 version;
1415 int irq;
1416 int err;
1417 int i;
1418
1419 dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL);
1420 if (dev == NULL) {
1421 dev_err(&pdev->dev, "unable to alloc data struct.\n");
1422 return -ENOMEM;
1423 }
1424
1425 dev->device = &pdev->dev;
1426 platform_set_drvdata(pdev, dev);
1427
1428 /* Get the base address */
1429 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
9e95275c
JH		 1430 dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
1431 if (IS_ERR(dev->regs_base))
1432 return PTR_ERR(dev->regs_base);
5de88752
JM		 1433
1434 /* Get the IRQ */
1435 irq = platform_get_irq(pdev, 0);
1436 if (irq < 0) {
1437 dev_err(&pdev->dev, "failed to get irq resource\n");
1438 return irq;
1439 }
1440
3d6f1d12
AS		 1441 err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
1442 0, dev_name(&pdev->dev), dev);
1443 if (err) {
5de88752 1444 dev_err(&pdev->dev, "failed to request irq\n");
3d6f1d12 1445 return err;
5de88752
JM		 1446 }
1447
1448 /* clocks */
1449 dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1450 if (IS_ERR(dev->clk_ipg)) {
1451 dev_err(&pdev->dev, "Could not get ipg clock\n");
1452 return PTR_ERR(dev->clk_ipg);
1453 }
1454
1455 dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
1456 if (IS_ERR(dev->clk_ahb)) {
1457 dev_err(&pdev->dev, "Could not get ahb clock\n");
1458 return PTR_ERR(dev->clk_ahb);
1459 }
1460
1461 /* Allocate HW descriptors */
66c9a04e 1462 dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev,
5de88752
JM		 1463 SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1464 &dev->hw_phys_desc[0], GFP_KERNEL);
1465 if (!dev->hw_desc[0]) {
1466 dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
1467 return -ENOMEM;
1468 }
1469 dev->hw_desc[1] = dev->hw_desc[0] + 1;
1470 dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
1471 sizeof(struct sahara_hw_desc);
1472
1473 /* Allocate space for iv and key */
66c9a04e 1474 dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
5de88752
JM		 1475 &dev->key_phys_base, GFP_KERNEL);
1476 if (!dev->key_base) {
1477 dev_err(&pdev->dev, "Could not allocate memory for key\n");
66c9a04e 1478 return -ENOMEM;
5de88752
JM		 1479 }
1480 dev->iv_base = dev->key_base + AES_KEYSIZE_128;
1481 dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
1482
5a2bb93f 1483 /* Allocate space for context: largest digest + message length field */
66c9a04e 1484 dev->context_base = dmam_alloc_coherent(&pdev->dev,
5a2bb93f
ST		 1485 SHA256_DIGEST_SIZE + 4,
1486 &dev->context_phys_base, GFP_KERNEL);
1487 if (!dev->context_base) {
1488 dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n");
66c9a04e 1489 return -ENOMEM;
5a2bb93f
ST		 1490 }
1491
5de88752 1492 /* Allocate space for HW links */
66c9a04e 1493 dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev,
5de88752
JM		 1494 SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1495 &dev->hw_phys_link[0], GFP_KERNEL);
393e661d 1496 if (!dev->hw_link[0]) {
5de88752 1497 dev_err(&pdev->dev, "Could not allocate hw links\n");
66c9a04e 1498 return -ENOMEM;
5de88752
JM		 1499 }
1500 for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
1501 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
1502 sizeof(struct sahara_hw_link);
1503 dev->hw_link[i] = dev->hw_link[i - 1] + 1;
1504 }
1505
1506 crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1507
20ec9d81 1508 spin_lock_init(&dev->lock);
c0c3c89a 1509 mutex_init(&dev->queue_mutex);
20ec9d81 1510
5de88752
JM		 1511 dev_ptr = dev;
1512
c0c3c89a
ST		 1513 dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto");
1514 if (IS_ERR(dev->kthread)) {
66c9a04e 1515 return PTR_ERR(dev->kthread);
c0c3c89a 1516 }
5de88752 1517
c0c3c89a 1518 init_completion(&dev->dma_completion);
5de88752 1519
7eac7144
FE		 1520 err = clk_prepare_enable(dev->clk_ipg);
1521 if (err)
66c9a04e 1522 return err;
7eac7144
FE		 1523 err = clk_prepare_enable(dev->clk_ahb);
1524 if (err)
1525 goto clk_ipg_disable;
5de88752
JM		 1526
1527 version = sahara_read(dev, SAHARA_REG_VERSION);
5ed903b3
ST		 1528 if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {
1529 if (version != SAHARA_VERSION_3)
1530 err = -ENODEV;
1531 } else if (of_device_is_compatible(pdev->dev.of_node,
1532 "fsl,imx53-sahara")) {
1533 if (((version >> 8) & 0xff) != SAHARA_VERSION_4)
1534 err = -ENODEV;
1535 version = (version >> 8) & 0xff;
1536 }
1537 if (err == -ENODEV) {
5de88752 1538 dev_err(&pdev->dev, "SAHARA version %d not supported\n",
5ed903b3 1539 version);
5de88752
JM		 1540 goto err_algs;
1541 }
1542
5ed903b3
ST		 1543 dev->version = version;
1544
5de88752
JM		 1545 sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
1546 SAHARA_REG_CMD);
1547 sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
1548 SAHARA_CONTROL_SET_MAXBURST(8) |
1549 SAHARA_CONTROL_RNG_AUTORSD |
1550 SAHARA_CONTROL_ENABLE_INT,
1551 SAHARA_REG_CONTROL);
1552
1553 err = sahara_register_algs(dev);
1554 if (err)
1555 goto err_algs;
1556
1557 dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1558
1559 return 0;
1560
1561err_algs:
c0c3c89a 1562 kthread_stop(dev->kthread);
5de88752 1563 dev_ptr = NULL;
7eac7144
FE		 1564 clk_disable_unprepare(dev->clk_ahb);
1565clk_ipg_disable:
1566 clk_disable_unprepare(dev->clk_ipg);
5de88752
JM		 1567
1568 return err;
1569}
1570
1571static int sahara_remove(struct platform_device *pdev)
1572{
1573 struct sahara_dev *dev = platform_get_drvdata(pdev);
1574
c0c3c89a 1575 kthread_stop(dev->kthread);
5de88752
JM		 1576
1577 sahara_unregister_algs(dev);
1578
1579 clk_disable_unprepare(dev->clk_ipg);
1580 clk_disable_unprepare(dev->clk_ahb);
1581
1582 dev_ptr = NULL;
1583
1584 return 0;
1585}
1586
1587static struct platform_driver sahara_driver = {
1588 .probe = sahara_probe,
1589 .remove = sahara_remove,
1590 .driver = {
1591 .name = SAHARA_NAME,
1b0b2605 1592 .of_match_table = sahara_dt_ids,
5de88752
JM		 1593 },
1594 .id_table = sahara_platform_ids,
1595};
1596
1597module_platform_driver(sahara_driver);
1598
1599MODULE_LICENSE("GPL");
1600MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
5a2bb93f 1601MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
5de88752 1602MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
Ubuntu Artful kernel mirror