]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/crypto/atmel-tdes.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: atmel-sha - Retire dma_request_slave_channel_compat()
[mirror_ubuntu-hirsute-kernel.git] / drivers / crypto / atmel-tdes.c
CommitLineData
820684cc 1// SPDX-License-Identifier: GPL-2.0
13802005
NR		 2/*
3 * Cryptographic API.
4 *
5 * Support for ATMEL DES/TDES HW acceleration.
6 *
7 * Copyright (c) 2012 Eukréa Electromatique - ATMEL
8 * Author: Nicolas Royer <nicolas@eukrea.com>
9 *
13802005
NR		 10 * Some ideas are from omap-aes.c drivers.
11 */
12
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/slab.h>
17#include <linux/err.h>
18#include <linux/clk.h>
19#include <linux/io.h>
20#include <linux/hw_random.h>
21#include <linux/platform_device.h>
22
23#include <linux/device.h>
13802005
NR		 24#include <linux/init.h>
25#include <linux/errno.h>
26#include <linux/interrupt.h>
13802005 27#include <linux/irq.h>
13802005
NR		 28#include <linux/scatterlist.h>
29#include <linux/dma-mapping.h>
84c8976b 30#include <linux/of_device.h>
13802005
NR		 31#include <linux/delay.h>
32#include <linux/crypto.h>
33#include <linux/cryptohash.h>
34#include <crypto/scatterwalk.h>
35#include <crypto/algapi.h>
92c203e2 36#include <crypto/internal/des.h>
13802005
NR		 37#include <crypto/hash.h>
38#include <crypto/internal/hash.h>
967d4910 39#include <crypto/internal/skcipher.h>
1f858040 40#include <linux/platform_data/crypto-atmel.h>
13802005
NR		 41#include "atmel-tdes-regs.h"
42
43/* TDES flags */
1f858040 44#define TDES_FLAGS_MODE_MASK 0x00ff
13802005
NR		 45#define TDES_FLAGS_ENCRYPT BIT(0)
46#define TDES_FLAGS_CBC BIT(1)
47#define TDES_FLAGS_CFB BIT(2)
48#define TDES_FLAGS_CFB8 BIT(3)
49#define TDES_FLAGS_CFB16 BIT(4)
50#define TDES_FLAGS_CFB32 BIT(5)
1f858040
NR		 51#define TDES_FLAGS_CFB64 BIT(6)
52#define TDES_FLAGS_OFB BIT(7)
13802005
NR		 53
54#define TDES_FLAGS_INIT BIT(16)
55#define TDES_FLAGS_FAST BIT(17)
56#define TDES_FLAGS_BUSY BIT(18)
1f858040 57#define TDES_FLAGS_DMA BIT(19)
13802005 58
1f858040 59#define ATMEL_TDES_QUEUE_LENGTH 50
13802005
NR		 60
61#define CFB8_BLOCK_SIZE 1
62#define CFB16_BLOCK_SIZE 2
63#define CFB32_BLOCK_SIZE 4
13802005 64
1f858040
NR		 65struct atmel_tdes_caps {
66 bool has_dma;
67 u32 has_cfb_3keys;
68};
13802005
NR		 69
70struct atmel_tdes_dev;
71
72struct atmel_tdes_ctx {
73 struct atmel_tdes_dev *dd;
74
75 int keylen;
967d4910 76 u32 key[DES3_EDE_KEY_SIZE / sizeof(u32)];
13802005 77 unsigned long flags;
1f858040
NR		 78
79 u16 block_size;
13802005
NR		 80};
81
82struct atmel_tdes_reqctx {
83 unsigned long mode;
61b0dd66 84 u8 lastc[DES_BLOCK_SIZE];
13802005
NR		 85};
86
1f858040
NR		 87struct atmel_tdes_dma {
88 struct dma_chan *chan;
89 struct dma_slave_config dma_conf;
90};
91
13802005
NR		 92struct atmel_tdes_dev {
93 struct list_head list;
94 unsigned long phys_base;
95 void __iomem *io_base;
96
97 struct atmel_tdes_ctx *ctx;
98 struct device *dev;
99 struct clk *iclk;
100 int irq;
101
102 unsigned long flags;
103 int err;
104
105 spinlock_t lock;
106 struct crypto_queue queue;
107
108 struct tasklet_struct done_task;
109 struct tasklet_struct queue_task;
110
967d4910 111 struct skcipher_request *req;
13802005
NR		 112 size_t total;
113
114 struct scatterlist *in_sg;
1f858040 115 unsigned int nb_in_sg;
13802005
NR		 116 size_t in_offset;
117 struct scatterlist *out_sg;
1f858040 118 unsigned int nb_out_sg;
13802005
NR		 119 size_t out_offset;
120
121 size_t buflen;
122 size_t dma_size;
123
124 void *buf_in;
125 int dma_in;
126 dma_addr_t dma_addr_in;
1f858040 127 struct atmel_tdes_dma dma_lch_in;
13802005
NR		 128
129 void *buf_out;
130 int dma_out;
131 dma_addr_t dma_addr_out;
1f858040
NR		 132 struct atmel_tdes_dma dma_lch_out;
133
134 struct atmel_tdes_caps caps;
135
136 u32 hw_version;
13802005
NR		 137};
138
139struct atmel_tdes_drv {
140 struct list_head dev_list;
141 spinlock_t lock;
142};
143
144static struct atmel_tdes_drv atmel_tdes = {
145 .dev_list = LIST_HEAD_INIT(atmel_tdes.dev_list),
146 .lock = __SPIN_LOCK_UNLOCKED(atmel_tdes.lock),
147};
148
149static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset,
150 void *buf, size_t buflen, size_t total, int out)
151{
4c147bcf 152 size_t count, off = 0;
13802005
NR		 153
154 while (buflen && total) {
155 count = min((*sg)->length - *offset, total);
156 count = min(count, buflen);
157
158 if (!count)
159 return off;
160
161 scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out);
162
163 off += count;
164 buflen -= count;
165 *offset += count;
166 total -= count;
167
168 if (*offset == (*sg)->length) {
169 *sg = sg_next(*sg);
170 if (*sg)
171 *offset = 0;
172 else
173 total = 0;
174 }
175 }
176
177 return off;
178}
179
180static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset)
181{
182 return readl_relaxed(dd->io_base + offset);
183}
184
185static inline void atmel_tdes_write(struct atmel_tdes_dev *dd,
186 u32 offset, u32 value)
187{
188 writel_relaxed(value, dd->io_base + offset);
189}
190
191static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset,
192 u32 *value, int count)
193{
194 for (; count--; value++, offset += 4)
195 atmel_tdes_write(dd, offset, *value);
196}
197
198static struct atmel_tdes_dev *atmel_tdes_find_dev(struct atmel_tdes_ctx *ctx)
199{
200 struct atmel_tdes_dev *tdes_dd = NULL;
201 struct atmel_tdes_dev *tmp;
202
203 spin_lock_bh(&atmel_tdes.lock);
204 if (!ctx->dd) {
205 list_for_each_entry(tmp, &atmel_tdes.dev_list, list) {
206 tdes_dd = tmp;
207 break;
208 }
209 ctx->dd = tdes_dd;
210 } else {
211 tdes_dd = ctx->dd;
212 }
213 spin_unlock_bh(&atmel_tdes.lock);
214
215 return tdes_dd;
216}
217
218static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd)
219{
9d83d299
LC		 220 int err;
221
222 err = clk_prepare_enable(dd->iclk);
223 if (err)
224 return err;
13802005
NR		 225
226 if (!(dd->flags & TDES_FLAGS_INIT)) {
227 atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST);
228 dd->flags |= TDES_FLAGS_INIT;
229 dd->err = 0;
230 }
231
232 return 0;
233}
234
1f858040
NR		 235static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd)
236{
237 return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff;
238}
239
240static void atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd)
241{
242 atmel_tdes_hw_init(dd);
243
244 dd->hw_version = atmel_tdes_get_version(dd);
245
246 dev_info(dd->dev,
247 "version: 0x%x\n", dd->hw_version);
248
249 clk_disable_unprepare(dd->iclk);
250}
251
252static void atmel_tdes_dma_callback(void *data)
253{
254 struct atmel_tdes_dev *dd = data;
255
256 /* dma_lch_out - completed */
257 tasklet_schedule(&dd->done_task);
258}
259
13802005
NR		 260static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd)
261{
262 int err;
263 u32 valcr = 0, valmr = TDES_MR_SMOD_PDC;
264
265 err = atmel_tdes_hw_init(dd);
266
267 if (err)
268 return err;
269
1f858040
NR		 270 if (!dd->caps.has_dma)
271 atmel_tdes_write(dd, TDES_PTCR,
272 TDES_PTCR_TXTDIS | TDES_PTCR_RXTDIS);
13802005
NR		 273
274 /* MR register must be set before IV registers */
275 if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) {
276 valmr |= TDES_MR_KEYMOD_3KEY;
277 valmr |= TDES_MR_TDESMOD_TDES;
278 } else if (dd->ctx->keylen > DES_KEY_SIZE) {
279 valmr |= TDES_MR_KEYMOD_2KEY;
280 valmr |= TDES_MR_TDESMOD_TDES;
281 } else {
282 valmr |= TDES_MR_TDESMOD_DES;
283 }
284
285 if (dd->flags & TDES_FLAGS_CBC) {
286 valmr |= TDES_MR_OPMOD_CBC;
287 } else if (dd->flags & TDES_FLAGS_CFB) {
288 valmr |= TDES_MR_OPMOD_CFB;
289
290 if (dd->flags & TDES_FLAGS_CFB8)
291 valmr |= TDES_MR_CFBS_8b;
292 else if (dd->flags & TDES_FLAGS_CFB16)
293 valmr |= TDES_MR_CFBS_16b;
294 else if (dd->flags & TDES_FLAGS_CFB32)
295 valmr |= TDES_MR_CFBS_32b;
1f858040
NR		 296 else if (dd->flags & TDES_FLAGS_CFB64)
297 valmr |= TDES_MR_CFBS_64b;
13802005
NR		 298 } else if (dd->flags & TDES_FLAGS_OFB) {
299 valmr |= TDES_MR_OPMOD_OFB;
300 }
301
302 if ((dd->flags & TDES_FLAGS_ENCRYPT) || (dd->flags & TDES_FLAGS_OFB))
303 valmr |= TDES_MR_CYPHER_ENC;
304
305 atmel_tdes_write(dd, TDES_CR, valcr);
306 atmel_tdes_write(dd, TDES_MR, valmr);
307
308 atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key,
309 dd->ctx->keylen >> 2);
310
311 if (((dd->flags & TDES_FLAGS_CBC) || (dd->flags & TDES_FLAGS_CFB) ||
967d4910
AB		 312 (dd->flags & TDES_FLAGS_OFB)) && dd->req->iv) {
313 atmel_tdes_write_n(dd, TDES_IV1R, (void *)dd->req->iv, 2);
13802005
NR		 314 }
315
316 return 0;
317}
318
1f858040 319static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)
13802005
NR		 320{
321 int err = 0;
322 size_t count;
323
324 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);
325
326 if (dd->flags & TDES_FLAGS_FAST) {
327 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
328 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
329 } else {
330 dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
331 dd->dma_size, DMA_FROM_DEVICE);
332
333 /* copy data */
334 count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
335 dd->buf_out, dd->buflen, dd->dma_size, 1);
336 if (count != dd->dma_size) {
337 err = -EINVAL;
4c147bcf 338 pr_err("not all data converted: %zu\n", count);
13802005
NR		 339 }
340 }
341
342 return err;
343}
344
1f858040 345static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)
13802005
NR		 346{
347 int err = -ENOMEM;
348
349 dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0);
350 dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0);
351 dd->buflen = PAGE_SIZE;
352 dd->buflen &= ~(DES_BLOCK_SIZE - 1);
353
354 if (!dd->buf_in || !dd->buf_out) {
355 dev_err(dd->dev, "unable to alloc pages.\n");
356 goto err_alloc;
357 }
358
359 /* MAP here */
360 dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in,
361 dd->buflen, DMA_TO_DEVICE);
362 if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
4c147bcf 363 dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen);
13802005
NR		 364 err = -EINVAL;
365 goto err_map_in;
366 }
367
368 dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out,
369 dd->buflen, DMA_FROM_DEVICE);
370 if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
4c147bcf 371 dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen);
13802005
NR		 372 err = -EINVAL;
373 goto err_map_out;
374 }
375
376 return 0;
377
378err_map_out:
379 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
380 DMA_TO_DEVICE);
381err_map_in:
088f628c 382err_alloc:
13802005
NR		 383 free_page((unsigned long)dd->buf_out);
384 free_page((unsigned long)dd->buf_in);
13802005
NR		 385 if (err)
386 pr_err("error: %d\n", err);
387 return err;
388}
389
1f858040 390static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd)
13802005
NR		 391{
392 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
393 DMA_FROM_DEVICE);
394 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
395 DMA_TO_DEVICE);
396 free_page((unsigned long)dd->buf_out);
397 free_page((unsigned long)dd->buf_in);
398}
399
1f858040 400static int atmel_tdes_crypt_pdc(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
13802005
NR		 401 dma_addr_t dma_addr_out, int length)
402{
403 struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm);
404 struct atmel_tdes_dev *dd = ctx->dd;
405 int len32;
406
407 dd->dma_size = length;
408
409 if (!(dd->flags & TDES_FLAGS_FAST)) {
410 dma_sync_single_for_device(dd->dev, dma_addr_in, length,
411 DMA_TO_DEVICE);
412 }
413
414 if ((dd->flags & TDES_FLAGS_CFB) && (dd->flags & TDES_FLAGS_CFB8))
415 len32 = DIV_ROUND_UP(length, sizeof(u8));
416 else if ((dd->flags & TDES_FLAGS_CFB) && (dd->flags & TDES_FLAGS_CFB16))
417 len32 = DIV_ROUND_UP(length, sizeof(u16));
418 else
419 len32 = DIV_ROUND_UP(length, sizeof(u32));
420
421 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);
422 atmel_tdes_write(dd, TDES_TPR, dma_addr_in);
423 atmel_tdes_write(dd, TDES_TCR, len32);
424 atmel_tdes_write(dd, TDES_RPR, dma_addr_out);
425 atmel_tdes_write(dd, TDES_RCR, len32);
426
427 /* Enable Interrupt */
428 atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX);
429
430 /* Start DMA transfer */
431 atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN);
432
433 return 0;
434}
435
1f858040
NR		 436static int atmel_tdes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
437 dma_addr_t dma_addr_out, int length)
438{
439 struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm);
440 struct atmel_tdes_dev *dd = ctx->dd;
441 struct scatterlist sg[2];
442 struct dma_async_tx_descriptor *in_desc, *out_desc;
443
444 dd->dma_size = length;
445
446 if (!(dd->flags & TDES_FLAGS_FAST)) {
447 dma_sync_single_for_device(dd->dev, dma_addr_in, length,
448 DMA_TO_DEVICE);
449 }
450
451 if (dd->flags & TDES_FLAGS_CFB8) {
452 dd->dma_lch_in.dma_conf.dst_addr_width =
453 DMA_SLAVE_BUSWIDTH_1_BYTE;
454 dd->dma_lch_out.dma_conf.src_addr_width =
455 DMA_SLAVE_BUSWIDTH_1_BYTE;
456 } else if (dd->flags & TDES_FLAGS_CFB16) {
457 dd->dma_lch_in.dma_conf.dst_addr_width =
458 DMA_SLAVE_BUSWIDTH_2_BYTES;
459 dd->dma_lch_out.dma_conf.src_addr_width =
460 DMA_SLAVE_BUSWIDTH_2_BYTES;
461 } else {
462 dd->dma_lch_in.dma_conf.dst_addr_width =
463 DMA_SLAVE_BUSWIDTH_4_BYTES;
464 dd->dma_lch_out.dma_conf.src_addr_width =
465 DMA_SLAVE_BUSWIDTH_4_BYTES;
466 }
467
468 dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
469 dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf);
470
471 dd->flags |= TDES_FLAGS_DMA;
472
473 sg_init_table(&sg[0], 1);
474 sg_dma_address(&sg[0]) = dma_addr_in;
475 sg_dma_len(&sg[0]) = length;
476
477 sg_init_table(&sg[1], 1);
478 sg_dma_address(&sg[1]) = dma_addr_out;
479 sg_dma_len(&sg[1]) = length;
480
481 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0],
482 1, DMA_MEM_TO_DEV,
483 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
484 if (!in_desc)
485 return -EINVAL;
486
487 out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1],
488 1, DMA_DEV_TO_MEM,
489 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
490 if (!out_desc)
491 return -EINVAL;
492
493 out_desc->callback = atmel_tdes_dma_callback;
494 out_desc->callback_param = dd;
495
496 dmaengine_submit(out_desc);
497 dma_async_issue_pending(dd->dma_lch_out.chan);
498
499 dmaengine_submit(in_desc);
500 dma_async_issue_pending(dd->dma_lch_in.chan);
501
502 return 0;
503}
504
505static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd)
13802005 506{
967d4910
AB		 507 struct crypto_tfm *tfm = crypto_skcipher_tfm(
508 crypto_skcipher_reqtfm(dd->req));
13802005
NR		 509 int err, fast = 0, in, out;
510 size_t count;
511 dma_addr_t addr_in, addr_out;
512
1f858040 513 if ((!dd->in_offset) && (!dd->out_offset)) {
13802005 514 /* check for alignment */
1f858040
NR		 515 in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) &&
516 IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size);
517 out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) &&
518 IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size);
13802005 519 fast = in && out;
1f858040
NR		 520
521 if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg))
522 fast = 0;
13802005
NR		 523 }
524
1f858040 525
13802005 526 if (fast) {
4c147bcf
AB		 527 count = min_t(size_t, dd->total, sg_dma_len(dd->in_sg));
528 count = min_t(size_t, count, sg_dma_len(dd->out_sg));
13802005 529
13802005
NR		 530 err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
531 if (!err) {
532 dev_err(dd->dev, "dma_map_sg() error\n");
533 return -EINVAL;
534 }
535
536 err = dma_map_sg(dd->dev, dd->out_sg, 1,
537 DMA_FROM_DEVICE);
538 if (!err) {
539 dev_err(dd->dev, "dma_map_sg() error\n");
540 dma_unmap_sg(dd->dev, dd->in_sg, 1,
541 DMA_TO_DEVICE);
542 return -EINVAL;
543 }
544
545 addr_in = sg_dma_address(dd->in_sg);
546 addr_out = sg_dma_address(dd->out_sg);
547
548 dd->flags |= TDES_FLAGS_FAST;
549
550 } else {
551 /* use cache buffers */
552 count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset,
553 dd->buf_in, dd->buflen, dd->total, 0);
554
555 addr_in = dd->dma_addr_in;
556 addr_out = dd->dma_addr_out;
557
558 dd->flags &= ~TDES_FLAGS_FAST;
13802005
NR		 559 }
560
561 dd->total -= count;
562
1f858040
NR		 563 if (dd->caps.has_dma)
564 err = atmel_tdes_crypt_dma(tfm, addr_in, addr_out, count);
565 else
566 err = atmel_tdes_crypt_pdc(tfm, addr_in, addr_out, count);
567
568 if (err && (dd->flags & TDES_FLAGS_FAST)) {
13802005
NR		 569 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
570 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
571 }
572
573 return err;
574}
575
61b0dd66
TA		 576static void
577atmel_tdes_set_iv_as_last_ciphertext_block(struct atmel_tdes_dev *dd)
578{
579 struct skcipher_request *req = dd->req;
580 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req);
581 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
582 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
583
584 if (req->cryptlen < ivsize)
585 return;
586
587 if (rctx->mode & TDES_FLAGS_ENCRYPT) {
588 scatterwalk_map_and_copy(req->iv, req->dst,
589 req->cryptlen - ivsize, ivsize, 0);
590 } else {
591 if (req->src == req->dst)
592 memcpy(req->iv, rctx->lastc, ivsize);
593 else
594 scatterwalk_map_and_copy(req->iv, req->src,
595 req->cryptlen - ivsize,
596 ivsize, 0);
597 }
598}
599
13802005
NR		 600static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err)
601{
967d4910 602 struct skcipher_request *req = dd->req;
13802005
NR		 603
604 clk_disable_unprepare(dd->iclk);
605
606 dd->flags &= ~TDES_FLAGS_BUSY;
607
61b0dd66
TA		 608 atmel_tdes_set_iv_as_last_ciphertext_block(dd);
609
13802005
NR		 610 req->base.complete(&req->base, err);
611}
612
613static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd,
967d4910 614 struct skcipher_request *req)
13802005
NR		 615{
616 struct crypto_async_request *async_req, *backlog;
617 struct atmel_tdes_ctx *ctx;
618 struct atmel_tdes_reqctx *rctx;
619 unsigned long flags;
620 int err, ret = 0;
621
622 spin_lock_irqsave(&dd->lock, flags);
623 if (req)
967d4910 624 ret = crypto_enqueue_request(&dd->queue, &req->base);
13802005
NR		 625 if (dd->flags & TDES_FLAGS_BUSY) {
626 spin_unlock_irqrestore(&dd->lock, flags);
627 return ret;
628 }
629 backlog = crypto_get_backlog(&dd->queue);
630 async_req = crypto_dequeue_request(&dd->queue);
631 if (async_req)
632 dd->flags |= TDES_FLAGS_BUSY;
633 spin_unlock_irqrestore(&dd->lock, flags);
634
635 if (!async_req)
636 return ret;
637
638 if (backlog)
639 backlog->complete(backlog, -EINPROGRESS);
640
967d4910 641 req = skcipher_request_cast(async_req);
13802005
NR		 642
643 /* assign new request to device */
644 dd->req = req;
967d4910 645 dd->total = req->cryptlen;
13802005
NR		 646 dd->in_offset = 0;
647 dd->in_sg = req->src;
648 dd->out_offset = 0;
649 dd->out_sg = req->dst;
650
967d4910
AB		 651 rctx = skcipher_request_ctx(req);
652 ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
13802005
NR		 653 rctx->mode &= TDES_FLAGS_MODE_MASK;
654 dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode;
655 dd->ctx = ctx;
656 ctx->dd = dd;
657
658 err = atmel_tdes_write_ctrl(dd);
659 if (!err)
1f858040 660 err = atmel_tdes_crypt_start(dd);
13802005
NR		 661 if (err) {
662 /* des_task will not finish it, so do it here */
663 atmel_tdes_finish_req(dd, err);
664 tasklet_schedule(&dd->queue_task);
665 }
666
667 return ret;
668}
669
1f858040
NR		 670static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd)
671{
672 int err = -EINVAL;
673 size_t count;
674
675 if (dd->flags & TDES_FLAGS_DMA) {
676 err = 0;
677 if (dd->flags & TDES_FLAGS_FAST) {
678 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
679 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
680 } else {
681 dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
682 dd->dma_size, DMA_FROM_DEVICE);
683
684 /* copy data */
685 count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
686 dd->buf_out, dd->buflen, dd->dma_size, 1);
687 if (count != dd->dma_size) {
688 err = -EINVAL;
4c147bcf 689 pr_err("not all data converted: %zu\n", count);
1f858040
NR		 690 }
691 }
692 }
693 return err;
694}
13802005 695
967d4910 696static int atmel_tdes_crypt(struct skcipher_request *req, unsigned long mode)
13802005 697{
61b0dd66
TA		 698 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
699 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(skcipher);
967d4910 700 struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(req);
13802005
NR		 701
702 if (mode & TDES_FLAGS_CFB8) {
967d4910 703 if (!IS_ALIGNED(req->cryptlen, CFB8_BLOCK_SIZE)) {
13802005
NR		 704 pr_err("request size is not exact amount of CFB8 blocks\n");
705 return -EINVAL;
706 }
1f858040 707 ctx->block_size = CFB8_BLOCK_SIZE;
13802005 708 } else if (mode & TDES_FLAGS_CFB16) {
967d4910 709 if (!IS_ALIGNED(req->cryptlen, CFB16_BLOCK_SIZE)) {
13802005
NR		 710 pr_err("request size is not exact amount of CFB16 blocks\n");
711 return -EINVAL;
712 }
1f858040 713 ctx->block_size = CFB16_BLOCK_SIZE;
13802005 714 } else if (mode & TDES_FLAGS_CFB32) {
967d4910 715 if (!IS_ALIGNED(req->cryptlen, CFB32_BLOCK_SIZE)) {
13802005
NR		 716 pr_err("request size is not exact amount of CFB32 blocks\n");
717 return -EINVAL;
718 }
1f858040
NR		 719 ctx->block_size = CFB32_BLOCK_SIZE;
720 } else {
967d4910 721 if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) {
1f858040
NR		 722 pr_err("request size is not exact amount of DES blocks\n");
723 return -EINVAL;
724 }
725 ctx->block_size = DES_BLOCK_SIZE;
13802005
NR		 726 }
727
1f858040
NR		 728 rctx->mode = mode;
729
61b0dd66
TA		 730 if (!(mode & TDES_FLAGS_ENCRYPT) && req->src == req->dst) {
731 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
732
733 if (req->cryptlen >= ivsize)
734 scatterwalk_map_and_copy(rctx->lastc, req->src,
735 req->cryptlen - ivsize,
736 ivsize, 0);
737 }
738
1f858040
NR		 739 return atmel_tdes_handle_queue(ctx->dd, req);
740}
741
742static bool atmel_tdes_filter(struct dma_chan *chan, void *slave)
743{
744 struct at_dma_slave *sl = slave;
745
746 if (sl && sl->dma_dev == chan->device->dev) {
747 chan->private = sl;
748 return true;
749 } else {
750 return false;
751 }
752}
753
754static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd,
755 struct crypto_platform_data *pdata)
756{
84c8976b
NF		 757 dma_cap_mask_t mask;
758
759 dma_cap_zero(mask);
760 dma_cap_set(DMA_SLAVE, mask);
761
762 /* Try to grab 2 DMA channels */
763 dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask,
764 atmel_tdes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx");
765 if (!dd->dma_lch_in.chan)
766 goto err_dma_in;
767
768 dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV;
769 dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +
770 TDES_IDATA1R;
771 dd->dma_lch_in.dma_conf.src_maxburst = 1;
772 dd->dma_lch_in.dma_conf.src_addr_width =
773 DMA_SLAVE_BUSWIDTH_4_BYTES;
774 dd->dma_lch_in.dma_conf.dst_maxburst = 1;
775 dd->dma_lch_in.dma_conf.dst_addr_width =
776 DMA_SLAVE_BUSWIDTH_4_BYTES;
777 dd->dma_lch_in.dma_conf.device_fc = false;
778
779 dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask,
780 atmel_tdes_filter, &pdata->dma_slave->txdata, dd->dev, "rx");
781 if (!dd->dma_lch_out.chan)
782 goto err_dma_out;
783
784 dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM;
785 dd->dma_lch_out.dma_conf.src_addr = dd->phys_base +
786 TDES_ODATA1R;
787 dd->dma_lch_out.dma_conf.src_maxburst = 1;
788 dd->dma_lch_out.dma_conf.src_addr_width =
789 DMA_SLAVE_BUSWIDTH_4_BYTES;
790 dd->dma_lch_out.dma_conf.dst_maxburst = 1;
791 dd->dma_lch_out.dma_conf.dst_addr_width =
792 DMA_SLAVE_BUSWIDTH_4_BYTES;
793 dd->dma_lch_out.dma_conf.device_fc = false;
1f858040 794
84c8976b 795 return 0;
13802005 796
1f858040
NR		 797err_dma_out:
798 dma_release_channel(dd->dma_lch_in.chan);
799err_dma_in:
84c8976b 800 dev_warn(dd->dev, "no DMA channel available\n");
3c88761e 801 return -ENODEV;
1f858040 802}
13802005 803
1f858040
NR		 804static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd)
805{
806 dma_release_channel(dd->dma_lch_in.chan);
807 dma_release_channel(dd->dma_lch_out.chan);
13802005
NR		 808}
809
967d4910 810static int atmel_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
13802005
NR		 811 unsigned int keylen)
812{
967d4910 813 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);
92c203e2 814 int err;
13802005 815
967d4910 816 err = verify_skcipher_des_key(tfm, key);
92c203e2
AB		 817 if (err)
818 return err;
13802005
NR		 819
820 memcpy(ctx->key, key, keylen);
821 ctx->keylen = keylen;
822
823 return 0;
824}
825
967d4910 826static int atmel_tdes_setkey(struct crypto_skcipher *tfm, const u8 *key,
13802005
NR		 827 unsigned int keylen)
828{
967d4910 829 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);
52ea3cd2 830 int err;
13802005 831
967d4910 832 err = verify_skcipher_des3_key(tfm, key);
92c203e2 833 if (err)
52ea3cd2 834 return err;
13802005
NR		 835
836 memcpy(ctx->key, key, keylen);
837 ctx->keylen = keylen;
838
839 return 0;
840}
841
967d4910 842static int atmel_tdes_ecb_encrypt(struct skcipher_request *req)
13802005
NR		 843{
844 return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT);
845}
846
967d4910 847static int atmel_tdes_ecb_decrypt(struct skcipher_request *req)
13802005
NR		 848{
849 return atmel_tdes_crypt(req, 0);
850}
851
967d4910 852static int atmel_tdes_cbc_encrypt(struct skcipher_request *req)
13802005
NR		 853{
854 return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CBC);
855}
856
967d4910 857static int atmel_tdes_cbc_decrypt(struct skcipher_request *req)
13802005
NR		 858{
859 return atmel_tdes_crypt(req, TDES_FLAGS_CBC);
860}
967d4910 861static int atmel_tdes_cfb_encrypt(struct skcipher_request *req)
13802005
NR		 862{
863 return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB);
864}
865
967d4910 866static int atmel_tdes_cfb_decrypt(struct skcipher_request *req)
13802005
NR		 867{
868 return atmel_tdes_crypt(req, TDES_FLAGS_CFB);
869}
870
967d4910 871static int atmel_tdes_cfb8_encrypt(struct skcipher_request *req)
13802005
NR		 872{
873 return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
874 TDES_FLAGS_CFB8);
875}
876
967d4910 877static int atmel_tdes_cfb8_decrypt(struct skcipher_request *req)
13802005
NR		 878{
879 return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB8);
880}
881
967d4910 882static int atmel_tdes_cfb16_encrypt(struct skcipher_request *req)
13802005
NR		 883{
884 return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
885 TDES_FLAGS_CFB16);
886}
887
967d4910 888static int atmel_tdes_cfb16_decrypt(struct skcipher_request *req)
13802005
NR		 889{
890 return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB16);
891}
892
967d4910 893static int atmel_tdes_cfb32_encrypt(struct skcipher_request *req)
13802005
NR		 894{
895 return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
896 TDES_FLAGS_CFB32);
897}
898
967d4910 899static int atmel_tdes_cfb32_decrypt(struct skcipher_request *req)
13802005
NR		 900{
901 return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB32);
902}
903
967d4910 904static int atmel_tdes_ofb_encrypt(struct skcipher_request *req)
13802005
NR		 905{
906 return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_OFB);
907}
908
967d4910 909static int atmel_tdes_ofb_decrypt(struct skcipher_request *req)
13802005
NR		 910{
911 return atmel_tdes_crypt(req, TDES_FLAGS_OFB);
912}
913
967d4910 914static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm)
13802005 915{
967d4910 916 struct atmel_tdes_ctx *ctx = crypto_skcipher_ctx(tfm);
1f858040
NR		 917 struct atmel_tdes_dev *dd;
918
967d4910 919 crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_tdes_reqctx));
13802005 920
1f858040
NR		 921 dd = atmel_tdes_find_dev(ctx);
922 if (!dd)
923 return -ENODEV;
924
13802005
NR		 925 return 0;
926}
927
967d4910 928static struct skcipher_alg tdes_algs[] = {
13802005 929{
967d4910
AB		 930 .base.cra_name = "ecb(des)",
931 .base.cra_driver_name = "atmel-ecb-des",
932 .base.cra_priority = 100,
933 .base.cra_flags = CRYPTO_ALG_ASYNC,
934 .base.cra_blocksize = DES_BLOCK_SIZE,
935 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
936 .base.cra_alignmask = 0x7,
937 .base.cra_module = THIS_MODULE,
938
939 .init = atmel_tdes_init_tfm,
940 .min_keysize = DES_KEY_SIZE,
941 .max_keysize = DES_KEY_SIZE,
942 .setkey = atmel_des_setkey,
943 .encrypt = atmel_tdes_ecb_encrypt,
944 .decrypt = atmel_tdes_ecb_decrypt,
13802005
NR		 945},
946{
967d4910
AB		 947 .base.cra_name = "cbc(des)",
948 .base.cra_driver_name = "atmel-cbc-des",
949 .base.cra_priority = 100,
950 .base.cra_flags = CRYPTO_ALG_ASYNC,
951 .base.cra_blocksize = DES_BLOCK_SIZE,
952 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
953 .base.cra_alignmask = 0x7,
954 .base.cra_module = THIS_MODULE,
955
956 .init = atmel_tdes_init_tfm,
957 .min_keysize = DES_KEY_SIZE,
958 .max_keysize = DES_KEY_SIZE,
959 .ivsize = DES_BLOCK_SIZE,
960 .setkey = atmel_des_setkey,
961 .encrypt = atmel_tdes_cbc_encrypt,
962 .decrypt = atmel_tdes_cbc_decrypt,
13802005
NR		 963},
964{
967d4910
AB		 965 .base.cra_name = "cfb(des)",
966 .base.cra_driver_name = "atmel-cfb-des",
967 .base.cra_priority = 100,
968 .base.cra_flags = CRYPTO_ALG_ASYNC,
969 .base.cra_blocksize = DES_BLOCK_SIZE,
970 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
971 .base.cra_alignmask = 0x7,
972 .base.cra_module = THIS_MODULE,
973
974 .init = atmel_tdes_init_tfm,
975 .min_keysize = DES_KEY_SIZE,
976 .max_keysize = DES_KEY_SIZE,
977 .ivsize = DES_BLOCK_SIZE,
978 .setkey = atmel_des_setkey,
979 .encrypt = atmel_tdes_cfb_encrypt,
980 .decrypt = atmel_tdes_cfb_decrypt,
13802005
NR		 981},
982{
967d4910
AB		 983 .base.cra_name = "cfb8(des)",
984 .base.cra_driver_name = "atmel-cfb8-des",
985 .base.cra_priority = 100,
986 .base.cra_flags = CRYPTO_ALG_ASYNC,
987 .base.cra_blocksize = CFB8_BLOCK_SIZE,
988 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
989 .base.cra_alignmask = 0,
990 .base.cra_module = THIS_MODULE,
991
992 .init = atmel_tdes_init_tfm,
993 .min_keysize = DES_KEY_SIZE,
994 .max_keysize = DES_KEY_SIZE,
995 .ivsize = DES_BLOCK_SIZE,
996 .setkey = atmel_des_setkey,
997 .encrypt = atmel_tdes_cfb8_encrypt,
998 .decrypt = atmel_tdes_cfb8_decrypt,
13802005
NR		 999},
1000{
967d4910
AB		 1001 .base.cra_name = "cfb16(des)",
1002 .base.cra_driver_name = "atmel-cfb16-des",
1003 .base.cra_priority = 100,
1004 .base.cra_flags = CRYPTO_ALG_ASYNC,
1005 .base.cra_blocksize = CFB16_BLOCK_SIZE,
1006 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
1007 .base.cra_alignmask = 0x1,
1008 .base.cra_module = THIS_MODULE,
1009
1010 .init = atmel_tdes_init_tfm,
1011 .min_keysize = DES_KEY_SIZE,
1012 .max_keysize = DES_KEY_SIZE,
1013 .ivsize = DES_BLOCK_SIZE,
1014 .setkey = atmel_des_setkey,
1015 .encrypt = atmel_tdes_cfb16_encrypt,
1016 .decrypt = atmel_tdes_cfb16_decrypt,
13802005
NR		 1017},
1018{
967d4910
AB		 1019 .base.cra_name = "cfb32(des)",
1020 .base.cra_driver_name = "atmel-cfb32-des",
1021 .base.cra_priority = 100,
1022 .base.cra_flags = CRYPTO_ALG_ASYNC,
1023 .base.cra_blocksize = CFB32_BLOCK_SIZE,
1024 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
1025 .base.cra_alignmask = 0x3,
1026 .base.cra_module = THIS_MODULE,
1027
1028 .init = atmel_tdes_init_tfm,
1029 .min_keysize = DES_KEY_SIZE,
1030 .max_keysize = DES_KEY_SIZE,
1031 .ivsize = DES_BLOCK_SIZE,
1032 .setkey = atmel_des_setkey,
1033 .encrypt = atmel_tdes_cfb32_encrypt,
1034 .decrypt = atmel_tdes_cfb32_decrypt,
13802005
NR		 1035},
1036{
967d4910
AB		 1037 .base.cra_name = "ofb(des)",
1038 .base.cra_driver_name = "atmel-ofb-des",
1039 .base.cra_priority = 100,
1040 .base.cra_flags = CRYPTO_ALG_ASYNC,
1041 .base.cra_blocksize = DES_BLOCK_SIZE,
1042 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
1043 .base.cra_alignmask = 0x7,
1044 .base.cra_module = THIS_MODULE,
1045
1046 .init = atmel_tdes_init_tfm,
1047 .min_keysize = DES_KEY_SIZE,
1048 .max_keysize = DES_KEY_SIZE,
1049 .ivsize = DES_BLOCK_SIZE,
1050 .setkey = atmel_des_setkey,
1051 .encrypt = atmel_tdes_ofb_encrypt,
1052 .decrypt = atmel_tdes_ofb_decrypt,
13802005
NR		 1053},
1054{
967d4910
AB		 1055 .base.cra_name = "ecb(des3_ede)",
1056 .base.cra_driver_name = "atmel-ecb-tdes",
1057 .base.cra_priority = 100,
1058 .base.cra_flags = CRYPTO_ALG_ASYNC,
1059 .base.cra_blocksize = DES_BLOCK_SIZE,
1060 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
1061 .base.cra_alignmask = 0x7,
1062 .base.cra_module = THIS_MODULE,
1063
1064 .init = atmel_tdes_init_tfm,
1065 .min_keysize = DES3_EDE_KEY_SIZE,
1066 .max_keysize = DES3_EDE_KEY_SIZE,
1067 .setkey = atmel_tdes_setkey,
1068 .encrypt = atmel_tdes_ecb_encrypt,
1069 .decrypt = atmel_tdes_ecb_decrypt,
13802005
NR		 1070},
1071{
967d4910
AB		 1072 .base.cra_name = "cbc(des3_ede)",
1073 .base.cra_driver_name = "atmel-cbc-tdes",
1074 .base.cra_priority = 100,
1075 .base.cra_flags = CRYPTO_ALG_ASYNC,
1076 .base.cra_blocksize = DES_BLOCK_SIZE,
1077 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
1078 .base.cra_alignmask = 0x7,
1079 .base.cra_module = THIS_MODULE,
1080
1081 .init = atmel_tdes_init_tfm,
1082 .min_keysize = DES3_EDE_KEY_SIZE,
1083 .max_keysize = DES3_EDE_KEY_SIZE,
1084 .setkey = atmel_tdes_setkey,
1085 .encrypt = atmel_tdes_cbc_encrypt,
1086 .decrypt = atmel_tdes_cbc_decrypt,
1087 .ivsize = DES_BLOCK_SIZE,
13802005 1088},
13802005 1089{
967d4910
AB		 1090 .base.cra_name = "ofb(des3_ede)",
1091 .base.cra_driver_name = "atmel-ofb-tdes",
1092 .base.cra_priority = 100,
1093 .base.cra_flags = CRYPTO_ALG_ASYNC,
1094 .base.cra_blocksize = DES_BLOCK_SIZE,
1095 .base.cra_ctxsize = sizeof(struct atmel_tdes_ctx),
1096 .base.cra_alignmask = 0x7,
1097 .base.cra_module = THIS_MODULE,
1098
1099 .init = atmel_tdes_init_tfm,
1100 .min_keysize = DES3_EDE_KEY_SIZE,
1101 .max_keysize = DES3_EDE_KEY_SIZE,
1102 .setkey = atmel_tdes_setkey,
1103 .encrypt = atmel_tdes_ofb_encrypt,
1104 .decrypt = atmel_tdes_ofb_decrypt,
1105 .ivsize = DES_BLOCK_SIZE,
13802005
NR		 1106},
1107};
1108
1109static void atmel_tdes_queue_task(unsigned long data)
1110{
1111 struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data;
1112
1113 atmel_tdes_handle_queue(dd, NULL);
1114}
1115
1116static void atmel_tdes_done_task(unsigned long data)
1117{
1118 struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data;
1119 int err;
1120
1f858040
NR		 1121 if (!(dd->flags & TDES_FLAGS_DMA))
1122 err = atmel_tdes_crypt_pdc_stop(dd);
1123 else
1124 err = atmel_tdes_crypt_dma_stop(dd);
13802005
NR		 1125
1126 err = dd->err ? : err;
1127
1128 if (dd->total && !err) {
1f858040
NR		 1129 if (dd->flags & TDES_FLAGS_FAST) {
1130 dd->in_sg = sg_next(dd->in_sg);
1131 dd->out_sg = sg_next(dd->out_sg);
1132 if (!dd->in_sg || !dd->out_sg)
1133 err = -EINVAL;
1134 }
13802005 1135 if (!err)
1f858040
NR		 1136 err = atmel_tdes_crypt_start(dd);
1137 if (!err)
1138 return; /* DMA started. Not fininishing. */
13802005
NR		 1139 }
1140
1141 atmel_tdes_finish_req(dd, err);
1142 atmel_tdes_handle_queue(dd, NULL);
1143}
1144
1145static irqreturn_t atmel_tdes_irq(int irq, void *dev_id)
1146{
1147 struct atmel_tdes_dev *tdes_dd = dev_id;
1148 u32 reg;
1149
1150 reg = atmel_tdes_read(tdes_dd, TDES_ISR);
1151 if (reg & atmel_tdes_read(tdes_dd, TDES_IMR)) {
1152 atmel_tdes_write(tdes_dd, TDES_IDR, reg);
1153 if (TDES_FLAGS_BUSY & tdes_dd->flags)
1154 tasklet_schedule(&tdes_dd->done_task);
1155 else
1156 dev_warn(tdes_dd->dev, "TDES interrupt when no active requests.\n");
1157 return IRQ_HANDLED;
1158 }
1159
1160 return IRQ_NONE;
1161}
1162
1163static void atmel_tdes_unregister_algs(struct atmel_tdes_dev *dd)
1164{
1165 int i;
1166
1167 for (i = 0; i < ARRAY_SIZE(tdes_algs); i++)
967d4910 1168 crypto_unregister_skcipher(&tdes_algs[i]);
13802005
NR		 1169}
1170
1171static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd)
1172{
1173 int err, i, j;
1174
1175 for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) {
967d4910 1176 err = crypto_register_skcipher(&tdes_algs[i]);
13802005
NR		 1177 if (err)
1178 goto err_tdes_algs;
1179 }
1180
1181 return 0;
1182
1183err_tdes_algs:
1184 for (j = 0; j < i; j++)
967d4910 1185 crypto_unregister_skcipher(&tdes_algs[j]);
13802005
NR		 1186
1187 return err;
1188}
1189
1f858040
NR		 1190static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd)
1191{
1192
1193 dd->caps.has_dma = 0;
1194 dd->caps.has_cfb_3keys = 0;
1195
1196 /* keep only major version number */
1197 switch (dd->hw_version & 0xf00) {
1198 case 0x700:
1199 dd->caps.has_dma = 1;
1200 dd->caps.has_cfb_3keys = 1;
1201 break;
1202 case 0x600:
1203 break;
1204 default:
1205 dev_warn(dd->dev,
1206 "Unmanaged tdes version, set minimum capabilities\n");
1207 break;
1208 }
1209}
1210
84c8976b
NF		 1211#if defined(CONFIG_OF)
1212static const struct of_device_id atmel_tdes_dt_ids[] = {
1213 { .compatible = "atmel,at91sam9g46-tdes" },
1214 { /* sentinel */ }
1215};
1216MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids);
1217
1218static struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev)
1219{
1220 struct device_node *np = pdev->dev.of_node;
1221 struct crypto_platform_data *pdata;
1222
1223 if (!np) {
1224 dev_err(&pdev->dev, "device node not found\n");
1225 return ERR_PTR(-EINVAL);
1226 }
1227
1228 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
02684839 1229 if (!pdata)
84c8976b 1230 return ERR_PTR(-ENOMEM);
84c8976b
NF		 1231
1232 pdata->dma_slave = devm_kzalloc(&pdev->dev,
1233 sizeof(*(pdata->dma_slave)),
1234 GFP_KERNEL);
02684839 1235 if (!pdata->dma_slave)
84c8976b 1236 return ERR_PTR(-ENOMEM);
84c8976b
NF		 1237
1238 return pdata;
1239}
1240#else /* CONFIG_OF */
1241static inline struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev)
1242{
1243 return ERR_PTR(-EINVAL);
1244}
1245#endif
1246
49cfe4db 1247static int atmel_tdes_probe(struct platform_device *pdev)
13802005
NR		 1248{
1249 struct atmel_tdes_dev *tdes_dd;
1f858040 1250 struct crypto_platform_data *pdata;
13802005
NR		 1251 struct device *dev = &pdev->dev;
1252 struct resource *tdes_res;
13802005
NR		 1253 int err;
1254
c659d07f 1255 tdes_dd = devm_kmalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL);
13802005 1256 if (tdes_dd == NULL) {
13802005
NR		 1257 err = -ENOMEM;
1258 goto tdes_dd_err;
1259 }
1260
1261 tdes_dd->dev = dev;
1262
1263 platform_set_drvdata(pdev, tdes_dd);
1264
1265 INIT_LIST_HEAD(&tdes_dd->list);
1d1b9163 1266 spin_lock_init(&tdes_dd->lock);
13802005
NR		 1267
1268 tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task,
1269 (unsigned long)tdes_dd);
1270 tasklet_init(&tdes_dd->queue_task, atmel_tdes_queue_task,
1271 (unsigned long)tdes_dd);
1272
1273 crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH);
1274
13802005
NR		 1275 /* Get the base address */
1276 tdes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1277 if (!tdes_res) {
1278 dev_err(dev, "no MEM resource info\n");
1279 err = -ENODEV;
1280 goto res_err;
1281 }
1282 tdes_dd->phys_base = tdes_res->start;
13802005
NR		 1283
1284 /* Get the IRQ */
1285 tdes_dd->irq = platform_get_irq(pdev, 0);
1286 if (tdes_dd->irq < 0) {
13802005
NR		 1287 err = tdes_dd->irq;
1288 goto res_err;
1289 }
1290
b0e8b341
LC		 1291 err = devm_request_irq(&pdev->dev, tdes_dd->irq, atmel_tdes_irq,
1292 IRQF_SHARED, "atmel-tdes", tdes_dd);
13802005
NR		 1293 if (err) {
1294 dev_err(dev, "unable to request tdes irq.\n");
b0e8b341 1295 goto res_err;
13802005
NR		 1296 }
1297
1298 /* Initializing the clock */
b0e8b341 1299 tdes_dd->iclk = devm_clk_get(&pdev->dev, "tdes_clk");
13802005 1300 if (IS_ERR(tdes_dd->iclk)) {
be208356 1301 dev_err(dev, "clock initialization failed.\n");
13802005 1302 err = PTR_ERR(tdes_dd->iclk);
b0e8b341 1303 goto res_err;
13802005
NR		 1304 }
1305
b0e8b341 1306 tdes_dd->io_base = devm_ioremap_resource(&pdev->dev, tdes_res);
9b52d55f 1307 if (IS_ERR(tdes_dd->io_base)) {
13802005 1308 dev_err(dev, "can't ioremap\n");
9b52d55f 1309 err = PTR_ERR(tdes_dd->io_base);
b0e8b341 1310 goto res_err;
13802005
NR		 1311 }
1312
1f858040
NR		 1313 atmel_tdes_hw_version_init(tdes_dd);
1314
1315 atmel_tdes_get_cap(tdes_dd);
1316
1317 err = atmel_tdes_buff_init(tdes_dd);
13802005 1318 if (err)
1f858040
NR		 1319 goto err_tdes_buff;
1320
1321 if (tdes_dd->caps.has_dma) {
1322 pdata = pdev->dev.platform_data;
1323 if (!pdata) {
84c8976b
NF		 1324 pdata = atmel_tdes_of_init(pdev);
1325 if (IS_ERR(pdata)) {
1326 dev_err(&pdev->dev, "platform data not available\n");
1327 err = PTR_ERR(pdata);
1328 goto err_pdata;
1329 }
1330 }
1331 if (!pdata->dma_slave) {
1f858040
NR		 1332 err = -ENXIO;
1333 goto err_pdata;
1334 }
1335 err = atmel_tdes_dma_init(tdes_dd, pdata);
1336 if (err)
1337 goto err_tdes_dma;
84c8976b
NF		 1338
1339 dev_info(dev, "using %s, %s for DMA transfers\n",
1340 dma_chan_name(tdes_dd->dma_lch_in.chan),
1341 dma_chan_name(tdes_dd->dma_lch_out.chan));
1f858040 1342 }
13802005
NR		 1343
1344 spin_lock(&atmel_tdes.lock);
1345 list_add_tail(&tdes_dd->list, &atmel_tdes.dev_list);
1346 spin_unlock(&atmel_tdes.lock);
1347
1348 err = atmel_tdes_register_algs(tdes_dd);
1349 if (err)
1350 goto err_algs;
1351
1352 dev_info(dev, "Atmel DES/TDES\n");
1353
1354 return 0;
1355
1356err_algs:
1357 spin_lock(&atmel_tdes.lock);
1358 list_del(&tdes_dd->list);
1359 spin_unlock(&atmel_tdes.lock);
1f858040
NR		 1360 if (tdes_dd->caps.has_dma)
1361 atmel_tdes_dma_cleanup(tdes_dd);
13802005 1362err_tdes_dma:
1f858040
NR		 1363err_pdata:
1364 atmel_tdes_buff_cleanup(tdes_dd);
1365err_tdes_buff:
13802005
NR		 1366res_err:
1367 tasklet_kill(&tdes_dd->done_task);
1368 tasklet_kill(&tdes_dd->queue_task);
13802005
NR		 1369tdes_dd_err:
1370 dev_err(dev, "initialization failed.\n");
1371
1372 return err;
1373}
1374
49cfe4db 1375static int atmel_tdes_remove(struct platform_device *pdev)
13802005 1376{
47f1241e 1377 struct atmel_tdes_dev *tdes_dd;
13802005
NR		 1378
1379 tdes_dd = platform_get_drvdata(pdev);
1380 if (!tdes_dd)
1381 return -ENODEV;
1382 spin_lock(&atmel_tdes.lock);
1383 list_del(&tdes_dd->list);
1384 spin_unlock(&atmel_tdes.lock);
1385
1386 atmel_tdes_unregister_algs(tdes_dd);
1387
1388 tasklet_kill(&tdes_dd->done_task);
1389 tasklet_kill(&tdes_dd->queue_task);
1390
1f858040
NR		 1391 if (tdes_dd->caps.has_dma)
1392 atmel_tdes_dma_cleanup(tdes_dd);
1393
1394 atmel_tdes_buff_cleanup(tdes_dd);
13802005 1395
13802005
NR		 1396 return 0;
1397}
1398
1399static struct platform_driver atmel_tdes_driver = {
1400 .probe = atmel_tdes_probe,
49cfe4db 1401 .remove = atmel_tdes_remove,
13802005
NR		 1402 .driver = {
1403 .name = "atmel_tdes",
84c8976b 1404 .of_match_table = of_match_ptr(atmel_tdes_dt_ids),
13802005
NR		 1405 },
1406};
1407
1408module_platform_driver(atmel_tdes_driver);
1409
1410MODULE_DESCRIPTION("Atmel DES/TDES hw acceleration support.");
1411MODULE_LICENSE("GPL v2");
1412MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");
Ubuntu Hirsute Linux kernel mirror