]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/crypto/omap-des.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: omap - add base support library for common routines
[mirror_ubuntu-hirsute-kernel.git] / drivers / crypto / omap-des.c
CommitLineData
e91aa9d5
JF		 1/*
2 * Support for OMAP DES and Triple DES HW acceleration.
3 *
4 * Copyright (c) 2013 Texas Instruments Incorporated
5 * Author: Joel Fernandes <joelf@ti.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as published
9 * by the Free Software Foundation.
10 *
11 */
12
13#define pr_fmt(fmt) "%s: " fmt, __func__
14
15#ifdef DEBUG
16#define prn(num) printk(#num "=%d\n", num)
17#define prx(num) printk(#num "=%x\n", num)
18#else
19#define prn(num) do { } while (0)
20#define prx(num) do { } while (0)
21#endif
22
23#include <linux/err.h>
24#include <linux/module.h>
25#include <linux/init.h>
26#include <linux/errno.h>
27#include <linux/kernel.h>
28#include <linux/platform_device.h>
29#include <linux/scatterlist.h>
30#include <linux/dma-mapping.h>
31#include <linux/dmaengine.h>
e91aa9d5
JF		 32#include <linux/pm_runtime.h>
33#include <linux/of.h>
34#include <linux/of_device.h>
35#include <linux/of_address.h>
36#include <linux/io.h>
37#include <linux/crypto.h>
38#include <linux/interrupt.h>
39#include <crypto/scatterwalk.h>
40#include <crypto/des.h>
f1b77aac 41#include <crypto/algapi.h>
2589ad84 42#include <crypto/engine.h>
e91aa9d5
JF		 43
44#define DST_MAXBURST 2
45
46#define DES_BLOCK_WORDS (DES_BLOCK_SIZE >> 2)
47
48#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
49
50#define DES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \
51 ((x ^ 0x01) * 0x04))
52
53#define DES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04))
54
55#define DES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs)
56#define DES_REG_CTRL_CBC BIT(4)
57#define DES_REG_CTRL_TDES BIT(3)
58#define DES_REG_CTRL_DIRECTION BIT(2)
59#define DES_REG_CTRL_INPUT_READY BIT(1)
60#define DES_REG_CTRL_OUTPUT_READY BIT(0)
61
62#define DES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04))
63
64#define DES_REG_REV(dd) ((dd)->pdata->rev_ofs)
65
66#define DES_REG_MASK(dd) ((dd)->pdata->mask_ofs)
67
68#define DES_REG_LENGTH_N(x) (0x24 + ((x) * 0x04))
69
70#define DES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs)
71#define DES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs)
72#define DES_REG_IRQ_DATA_IN BIT(1)
73#define DES_REG_IRQ_DATA_OUT BIT(2)
74
75#define FLAGS_MODE_MASK 0x000f
76#define FLAGS_ENCRYPT BIT(0)
77#define FLAGS_CBC BIT(1)
78#define FLAGS_INIT BIT(4)
79#define FLAGS_BUSY BIT(6)
80
418f2a8c
TK		 81#define DEFAULT_AUTOSUSPEND_DELAY 1000
82
e91aa9d5
JF		 83struct omap_des_ctx {
84 struct omap_des_dev *dd;
85
86 int keylen;
87 u32 key[(3 * DES_KEY_SIZE) / sizeof(u32)];
88 unsigned long flags;
89};
90
91struct omap_des_reqctx {
92 unsigned long mode;
93};
94
95#define OMAP_DES_QUEUE_LENGTH 1
96#define OMAP_DES_CACHE_SIZE 0
97
98struct omap_des_algs_info {
99 struct crypto_alg *algs_list;
100 unsigned int size;
101 unsigned int registered;
102};
103
104struct omap_des_pdata {
105 struct omap_des_algs_info *algs_info;
106 unsigned int algs_info_size;
107
108 void (*trigger)(struct omap_des_dev *dd, int length);
109
110 u32 key_ofs;
111 u32 iv_ofs;
112 u32 ctrl_ofs;
113 u32 data_ofs;
114 u32 rev_ofs;
115 u32 mask_ofs;
116 u32 irq_enable_ofs;
117 u32 irq_status_ofs;
118
119 u32 dma_enable_in;
120 u32 dma_enable_out;
121 u32 dma_start;
122
123 u32 major_mask;
124 u32 major_shift;
125 u32 minor_mask;
126 u32 minor_shift;
127};
128
129struct omap_des_dev {
130 struct list_head list;
131 unsigned long phys_base;
132 void __iomem *io_base;
133 struct omap_des_ctx *ctx;
134 struct device *dev;
135 unsigned long flags;
136 int err;
137
e91aa9d5 138 struct tasklet_struct done_task;
e91aa9d5
JF		 139
140 struct ablkcipher_request *req;
f1b77aac 141 struct crypto_engine *engine;
e91aa9d5
JF		 142 /*
143 * total is used by PIO mode for book keeping so introduce
144 * variable total_save as need it to calc page_order
145 */
146 size_t total;
147 size_t total_save;
148
149 struct scatterlist *in_sg;
150 struct scatterlist *out_sg;
151
152 /* Buffers for copying for unaligned cases */
153 struct scatterlist in_sgl;
154 struct scatterlist out_sgl;
155 struct scatterlist *orig_out;
156 int sgs_copied;
157
158 struct scatter_walk in_walk;
159 struct scatter_walk out_walk;
e91aa9d5 160 struct dma_chan *dma_lch_in;
e91aa9d5
JF		 161 struct dma_chan *dma_lch_out;
162 int in_sg_len;
163 int out_sg_len;
164 int pio_only;
165 const struct omap_des_pdata *pdata;
166};
167
168/* keep registered devices data here */
169static LIST_HEAD(dev_list);
170static DEFINE_SPINLOCK(list_lock);
171
172#ifdef DEBUG
173#define omap_des_read(dd, offset) \
174 ({ \
175 int _read_ret; \
176 _read_ret = __raw_readl(dd->io_base + offset); \
177 pr_err("omap_des_read(" #offset "=%#x)= %#x\n", \
178 offset, _read_ret); \
179 _read_ret; \
180 })
181#else
182static inline u32 omap_des_read(struct omap_des_dev *dd, u32 offset)
183{
184 return __raw_readl(dd->io_base + offset);
185}
186#endif
187
188#ifdef DEBUG
189#define omap_des_write(dd, offset, value) \
190 do { \
191 pr_err("omap_des_write(" #offset "=%#x) value=%#x\n", \
192 offset, value); \
193 __raw_writel(value, dd->io_base + offset); \
194 } while (0)
195#else
196static inline void omap_des_write(struct omap_des_dev *dd, u32 offset,
197 u32 value)
198{
199 __raw_writel(value, dd->io_base + offset);
200}
201#endif
202
203static inline void omap_des_write_mask(struct omap_des_dev *dd, u32 offset,
204 u32 value, u32 mask)
205{
206 u32 val;
207
208 val = omap_des_read(dd, offset);
209 val &= ~mask;
210 val |= value;
211 omap_des_write(dd, offset, val);
212}
213
214static void omap_des_write_n(struct omap_des_dev *dd, u32 offset,
215 u32 *value, int count)
216{
217 for (; count--; value++, offset += 4)
218 omap_des_write(dd, offset, *value);
219}
220
221static int omap_des_hw_init(struct omap_des_dev *dd)
222{
f51f593b
NM		 223 int err;
224
e91aa9d5
JF		 225 /*
226 * clocks are enabled when request starts and disabled when finished.
227 * It may be long delays between requests.
228 * Device might go to off mode to save power.
229 */
f51f593b
NM		 230 err = pm_runtime_get_sync(dd->dev);
231 if (err < 0) {
232 pm_runtime_put_noidle(dd->dev);
233 dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);
234 return err;
235 }
e91aa9d5
JF		 236
237 if (!(dd->flags & FLAGS_INIT)) {
238 dd->flags |= FLAGS_INIT;
239 dd->err = 0;
240 }
241
242 return 0;
243}
244
245static int omap_des_write_ctrl(struct omap_des_dev *dd)
246{
247 unsigned int key32;
248 int i, err;
249 u32 val = 0, mask = 0;
250
251 err = omap_des_hw_init(dd);
252 if (err)
253 return err;
254
255 key32 = dd->ctx->keylen / sizeof(u32);
256
257 /* it seems a key should always be set even if it has not changed */
258 for (i = 0; i < key32; i++) {
259 omap_des_write(dd, DES_REG_KEY(dd, i),
260 __le32_to_cpu(dd->ctx->key[i]));
261 }
262
263 if ((dd->flags & FLAGS_CBC) && dd->req->info)
264 omap_des_write_n(dd, DES_REG_IV(dd, 0), dd->req->info, 2);
265
266 if (dd->flags & FLAGS_CBC)
267 val |= DES_REG_CTRL_CBC;
268 if (dd->flags & FLAGS_ENCRYPT)
269 val |= DES_REG_CTRL_DIRECTION;
270 if (key32 == 6)
271 val |= DES_REG_CTRL_TDES;
272
273 mask |= DES_REG_CTRL_CBC | DES_REG_CTRL_DIRECTION | DES_REG_CTRL_TDES;
274
275 omap_des_write_mask(dd, DES_REG_CTRL(dd), val, mask);
276
277 return 0;
278}
279
280static void omap_des_dma_trigger_omap4(struct omap_des_dev *dd, int length)
281{
282 u32 mask, val;
283
284 omap_des_write(dd, DES_REG_LENGTH_N(0), length);
285
286 val = dd->pdata->dma_start;
287
288 if (dd->dma_lch_out != NULL)
289 val |= dd->pdata->dma_enable_out;
290 if (dd->dma_lch_in != NULL)
291 val |= dd->pdata->dma_enable_in;
292
293 mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
294 dd->pdata->dma_start;
295
296 omap_des_write_mask(dd, DES_REG_MASK(dd), val, mask);
297}
298
299static void omap_des_dma_stop(struct omap_des_dev *dd)
300{
301 u32 mask;
302
303 mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
304 dd->pdata->dma_start;
305
306 omap_des_write_mask(dd, DES_REG_MASK(dd), 0, mask);
307}
308
309static struct omap_des_dev *omap_des_find_dev(struct omap_des_ctx *ctx)
310{
311 struct omap_des_dev *dd = NULL, *tmp;
312
313 spin_lock_bh(&list_lock);
314 if (!ctx->dd) {
315 list_for_each_entry(tmp, &dev_list, list) {
316 /* FIXME: take fist available des core */
317 dd = tmp;
318 break;
319 }
320 ctx->dd = dd;
321 } else {
322 /* already found before */
323 dd = ctx->dd;
324 }
325 spin_unlock_bh(&list_lock);
326
327 return dd;
328}
329
330static void omap_des_dma_out_callback(void *data)
331{
332 struct omap_des_dev *dd = data;
333
334 /* dma_lch_out - completed */
335 tasklet_schedule(&dd->done_task);
336}
337
338static int omap_des_dma_init(struct omap_des_dev *dd)
339{
2f6f0680 340 int err;
e91aa9d5
JF		 341
342 dd->dma_lch_out = NULL;
343 dd->dma_lch_in = NULL;
344
2f6f0680
PU		 345 dd->dma_lch_in = dma_request_chan(dd->dev, "rx");
346 if (IS_ERR(dd->dma_lch_in)) {
e91aa9d5 347 dev_err(dd->dev, "Unable to request in DMA channel\n");
2f6f0680 348 return PTR_ERR(dd->dma_lch_in);
e91aa9d5
JF		 349 }
350
2f6f0680
PU		 351 dd->dma_lch_out = dma_request_chan(dd->dev, "tx");
352 if (IS_ERR(dd->dma_lch_out)) {
e91aa9d5 353 dev_err(dd->dev, "Unable to request out DMA channel\n");
2f6f0680 354 err = PTR_ERR(dd->dma_lch_out);
e91aa9d5
JF		 355 goto err_dma_out;
356 }
357
358 return 0;
359
360err_dma_out:
361 dma_release_channel(dd->dma_lch_in);
2f6f0680 362
e91aa9d5
JF		 363 return err;
364}
365
366static void omap_des_dma_cleanup(struct omap_des_dev *dd)
367{
2f6f0680
PU		 368 if (dd->pio_only)
369 return;
370
e91aa9d5
JF		 371 dma_release_channel(dd->dma_lch_out);
372 dma_release_channel(dd->dma_lch_in);
373}
374
375static void sg_copy_buf(void *buf, struct scatterlist *sg,
376 unsigned int start, unsigned int nbytes, int out)
377{
378 struct scatter_walk walk;
379
380 if (!nbytes)
381 return;
382
383 scatterwalk_start(&walk, sg);
384 scatterwalk_advance(&walk, start);
385 scatterwalk_copychunks(buf, &walk, nbytes, out);
386 scatterwalk_done(&walk, out, 0);
387}
388
389static int omap_des_crypt_dma(struct crypto_tfm *tfm,
390 struct scatterlist *in_sg, struct scatterlist *out_sg,
391 int in_sg_len, int out_sg_len)
392{
393 struct omap_des_ctx *ctx = crypto_tfm_ctx(tfm);
394 struct omap_des_dev *dd = ctx->dd;
395 struct dma_async_tx_descriptor *tx_in, *tx_out;
396 struct dma_slave_config cfg;
397 int ret;
398
399 if (dd->pio_only) {
400 scatterwalk_start(&dd->in_walk, dd->in_sg);
401 scatterwalk_start(&dd->out_walk, dd->out_sg);
402
403 /* Enable DATAIN interrupt and let it take
404 care of the rest */
405 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2);
406 return 0;
407 }
408
409 dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);
410
411 memset(&cfg, 0, sizeof(cfg));
412
413 cfg.src_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);
414 cfg.dst_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);
415 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
416 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
417 cfg.src_maxburst = DST_MAXBURST;
418 cfg.dst_maxburst = DST_MAXBURST;
419
420 /* IN */
421 ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);
422 if (ret) {
423 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
424 ret);
425 return ret;
426 }
427
428 tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,
429 DMA_MEM_TO_DEV,
430 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
431 if (!tx_in) {
432 dev_err(dd->dev, "IN prep_slave_sg() failed\n");
433 return -EINVAL;
434 }
435
436 /* No callback necessary */
437 tx_in->callback_param = dd;
438
439 /* OUT */
440 ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);
441 if (ret) {
442 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
443 ret);
444 return ret;
445 }
446
447 tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,
448 DMA_DEV_TO_MEM,
449 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
450 if (!tx_out) {
451 dev_err(dd->dev, "OUT prep_slave_sg() failed\n");
452 return -EINVAL;
453 }
454
455 tx_out->callback = omap_des_dma_out_callback;
456 tx_out->callback_param = dd;
457
458 dmaengine_submit(tx_in);
459 dmaengine_submit(tx_out);
460
461 dma_async_issue_pending(dd->dma_lch_in);
462 dma_async_issue_pending(dd->dma_lch_out);
463
464 /* start DMA */
465 dd->pdata->trigger(dd, dd->total);
466
467 return 0;
468}
469
470static int omap_des_crypt_dma_start(struct omap_des_dev *dd)
471{
472 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
473 crypto_ablkcipher_reqtfm(dd->req));
474 int err;
475
476 pr_debug("total: %d\n", dd->total);
477
478 if (!dd->pio_only) {
479 err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,
480 DMA_TO_DEVICE);
481 if (!err) {
482 dev_err(dd->dev, "dma_map_sg() error\n");
483 return -EINVAL;
484 }
485
486 err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,
487 DMA_FROM_DEVICE);
488 if (!err) {
489 dev_err(dd->dev, "dma_map_sg() error\n");
490 return -EINVAL;
491 }
492 }
493
494 err = omap_des_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,
495 dd->out_sg_len);
496 if (err && !dd->pio_only) {
497 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
498 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
499 DMA_FROM_DEVICE);
500 }
501
502 return err;
503}
504
505static void omap_des_finish_req(struct omap_des_dev *dd, int err)
506{
507 struct ablkcipher_request *req = dd->req;
508
509 pr_debug("err: %d\n", err);
510
4cba7cf0 511 crypto_finalize_cipher_request(dd->engine, req, err);
418f2a8c
TK		 512
513 pm_runtime_mark_last_busy(dd->dev);
514 pm_runtime_put_autosuspend(dd->dev);
e91aa9d5
JF		 515}
516
517static int omap_des_crypt_dma_stop(struct omap_des_dev *dd)
518{
e91aa9d5
JF		 519 pr_debug("total: %d\n", dd->total);
520
521 omap_des_dma_stop(dd);
522
523 dmaengine_terminate_all(dd->dma_lch_in);
524 dmaengine_terminate_all(dd->dma_lch_out);
525
16f080aa 526 return 0;
e91aa9d5
JF		 527}
528
26f25b26 529static int omap_des_copy_needed(struct scatterlist *sg)
e91aa9d5
JF		 530{
531 while (sg) {
532 if (!IS_ALIGNED(sg->offset, 4))
533 return -1;
534 if (!IS_ALIGNED(sg->length, DES_BLOCK_SIZE))
535 return -1;
536 sg = sg_next(sg);
537 }
538 return 0;
539}
540
26f25b26 541static int omap_des_copy_sgs(struct omap_des_dev *dd)
e91aa9d5
JF		 542{
543 void *buf_in, *buf_out;
544 int pages;
545
546 pages = dd->total >> PAGE_SHIFT;
547
548 if (dd->total & (PAGE_SIZE-1))
549 pages++;
550
551 BUG_ON(!pages);
552
553 buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
554 buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
555
556 if (!buf_in || !buf_out) {
557 pr_err("Couldn't allocated pages for unaligned cases.\n");
558 return -1;
559 }
560
561 dd->orig_out = dd->out_sg;
562
563 sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0);
564
565 sg_init_table(&dd->in_sgl, 1);
566 sg_set_buf(&dd->in_sgl, buf_in, dd->total);
567 dd->in_sg = &dd->in_sgl;
7c001a86 568 dd->in_sg_len = 1;
e91aa9d5
JF		 569
570 sg_init_table(&dd->out_sgl, 1);
571 sg_set_buf(&dd->out_sgl, buf_out, dd->total);
572 dd->out_sg = &dd->out_sgl;
7c001a86 573 dd->out_sg_len = 1;
e91aa9d5
JF		 574
575 return 0;
576}
577
578static int omap_des_handle_queue(struct omap_des_dev *dd,
f1b77aac 579 struct ablkcipher_request *req)
e91aa9d5 580{
e91aa9d5 581 if (req)
4cba7cf0 582 return crypto_transfer_cipher_request_to_engine(dd->engine, req);
e91aa9d5 583
f1b77aac
BW		 584 return 0;
585}
e91aa9d5 586
f1b77aac
BW		 587static int omap_des_prepare_req(struct crypto_engine *engine,
588 struct ablkcipher_request *req)
589{
590 struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(
591 crypto_ablkcipher_reqtfm(req));
592 struct omap_des_dev *dd = omap_des_find_dev(ctx);
593 struct omap_des_reqctx *rctx;
e91aa9d5 594
f1b77aac
BW		 595 if (!dd)
596 return -ENODEV;
e91aa9d5
JF		 597
598 /* assign new request to device */
599 dd->req = req;
600 dd->total = req->nbytes;
601 dd->total_save = req->nbytes;
602 dd->in_sg = req->src;
603 dd->out_sg = req->dst;
604
7c001a86
HX		 605 dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);
606 if (dd->in_sg_len < 0)
607 return dd->in_sg_len;
608
609 dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total);
610 if (dd->out_sg_len < 0)
611 return dd->out_sg_len;
612
e91aa9d5
JF		 613 if (omap_des_copy_needed(dd->in_sg) ||
614 omap_des_copy_needed(dd->out_sg)) {
615 if (omap_des_copy_sgs(dd))
616 pr_err("Failed to copy SGs for unaligned cases\n");
617 dd->sgs_copied = 1;
618 } else {
619 dd->sgs_copied = 0;
620 }
621
e91aa9d5
JF		 622 rctx = ablkcipher_request_ctx(req);
623 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
624 rctx->mode &= FLAGS_MODE_MASK;
625 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
626
627 dd->ctx = ctx;
628 ctx->dd = dd;
629
f1b77aac
BW		 630 return omap_des_write_ctrl(dd);
631}
e91aa9d5 632
f1b77aac
BW		 633static int omap_des_crypt_req(struct crypto_engine *engine,
634 struct ablkcipher_request *req)
635{
636 struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(
637 crypto_ablkcipher_reqtfm(req));
638 struct omap_des_dev *dd = omap_des_find_dev(ctx);
639
640 if (!dd)
641 return -ENODEV;
642
643 return omap_des_crypt_dma_start(dd);
e91aa9d5
JF		 644}
645
646static void omap_des_done_task(unsigned long data)
647{
648 struct omap_des_dev *dd = (struct omap_des_dev *)data;
649 void *buf_in, *buf_out;
650 int pages;
651
652 pr_debug("enter done_task\n");
653
654 if (!dd->pio_only) {
655 dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
656 DMA_FROM_DEVICE);
657 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
658 dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,
659 DMA_FROM_DEVICE);
660 omap_des_crypt_dma_stop(dd);
661 }
662
663 if (dd->sgs_copied) {
664 buf_in = sg_virt(&dd->in_sgl);
665 buf_out = sg_virt(&dd->out_sgl);
666
667 sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1);
668
669 pages = get_order(dd->total_save);
670 free_pages((unsigned long)buf_in, pages);
671 free_pages((unsigned long)buf_out, pages);
672 }
673
674 omap_des_finish_req(dd, 0);
e91aa9d5
JF		 675
676 pr_debug("exit\n");
677}
678
e91aa9d5
JF		 679static int omap_des_crypt(struct ablkcipher_request *req, unsigned long mode)
680{
681 struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(
682 crypto_ablkcipher_reqtfm(req));
683 struct omap_des_reqctx *rctx = ablkcipher_request_ctx(req);
684 struct omap_des_dev *dd;
685
686 pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
687 !!(mode & FLAGS_ENCRYPT),
688 !!(mode & FLAGS_CBC));
689
690 if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) {
691 pr_err("request size is not exact amount of DES blocks\n");
692 return -EINVAL;
693 }
694
695 dd = omap_des_find_dev(ctx);
696 if (!dd)
697 return -ENODEV;
698
699 rctx->mode = mode;
700
701 return omap_des_handle_queue(dd, req);
702}
703
704/* ********************** ALG API ************************************ */
705
a636fdce 706static int omap_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
e91aa9d5
JF		 707 unsigned int keylen)
708{
a636fdce
TK		 709 struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(cipher);
710 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
e91aa9d5
JF		 711
712 if (keylen != DES_KEY_SIZE && keylen != (3*DES_KEY_SIZE))
713 return -EINVAL;
714
715 pr_debug("enter, keylen: %d\n", keylen);
716
a636fdce
TK		 717 /* Do we need to test against weak key? */
718 if (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY) {
719 u32 tmp[DES_EXPKEY_WORDS];
720 int ret = des_ekey(tmp, key);
721
722 if (!ret) {
723 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
724 return -EINVAL;
725 }
726 }
727
e91aa9d5
JF		 728 memcpy(ctx->key, key, keylen);
729 ctx->keylen = keylen;
730
731 return 0;
732}
733
734static int omap_des_ecb_encrypt(struct ablkcipher_request *req)
735{
736 return omap_des_crypt(req, FLAGS_ENCRYPT);
737}
738
739static int omap_des_ecb_decrypt(struct ablkcipher_request *req)
740{
741 return omap_des_crypt(req, 0);
742}
743
744static int omap_des_cbc_encrypt(struct ablkcipher_request *req)
745{
746 return omap_des_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
747}
748
749static int omap_des_cbc_decrypt(struct ablkcipher_request *req)
750{
751 return omap_des_crypt(req, FLAGS_CBC);
752}
753
754static int omap_des_cra_init(struct crypto_tfm *tfm)
755{
756 pr_debug("enter\n");
757
758 tfm->crt_ablkcipher.reqsize = sizeof(struct omap_des_reqctx);
759
760 return 0;
761}
762
763static void omap_des_cra_exit(struct crypto_tfm *tfm)
764{
765 pr_debug("enter\n");
766}
767
768/* ********************** ALGS ************************************ */
769
770static struct crypto_alg algs_ecb_cbc[] = {
771{
772 .cra_name = "ecb(des)",
773 .cra_driver_name = "ecb-des-omap",
774 .cra_priority = 100,
775 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
776 CRYPTO_ALG_KERN_DRIVER_ONLY |
777 CRYPTO_ALG_ASYNC,
778 .cra_blocksize = DES_BLOCK_SIZE,
779 .cra_ctxsize = sizeof(struct omap_des_ctx),
780 .cra_alignmask = 0,
781 .cra_type = &crypto_ablkcipher_type,
782 .cra_module = THIS_MODULE,
783 .cra_init = omap_des_cra_init,
784 .cra_exit = omap_des_cra_exit,
785 .cra_u.ablkcipher = {
786 .min_keysize = DES_KEY_SIZE,
787 .max_keysize = DES_KEY_SIZE,
788 .setkey = omap_des_setkey,
789 .encrypt = omap_des_ecb_encrypt,
790 .decrypt = omap_des_ecb_decrypt,
791 }
792},
793{
794 .cra_name = "cbc(des)",
795 .cra_driver_name = "cbc-des-omap",
796 .cra_priority = 100,
797 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
798 CRYPTO_ALG_KERN_DRIVER_ONLY |
799 CRYPTO_ALG_ASYNC,
800 .cra_blocksize = DES_BLOCK_SIZE,
801 .cra_ctxsize = sizeof(struct omap_des_ctx),
802 .cra_alignmask = 0,
803 .cra_type = &crypto_ablkcipher_type,
804 .cra_module = THIS_MODULE,
805 .cra_init = omap_des_cra_init,
806 .cra_exit = omap_des_cra_exit,
807 .cra_u.ablkcipher = {
808 .min_keysize = DES_KEY_SIZE,
809 .max_keysize = DES_KEY_SIZE,
810 .ivsize = DES_BLOCK_SIZE,
811 .setkey = omap_des_setkey,
812 .encrypt = omap_des_cbc_encrypt,
813 .decrypt = omap_des_cbc_decrypt,
814 }
815},
816{
817 .cra_name = "ecb(des3_ede)",
818 .cra_driver_name = "ecb-des3-omap",
819 .cra_priority = 100,
820 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
821 CRYPTO_ALG_KERN_DRIVER_ONLY |
822 CRYPTO_ALG_ASYNC,
823 .cra_blocksize = DES_BLOCK_SIZE,
824 .cra_ctxsize = sizeof(struct omap_des_ctx),
825 .cra_alignmask = 0,
826 .cra_type = &crypto_ablkcipher_type,
827 .cra_module = THIS_MODULE,
828 .cra_init = omap_des_cra_init,
829 .cra_exit = omap_des_cra_exit,
830 .cra_u.ablkcipher = {
831 .min_keysize = 3*DES_KEY_SIZE,
832 .max_keysize = 3*DES_KEY_SIZE,
833 .setkey = omap_des_setkey,
834 .encrypt = omap_des_ecb_encrypt,
835 .decrypt = omap_des_ecb_decrypt,
836 }
837},
838{
839 .cra_name = "cbc(des3_ede)",
840 .cra_driver_name = "cbc-des3-omap",
841 .cra_priority = 100,
842 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
843 CRYPTO_ALG_KERN_DRIVER_ONLY |
844 CRYPTO_ALG_ASYNC,
845 .cra_blocksize = DES_BLOCK_SIZE,
846 .cra_ctxsize = sizeof(struct omap_des_ctx),
847 .cra_alignmask = 0,
848 .cra_type = &crypto_ablkcipher_type,
849 .cra_module = THIS_MODULE,
850 .cra_init = omap_des_cra_init,
851 .cra_exit = omap_des_cra_exit,
852 .cra_u.ablkcipher = {
853 .min_keysize = 3*DES_KEY_SIZE,
854 .max_keysize = 3*DES_KEY_SIZE,
855 .ivsize = DES_BLOCK_SIZE,
856 .setkey = omap_des_setkey,
857 .encrypt = omap_des_cbc_encrypt,
858 .decrypt = omap_des_cbc_decrypt,
859 }
860}
861};
862
863static struct omap_des_algs_info omap_des_algs_info_ecb_cbc[] = {
864 {
865 .algs_list = algs_ecb_cbc,
866 .size = ARRAY_SIZE(algs_ecb_cbc),
867 },
868};
869
870#ifdef CONFIG_OF
871static const struct omap_des_pdata omap_des_pdata_omap4 = {
872 .algs_info = omap_des_algs_info_ecb_cbc,
873 .algs_info_size = ARRAY_SIZE(omap_des_algs_info_ecb_cbc),
874 .trigger = omap_des_dma_trigger_omap4,
875 .key_ofs = 0x14,
876 .iv_ofs = 0x18,
877 .ctrl_ofs = 0x20,
878 .data_ofs = 0x28,
879 .rev_ofs = 0x30,
880 .mask_ofs = 0x34,
881 .irq_status_ofs = 0x3c,
882 .irq_enable_ofs = 0x40,
883 .dma_enable_in = BIT(5),
884 .dma_enable_out = BIT(6),
885 .major_mask = 0x0700,
886 .major_shift = 8,
887 .minor_mask = 0x003f,
888 .minor_shift = 0,
889};
890
891static irqreturn_t omap_des_irq(int irq, void *dev_id)
892{
893 struct omap_des_dev *dd = dev_id;
894 u32 status, i;
895 u32 *src, *dst;
896
897 status = omap_des_read(dd, DES_REG_IRQ_STATUS(dd));
898 if (status & DES_REG_IRQ_DATA_IN) {
899 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0);
900
901 BUG_ON(!dd->in_sg);
902
903 BUG_ON(_calc_walked(in) > dd->in_sg->length);
904
905 src = sg_virt(dd->in_sg) + _calc_walked(in);
906
907 for (i = 0; i < DES_BLOCK_WORDS; i++) {
908 omap_des_write(dd, DES_REG_DATA_N(dd, i), *src);
909
910 scatterwalk_advance(&dd->in_walk, 4);
911 if (dd->in_sg->length == _calc_walked(in)) {
5be4d4c9 912 dd->in_sg = sg_next(dd->in_sg);
e91aa9d5
JF		 913 if (dd->in_sg) {
914 scatterwalk_start(&dd->in_walk,
915 dd->in_sg);
916 src = sg_virt(dd->in_sg) +
917 _calc_walked(in);
918 }
919 } else {
920 src++;
921 }
922 }
923
924 /* Clear IRQ status */
925 status &= ~DES_REG_IRQ_DATA_IN;
926 omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status);
927
928 /* Enable DATA_OUT interrupt */
929 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x4);
930
931 } else if (status & DES_REG_IRQ_DATA_OUT) {
932 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0);
933
934 BUG_ON(!dd->out_sg);
935
936 BUG_ON(_calc_walked(out) > dd->out_sg->length);
937
938 dst = sg_virt(dd->out_sg) + _calc_walked(out);
939
940 for (i = 0; i < DES_BLOCK_WORDS; i++) {
941 *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i));
942 scatterwalk_advance(&dd->out_walk, 4);
943 if (dd->out_sg->length == _calc_walked(out)) {
5be4d4c9 944 dd->out_sg = sg_next(dd->out_sg);
e91aa9d5
JF		 945 if (dd->out_sg) {
946 scatterwalk_start(&dd->out_walk,
947 dd->out_sg);
948 dst = sg_virt(dd->out_sg) +
949 _calc_walked(out);
950 }
951 } else {
952 dst++;
953 }
954 }
955
42d2e780 956 BUG_ON(dd->total < DES_BLOCK_SIZE);
e91aa9d5 957
42d2e780 958 dd->total -= DES_BLOCK_SIZE;
e91aa9d5
JF		 959
960 /* Clear IRQ status */
961 status &= ~DES_REG_IRQ_DATA_OUT;
962 omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status);
963
964 if (!dd->total)
965 /* All bytes read! */
966 tasklet_schedule(&dd->done_task);
967 else
968 /* Enable DATA_IN interrupt for next block */
969 omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2);
970 }
971
972 return IRQ_HANDLED;
973}
974
975static const struct of_device_id omap_des_of_match[] = {
976 {
977 .compatible = "ti,omap4-des",
978 .data = &omap_des_pdata_omap4,
979 },
980 {},
981};
982MODULE_DEVICE_TABLE(of, omap_des_of_match);
983
984static int omap_des_get_of(struct omap_des_dev *dd,
985 struct platform_device *pdev)
986{
987 const struct of_device_id *match;
988
989 match = of_match_device(of_match_ptr(omap_des_of_match), &pdev->dev);
990 if (!match) {
991 dev_err(&pdev->dev, "no compatible OF match\n");
992 return -EINVAL;
993 }
994
e91aa9d5
JF		 995 dd->pdata = match->data;
996
997 return 0;
998}
999#else
1000static int omap_des_get_of(struct omap_des_dev *dd,
1001 struct device *dev)
1002{
1003 return -EINVAL;
1004}
1005#endif
1006
1007static int omap_des_get_pdev(struct omap_des_dev *dd,
1008 struct platform_device *pdev)
1009{
e91aa9d5
JF		 1010 /* non-DT devices get pdata from pdev */
1011 dd->pdata = pdev->dev.platform_data;
1012
2f6f0680 1013 return 0;
e91aa9d5
JF		 1014}
1015
1016static int omap_des_probe(struct platform_device *pdev)
1017{
1018 struct device *dev = &pdev->dev;
1019 struct omap_des_dev *dd;
1020 struct crypto_alg *algp;
1021 struct resource *res;
1022 int err = -ENOMEM, i, j, irq = -1;
1023 u32 reg;
1024
1025 dd = devm_kzalloc(dev, sizeof(struct omap_des_dev), GFP_KERNEL);
1026 if (dd == NULL) {
1027 dev_err(dev, "unable to alloc data struct.\n");
1028 goto err_data;
1029 }
1030 dd->dev = dev;
1031 platform_set_drvdata(pdev, dd);
1032
e91aa9d5
JF		 1033 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1034 if (!res) {
1035 dev_err(dev, "no MEM resource info\n");
1036 goto err_res;
1037 }
1038
1039 err = (dev->of_node) ? omap_des_get_of(dd, pdev) :
1040 omap_des_get_pdev(dd, pdev);
1041 if (err)
1042 goto err_res;
1043
2496be2e
JH		 1044 dd->io_base = devm_ioremap_resource(dev, res);
1045 if (IS_ERR(dd->io_base)) {
1046 err = PTR_ERR(dd->io_base);
e91aa9d5
JF		 1047 goto err_res;
1048 }
1049 dd->phys_base = res->start;
1050
418f2a8c
TK		 1051 pm_runtime_use_autosuspend(dev);
1052 pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
1053
e91aa9d5 1054 pm_runtime_enable(dev);
f51f593b
NM		 1055 err = pm_runtime_get_sync(dev);
1056 if (err < 0) {
1057 pm_runtime_put_noidle(dev);
1058 dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);
1059 goto err_get;
1060 }
e91aa9d5
JF		 1061
1062 omap_des_dma_stop(dd);
1063
1064 reg = omap_des_read(dd, DES_REG_REV(dd));
1065
1066 pm_runtime_put_sync(dev);
1067
1068 dev_info(dev, "OMAP DES hw accel rev: %u.%u\n",
1069 (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,
1070 (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
1071
1072 tasklet_init(&dd->done_task, omap_des_done_task, (unsigned long)dd);
e91aa9d5
JF		 1073
1074 err = omap_des_dma_init(dd);
2f6f0680
PU		 1075 if (err == -EPROBE_DEFER) {
1076 goto err_irq;
1077 } else if (err && DES_REG_IRQ_STATUS(dd) && DES_REG_IRQ_ENABLE(dd)) {
e91aa9d5
JF		 1078 dd->pio_only = 1;
1079
1080 irq = platform_get_irq(pdev, 0);
1081 if (irq < 0) {
1082 dev_err(dev, "can't get IRQ resource\n");
1083 goto err_irq;
1084 }
1085
1086 err = devm_request_irq(dev, irq, omap_des_irq, 0,
1087 dev_name(dev), dd);
1088 if (err) {
1089 dev_err(dev, "Unable to grab omap-des IRQ\n");
1090 goto err_irq;
1091 }
1092 }
1093
1094
1095 INIT_LIST_HEAD(&dd->list);
1096 spin_lock(&list_lock);
1097 list_add_tail(&dd->list, &dev_list);
1098 spin_unlock(&list_lock);
1099
1d1f98d1
TK		 1100 /* Initialize des crypto engine */
1101 dd->engine = crypto_engine_alloc_init(dev, 1);
59af1566
WY		 1102 if (!dd->engine) {
1103 err = -ENOMEM;
1d1f98d1 1104 goto err_engine;
59af1566 1105 }
1d1f98d1
TK		 1106
1107 dd->engine->prepare_cipher_request = omap_des_prepare_req;
1108 dd->engine->cipher_one_request = omap_des_crypt_req;
1109 err = crypto_engine_start(dd->engine);
1110 if (err)
1111 goto err_engine;
1112
e91aa9d5
JF		 1113 for (i = 0; i < dd->pdata->algs_info_size; i++) {
1114 for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
1115 algp = &dd->pdata->algs_info[i].algs_list[j];
1116
1117 pr_debug("reg alg: %s\n", algp->cra_name);
1118 INIT_LIST_HEAD(&algp->cra_list);
1119
1120 err = crypto_register_alg(algp);
1121 if (err)
1122 goto err_algs;
1123
1124 dd->pdata->algs_info[i].registered++;
1125 }
1126 }
1127
1128 return 0;
f1b77aac 1129
e91aa9d5
JF		 1130err_algs:
1131 for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1132 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1133 crypto_unregister_alg(
1134 &dd->pdata->algs_info[i].algs_list[j]);
2f6f0680 1135
1d1f98d1
TK		 1136err_engine:
1137 if (dd->engine)
1138 crypto_engine_exit(dd->engine);
1139
2f6f0680 1140 omap_des_dma_cleanup(dd);
e91aa9d5
JF		 1141err_irq:
1142 tasklet_kill(&dd->done_task);
f51f593b 1143err_get:
e91aa9d5
JF		 1144 pm_runtime_disable(dev);
1145err_res:
1146 dd = NULL;
1147err_data:
1148 dev_err(dev, "initialization failed.\n");
1149 return err;
1150}
1151
1152static int omap_des_remove(struct platform_device *pdev)
1153{
1154 struct omap_des_dev *dd = platform_get_drvdata(pdev);
1155 int i, j;
1156
1157 if (!dd)
1158 return -ENODEV;
1159
1160 spin_lock(&list_lock);
1161 list_del(&dd->list);
1162 spin_unlock(&list_lock);
1163
1164 for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1165 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1166 crypto_unregister_alg(
1167 &dd->pdata->algs_info[i].algs_list[j]);
1168
1169 tasklet_kill(&dd->done_task);
e91aa9d5
JF		 1170 omap_des_dma_cleanup(dd);
1171 pm_runtime_disable(dd->dev);
1172 dd = NULL;
1173
1174 return 0;
1175}
1176
1177#ifdef CONFIG_PM_SLEEP
1178static int omap_des_suspend(struct device *dev)
1179{
1180 pm_runtime_put_sync(dev);
1181 return 0;
1182}
1183
1184static int omap_des_resume(struct device *dev)
1185{
f51f593b
NM		 1186 int err;
1187
1188 err = pm_runtime_get_sync(dev);
1189 if (err < 0) {
1190 pm_runtime_put_noidle(dev);
1191 dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err);
1192 return err;
1193 }
e91aa9d5
JF		 1194 return 0;
1195}
1196#endif
1197
e78f9193 1198static SIMPLE_DEV_PM_OPS(omap_des_pm_ops, omap_des_suspend, omap_des_resume);
e91aa9d5
JF		 1199
1200static struct platform_driver omap_des_driver = {
1201 .probe = omap_des_probe,
1202 .remove = omap_des_remove,
1203 .driver = {
1204 .name = "omap-des",
e91aa9d5
JF		 1205 .pm = &omap_des_pm_ops,
1206 .of_match_table = of_match_ptr(omap_des_of_match),
1207 },
1208};
1209
1210module_platform_driver(omap_des_driver);
1211
1212MODULE_DESCRIPTION("OMAP DES hw acceleration support.");
1213MODULE_LICENSE("GPL v2");
1214MODULE_AUTHOR("Joel Fernandes <joelf@ti.com>");
Ubuntu Hirsute Linux kernel mirror