2 * Support for Marvell's Cryptographic Engine and Security Accelerator (CESA)
3 * that can be found on the following platform: Orion, Kirkwood, Armada. This
4 * driver supports the TDMA engine on platforms on which it is available.
6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
7 * Author: Arnaud Ebalard <arno@natisbad.org>
9 * This work is based on an initial version written by
10 * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc >
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms of the GNU General Public License version 2 as published
14 * by the Free Software Foundation.
17 #include <linux/delay.h>
18 #include <linux/genalloc.h>
19 #include <linux/interrupt.h>
21 #include <linux/kthread.h>
22 #include <linux/mbus.h>
23 #include <linux/platform_device.h>
24 #include <linux/scatterlist.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/clk.h>
29 #include <linux/of_platform.h>
30 #include <linux/of_irq.h>
34 /* Limit of the crypto queue before reaching the backlog */
35 #define CESA_CRYPTO_DEFAULT_MAX_QLEN 128
37 static int allhwsupport
= !IS_ENABLED(CONFIG_CRYPTO_DEV_MV_CESA
);
38 module_param_named(allhwsupport
, allhwsupport
, int, 0444);
39 MODULE_PARM_DESC(allhwsupport
, "Enable support for all hardware (even it if overlaps with the mv_cesa driver)");
41 struct mv_cesa_dev
*cesa_dev
;
43 struct crypto_async_request
*
44 mv_cesa_dequeue_req_locked(struct mv_cesa_engine
*engine
,
45 struct crypto_async_request
**backlog
)
47 struct crypto_async_request
*req
;
49 *backlog
= crypto_get_backlog(&engine
->queue
);
50 req
= crypto_dequeue_request(&engine
->queue
);
58 static void mv_cesa_rearm_engine(struct mv_cesa_engine
*engine
)
60 struct crypto_async_request
*req
= NULL
, *backlog
= NULL
;
61 struct mv_cesa_ctx
*ctx
;
64 spin_lock_bh(&engine
->lock
);
66 req
= mv_cesa_dequeue_req_locked(engine
, &backlog
);
69 spin_unlock_bh(&engine
->lock
);
75 backlog
->complete(backlog
, -EINPROGRESS
);
77 ctx
= crypto_tfm_ctx(req
->tfm
);
81 static int mv_cesa_std_process(struct mv_cesa_engine
*engine
, u32 status
)
83 struct crypto_async_request
*req
;
84 struct mv_cesa_ctx
*ctx
;
88 ctx
= crypto_tfm_ctx(req
->tfm
);
89 res
= ctx
->ops
->process(req
, status
);
92 ctx
->ops
->complete(req
);
93 mv_cesa_engine_enqueue_complete_request(engine
, req
);
94 } else if (res
== -EINPROGRESS
) {
101 static int mv_cesa_int_process(struct mv_cesa_engine
*engine
, u32 status
)
103 if (engine
->chain
.first
&& engine
->chain
.last
)
104 return mv_cesa_tdma_process(engine
, status
);
106 return mv_cesa_std_process(engine
, status
);
110 mv_cesa_complete_req(struct mv_cesa_ctx
*ctx
, struct crypto_async_request
*req
,
113 ctx
->ops
->cleanup(req
);
115 req
->complete(req
, res
);
119 static irqreturn_t
mv_cesa_int(int irq
, void *priv
)
121 struct mv_cesa_engine
*engine
= priv
;
122 struct crypto_async_request
*req
;
123 struct mv_cesa_ctx
*ctx
;
125 irqreturn_t ret
= IRQ_NONE
;
130 mask
= mv_cesa_get_int_mask(engine
);
131 status
= readl(engine
->regs
+ CESA_SA_INT_STATUS
);
133 if (!(status
& mask
))
137 * TODO: avoid clearing the FPGA_INT_STATUS if this not
138 * relevant on some platforms.
140 writel(~status
, engine
->regs
+ CESA_SA_FPGA_INT_STATUS
);
141 writel(~status
, engine
->regs
+ CESA_SA_INT_STATUS
);
143 /* Process fetched requests */
144 res
= mv_cesa_int_process(engine
, status
& mask
);
147 spin_lock_bh(&engine
->lock
);
149 if (res
!= -EINPROGRESS
)
151 spin_unlock_bh(&engine
->lock
);
153 ctx
= crypto_tfm_ctx(req
->tfm
);
155 if (res
&& res
!= -EINPROGRESS
)
156 mv_cesa_complete_req(ctx
, req
, res
);
158 /* Launch the next pending request */
159 mv_cesa_rearm_engine(engine
);
161 /* Iterate over the complete queue */
163 req
= mv_cesa_engine_dequeue_complete_request(engine
);
167 ctx
= crypto_tfm_ctx(req
->tfm
);
168 mv_cesa_complete_req(ctx
, req
, 0);
175 int mv_cesa_queue_req(struct crypto_async_request
*req
,
176 struct mv_cesa_req
*creq
)
179 struct mv_cesa_engine
*engine
= creq
->engine
;
181 spin_lock_bh(&engine
->lock
);
182 ret
= crypto_enqueue_request(&engine
->queue
, req
);
183 if ((mv_cesa_req_get_type(creq
) == CESA_DMA_REQ
) &&
184 (ret
== -EINPROGRESS
||
185 (ret
== -EBUSY
&& req
->flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)))
186 mv_cesa_tdma_chain(engine
, creq
);
187 spin_unlock_bh(&engine
->lock
);
189 if (ret
!= -EINPROGRESS
)
192 mv_cesa_rearm_engine(engine
);
197 static int mv_cesa_add_algs(struct mv_cesa_dev
*cesa
)
202 for (i
= 0; i
< cesa
->caps
->ncipher_algs
; i
++) {
203 ret
= crypto_register_alg(cesa
->caps
->cipher_algs
[i
]);
205 goto err_unregister_crypto
;
208 for (i
= 0; i
< cesa
->caps
->nahash_algs
; i
++) {
209 ret
= crypto_register_ahash(cesa
->caps
->ahash_algs
[i
]);
211 goto err_unregister_ahash
;
216 err_unregister_ahash
:
217 for (j
= 0; j
< i
; j
++)
218 crypto_unregister_ahash(cesa
->caps
->ahash_algs
[j
]);
219 i
= cesa
->caps
->ncipher_algs
;
221 err_unregister_crypto
:
222 for (j
= 0; j
< i
; j
++)
223 crypto_unregister_alg(cesa
->caps
->cipher_algs
[j
]);
228 static void mv_cesa_remove_algs(struct mv_cesa_dev
*cesa
)
232 for (i
= 0; i
< cesa
->caps
->nahash_algs
; i
++)
233 crypto_unregister_ahash(cesa
->caps
->ahash_algs
[i
]);
235 for (i
= 0; i
< cesa
->caps
->ncipher_algs
; i
++)
236 crypto_unregister_alg(cesa
->caps
->cipher_algs
[i
]);
239 static struct crypto_alg
*orion_cipher_algs
[] = {
240 &mv_cesa_ecb_des_alg
,
241 &mv_cesa_cbc_des_alg
,
242 &mv_cesa_ecb_des3_ede_alg
,
243 &mv_cesa_cbc_des3_ede_alg
,
244 &mv_cesa_ecb_aes_alg
,
245 &mv_cesa_cbc_aes_alg
,
248 static struct ahash_alg
*orion_ahash_algs
[] = {
255 static struct crypto_alg
*armada_370_cipher_algs
[] = {
256 &mv_cesa_ecb_des_alg
,
257 &mv_cesa_cbc_des_alg
,
258 &mv_cesa_ecb_des3_ede_alg
,
259 &mv_cesa_cbc_des3_ede_alg
,
260 &mv_cesa_ecb_aes_alg
,
261 &mv_cesa_cbc_aes_alg
,
264 static struct ahash_alg
*armada_370_ahash_algs
[] = {
270 &mv_ahmac_sha256_alg
,
273 static const struct mv_cesa_caps orion_caps
= {
275 .cipher_algs
= orion_cipher_algs
,
276 .ncipher_algs
= ARRAY_SIZE(orion_cipher_algs
),
277 .ahash_algs
= orion_ahash_algs
,
278 .nahash_algs
= ARRAY_SIZE(orion_ahash_algs
),
282 static const struct mv_cesa_caps kirkwood_caps
= {
284 .cipher_algs
= orion_cipher_algs
,
285 .ncipher_algs
= ARRAY_SIZE(orion_cipher_algs
),
286 .ahash_algs
= orion_ahash_algs
,
287 .nahash_algs
= ARRAY_SIZE(orion_ahash_algs
),
291 static const struct mv_cesa_caps armada_370_caps
= {
293 .cipher_algs
= armada_370_cipher_algs
,
294 .ncipher_algs
= ARRAY_SIZE(armada_370_cipher_algs
),
295 .ahash_algs
= armada_370_ahash_algs
,
296 .nahash_algs
= ARRAY_SIZE(armada_370_ahash_algs
),
300 static const struct mv_cesa_caps armada_xp_caps
= {
302 .cipher_algs
= armada_370_cipher_algs
,
303 .ncipher_algs
= ARRAY_SIZE(armada_370_cipher_algs
),
304 .ahash_algs
= armada_370_ahash_algs
,
305 .nahash_algs
= ARRAY_SIZE(armada_370_ahash_algs
),
309 static const struct of_device_id mv_cesa_of_match_table
[] = {
310 { .compatible
= "marvell,orion-crypto", .data
= &orion_caps
},
311 { .compatible
= "marvell,kirkwood-crypto", .data
= &kirkwood_caps
},
312 { .compatible
= "marvell,dove-crypto", .data
= &kirkwood_caps
},
313 { .compatible
= "marvell,armada-370-crypto", .data
= &armada_370_caps
},
314 { .compatible
= "marvell,armada-xp-crypto", .data
= &armada_xp_caps
},
315 { .compatible
= "marvell,armada-375-crypto", .data
= &armada_xp_caps
},
316 { .compatible
= "marvell,armada-38x-crypto", .data
= &armada_xp_caps
},
319 MODULE_DEVICE_TABLE(of
, mv_cesa_of_match_table
);
322 mv_cesa_conf_mbus_windows(struct mv_cesa_engine
*engine
,
323 const struct mbus_dram_target_info
*dram
)
325 void __iomem
*iobase
= engine
->regs
;
328 for (i
= 0; i
< 4; i
++) {
329 writel(0, iobase
+ CESA_TDMA_WINDOW_CTRL(i
));
330 writel(0, iobase
+ CESA_TDMA_WINDOW_BASE(i
));
333 for (i
= 0; i
< dram
->num_cs
; i
++) {
334 const struct mbus_dram_window
*cs
= dram
->cs
+ i
;
336 writel(((cs
->size
- 1) & 0xffff0000) |
337 (cs
->mbus_attr
<< 8) |
338 (dram
->mbus_dram_target_id
<< 4) | 1,
339 iobase
+ CESA_TDMA_WINDOW_CTRL(i
));
340 writel(cs
->base
, iobase
+ CESA_TDMA_WINDOW_BASE(i
));
344 static int mv_cesa_dev_dma_init(struct mv_cesa_dev
*cesa
)
346 struct device
*dev
= cesa
->dev
;
347 struct mv_cesa_dev_dma
*dma
;
349 if (!cesa
->caps
->has_tdma
)
352 dma
= devm_kzalloc(dev
, sizeof(*dma
), GFP_KERNEL
);
356 dma
->tdma_desc_pool
= dmam_pool_create("tdma_desc", dev
,
357 sizeof(struct mv_cesa_tdma_desc
),
359 if (!dma
->tdma_desc_pool
)
362 dma
->op_pool
= dmam_pool_create("cesa_op", dev
,
363 sizeof(struct mv_cesa_op_ctx
), 16, 0);
367 dma
->cache_pool
= dmam_pool_create("cesa_cache", dev
,
368 CESA_MAX_HASH_BLOCK_SIZE
, 1, 0);
369 if (!dma
->cache_pool
)
372 dma
->padding_pool
= dmam_pool_create("cesa_padding", dev
, 72, 1, 0);
373 if (!dma
->padding_pool
)
381 static int mv_cesa_get_sram(struct platform_device
*pdev
, int idx
)
383 struct mv_cesa_dev
*cesa
= platform_get_drvdata(pdev
);
384 struct mv_cesa_engine
*engine
= &cesa
->engines
[idx
];
385 const char *res_name
= "sram";
386 struct resource
*res
;
388 engine
->pool
= of_gen_pool_get(cesa
->dev
->of_node
,
389 "marvell,crypto-srams", idx
);
391 engine
->sram
= gen_pool_dma_alloc(engine
->pool
,
401 if (cesa
->caps
->nengines
> 1) {
408 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
,
410 if (!res
|| resource_size(res
) < cesa
->sram_size
)
413 engine
->sram
= devm_ioremap_resource(cesa
->dev
, res
);
414 if (IS_ERR(engine
->sram
))
415 return PTR_ERR(engine
->sram
);
417 engine
->sram_dma
= phys_to_dma(cesa
->dev
,
418 (phys_addr_t
)res
->start
);
423 static void mv_cesa_put_sram(struct platform_device
*pdev
, int idx
)
425 struct mv_cesa_dev
*cesa
= platform_get_drvdata(pdev
);
426 struct mv_cesa_engine
*engine
= &cesa
->engines
[idx
];
431 gen_pool_free(engine
->pool
, (unsigned long)engine
->sram
,
435 static int mv_cesa_probe(struct platform_device
*pdev
)
437 const struct mv_cesa_caps
*caps
= &orion_caps
;
438 const struct mbus_dram_target_info
*dram
;
439 const struct of_device_id
*match
;
440 struct device
*dev
= &pdev
->dev
;
441 struct mv_cesa_dev
*cesa
;
442 struct mv_cesa_engine
*engines
;
443 struct resource
*res
;
448 dev_err(&pdev
->dev
, "Only one CESA device authorized\n");
453 match
= of_match_node(mv_cesa_of_match_table
, dev
->of_node
);
454 if (!match
|| !match
->data
)
460 if ((caps
== &orion_caps
|| caps
== &kirkwood_caps
) && !allhwsupport
)
463 cesa
= devm_kzalloc(dev
, sizeof(*cesa
), GFP_KERNEL
);
470 sram_size
= CESA_SA_DEFAULT_SRAM_SIZE
;
471 of_property_read_u32(cesa
->dev
->of_node
, "marvell,crypto-sram-size",
473 if (sram_size
< CESA_SA_MIN_SRAM_SIZE
)
474 sram_size
= CESA_SA_MIN_SRAM_SIZE
;
476 cesa
->sram_size
= sram_size
;
477 cesa
->engines
= devm_kzalloc(dev
, caps
->nengines
* sizeof(*engines
),
482 spin_lock_init(&cesa
->lock
);
484 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "regs");
485 cesa
->regs
= devm_ioremap_resource(dev
, res
);
486 if (IS_ERR(cesa
->regs
))
487 return PTR_ERR(cesa
->regs
);
489 ret
= mv_cesa_dev_dma_init(cesa
);
493 dram
= mv_mbus_dram_info_nooverlap();
495 platform_set_drvdata(pdev
, cesa
);
497 for (i
= 0; i
< caps
->nengines
; i
++) {
498 struct mv_cesa_engine
*engine
= &cesa
->engines
[i
];
502 spin_lock_init(&engine
->lock
);
504 ret
= mv_cesa_get_sram(pdev
, i
);
508 irq
= platform_get_irq(pdev
, i
);
515 * Not all platforms can gate the CESA clocks: do not complain
516 * if the clock does not exist.
518 snprintf(res_name
, sizeof(res_name
), "cesa%d", i
);
519 engine
->clk
= devm_clk_get(dev
, res_name
);
520 if (IS_ERR(engine
->clk
)) {
521 engine
->clk
= devm_clk_get(dev
, NULL
);
522 if (IS_ERR(engine
->clk
))
526 snprintf(res_name
, sizeof(res_name
), "cesaz%d", i
);
527 engine
->zclk
= devm_clk_get(dev
, res_name
);
528 if (IS_ERR(engine
->zclk
))
531 ret
= clk_prepare_enable(engine
->clk
);
535 ret
= clk_prepare_enable(engine
->zclk
);
539 engine
->regs
= cesa
->regs
+ CESA_ENGINE_OFF(i
);
541 if (dram
&& cesa
->caps
->has_tdma
)
542 mv_cesa_conf_mbus_windows(engine
, dram
);
544 writel(0, engine
->regs
+ CESA_SA_INT_STATUS
);
545 writel(CESA_SA_CFG_STOP_DIG_ERR
,
546 engine
->regs
+ CESA_SA_CFG
);
547 writel(engine
->sram_dma
& CESA_SA_SRAM_MSK
,
548 engine
->regs
+ CESA_SA_DESC_P0
);
550 ret
= devm_request_threaded_irq(dev
, irq
, NULL
, mv_cesa_int
,
552 dev_name(&pdev
->dev
),
557 crypto_init_queue(&engine
->queue
, CESA_CRYPTO_DEFAULT_MAX_QLEN
);
558 atomic_set(&engine
->load
, 0);
559 INIT_LIST_HEAD(&engine
->complete_queue
);
564 ret
= mv_cesa_add_algs(cesa
);
570 dev_info(dev
, "CESA device successfully registered\n");
575 for (i
= 0; i
< caps
->nengines
; i
++) {
576 clk_disable_unprepare(cesa
->engines
[i
].zclk
);
577 clk_disable_unprepare(cesa
->engines
[i
].clk
);
578 mv_cesa_put_sram(pdev
, i
);
584 static int mv_cesa_remove(struct platform_device
*pdev
)
586 struct mv_cesa_dev
*cesa
= platform_get_drvdata(pdev
);
589 mv_cesa_remove_algs(cesa
);
591 for (i
= 0; i
< cesa
->caps
->nengines
; i
++) {
592 clk_disable_unprepare(cesa
->engines
[i
].zclk
);
593 clk_disable_unprepare(cesa
->engines
[i
].clk
);
594 mv_cesa_put_sram(pdev
, i
);
600 static struct platform_driver marvell_cesa
= {
601 .probe
= mv_cesa_probe
,
602 .remove
= mv_cesa_remove
,
604 .name
= "marvell-cesa",
605 .of_match_table
= mv_cesa_of_match_table
,
608 module_platform_driver(marvell_cesa
);
610 MODULE_ALIAS("platform:mv_crypto");
611 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
612 MODULE_AUTHOR("Arnaud Ebalard <arno@natisbad.org>");
613 MODULE_DESCRIPTION("Support for Marvell's cryptographic engine");
614 MODULE_LICENSE("GPL v2");