]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/crypto/atmel-aes.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: atmel-aes - properly set IV after {en,de}crypt
[mirror_ubuntu-hirsute-kernel.git] / drivers / crypto / atmel-aes.c
CommitLineData
bd3c7b5c
NR		 1/*
2 * Cryptographic API.
3 *
4 * Support for ATMEL AES HW acceleration.
5 *
6 * Copyright (c) 2012 Eukréa Electromatique - ATMEL
7 * Author: Nicolas Royer <nicolas@eukrea.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as published
11 * by the Free Software Foundation.
12 *
13 * Some ideas are from omap-aes.c driver.
14 */
15
16
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/slab.h>
20#include <linux/err.h>
21#include <linux/clk.h>
22#include <linux/io.h>
23#include <linux/hw_random.h>
24#include <linux/platform_device.h>
25
26#include <linux/device.h>
bd3c7b5c
NR		 27#include <linux/init.h>
28#include <linux/errno.h>
29#include <linux/interrupt.h>
bd3c7b5c 30#include <linux/irq.h>
bd3c7b5c
NR		 31#include <linux/scatterlist.h>
32#include <linux/dma-mapping.h>
be943c7d 33#include <linux/of_device.h>
bd3c7b5c
NR		 34#include <linux/delay.h>
35#include <linux/crypto.h>
bd3c7b5c
NR		 36#include <crypto/scatterwalk.h>
37#include <crypto/algapi.h>
38#include <crypto/aes.h>
219d51c7 39#include <crypto/gcm.h>
d52db518 40#include <crypto/xts.h>
d4419548 41#include <crypto/internal/aead.h>
cadc4ab8 42#include <linux/platform_data/crypto-atmel.h>
be943c7d 43#include <dt-bindings/dma/at91.h>
bd3c7b5c 44#include "atmel-aes-regs.h"
89a82ef8 45#include "atmel-authenc.h"
bd3c7b5c 46
88efd9a9
CP		 47#define ATMEL_AES_PRIORITY 300
48
bbe628ed
CP		 49#define ATMEL_AES_BUFFER_ORDER 2
50#define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER)
51
bd3c7b5c
NR		 52#define CFB8_BLOCK_SIZE 1
53#define CFB16_BLOCK_SIZE 2
54#define CFB32_BLOCK_SIZE 4
55#define CFB64_BLOCK_SIZE 8
56
bbe628ed
CP		 57#define SIZE_IN_WORDS(x) ((x) >> 2)
58
bd3c7b5c 59/* AES flags */
d4419548 60/* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */
77dacf5f 61#define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC
d4419548 62#define AES_FLAGS_GTAGEN AES_MR_GTAGEN
77dacf5f
CP		 63#define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK)
64#define AES_FLAGS_ECB AES_MR_OPMOD_ECB
65#define AES_FLAGS_CBC AES_MR_OPMOD_CBC
66#define AES_FLAGS_OFB AES_MR_OPMOD_OFB
67#define AES_FLAGS_CFB128 (AES_MR_OPMOD_CFB | AES_MR_CFBS_128b)
68#define AES_FLAGS_CFB64 (AES_MR_OPMOD_CFB | AES_MR_CFBS_64b)
69#define AES_FLAGS_CFB32 (AES_MR_OPMOD_CFB | AES_MR_CFBS_32b)
70#define AES_FLAGS_CFB16 (AES_MR_OPMOD_CFB | AES_MR_CFBS_16b)
71#define AES_FLAGS_CFB8 (AES_MR_OPMOD_CFB | AES_MR_CFBS_8b)
72#define AES_FLAGS_CTR AES_MR_OPMOD_CTR
d4419548 73#define AES_FLAGS_GCM AES_MR_OPMOD_GCM
d52db518 74#define AES_FLAGS_XTS AES_MR_OPMOD_XTS
77dacf5f
CP		 75
76#define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \
d4419548
CP		 77 AES_FLAGS_ENCRYPT | \
78 AES_FLAGS_GTAGEN)
77dacf5f
CP		 79
80#define AES_FLAGS_INIT BIT(2)
81#define AES_FLAGS_BUSY BIT(3)
4537992b 82#define AES_FLAGS_DUMP_REG BIT(4)
89a82ef8 83#define AES_FLAGS_OWN_SHA BIT(5)
77dacf5f
CP		 84
85#define AES_FLAGS_PERSISTENT (AES_FLAGS_INIT | AES_FLAGS_BUSY)
bd3c7b5c 86
cadc4ab8 87#define ATMEL_AES_QUEUE_LENGTH 50
bd3c7b5c 88
129f8bb6 89#define ATMEL_AES_DMA_THRESHOLD 256
bd3c7b5c
NR		 90
91
cadc4ab8 92struct atmel_aes_caps {
afbac17e
CP		 93 bool has_dualbuff;
94 bool has_cfb64;
fcac8365 95 bool has_ctr32;
d4419548 96 bool has_gcm;
d52db518 97 bool has_xts;
89a82ef8 98 bool has_authenc;
afbac17e 99 u32 max_burst_size;
cadc4ab8
NR		 100};
101
bd3c7b5c
NR		 102struct atmel_aes_dev;
103
ccbf7298
CP		 104
105typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *);
106
107
108struct atmel_aes_base_ctx {
afbac17e
CP		 109 struct atmel_aes_dev *dd;
110 atmel_aes_fn_t start;
111 int keylen;
112 u32 key[AES_KEYSIZE_256 / sizeof(u32)];
113 u16 block_size;
91308019 114 bool is_aead;
bd3c7b5c
NR		 115};
116
ccbf7298
CP		 117struct atmel_aes_ctx {
118 struct atmel_aes_base_ctx base;
119};
120
fcac8365
CP		 121struct atmel_aes_ctr_ctx {
122 struct atmel_aes_base_ctx base;
123
124 u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
125 size_t offset;
126 struct scatterlist src[2];
127 struct scatterlist dst[2];
128};
129
d4419548
CP		 130struct atmel_aes_gcm_ctx {
131 struct atmel_aes_base_ctx base;
132
133 struct scatterlist src[2];
134 struct scatterlist dst[2];
135
136 u32 j0[AES_BLOCK_SIZE / sizeof(u32)];
137 u32 tag[AES_BLOCK_SIZE / sizeof(u32)];
138 u32 ghash[AES_BLOCK_SIZE / sizeof(u32)];
139 size_t textlen;
140
141 const u32 *ghash_in;
142 u32 *ghash_out;
143 atmel_aes_fn_t ghash_resume;
144};
145
d52db518
CP		 146struct atmel_aes_xts_ctx {
147 struct atmel_aes_base_ctx base;
148
149 u32 key2[AES_KEYSIZE_256 / sizeof(u32)];
150};
151
89a82ef8
CP		 152#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
153struct atmel_aes_authenc_ctx {
154 struct atmel_aes_base_ctx base;
155 struct atmel_sha_authenc_ctx *auth;
156};
157#endif
158
bd3c7b5c 159struct atmel_aes_reqctx {
afbac17e 160 unsigned long mode;
91308019 161 u32 lastc[AES_BLOCK_SIZE / sizeof(u32)];
bd3c7b5c
NR		 162};
163
89a82ef8
CP		 164#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
165struct atmel_aes_authenc_reqctx {
166 struct atmel_aes_reqctx base;
167
168 struct scatterlist src[2];
169 struct scatterlist dst[2];
170 size_t textlen;
171 u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)];
172
173 /* auth_req MUST be place last. */
174 struct ahash_request auth_req;
175};
176#endif
177
bd3c7b5c 178struct atmel_aes_dma {
bbe628ed
CP		 179 struct dma_chan *chan;
180 struct scatterlist *sg;
181 int nents;
182 unsigned int remainder;
183 unsigned int sg_len;
bd3c7b5c
NR		 184};
185
186struct atmel_aes_dev {
187 struct list_head list;
188 unsigned long phys_base;
189 void __iomem *io_base;
190
ccbf7298
CP		 191 struct crypto_async_request *areq;
192 struct atmel_aes_base_ctx *ctx;
193
10f12c1b
CP		 194 bool is_async;
195 atmel_aes_fn_t resume;
bbe628ed 196 atmel_aes_fn_t cpu_transfer_complete;
10f12c1b 197
bd3c7b5c
NR		 198 struct device *dev;
199 struct clk *iclk;
afbac17e 200 int irq;
bd3c7b5c
NR		 201
202 unsigned long flags;
bd3c7b5c
NR		 203
204 spinlock_t lock;
205 struct crypto_queue queue;
206
207 struct tasklet_struct done_task;
208 struct tasklet_struct queue_task;
209
bbe628ed
CP		 210 size_t total;
211 size_t datalen;
212 u32 *data;
bd3c7b5c 213
bbe628ed
CP		 214 struct atmel_aes_dma src;
215 struct atmel_aes_dma dst;
bd3c7b5c 216
bbe628ed
CP		 217 size_t buflen;
218 void *buf;
219 struct scatterlist aligned_sg;
220 struct scatterlist *real_dst;
bd3c7b5c 221
cadc4ab8
NR		 222 struct atmel_aes_caps caps;
223
afbac17e 224 u32 hw_version;
bd3c7b5c
NR		 225};
226
227struct atmel_aes_drv {
228 struct list_head dev_list;
229 spinlock_t lock;
230};
231
232static struct atmel_aes_drv atmel_aes = {
233 .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list),
234 .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock),
235};
236
4537992b
CP		 237#ifdef VERBOSE_DEBUG
238static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz)
239{
240 switch (offset) {
241 case AES_CR:
242 return "CR";
243
244 case AES_MR:
245 return "MR";
246
247 case AES_ISR:
248 return "ISR";
249
250 case AES_IMR:
251 return "IMR";
252
253 case AES_IER:
254 return "IER";
255
256 case AES_IDR:
257 return "IDR";
258
259 case AES_KEYWR(0):
260 case AES_KEYWR(1):
261 case AES_KEYWR(2):
262 case AES_KEYWR(3):
263 case AES_KEYWR(4):
264 case AES_KEYWR(5):
265 case AES_KEYWR(6):
266 case AES_KEYWR(7):
267 snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2);
268 break;
269
270 case AES_IDATAR(0):
271 case AES_IDATAR(1):
272 case AES_IDATAR(2):
273 case AES_IDATAR(3):
274 snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2);
275 break;
276
277 case AES_ODATAR(0):
278 case AES_ODATAR(1):
279 case AES_ODATAR(2):
280 case AES_ODATAR(3):
281 snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2);
282 break;
283
284 case AES_IVR(0):
285 case AES_IVR(1):
286 case AES_IVR(2):
287 case AES_IVR(3):
288 snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2);
289 break;
290
291 case AES_AADLENR:
292 return "AADLENR";
293
294 case AES_CLENR:
295 return "CLENR";
296
297 case AES_GHASHR(0):
298 case AES_GHASHR(1):
299 case AES_GHASHR(2):
300 case AES_GHASHR(3):
301 snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2);
302 break;
303
304 case AES_TAGR(0):
305 case AES_TAGR(1):
306 case AES_TAGR(2):
307 case AES_TAGR(3):
308 snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2);
309 break;
310
311 case AES_CTRR:
312 return "CTRR";
313
314 case AES_GCMHR(0):
315 case AES_GCMHR(1):
316 case AES_GCMHR(2):
317 case AES_GCMHR(3):
318 snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2);
e31835ad 319 break;
4537992b 320
89a82ef8
CP		 321 case AES_EMR:
322 return "EMR";
323
d52db518
CP		 324 case AES_TWR(0):
325 case AES_TWR(1):
326 case AES_TWR(2):
327 case AES_TWR(3):
328 snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2);
329 break;
330
331 case AES_ALPHAR(0):
332 case AES_ALPHAR(1):
333 case AES_ALPHAR(2):
334 case AES_ALPHAR(3):
335 snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2);
336 break;
337
4537992b
CP		 338 default:
339 snprintf(tmp, sz, "0x%02x", offset);
340 break;
341 }
342
343 return tmp;
344}
345#endif /* VERBOSE_DEBUG */
346
e37a7e55 347/* Shared functions */
cadc4ab8 348
bd3c7b5c
NR		 349static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset)
350{
4537992b
CP		 351 u32 value = readl_relaxed(dd->io_base + offset);
352
353#ifdef VERBOSE_DEBUG
354 if (dd->flags & AES_FLAGS_DUMP_REG) {
355 char tmp[16];
356
357 dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,
358 atmel_aes_reg_name(offset, tmp, sizeof(tmp)));
359 }
360#endif /* VERBOSE_DEBUG */
361
362 return value;
bd3c7b5c
NR		 363}
364
365static inline void atmel_aes_write(struct atmel_aes_dev *dd,
366 u32 offset, u32 value)
367{
4537992b
CP		 368#ifdef VERBOSE_DEBUG
369 if (dd->flags & AES_FLAGS_DUMP_REG) {
370 char tmp[16];
371
372 dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,
f709dc86 373 atmel_aes_reg_name(offset, tmp, sizeof(tmp)));
4537992b
CP		 374 }
375#endif /* VERBOSE_DEBUG */
376
bd3c7b5c
NR		 377 writel_relaxed(value, dd->io_base + offset);
378}
379
380static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset,
381 u32 *value, int count)
382{
383 for (; count--; value++, offset += 4)
384 *value = atmel_aes_read(dd, offset);
385}
386
387static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset,
c0b28d8c 388 const u32 *value, int count)
bd3c7b5c
NR		 389{
390 for (; count--; value++, offset += 4)
391 atmel_aes_write(dd, offset, *value);
392}
393
bbe628ed
CP		 394static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset,
395 u32 *value)
396{
397 atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));
398}
399
400static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset,
401 const u32 *value)
402{
403 atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));
404}
405
406static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd,
407 atmel_aes_fn_t resume)
408{
409 u32 isr = atmel_aes_read(dd, AES_ISR);
410
411 if (unlikely(isr & AES_INT_DATARDY))
412 return resume(dd);
413
414 dd->resume = resume;
415 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
416 return -EINPROGRESS;
417}
418
419static inline size_t atmel_aes_padlen(size_t len, size_t block_size)
420{
421 len &= block_size - 1;
422 return len ? block_size - len : 0;
423}
424
ccbf7298 425static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_base_ctx *ctx)
bd3c7b5c
NR		 426{
427 struct atmel_aes_dev *aes_dd = NULL;
428 struct atmel_aes_dev *tmp;
429
430 spin_lock_bh(&atmel_aes.lock);
431 if (!ctx->dd) {
432 list_for_each_entry(tmp, &atmel_aes.dev_list, list) {
433 aes_dd = tmp;
434 break;
435 }
436 ctx->dd = aes_dd;
437 } else {
438 aes_dd = ctx->dd;
439 }
440
441 spin_unlock_bh(&atmel_aes.lock);
442
443 return aes_dd;
444}
445
446static int atmel_aes_hw_init(struct atmel_aes_dev *dd)
447{
9d83d299
LC		 448 int err;
449
49a20454 450 err = clk_enable(dd->iclk);
9d83d299
LC		 451 if (err)
452 return err;
bd3c7b5c
NR		 453
454 if (!(dd->flags & AES_FLAGS_INIT)) {
455 atmel_aes_write(dd, AES_CR, AES_CR_SWRST);
cadc4ab8 456 atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET);
bd3c7b5c 457 dd->flags |= AES_FLAGS_INIT;
bd3c7b5c
NR		 458 }
459
460 return 0;
461}
462
cadc4ab8
NR		 463static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd)
464{
465 return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff;
466}
467
aab0a39b 468static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd)
bd3c7b5c 469{
aab0a39b
CP		 470 int err;
471
472 err = atmel_aes_hw_init(dd);
473 if (err)
474 return err;
bd3c7b5c 475
cadc4ab8
NR		 476 dd->hw_version = atmel_aes_get_version(dd);
477
aab0a39b 478 dev_info(dd->dev, "version: 0x%x\n", dd->hw_version);
bd3c7b5c 479
49a20454 480 clk_disable(dd->iclk);
aab0a39b 481 return 0;
bd3c7b5c
NR		 482}
483
77dacf5f
CP		 484static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd,
485 const struct atmel_aes_reqctx *rctx)
486{
487 /* Clear all but persistent flags and set request flags. */
488 dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode;
489}
490
d4419548
CP		 491static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd)
492{
493 return (dd->flags & AES_FLAGS_ENCRYPT);
494}
495
89a82ef8
CP		 496#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
497static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err);
498#endif
499
10f12c1b 500static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)
bd3c7b5c 501{
89a82ef8 502#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
91308019
RI		 503 if (dd->ctx->is_aead)
504 atmel_aes_authenc_complete(dd, err);
89a82ef8
CP		 505#endif
506
49a20454 507 clk_disable(dd->iclk);
bd3c7b5c
NR		 508 dd->flags &= ~AES_FLAGS_BUSY;
509
91308019
RI		 510 if (!dd->ctx->is_aead) {
511 struct ablkcipher_request *req =
512 ablkcipher_request_cast(dd->areq);
513 struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req);
514 struct crypto_ablkcipher *ablkcipher =
515 crypto_ablkcipher_reqtfm(req);
516 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
517
518 if (rctx->mode & AES_FLAGS_ENCRYPT) {
519 scatterwalk_map_and_copy(req->info, req->dst,
520 req->nbytes - ivsize, ivsize, 0);
521 } else {
522 if (req->src == req->dst) {
523 memcpy(req->info, rctx->lastc, ivsize);
524 } else {
525 scatterwalk_map_and_copy(req->info, req->src,
526 req->nbytes - ivsize, ivsize, 0);
527 }
528 }
529 }
530
10f12c1b
CP		 531 if (dd->is_async)
532 dd->areq->complete(dd->areq, err);
533
534 tasklet_schedule(&dd->queue_task);
535
536 return err;
bd3c7b5c
NR		 537}
538
d52db518
CP		 539static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma,
540 const u32 *iv, const u32 *key, int keylen)
e37a7e55
CP		 541{
542 u32 valmr = 0;
543
544 /* MR register must be set before IV registers */
d52db518 545 if (keylen == AES_KEYSIZE_128)
e37a7e55 546 valmr |= AES_MR_KEYSIZE_128;
d52db518 547 else if (keylen == AES_KEYSIZE_192)
e37a7e55
CP		 548 valmr |= AES_MR_KEYSIZE_192;
549 else
550 valmr |= AES_MR_KEYSIZE_256;
551
552 valmr |= dd->flags & AES_FLAGS_MODE_MASK;
553
554 if (use_dma) {
555 valmr |= AES_MR_SMOD_IDATAR0;
556 if (dd->caps.has_dualbuff)
557 valmr |= AES_MR_DUALBUFF;
558 } else {
559 valmr |= AES_MR_SMOD_AUTO;
560 }
561
562 atmel_aes_write(dd, AES_MR, valmr);
563
d52db518 564 atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen));
e37a7e55
CP		 565
566 if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB)
567 atmel_aes_write_block(dd, AES_IVR(0), iv);
568}
569
d52db518
CP		 570static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,
571 const u32 *iv)
572
573{
574 atmel_aes_write_ctrl_key(dd, use_dma, iv,
575 dd->ctx->key, dd->ctx->keylen);
576}
bbe628ed
CP		 577
578/* CPU transfer */
579
580static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd)
bd3c7b5c 581{
bbe628ed
CP		 582 int err = 0;
583 u32 isr;
bd3c7b5c 584
bbe628ed
CP		 585 for (;;) {
586 atmel_aes_read_block(dd, AES_ODATAR(0), dd->data);
587 dd->data += 4;
588 dd->datalen -= AES_BLOCK_SIZE;
589
590 if (dd->datalen < AES_BLOCK_SIZE)
591 break;
592
593 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
594
595 isr = atmel_aes_read(dd, AES_ISR);
596 if (!(isr & AES_INT_DATARDY)) {
597 dd->resume = atmel_aes_cpu_transfer;
598 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
599 return -EINPROGRESS;
600 }
601 }
602
603 if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),
604 dd->buf, dd->total))
605 err = -EINVAL;
606
607 if (err)
608 return atmel_aes_complete(dd, err);
609
610 return dd->cpu_transfer_complete(dd);
bd3c7b5c
NR		 611}
612
bbe628ed
CP		 613static int atmel_aes_cpu_start(struct atmel_aes_dev *dd,
614 struct scatterlist *src,
615 struct scatterlist *dst,
616 size_t len,
617 atmel_aes_fn_t resume)
bd3c7b5c 618{
bbe628ed 619 size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE);
77dacf5f 620
bbe628ed
CP		 621 if (unlikely(len == 0))
622 return -EINVAL;
77dacf5f 623
bbe628ed 624 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);
77dacf5f 625
bbe628ed
CP		 626 dd->total = len;
627 dd->real_dst = dst;
628 dd->cpu_transfer_complete = resume;
629 dd->datalen = len + padlen;
630 dd->data = (u32 *)dd->buf;
631 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
632 return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer);
633}
77dacf5f 634
77dacf5f 635
bbe628ed
CP		 636/* DMA transfer */
637
638static void atmel_aes_dma_callback(void *data);
639
640static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd,
641 struct scatterlist *sg,
642 size_t len,
643 struct atmel_aes_dma *dma)
644{
645 int nents;
646
647 if (!IS_ALIGNED(len, dd->ctx->block_size))
648 return false;
649
650 for (nents = 0; sg; sg = sg_next(sg), ++nents) {
651 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
652 return false;
653
654 if (len <= sg->length) {
655 if (!IS_ALIGNED(len, dd->ctx->block_size))
656 return false;
657
658 dma->nents = nents+1;
659 dma->remainder = sg->length - len;
660 sg->length = len;
661 return true;
662 }
663
664 if (!IS_ALIGNED(sg->length, dd->ctx->block_size))
665 return false;
666
667 len -= sg->length;
77dacf5f 668 }
bd3c7b5c 669
bbe628ed
CP		 670 return false;
671}
bd3c7b5c 672
bbe628ed
CP		 673static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma)
674{
675 struct scatterlist *sg = dma->sg;
676 int nents = dma->nents;
bd3c7b5c 677
bbe628ed
CP		 678 if (!dma->remainder)
679 return;
bd3c7b5c 680
bbe628ed
CP		 681 while (--nents > 0 && sg)
682 sg = sg_next(sg);
bd3c7b5c 683
bbe628ed
CP		 684 if (!sg)
685 return;
bd3c7b5c 686
bbe628ed
CP		 687 sg->length += dma->remainder;
688}
bd3c7b5c 689
bbe628ed
CP		 690static int atmel_aes_map(struct atmel_aes_dev *dd,
691 struct scatterlist *src,
692 struct scatterlist *dst,
693 size_t len)
694{
695 bool src_aligned, dst_aligned;
696 size_t padlen;
cadc4ab8 697
bbe628ed
CP		 698 dd->total = len;
699 dd->src.sg = src;
700 dd->dst.sg = dst;
701 dd->real_dst = dst;
bd3c7b5c 702
bbe628ed
CP		 703 src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src);
704 if (src == dst)
705 dst_aligned = src_aligned;
706 else
707 dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst);
708 if (!src_aligned || !dst_aligned) {
709 padlen = atmel_aes_padlen(len, dd->ctx->block_size);
710
711 if (dd->buflen < len + padlen)
712 return -ENOMEM;
713
714 if (!src_aligned) {
715 sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);
716 dd->src.sg = &dd->aligned_sg;
717 dd->src.nents = 1;
718 dd->src.remainder = 0;
719 }
bd3c7b5c 720
bbe628ed
CP		 721 if (!dst_aligned) {
722 dd->dst.sg = &dd->aligned_sg;
723 dd->dst.nents = 1;
724 dd->dst.remainder = 0;
725 }
bd3c7b5c 726
bbe628ed
CP		 727 sg_init_table(&dd->aligned_sg, 1);
728 sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen);
729 }
bd3c7b5c 730
bbe628ed
CP		 731 if (dd->src.sg == dd->dst.sg) {
732 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,
733 DMA_BIDIRECTIONAL);
734 dd->dst.sg_len = dd->src.sg_len;
735 if (!dd->src.sg_len)
736 return -EFAULT;
737 } else {
738 dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,
739 DMA_TO_DEVICE);
740 if (!dd->src.sg_len)
741 return -EFAULT;
742
743 dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents,
744 DMA_FROM_DEVICE);
745 if (!dd->dst.sg_len) {
746 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
747 DMA_TO_DEVICE);
748 return -EFAULT;
749 }
750 }
bd3c7b5c
NR		 751
752 return 0;
bd3c7b5c
NR		 753}
754
bbe628ed 755static void atmel_aes_unmap(struct atmel_aes_dev *dd)
bd3c7b5c 756{
bbe628ed
CP		 757 if (dd->src.sg == dd->dst.sg) {
758 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
759 DMA_BIDIRECTIONAL);
ccbf7298 760
bbe628ed
CP		 761 if (dd->src.sg != &dd->aligned_sg)
762 atmel_aes_restore_sg(&dd->src);
763 } else {
764 dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents,
765 DMA_FROM_DEVICE);
289b2623 766
bbe628ed
CP		 767 if (dd->dst.sg != &dd->aligned_sg)
768 atmel_aes_restore_sg(&dd->dst);
bd3c7b5c 769
bbe628ed
CP		 770 dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
771 DMA_TO_DEVICE);
772
773 if (dd->src.sg != &dd->aligned_sg)
774 atmel_aes_restore_sg(&dd->src);
775 }
776
777 if (dd->dst.sg == &dd->aligned_sg)
778 sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),
779 dd->buf, dd->total);
780}
bd3c7b5c 781
bbe628ed
CP		 782static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,
783 enum dma_slave_buswidth addr_width,
784 enum dma_transfer_direction dir,
785 u32 maxburst)
786{
787 struct dma_async_tx_descriptor *desc;
788 struct dma_slave_config config;
789 dma_async_tx_callback callback;
790 struct atmel_aes_dma *dma;
791 int err;
792
793 memset(&config, 0, sizeof(config));
794 config.direction = dir;
795 config.src_addr_width = addr_width;
796 config.dst_addr_width = addr_width;
797 config.src_maxburst = maxburst;
798 config.dst_maxburst = maxburst;
799
800 switch (dir) {
801 case DMA_MEM_TO_DEV:
802 dma = &dd->src;
803 callback = NULL;
804 config.dst_addr = dd->phys_base + AES_IDATAR(0);
805 break;
bd3c7b5c 806
bbe628ed
CP		 807 case DMA_DEV_TO_MEM:
808 dma = &dd->dst;
809 callback = atmel_aes_dma_callback;
810 config.src_addr = dd->phys_base + AES_ODATAR(0);
811 break;
812
813 default:
bd3c7b5c 814 return -EINVAL;
bbe628ed 815 }
bd3c7b5c 816
bbe628ed
CP		 817 err = dmaengine_slave_config(dma->chan, &config);
818 if (err)
819 return err;
bd3c7b5c 820
bbe628ed
CP		 821 desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir,
822 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
823 if (!desc)
824 return -ENOMEM;
bd3c7b5c 825
bbe628ed
CP		 826 desc->callback = callback;
827 desc->callback_param = dd;
828 dmaengine_submit(desc);
829 dma_async_issue_pending(dma->chan);
bd3c7b5c 830
bbe628ed
CP		 831 return 0;
832}
10f12c1b 833
bbe628ed
CP		 834static void atmel_aes_dma_transfer_stop(struct atmel_aes_dev *dd,
835 enum dma_transfer_direction dir)
bd3c7b5c 836{
bbe628ed 837 struct atmel_aes_dma *dma;
cadc4ab8 838
bbe628ed
CP		 839 switch (dir) {
840 case DMA_MEM_TO_DEV:
841 dma = &dd->src;
842 break;
843
844 case DMA_DEV_TO_MEM:
845 dma = &dd->dst;
846 break;
cadc4ab8 847
bbe628ed
CP		 848 default:
849 return;
cadc4ab8
NR		 850 }
851
bbe628ed
CP		 852 dmaengine_terminate_all(dma->chan);
853}
cadc4ab8 854
bbe628ed
CP		 855static int atmel_aes_dma_start(struct atmel_aes_dev *dd,
856 struct scatterlist *src,
857 struct scatterlist *dst,
858 size_t len,
859 atmel_aes_fn_t resume)
860{
861 enum dma_slave_buswidth addr_width;
862 u32 maxburst;
863 int err;
cadc4ab8 864
bbe628ed
CP		 865 switch (dd->ctx->block_size) {
866 case CFB8_BLOCK_SIZE:
867 addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
868 maxburst = 1;
869 break;
cadc4ab8 870
bbe628ed
CP		 871 case CFB16_BLOCK_SIZE:
872 addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
873 maxburst = 1;
874 break;
cadc4ab8 875
bbe628ed
CP		 876 case CFB32_BLOCK_SIZE:
877 case CFB64_BLOCK_SIZE:
878 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
879 maxburst = 1;
880 break;
cadc4ab8 881
bbe628ed
CP		 882 case AES_BLOCK_SIZE:
883 addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
884 maxburst = dd->caps.max_burst_size;
885 break;
bd3c7b5c 886
bbe628ed
CP		 887 default:
888 err = -EINVAL;
889 goto exit;
890 }
289b2623 891
bbe628ed
CP		 892 err = atmel_aes_map(dd, src, dst, len);
893 if (err)
894 goto exit;
cadc4ab8 895
bbe628ed 896 dd->resume = resume;
cadc4ab8 897
bbe628ed
CP		 898 /* Set output DMA transfer first */
899 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM,
900 maxburst);
901 if (err)
902 goto unmap;
bd3c7b5c 903
bbe628ed
CP		 904 /* Then set input DMA transfer */
905 err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV,
906 maxburst);
907 if (err)
908 goto output_transfer_stop;
bd3c7b5c 909
bbe628ed 910 return -EINPROGRESS;
cadc4ab8 911
bbe628ed
CP		 912output_transfer_stop:
913 atmel_aes_dma_transfer_stop(dd, DMA_DEV_TO_MEM);
914unmap:
915 atmel_aes_unmap(dd);
916exit:
917 return atmel_aes_complete(dd, err);
918}
bd3c7b5c 919
bbe628ed
CP		 920static void atmel_aes_dma_stop(struct atmel_aes_dev *dd)
921{
922 atmel_aes_dma_transfer_stop(dd, DMA_MEM_TO_DEV);
923 atmel_aes_dma_transfer_stop(dd, DMA_DEV_TO_MEM);
924 atmel_aes_unmap(dd);
925}
926
927static void atmel_aes_dma_callback(void *data)
928{
929 struct atmel_aes_dev *dd = data;
930
931 atmel_aes_dma_stop(dd);
932 dd->is_async = true;
933 (void)dd->resume(dd);
bd3c7b5c
NR		 934}
935
bd3c7b5c 936static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
ccbf7298 937 struct crypto_async_request *new_areq)
bd3c7b5c 938{
ccbf7298
CP		 939 struct crypto_async_request *areq, *backlog;
940 struct atmel_aes_base_ctx *ctx;
bd3c7b5c 941 unsigned long flags;
a1f613f1 942 bool start_async;
bd3c7b5c
NR		 943 int err, ret = 0;
944
945 spin_lock_irqsave(&dd->lock, flags);
ccbf7298
CP		 946 if (new_areq)
947 ret = crypto_enqueue_request(&dd->queue, new_areq);
bd3c7b5c
NR		 948 if (dd->flags & AES_FLAGS_BUSY) {
949 spin_unlock_irqrestore(&dd->lock, flags);
950 return ret;
951 }
952 backlog = crypto_get_backlog(&dd->queue);
ccbf7298
CP		 953 areq = crypto_dequeue_request(&dd->queue);
954 if (areq)
bd3c7b5c
NR		 955 dd->flags |= AES_FLAGS_BUSY;
956 spin_unlock_irqrestore(&dd->lock, flags);
957
ccbf7298 958 if (!areq)
bd3c7b5c
NR		 959 return ret;
960
961 if (backlog)
962 backlog->complete(backlog, -EINPROGRESS);
963
ccbf7298
CP		 964 ctx = crypto_tfm_ctx(areq->tfm);
965
966 dd->areq = areq;
967 dd->ctx = ctx;
a1f613f1
CP		 968 start_async = (areq != new_areq);
969 dd->is_async = start_async;
ccbf7298 970
a1f613f1 971 /* WARNING: ctx->start() MAY change dd->is_async. */
ccbf7298 972 err = ctx->start(dd);
a1f613f1 973 return (start_async) ? ret : err;
ccbf7298
CP		 974}
975
e37a7e55
CP		 976
977/* AES async block ciphers */
978
bbe628ed
CP		 979static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd)
980{
981 return atmel_aes_complete(dd, 0);
982}
983
ccbf7298
CP		 984static int atmel_aes_start(struct atmel_aes_dev *dd)
985{
986 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);
bbe628ed
CP		 987 struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req);
988 bool use_dma = (req->nbytes >= ATMEL_AES_DMA_THRESHOLD ||
989 dd->ctx->block_size != AES_BLOCK_SIZE);
ccbf7298 990 int err;
bd3c7b5c 991
77dacf5f 992 atmel_aes_set_mode(dd, rctx);
bd3c7b5c 993
cdfab4a7 994 err = atmel_aes_hw_init(dd);
bbe628ed 995 if (err)
10f12c1b 996 return atmel_aes_complete(dd, err);
bd3c7b5c 997
bbe628ed
CP		 998 atmel_aes_write_ctrl(dd, use_dma, req->info);
999 if (use_dma)
1000 return atmel_aes_dma_start(dd, req->src, req->dst, req->nbytes,
1001 atmel_aes_transfer_complete);
bd3c7b5c 1002
bbe628ed
CP		 1003 return atmel_aes_cpu_start(dd, req->src, req->dst, req->nbytes,
1004 atmel_aes_transfer_complete);
bd3c7b5c
NR		 1005}
1006
fcac8365
CP		 1007static inline struct atmel_aes_ctr_ctx *
1008atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx)
1009{
1010 return container_of(ctx, struct atmel_aes_ctr_ctx, base);
1011}
1012
1013static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd)
1014{
1015 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);
1016 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);
1017 struct scatterlist *src, *dst;
1018 u32 ctr, blocks;
1019 size_t datalen;
1020 bool use_dma, fragmented = false;
1021
1022 /* Check for transfer completion. */
1023 ctx->offset += dd->total;
1024 if (ctx->offset >= req->nbytes)
1025 return atmel_aes_transfer_complete(dd);
1026
1027 /* Compute data length. */
1028 datalen = req->nbytes - ctx->offset;
1029 blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE);
1030 ctr = be32_to_cpu(ctx->iv[3]);
1031 if (dd->caps.has_ctr32) {
1032 /* Check 32bit counter overflow. */
1033 u32 start = ctr;
1034 u32 end = start + blocks - 1;
1035
1036 if (end < start) {
1037 ctr |= 0xffffffff;
1038 datalen = AES_BLOCK_SIZE * -start;
1039 fragmented = true;
1040 }
1041 } else {
1042 /* Check 16bit counter overflow. */
1043 u16 start = ctr & 0xffff;
1044 u16 end = start + (u16)blocks - 1;
1045
1046 if (blocks >> 16 || end < start) {
1047 ctr |= 0xffff;
1048 datalen = AES_BLOCK_SIZE * (0x10000-start);
1049 fragmented = true;
1050 }
1051 }
1052 use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD);
1053
1054 /* Jump to offset. */
1055 src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset);
1056 dst = ((req->src == req->dst) ? src :
1057 scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset));
1058
1059 /* Configure hardware. */
1060 atmel_aes_write_ctrl(dd, use_dma, ctx->iv);
1061 if (unlikely(fragmented)) {
1062 /*
1063 * Increment the counter manually to cope with the hardware
1064 * counter overflow.
1065 */
1066 ctx->iv[3] = cpu_to_be32(ctr);
1067 crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE);
1068 }
1069
1070 if (use_dma)
1071 return atmel_aes_dma_start(dd, src, dst, datalen,
1072 atmel_aes_ctr_transfer);
1073
1074 return atmel_aes_cpu_start(dd, src, dst, datalen,
1075 atmel_aes_ctr_transfer);
1076}
1077
1078static int atmel_aes_ctr_start(struct atmel_aes_dev *dd)
1079{
1080 struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);
1081 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);
1082 struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req);
1083 int err;
1084
1085 atmel_aes_set_mode(dd, rctx);
1086
1087 err = atmel_aes_hw_init(dd);
1088 if (err)
1089 return atmel_aes_complete(dd, err);
1090
1091 memcpy(ctx->iv, req->info, AES_BLOCK_SIZE);
1092 ctx->offset = 0;
1093 dd->total = 0;
1094 return atmel_aes_ctr_transfer(dd);
1095}
1096
bd3c7b5c
NR		 1097static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
1098{
91308019
RI		 1099 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1100 struct atmel_aes_base_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
afbac17e 1101 struct atmel_aes_reqctx *rctx;
bd3c7b5c
NR		 1102 struct atmel_aes_dev *dd;
1103
77dacf5f
CP		 1104 switch (mode & AES_FLAGS_OPMODE_MASK) {
1105 case AES_FLAGS_CFB8:
cadc4ab8 1106 ctx->block_size = CFB8_BLOCK_SIZE;
77dacf5f
CP		 1107 break;
1108
1109 case AES_FLAGS_CFB16:
cadc4ab8 1110 ctx->block_size = CFB16_BLOCK_SIZE;
77dacf5f
CP		 1111 break;
1112
1113 case AES_FLAGS_CFB32:
cadc4ab8 1114 ctx->block_size = CFB32_BLOCK_SIZE;
77dacf5f
CP		 1115 break;
1116
1117 case AES_FLAGS_CFB64:
9f84951f 1118 ctx->block_size = CFB64_BLOCK_SIZE;
77dacf5f
CP		 1119 break;
1120
1121 default:
cadc4ab8 1122 ctx->block_size = AES_BLOCK_SIZE;
77dacf5f 1123 break;
bd3c7b5c 1124 }
91308019 1125 ctx->is_aead = false;
bd3c7b5c
NR		 1126
1127 dd = atmel_aes_find_dev(ctx);
1128 if (!dd)
1129 return -ENODEV;
1130
afbac17e 1131 rctx = ablkcipher_request_ctx(req);
bd3c7b5c
NR		 1132 rctx->mode = mode;
1133
91308019
RI		 1134 if (!(mode & AES_FLAGS_ENCRYPT) && (req->src == req->dst)) {
1135 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1136
1137 scatterwalk_map_and_copy(rctx->lastc, req->src,
1138 (req->nbytes - ivsize), ivsize, 0);
1139 }
1140
ccbf7298 1141 return atmel_aes_handle_queue(dd, &req->base);
bd3c7b5c
NR		 1142}
1143
bd3c7b5c
NR		 1144static int atmel_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
1145 unsigned int keylen)
1146{
ccbf7298 1147 struct atmel_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm);
bd3c7b5c 1148
afbac17e
CP		 1149 if (keylen != AES_KEYSIZE_128 &&
1150 keylen != AES_KEYSIZE_192 &&
1151 keylen != AES_KEYSIZE_256) {
bd3c7b5c
NR		 1152 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1153 return -EINVAL;
1154 }
1155
1156 memcpy(ctx->key, key, keylen);
1157 ctx->keylen = keylen;
1158
1159 return 0;
1160}
1161
1162static int atmel_aes_ecb_encrypt(struct ablkcipher_request *req)
1163{
77dacf5f 1164 return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1165}
1166
1167static int atmel_aes_ecb_decrypt(struct ablkcipher_request *req)
1168{
77dacf5f 1169 return atmel_aes_crypt(req, AES_FLAGS_ECB);
bd3c7b5c
NR		 1170}
1171
1172static int atmel_aes_cbc_encrypt(struct ablkcipher_request *req)
1173{
afbac17e 1174 return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1175}
1176
1177static int atmel_aes_cbc_decrypt(struct ablkcipher_request *req)
1178{
afbac17e 1179 return atmel_aes_crypt(req, AES_FLAGS_CBC);
bd3c7b5c
NR		 1180}
1181
1182static int atmel_aes_ofb_encrypt(struct ablkcipher_request *req)
1183{
afbac17e 1184 return atmel_aes_crypt(req, AES_FLAGS_OFB | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1185}
1186
1187static int atmel_aes_ofb_decrypt(struct ablkcipher_request *req)
1188{
afbac17e 1189 return atmel_aes_crypt(req, AES_FLAGS_OFB);
bd3c7b5c
NR		 1190}
1191
1192static int atmel_aes_cfb_encrypt(struct ablkcipher_request *req)
1193{
77dacf5f 1194 return atmel_aes_crypt(req, AES_FLAGS_CFB128 | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1195}
1196
1197static int atmel_aes_cfb_decrypt(struct ablkcipher_request *req)
1198{
77dacf5f 1199 return atmel_aes_crypt(req, AES_FLAGS_CFB128);
bd3c7b5c
NR		 1200}
1201
1202static int atmel_aes_cfb64_encrypt(struct ablkcipher_request *req)
1203{
77dacf5f 1204 return atmel_aes_crypt(req, AES_FLAGS_CFB64 | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1205}
1206
1207static int atmel_aes_cfb64_decrypt(struct ablkcipher_request *req)
1208{
77dacf5f 1209 return atmel_aes_crypt(req, AES_FLAGS_CFB64);
bd3c7b5c
NR		 1210}
1211
1212static int atmel_aes_cfb32_encrypt(struct ablkcipher_request *req)
1213{
77dacf5f 1214 return atmel_aes_crypt(req, AES_FLAGS_CFB32 | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1215}
1216
1217static int atmel_aes_cfb32_decrypt(struct ablkcipher_request *req)
1218{
77dacf5f 1219 return atmel_aes_crypt(req, AES_FLAGS_CFB32);
bd3c7b5c
NR		 1220}
1221
1222static int atmel_aes_cfb16_encrypt(struct ablkcipher_request *req)
1223{
77dacf5f 1224 return atmel_aes_crypt(req, AES_FLAGS_CFB16 | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1225}
1226
1227static int atmel_aes_cfb16_decrypt(struct ablkcipher_request *req)
1228{
77dacf5f 1229 return atmel_aes_crypt(req, AES_FLAGS_CFB16);
bd3c7b5c
NR		 1230}
1231
1232static int atmel_aes_cfb8_encrypt(struct ablkcipher_request *req)
1233{
77dacf5f 1234 return atmel_aes_crypt(req, AES_FLAGS_CFB8 | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1235}
1236
1237static int atmel_aes_cfb8_decrypt(struct ablkcipher_request *req)
1238{
77dacf5f 1239 return atmel_aes_crypt(req, AES_FLAGS_CFB8);
bd3c7b5c
NR		 1240}
1241
1242static int atmel_aes_ctr_encrypt(struct ablkcipher_request *req)
1243{
afbac17e 1244 return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT);
bd3c7b5c
NR		 1245}
1246
1247static int atmel_aes_ctr_decrypt(struct ablkcipher_request *req)
1248{
afbac17e 1249 return atmel_aes_crypt(req, AES_FLAGS_CTR);
bd3c7b5c
NR		 1250}
1251
1252static int atmel_aes_cra_init(struct crypto_tfm *tfm)
1253{
ccbf7298
CP		 1254 struct atmel_aes_ctx *ctx = crypto_tfm_ctx(tfm);
1255
bd3c7b5c 1256 tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx);
ccbf7298 1257 ctx->base.start = atmel_aes_start;
bd3c7b5c
NR		 1258
1259 return 0;
1260}
1261
fcac8365
CP		 1262static int atmel_aes_ctr_cra_init(struct crypto_tfm *tfm)
1263{
1264 struct atmel_aes_ctx *ctx = crypto_tfm_ctx(tfm);
1265
1266 tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx);
1267 ctx->base.start = atmel_aes_ctr_start;
1268
1269 return 0;
1270}
1271
bd3c7b5c
NR		 1272static void atmel_aes_cra_exit(struct crypto_tfm *tfm)
1273{
1274}
1275
1276static struct crypto_alg aes_algs[] = {
1277{
1278 .cra_name = "ecb(aes)",
1279 .cra_driver_name = "atmel-ecb-aes",
88efd9a9 1280 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c
NR		 1281 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1282 .cra_blocksize = AES_BLOCK_SIZE,
1283 .cra_ctxsize = sizeof(struct atmel_aes_ctx),
cadc4ab8 1284 .cra_alignmask = 0xf,
bd3c7b5c
NR		 1285 .cra_type = &crypto_ablkcipher_type,
1286 .cra_module = THIS_MODULE,
1287 .cra_init = atmel_aes_cra_init,
1288 .cra_exit = atmel_aes_cra_exit,
1289 .cra_u.ablkcipher = {
1290 .min_keysize = AES_MIN_KEY_SIZE,
1291 .max_keysize = AES_MAX_KEY_SIZE,
1292 .setkey = atmel_aes_setkey,
1293 .encrypt = atmel_aes_ecb_encrypt,
1294 .decrypt = atmel_aes_ecb_decrypt,
1295 }
1296},
1297{
1298 .cra_name = "cbc(aes)",
1299 .cra_driver_name = "atmel-cbc-aes",
88efd9a9 1300 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c
NR		 1301 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1302 .cra_blocksize = AES_BLOCK_SIZE,
1303 .cra_ctxsize = sizeof(struct atmel_aes_ctx),
cadc4ab8 1304 .cra_alignmask = 0xf,
bd3c7b5c
NR		 1305 .cra_type = &crypto_ablkcipher_type,
1306 .cra_module = THIS_MODULE,
1307 .cra_init = atmel_aes_cra_init,
1308 .cra_exit = atmel_aes_cra_exit,
1309 .cra_u.ablkcipher = {
1310 .min_keysize = AES_MIN_KEY_SIZE,
1311 .max_keysize = AES_MAX_KEY_SIZE,
1312 .ivsize = AES_BLOCK_SIZE,
1313 .setkey = atmel_aes_setkey,
1314 .encrypt = atmel_aes_cbc_encrypt,
1315 .decrypt = atmel_aes_cbc_decrypt,
1316 }
1317},
1318{
1319 .cra_name = "ofb(aes)",
1320 .cra_driver_name = "atmel-ofb-aes",
88efd9a9 1321 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c
NR		 1322 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1323 .cra_blocksize = AES_BLOCK_SIZE,
1324 .cra_ctxsize = sizeof(struct atmel_aes_ctx),
cadc4ab8 1325 .cra_alignmask = 0xf,
bd3c7b5c
NR		 1326 .cra_type = &crypto_ablkcipher_type,
1327 .cra_module = THIS_MODULE,
1328 .cra_init = atmel_aes_cra_init,
1329 .cra_exit = atmel_aes_cra_exit,
1330 .cra_u.ablkcipher = {
1331 .min_keysize = AES_MIN_KEY_SIZE,
1332 .max_keysize = AES_MAX_KEY_SIZE,
1333 .ivsize = AES_BLOCK_SIZE,
1334 .setkey = atmel_aes_setkey,
1335 .encrypt = atmel_aes_ofb_encrypt,
1336 .decrypt = atmel_aes_ofb_decrypt,
1337 }
1338},
1339{
1340 .cra_name = "cfb(aes)",
1341 .cra_driver_name = "atmel-cfb-aes",
88efd9a9 1342 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c
NR		 1343 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1344 .cra_blocksize = AES_BLOCK_SIZE,
1345 .cra_ctxsize = sizeof(struct atmel_aes_ctx),
cadc4ab8 1346 .cra_alignmask = 0xf,
bd3c7b5c
NR		 1347 .cra_type = &crypto_ablkcipher_type,
1348 .cra_module = THIS_MODULE,
1349 .cra_init = atmel_aes_cra_init,
1350 .cra_exit = atmel_aes_cra_exit,
1351 .cra_u.ablkcipher = {
1352 .min_keysize = AES_MIN_KEY_SIZE,
1353 .max_keysize = AES_MAX_KEY_SIZE,
1354 .ivsize = AES_BLOCK_SIZE,
1355 .setkey = atmel_aes_setkey,
1356 .encrypt = atmel_aes_cfb_encrypt,
1357 .decrypt = atmel_aes_cfb_decrypt,
1358 }
1359},
1360{
1361 .cra_name = "cfb32(aes)",
1362 .cra_driver_name = "atmel-cfb32-aes",
88efd9a9 1363 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c
NR		 1364 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1365 .cra_blocksize = CFB32_BLOCK_SIZE,
1366 .cra_ctxsize = sizeof(struct atmel_aes_ctx),
cadc4ab8 1367 .cra_alignmask = 0x3,
bd3c7b5c
NR		 1368 .cra_type = &crypto_ablkcipher_type,
1369 .cra_module = THIS_MODULE,
1370 .cra_init = atmel_aes_cra_init,
1371 .cra_exit = atmel_aes_cra_exit,
1372 .cra_u.ablkcipher = {
1373 .min_keysize = AES_MIN_KEY_SIZE,
1374 .max_keysize = AES_MAX_KEY_SIZE,
1375 .ivsize = AES_BLOCK_SIZE,
1376 .setkey = atmel_aes_setkey,
1377 .encrypt = atmel_aes_cfb32_encrypt,
1378 .decrypt = atmel_aes_cfb32_decrypt,
1379 }
1380},
1381{
1382 .cra_name = "cfb16(aes)",
1383 .cra_driver_name = "atmel-cfb16-aes",
88efd9a9 1384 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c
NR		 1385 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1386 .cra_blocksize = CFB16_BLOCK_SIZE,
1387 .cra_ctxsize = sizeof(struct atmel_aes_ctx),
cadc4ab8 1388 .cra_alignmask = 0x1,
bd3c7b5c
NR		 1389 .cra_type = &crypto_ablkcipher_type,
1390 .cra_module = THIS_MODULE,
1391 .cra_init = atmel_aes_cra_init,
1392 .cra_exit = atmel_aes_cra_exit,
1393 .cra_u.ablkcipher = {
1394 .min_keysize = AES_MIN_KEY_SIZE,
1395 .max_keysize = AES_MAX_KEY_SIZE,
1396 .ivsize = AES_BLOCK_SIZE,
1397 .setkey = atmel_aes_setkey,
1398 .encrypt = atmel_aes_cfb16_encrypt,
1399 .decrypt = atmel_aes_cfb16_decrypt,
1400 }
1401},
1402{
1403 .cra_name = "cfb8(aes)",
1404 .cra_driver_name = "atmel-cfb8-aes",
88efd9a9 1405 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c 1406 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
e5d8c961 1407 .cra_blocksize = CFB8_BLOCK_SIZE,
bd3c7b5c
NR		 1408 .cra_ctxsize = sizeof(struct atmel_aes_ctx),
1409 .cra_alignmask = 0x0,
1410 .cra_type = &crypto_ablkcipher_type,
1411 .cra_module = THIS_MODULE,
1412 .cra_init = atmel_aes_cra_init,
1413 .cra_exit = atmel_aes_cra_exit,
1414 .cra_u.ablkcipher = {
1415 .min_keysize = AES_MIN_KEY_SIZE,
1416 .max_keysize = AES_MAX_KEY_SIZE,
1417 .ivsize = AES_BLOCK_SIZE,
1418 .setkey = atmel_aes_setkey,
1419 .encrypt = atmel_aes_cfb8_encrypt,
1420 .decrypt = atmel_aes_cfb8_decrypt,
1421 }
1422},
1423{
1424 .cra_name = "ctr(aes)",
1425 .cra_driver_name = "atmel-ctr-aes",
88efd9a9 1426 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c 1427 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
da7b850e 1428 .cra_blocksize = 1,
fcac8365 1429 .cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx),
cadc4ab8 1430 .cra_alignmask = 0xf,
bd3c7b5c
NR		 1431 .cra_type = &crypto_ablkcipher_type,
1432 .cra_module = THIS_MODULE,
fcac8365 1433 .cra_init = atmel_aes_ctr_cra_init,
bd3c7b5c
NR		 1434 .cra_exit = atmel_aes_cra_exit,
1435 .cra_u.ablkcipher = {
1436 .min_keysize = AES_MIN_KEY_SIZE,
1437 .max_keysize = AES_MAX_KEY_SIZE,
1438 .ivsize = AES_BLOCK_SIZE,
1439 .setkey = atmel_aes_setkey,
1440 .encrypt = atmel_aes_ctr_encrypt,
1441 .decrypt = atmel_aes_ctr_decrypt,
1442 }
1443},
1444};
1445
cadc4ab8 1446static struct crypto_alg aes_cfb64_alg = {
bd3c7b5c
NR		 1447 .cra_name = "cfb64(aes)",
1448 .cra_driver_name = "atmel-cfb64-aes",
88efd9a9 1449 .cra_priority = ATMEL_AES_PRIORITY,
bd3c7b5c
NR		 1450 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1451 .cra_blocksize = CFB64_BLOCK_SIZE,
1452 .cra_ctxsize = sizeof(struct atmel_aes_ctx),
cadc4ab8 1453 .cra_alignmask = 0x7,
bd3c7b5c
NR		 1454 .cra_type = &crypto_ablkcipher_type,
1455 .cra_module = THIS_MODULE,
1456 .cra_init = atmel_aes_cra_init,
1457 .cra_exit = atmel_aes_cra_exit,
1458 .cra_u.ablkcipher = {
1459 .min_keysize = AES_MIN_KEY_SIZE,
1460 .max_keysize = AES_MAX_KEY_SIZE,
1461 .ivsize = AES_BLOCK_SIZE,
1462 .setkey = atmel_aes_setkey,
1463 .encrypt = atmel_aes_cfb64_encrypt,
1464 .decrypt = atmel_aes_cfb64_decrypt,
1465 }
bd3c7b5c
NR		 1466};
1467
e37a7e55 1468
d4419548
CP		 1469/* gcm aead functions */
1470
1471static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
1472 const u32 *data, size_t datalen,
1473 const u32 *ghash_in, u32 *ghash_out,
1474 atmel_aes_fn_t resume);
1475static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd);
1476static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd);
1477
1478static int atmel_aes_gcm_start(struct atmel_aes_dev *dd);
1479static int atmel_aes_gcm_process(struct atmel_aes_dev *dd);
1480static int atmel_aes_gcm_length(struct atmel_aes_dev *dd);
1481static int atmel_aes_gcm_data(struct atmel_aes_dev *dd);
1482static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd);
1483static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd);
1484static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd);
1485
1486static inline struct atmel_aes_gcm_ctx *
1487atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx)
1488{
1489 return container_of(ctx, struct atmel_aes_gcm_ctx, base);
1490}
1491
1492static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
1493 const u32 *data, size_t datalen,
1494 const u32 *ghash_in, u32 *ghash_out,
1495 atmel_aes_fn_t resume)
1496{
1497 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1498
1499 dd->data = (u32 *)data;
1500 dd->datalen = datalen;
1501 ctx->ghash_in = ghash_in;
1502 ctx->ghash_out = ghash_out;
1503 ctx->ghash_resume = resume;
1504
1505 atmel_aes_write_ctrl(dd, false, NULL);
1506 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init);
1507}
1508
1509static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd)
1510{
1511 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1512
1513 /* Set the data length. */
1514 atmel_aes_write(dd, AES_AADLENR, dd->total);
1515 atmel_aes_write(dd, AES_CLENR, 0);
1516
1517 /* If needed, overwrite the GCM Intermediate Hash Word Registers */
1518 if (ctx->ghash_in)
1519 atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in);
1520
1521 return atmel_aes_gcm_ghash_finalize(dd);
1522}
1523
1524static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd)
1525{
1526 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1527 u32 isr;
1528
1529 /* Write data into the Input Data Registers. */
1530 while (dd->datalen > 0) {
1531 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
1532 dd->data += 4;
1533 dd->datalen -= AES_BLOCK_SIZE;
1534
1535 isr = atmel_aes_read(dd, AES_ISR);
1536 if (!(isr & AES_INT_DATARDY)) {
1537 dd->resume = atmel_aes_gcm_ghash_finalize;
1538 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
1539 return -EINPROGRESS;
1540 }
1541 }
1542
1543 /* Read the computed hash from GHASHRx. */
1544 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out);
1545
1546 return ctx->ghash_resume(dd);
1547}
1548
1549
1550static int atmel_aes_gcm_start(struct atmel_aes_dev *dd)
1551{
1552 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1553 struct aead_request *req = aead_request_cast(dd->areq);
1554 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1555 struct atmel_aes_reqctx *rctx = aead_request_ctx(req);
1556 size_t ivsize = crypto_aead_ivsize(tfm);
1557 size_t datalen, padlen;
1558 const void *iv = req->iv;
1559 u8 *data = dd->buf;
1560 int err;
1561
1562 atmel_aes_set_mode(dd, rctx);
1563
1564 err = atmel_aes_hw_init(dd);
1565 if (err)
1566 return atmel_aes_complete(dd, err);
1567
219d51c7 1568 if (likely(ivsize == GCM_AES_IV_SIZE)) {
d4419548
CP		 1569 memcpy(ctx->j0, iv, ivsize);
1570 ctx->j0[3] = cpu_to_be32(1);
1571 return atmel_aes_gcm_process(dd);
1572 }
1573
1574 padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE);
1575 datalen = ivsize + padlen + AES_BLOCK_SIZE;
1576 if (datalen > dd->buflen)
1577 return atmel_aes_complete(dd, -EINVAL);
1578
1579 memcpy(data, iv, ivsize);
1580 memset(data + ivsize, 0, padlen + sizeof(u64));
1581 ((u64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8);
1582
1583 return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen,
1584 NULL, ctx->j0, atmel_aes_gcm_process);
1585}
1586
1587static int atmel_aes_gcm_process(struct atmel_aes_dev *dd)
1588{
1589 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1590 struct aead_request *req = aead_request_cast(dd->areq);
1591 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1592 bool enc = atmel_aes_is_encrypt(dd);
1593 u32 authsize;
1594
1595 /* Compute text length. */
1596 authsize = crypto_aead_authsize(tfm);
1597 ctx->textlen = req->cryptlen - (enc ? 0 : authsize);
1598
1599 /*
1600 * According to tcrypt test suite, the GCM Automatic Tag Generation
1601 * fails when both the message and its associated data are empty.
1602 */
1603 if (likely(req->assoclen != 0 || ctx->textlen != 0))
1604 dd->flags |= AES_FLAGS_GTAGEN;
1605
1606 atmel_aes_write_ctrl(dd, false, NULL);
1607 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length);
1608}
1609
1610static int atmel_aes_gcm_length(struct atmel_aes_dev *dd)
1611{
1612 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1613 struct aead_request *req = aead_request_cast(dd->areq);
1614 u32 j0_lsw, *j0 = ctx->j0;
1615 size_t padlen;
1616
1617 /* Write incr32(J0) into IV. */
1618 j0_lsw = j0[3];
1619 j0[3] = cpu_to_be32(be32_to_cpu(j0[3]) + 1);
1620 atmel_aes_write_block(dd, AES_IVR(0), j0);
1621 j0[3] = j0_lsw;
1622
1623 /* Set aad and text lengths. */
1624 atmel_aes_write(dd, AES_AADLENR, req->assoclen);
1625 atmel_aes_write(dd, AES_CLENR, ctx->textlen);
1626
1627 /* Check whether AAD are present. */
1628 if (unlikely(req->assoclen == 0)) {
1629 dd->datalen = 0;
1630 return atmel_aes_gcm_data(dd);
1631 }
1632
1633 /* Copy assoc data and add padding. */
1634 padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE);
1635 if (unlikely(req->assoclen + padlen > dd->buflen))
1636 return atmel_aes_complete(dd, -EINVAL);
1637 sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen);
1638
1639 /* Write assoc data into the Input Data register. */
1640 dd->data = (u32 *)dd->buf;
1641 dd->datalen = req->assoclen + padlen;
1642 return atmel_aes_gcm_data(dd);
1643}
1644
1645static int atmel_aes_gcm_data(struct atmel_aes_dev *dd)
1646{
1647 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1648 struct aead_request *req = aead_request_cast(dd->areq);
1649 bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD);
1650 struct scatterlist *src, *dst;
1651 u32 isr, mr;
1652
1653 /* Write AAD first. */
1654 while (dd->datalen > 0) {
1655 atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
1656 dd->data += 4;
1657 dd->datalen -= AES_BLOCK_SIZE;
1658
1659 isr = atmel_aes_read(dd, AES_ISR);
1660 if (!(isr & AES_INT_DATARDY)) {
1661 dd->resume = atmel_aes_gcm_data;
1662 atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
1663 return -EINPROGRESS;
1664 }
1665 }
1666
1667 /* GMAC only. */
1668 if (unlikely(ctx->textlen == 0))
1669 return atmel_aes_gcm_tag_init(dd);
1670
1671 /* Prepare src and dst scatter lists to transfer cipher/plain texts */
1672 src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen);
1673 dst = ((req->src == req->dst) ? src :
1674 scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen));
1675
1676 if (use_dma) {
1677 /* Update the Mode Register for DMA transfers. */
1678 mr = atmel_aes_read(dd, AES_MR);
1679 mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF);
1680 mr |= AES_MR_SMOD_IDATAR0;
1681 if (dd->caps.has_dualbuff)
1682 mr |= AES_MR_DUALBUFF;
1683 atmel_aes_write(dd, AES_MR, mr);
1684
1685 return atmel_aes_dma_start(dd, src, dst, ctx->textlen,
1686 atmel_aes_gcm_tag_init);
1687 }
1688
1689 return atmel_aes_cpu_start(dd, src, dst, ctx->textlen,
1690 atmel_aes_gcm_tag_init);
1691}
1692
1693static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd)
1694{
1695 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1696 struct aead_request *req = aead_request_cast(dd->areq);
1697 u64 *data = dd->buf;
1698
1699 if (likely(dd->flags & AES_FLAGS_GTAGEN)) {
1700 if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) {
1701 dd->resume = atmel_aes_gcm_tag_init;
1702 atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY);
1703 return -EINPROGRESS;
1704 }
1705
1706 return atmel_aes_gcm_finalize(dd);
1707 }
1708
1709 /* Read the GCM Intermediate Hash Word Registers. */
1710 atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash);
1711
1712 data[0] = cpu_to_be64(req->assoclen * 8);
1713 data[1] = cpu_to_be64(ctx->textlen * 8);
1714
1715 return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE,
1716 ctx->ghash, ctx->ghash, atmel_aes_gcm_tag);
1717}
1718
1719static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd)
1720{
1721 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1722 unsigned long flags;
1723
1724 /*
1725 * Change mode to CTR to complete the tag generation.
1726 * Use J0 as Initialization Vector.
1727 */
1728 flags = dd->flags;
1729 dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN);
1730 dd->flags |= AES_FLAGS_CTR;
1731 atmel_aes_write_ctrl(dd, false, ctx->j0);
1732 dd->flags = flags;
1733
1734 atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash);
1735 return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize);
1736}
1737
1738static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd)
1739{
1740 struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
1741 struct aead_request *req = aead_request_cast(dd->areq);
1742 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1743 bool enc = atmel_aes_is_encrypt(dd);
1744 u32 offset, authsize, itag[4], *otag = ctx->tag;
1745 int err;
1746
1747 /* Read the computed tag. */
1748 if (likely(dd->flags & AES_FLAGS_GTAGEN))
1749 atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag);
1750 else
1751 atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag);
1752
1753 offset = req->assoclen + ctx->textlen;
1754 authsize = crypto_aead_authsize(tfm);
1755 if (enc) {
1756 scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1);
1757 err = 0;
1758 } else {
1759 scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0);
1760 err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0;
1761 }
1762
1763 return atmel_aes_complete(dd, err);
1764}
1765
1766static int atmel_aes_gcm_crypt(struct aead_request *req,
1767 unsigned long mode)
1768{
1769 struct atmel_aes_base_ctx *ctx;
1770 struct atmel_aes_reqctx *rctx;
1771 struct atmel_aes_dev *dd;
1772
1773 ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
1774 ctx->block_size = AES_BLOCK_SIZE;
91308019 1775 ctx->is_aead = true;
d4419548
CP		 1776
1777 dd = atmel_aes_find_dev(ctx);
1778 if (!dd)
1779 return -ENODEV;
1780
1781 rctx = aead_request_ctx(req);
1782 rctx->mode = AES_FLAGS_GCM | mode;
1783
1784 return atmel_aes_handle_queue(dd, &req->base);
1785}
1786
1787static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
1788 unsigned int keylen)
1789{
1790 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm);
1791
1792 if (keylen != AES_KEYSIZE_256 &&
1793 keylen != AES_KEYSIZE_192 &&
1794 keylen != AES_KEYSIZE_128) {
1795 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1796 return -EINVAL;
1797 }
1798
1799 memcpy(ctx->key, key, keylen);
1800 ctx->keylen = keylen;
1801
1802 return 0;
1803}
1804
1805static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm,
1806 unsigned int authsize)
1807{
1808 /* Same as crypto_gcm_authsize() from crypto/gcm.c */
1809 switch (authsize) {
1810 case 4:
1811 case 8:
1812 case 12:
1813 case 13:
1814 case 14:
1815 case 15:
1816 case 16:
1817 break;
1818 default:
1819 return -EINVAL;
1820 }
1821
1822 return 0;
1823}
1824
1825static int atmel_aes_gcm_encrypt(struct aead_request *req)
1826{
1827 return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT);
1828}
1829
1830static int atmel_aes_gcm_decrypt(struct aead_request *req)
1831{
1832 return atmel_aes_gcm_crypt(req, 0);
1833}
1834
1835static int atmel_aes_gcm_init(struct crypto_aead *tfm)
1836{
1837 struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
1838
1839 crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
1840 ctx->base.start = atmel_aes_gcm_start;
1841
1842 return 0;
1843}
1844
1845static void atmel_aes_gcm_exit(struct crypto_aead *tfm)
1846{
1847
1848}
1849
1850static struct aead_alg aes_gcm_alg = {
1851 .setkey = atmel_aes_gcm_setkey,
1852 .setauthsize = atmel_aes_gcm_setauthsize,
1853 .encrypt = atmel_aes_gcm_encrypt,
1854 .decrypt = atmel_aes_gcm_decrypt,
1855 .init = atmel_aes_gcm_init,
1856 .exit = atmel_aes_gcm_exit,
219d51c7 1857 .ivsize = GCM_AES_IV_SIZE,
d4419548
CP		 1858 .maxauthsize = AES_BLOCK_SIZE,
1859
1860 .base = {
1861 .cra_name = "gcm(aes)",
1862 .cra_driver_name = "atmel-gcm-aes",
1863 .cra_priority = ATMEL_AES_PRIORITY,
1864 .cra_flags = CRYPTO_ALG_ASYNC,
1865 .cra_blocksize = 1,
1866 .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx),
1867 .cra_alignmask = 0xf,
1868 .cra_module = THIS_MODULE,
1869 },
1870};
1871
1872
d52db518
CP		 1873/* xts functions */
1874
1875static inline struct atmel_aes_xts_ctx *
1876atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx)
1877{
1878 return container_of(ctx, struct atmel_aes_xts_ctx, base);
1879}
1880
1881static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd);
1882
1883static int atmel_aes_xts_start(struct atmel_aes_dev *dd)
1884{
1885 struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx);
1886 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);
1887 struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req);
1888 unsigned long flags;
1889 int err;
1890
1891 atmel_aes_set_mode(dd, rctx);
1892
1893 err = atmel_aes_hw_init(dd);
1894 if (err)
1895 return atmel_aes_complete(dd, err);
1896
1897 /* Compute the tweak value from req->info with ecb(aes). */
1898 flags = dd->flags;
1899 dd->flags &= ~AES_FLAGS_MODE_MASK;
1900 dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);
1901 atmel_aes_write_ctrl_key(dd, false, NULL,
1902 ctx->key2, ctx->base.keylen);
1903 dd->flags = flags;
1904
1905 atmel_aes_write_block(dd, AES_IDATAR(0), req->info);
1906 return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data);
1907}
1908
1909static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd)
1910{
1911 struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq);
1912 bool use_dma = (req->nbytes >= ATMEL_AES_DMA_THRESHOLD);
1913 u32 tweak[AES_BLOCK_SIZE / sizeof(u32)];
1914 static const u32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), };
1915 u8 *tweak_bytes = (u8 *)tweak;
1916 int i;
1917
1918 /* Read the computed ciphered tweak value. */
1919 atmel_aes_read_block(dd, AES_ODATAR(0), tweak);
1920 /*
1921 * Hardware quirk:
1922 * the order of the ciphered tweak bytes need to be reversed before
1923 * writing them into the ODATARx registers.
1924 */
1925 for (i = 0; i < AES_BLOCK_SIZE/2; ++i) {
1926 u8 tmp = tweak_bytes[AES_BLOCK_SIZE - 1 - i];
1927
1928 tweak_bytes[AES_BLOCK_SIZE - 1 - i] = tweak_bytes[i];
1929 tweak_bytes[i] = tmp;
1930 }
1931
1932 /* Process the data. */
1933 atmel_aes_write_ctrl(dd, use_dma, NULL);
1934 atmel_aes_write_block(dd, AES_TWR(0), tweak);
1935 atmel_aes_write_block(dd, AES_ALPHAR(0), one);
1936 if (use_dma)
1937 return atmel_aes_dma_start(dd, req->src, req->dst, req->nbytes,
1938 atmel_aes_transfer_complete);
1939
1940 return atmel_aes_cpu_start(dd, req->src, req->dst, req->nbytes,
1941 atmel_aes_transfer_complete);
1942}
1943
1944static int atmel_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
1945 unsigned int keylen)
1946{
1947 struct atmel_aes_xts_ctx *ctx = crypto_ablkcipher_ctx(tfm);
1948 int err;
1949
1950 err = xts_check_key(crypto_ablkcipher_tfm(tfm), key, keylen);
1951 if (err)
1952 return err;
1953
1954 memcpy(ctx->base.key, key, keylen/2);
1955 memcpy(ctx->key2, key + keylen/2, keylen/2);
1956 ctx->base.keylen = keylen/2;
1957
1958 return 0;
1959}
1960
1961static int atmel_aes_xts_encrypt(struct ablkcipher_request *req)
1962{
1963 return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT);
1964}
1965
1966static int atmel_aes_xts_decrypt(struct ablkcipher_request *req)
1967{
1968 return atmel_aes_crypt(req, AES_FLAGS_XTS);
1969}
1970
1971static int atmel_aes_xts_cra_init(struct crypto_tfm *tfm)
1972{
1973 struct atmel_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
1974
1975 tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx);
1976 ctx->base.start = atmel_aes_xts_start;
1977
1978 return 0;
1979}
1980
1981static struct crypto_alg aes_xts_alg = {
1982 .cra_name = "xts(aes)",
1983 .cra_driver_name = "atmel-xts-aes",
1984 .cra_priority = ATMEL_AES_PRIORITY,
1985 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1986 .cra_blocksize = AES_BLOCK_SIZE,
1987 .cra_ctxsize = sizeof(struct atmel_aes_xts_ctx),
1988 .cra_alignmask = 0xf,
1989 .cra_type = &crypto_ablkcipher_type,
1990 .cra_module = THIS_MODULE,
1991 .cra_init = atmel_aes_xts_cra_init,
1992 .cra_exit = atmel_aes_cra_exit,
1993 .cra_u.ablkcipher = {
1994 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1995 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1996 .ivsize = AES_BLOCK_SIZE,
1997 .setkey = atmel_aes_xts_setkey,
1998 .encrypt = atmel_aes_xts_encrypt,
1999 .decrypt = atmel_aes_xts_decrypt,
2000 }
2001};
2002
89a82ef8
CP		 2003#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2004/* authenc aead functions */
2005
2006static int atmel_aes_authenc_start(struct atmel_aes_dev *dd);
2007static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
2008 bool is_async);
2009static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
2010 bool is_async);
2011static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd);
2012static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
2013 bool is_async);
2014
2015static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err)
2016{
2017 struct aead_request *req = aead_request_cast(dd->areq);
2018 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2019
2020 if (err && (dd->flags & AES_FLAGS_OWN_SHA))
2021 atmel_sha_authenc_abort(&rctx->auth_req);
2022 dd->flags &= ~AES_FLAGS_OWN_SHA;
2023}
2024
2025static int atmel_aes_authenc_start(struct atmel_aes_dev *dd)
2026{
2027 struct aead_request *req = aead_request_cast(dd->areq);
2028 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2029 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2030 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
2031 int err;
2032
2033 atmel_aes_set_mode(dd, &rctx->base);
2034
2035 err = atmel_aes_hw_init(dd);
2036 if (err)
2037 return atmel_aes_complete(dd, err);
2038
2039 return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth,
2040 atmel_aes_authenc_init, dd);
2041}
2042
2043static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
2044 bool is_async)
2045{
2046 struct aead_request *req = aead_request_cast(dd->areq);
2047 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2048
2049 if (is_async)
2050 dd->is_async = true;
2051 if (err)
2052 return atmel_aes_complete(dd, err);
2053
2054 /* If here, we've got the ownership of the SHA device. */
2055 dd->flags |= AES_FLAGS_OWN_SHA;
2056
2057 /* Configure the SHA device. */
2058 return atmel_sha_authenc_init(&rctx->auth_req,
2059 req->src, req->assoclen,
2060 rctx->textlen,
2061 atmel_aes_authenc_transfer, dd);
2062}
2063
2064static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
2065 bool is_async)
2066{
2067 struct aead_request *req = aead_request_cast(dd->areq);
2068 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2069 bool enc = atmel_aes_is_encrypt(dd);
2070 struct scatterlist *src, *dst;
2071 u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
2072 u32 emr;
2073
2074 if (is_async)
2075 dd->is_async = true;
2076 if (err)
2077 return atmel_aes_complete(dd, err);
2078
2079 /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */
2080 src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
2081 dst = src;
2082
2083 if (req->src != req->dst)
2084 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
2085
2086 /* Configure the AES device. */
2087 memcpy(iv, req->iv, sizeof(iv));
2088
2089 /*
2090 * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to
2091 * 'true' even if the data transfer is actually performed by the CPU (so
2092 * not by the DMA) because we must force the AES_MR_SMOD bitfield to the
2093 * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD
2094 * must be set to *_MR_SMOD_IDATAR0.
2095 */
2096 atmel_aes_write_ctrl(dd, true, iv);
2097 emr = AES_EMR_PLIPEN;
2098 if (!enc)
2099 emr |= AES_EMR_PLIPD;
2100 atmel_aes_write(dd, AES_EMR, emr);
2101
2102 /* Transfer data. */
2103 return atmel_aes_dma_start(dd, src, dst, rctx->textlen,
2104 atmel_aes_authenc_digest);
2105}
2106
2107static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd)
2108{
2109 struct aead_request *req = aead_request_cast(dd->areq);
2110 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2111
2112 /* atmel_sha_authenc_final() releases the SHA device. */
2113 dd->flags &= ~AES_FLAGS_OWN_SHA;
2114 return atmel_sha_authenc_final(&rctx->auth_req,
2115 rctx->digest, sizeof(rctx->digest),
2116 atmel_aes_authenc_final, dd);
2117}
2118
2119static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
2120 bool is_async)
2121{
2122 struct aead_request *req = aead_request_cast(dd->areq);
2123 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2124 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2125 bool enc = atmel_aes_is_encrypt(dd);
2126 u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest;
2127 u32 offs, authsize;
2128
2129 if (is_async)
2130 dd->is_async = true;
2131 if (err)
2132 goto complete;
2133
2134 offs = req->assoclen + rctx->textlen;
2135 authsize = crypto_aead_authsize(tfm);
2136 if (enc) {
2137 scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1);
2138 } else {
2139 scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0);
2140 if (crypto_memneq(idigest, odigest, authsize))
2141 err = -EBADMSG;
2142 }
2143
2144complete:
2145 return atmel_aes_complete(dd, err);
2146}
2147
2148static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key,
2149 unsigned int keylen)
2150{
2151 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
2152 struct crypto_authenc_keys keys;
2153 u32 flags;
2154 int err;
2155
2156 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
2157 goto badkey;
2158
2159 if (keys.enckeylen > sizeof(ctx->base.key))
2160 goto badkey;
2161
2162 /* Save auth key. */
2163 flags = crypto_aead_get_flags(tfm);
2164 err = atmel_sha_authenc_setkey(ctx->auth,
2165 keys.authkey, keys.authkeylen,
2166 &flags);
2167 crypto_aead_set_flags(tfm, flags & CRYPTO_TFM_RES_MASK);
2168 if (err) {
2169 memzero_explicit(&keys, sizeof(keys));
2170 return err;
2171 }
2172
2173 /* Save enc key. */
2174 ctx->base.keylen = keys.enckeylen;
2175 memcpy(ctx->base.key, keys.enckey, keys.enckeylen);
2176
2177 memzero_explicit(&keys, sizeof(keys));
2178 return 0;
2179
2180badkey:
2181 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2182 memzero_explicit(&key, sizeof(keys));
2183 return -EINVAL;
2184}
2185
2186static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
2187 unsigned long auth_mode)
2188{
2189 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
2190 unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize();
2191
2192 ctx->auth = atmel_sha_authenc_spawn(auth_mode);
2193 if (IS_ERR(ctx->auth))
2194 return PTR_ERR(ctx->auth);
2195
2196 crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
2197 auth_reqsize));
2198 ctx->base.start = atmel_aes_authenc_start;
2199
2200 return 0;
2201}
2202
2203static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm)
2204{
2205 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1);
2206}
2207
2208static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm)
2209{
2210 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224);
2211}
2212
2213static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm)
2214{
2215 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256);
2216}
2217
2218static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm)
2219{
2220 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384);
2221}
2222
2223static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm)
2224{
2225 return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512);
2226}
2227
2228static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm)
2229{
2230 struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
2231
2232 atmel_sha_authenc_free(ctx->auth);
2233}
2234
2235static int atmel_aes_authenc_crypt(struct aead_request *req,
2236 unsigned long mode)
2237{
2238 struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
2239 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2240 struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm);
2241 u32 authsize = crypto_aead_authsize(tfm);
2242 bool enc = (mode & AES_FLAGS_ENCRYPT);
2243 struct atmel_aes_dev *dd;
2244
2245 /* Compute text length. */
2246 if (!enc && req->cryptlen < authsize)
2247 return -EINVAL;
2248 rctx->textlen = req->cryptlen - (enc ? 0 : authsize);
2249
2250 /*
2251 * Currently, empty messages are not supported yet:
2252 * the SHA auto-padding can be used only on non-empty messages.
2253 * Hence a special case needs to be implemented for empty message.
2254 */
2255 if (!rctx->textlen && !req->assoclen)
2256 return -EINVAL;
2257
2258 rctx->base.mode = mode;
2259 ctx->block_size = AES_BLOCK_SIZE;
91308019 2260 ctx->is_aead = true;
89a82ef8
CP		 2261
2262 dd = atmel_aes_find_dev(ctx);
2263 if (!dd)
2264 return -ENODEV;
2265
2266 return atmel_aes_handle_queue(dd, &req->base);
2267}
2268
2269static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req)
2270{
2271 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT);
2272}
2273
2274static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req)
2275{
2276 return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC);
2277}
2278
2279static struct aead_alg aes_authenc_algs[] = {
2280{
2281 .setkey = atmel_aes_authenc_setkey,
2282 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2283 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2284 .init = atmel_aes_authenc_hmac_sha1_init_tfm,
2285 .exit = atmel_aes_authenc_exit_tfm,
2286 .ivsize = AES_BLOCK_SIZE,
2287 .maxauthsize = SHA1_DIGEST_SIZE,
2288
2289 .base = {
2290 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2291 .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes",
2292 .cra_priority = ATMEL_AES_PRIORITY,
2293 .cra_flags = CRYPTO_ALG_ASYNC,
2294 .cra_blocksize = AES_BLOCK_SIZE,
2295 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2296 .cra_alignmask = 0xf,
2297 .cra_module = THIS_MODULE,
2298 },
2299},
2300{
2301 .setkey = atmel_aes_authenc_setkey,
2302 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2303 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2304 .init = atmel_aes_authenc_hmac_sha224_init_tfm,
2305 .exit = atmel_aes_authenc_exit_tfm,
2306 .ivsize = AES_BLOCK_SIZE,
2307 .maxauthsize = SHA224_DIGEST_SIZE,
2308
2309 .base = {
2310 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2311 .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes",
2312 .cra_priority = ATMEL_AES_PRIORITY,
2313 .cra_flags = CRYPTO_ALG_ASYNC,
2314 .cra_blocksize = AES_BLOCK_SIZE,
2315 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2316 .cra_alignmask = 0xf,
2317 .cra_module = THIS_MODULE,
2318 },
2319},
2320{
2321 .setkey = atmel_aes_authenc_setkey,
2322 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2323 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2324 .init = atmel_aes_authenc_hmac_sha256_init_tfm,
2325 .exit = atmel_aes_authenc_exit_tfm,
2326 .ivsize = AES_BLOCK_SIZE,
2327 .maxauthsize = SHA256_DIGEST_SIZE,
2328
2329 .base = {
2330 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2331 .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes",
2332 .cra_priority = ATMEL_AES_PRIORITY,
2333 .cra_flags = CRYPTO_ALG_ASYNC,
2334 .cra_blocksize = AES_BLOCK_SIZE,
2335 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2336 .cra_alignmask = 0xf,
2337 .cra_module = THIS_MODULE,
2338 },
2339},
2340{
2341 .setkey = atmel_aes_authenc_setkey,
2342 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2343 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2344 .init = atmel_aes_authenc_hmac_sha384_init_tfm,
2345 .exit = atmel_aes_authenc_exit_tfm,
2346 .ivsize = AES_BLOCK_SIZE,
2347 .maxauthsize = SHA384_DIGEST_SIZE,
2348
2349 .base = {
2350 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2351 .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes",
2352 .cra_priority = ATMEL_AES_PRIORITY,
2353 .cra_flags = CRYPTO_ALG_ASYNC,
2354 .cra_blocksize = AES_BLOCK_SIZE,
2355 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2356 .cra_alignmask = 0xf,
2357 .cra_module = THIS_MODULE,
2358 },
2359},
2360{
2361 .setkey = atmel_aes_authenc_setkey,
2362 .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
2363 .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
2364 .init = atmel_aes_authenc_hmac_sha512_init_tfm,
2365 .exit = atmel_aes_authenc_exit_tfm,
2366 .ivsize = AES_BLOCK_SIZE,
2367 .maxauthsize = SHA512_DIGEST_SIZE,
2368
2369 .base = {
2370 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2371 .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes",
2372 .cra_priority = ATMEL_AES_PRIORITY,
2373 .cra_flags = CRYPTO_ALG_ASYNC,
2374 .cra_blocksize = AES_BLOCK_SIZE,
2375 .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
2376 .cra_alignmask = 0xf,
2377 .cra_module = THIS_MODULE,
2378 },
2379},
2380};
2381#endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
d52db518 2382
e37a7e55
CP		 2383/* Probe functions */
2384
2385static int atmel_aes_buff_init(struct atmel_aes_dev *dd)
2386{
2387 dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER);
2388 dd->buflen = ATMEL_AES_BUFFER_SIZE;
2389 dd->buflen &= ~(AES_BLOCK_SIZE - 1);
2390
2391 if (!dd->buf) {
2392 dev_err(dd->dev, "unable to alloc pages.\n");
2393 return -ENOMEM;
2394 }
2395
2396 return 0;
2397}
2398
2399static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd)
2400{
2401 free_page((unsigned long)dd->buf);
2402}
2403
2404static bool atmel_aes_filter(struct dma_chan *chan, void *slave)
2405{
2406 struct at_dma_slave *sl = slave;
2407
2408 if (sl && sl->dma_dev == chan->device->dev) {
2409 chan->private = sl;
2410 return true;
2411 } else {
2412 return false;
2413 }
2414}
2415
2416static int atmel_aes_dma_init(struct atmel_aes_dev *dd,
2417 struct crypto_platform_data *pdata)
2418{
2419 struct at_dma_slave *slave;
e37a7e55
CP		 2420 dma_cap_mask_t mask;
2421
2422 dma_cap_zero(mask);
2423 dma_cap_set(DMA_SLAVE, mask);
2424
2425 /* Try to grab 2 DMA channels */
2426 slave = &pdata->dma_slave->rxdata;
2427 dd->src.chan = dma_request_slave_channel_compat(mask, atmel_aes_filter,
2428 slave, dd->dev, "tx");
2429 if (!dd->src.chan)
2430 goto err_dma_in;
2431
2432 slave = &pdata->dma_slave->txdata;
2433 dd->dst.chan = dma_request_slave_channel_compat(mask, atmel_aes_filter,
2434 slave, dd->dev, "rx");
2435 if (!dd->dst.chan)
2436 goto err_dma_out;
2437
2438 return 0;
2439
2440err_dma_out:
2441 dma_release_channel(dd->src.chan);
2442err_dma_in:
2443 dev_warn(dd->dev, "no DMA channel available\n");
3c88761e 2444 return -ENODEV;
e37a7e55
CP		 2445}
2446
2447static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd)
2448{
2449 dma_release_channel(dd->dst.chan);
2450 dma_release_channel(dd->src.chan);
2451}
2452
bd3c7b5c
NR		 2453static void atmel_aes_queue_task(unsigned long data)
2454{
2455 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;
2456
2457 atmel_aes_handle_queue(dd, NULL);
2458}
2459
2460static void atmel_aes_done_task(unsigned long data)
2461{
afbac17e 2462 struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;
bd3c7b5c 2463
10f12c1b
CP		 2464 dd->is_async = true;
2465 (void)dd->resume(dd);
2466}
bd3c7b5c 2467
bd3c7b5c
NR		 2468static irqreturn_t atmel_aes_irq(int irq, void *dev_id)
2469{
2470 struct atmel_aes_dev *aes_dd = dev_id;
2471 u32 reg;
2472
2473 reg = atmel_aes_read(aes_dd, AES_ISR);
2474 if (reg & atmel_aes_read(aes_dd, AES_IMR)) {
2475 atmel_aes_write(aes_dd, AES_IDR, reg);
2476 if (AES_FLAGS_BUSY & aes_dd->flags)
2477 tasklet_schedule(&aes_dd->done_task);
2478 else
2479 dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n");
2480 return IRQ_HANDLED;
2481 }
2482
2483 return IRQ_NONE;
2484}
2485
2486static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)
2487{
2488 int i;
2489
89a82ef8
CP		 2490#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2491 if (dd->caps.has_authenc)
2492 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++)
2493 crypto_unregister_aead(&aes_authenc_algs[i]);
2494#endif
2495
d52db518
CP		 2496 if (dd->caps.has_xts)
2497 crypto_unregister_alg(&aes_xts_alg);
2498
d4419548
CP		 2499 if (dd->caps.has_gcm)
2500 crypto_unregister_aead(&aes_gcm_alg);
2501
cadc4ab8
NR		 2502 if (dd->caps.has_cfb64)
2503 crypto_unregister_alg(&aes_cfb64_alg);
924a8bc7
CP		 2504
2505 for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
2506 crypto_unregister_alg(&aes_algs[i]);
bd3c7b5c
NR		 2507}
2508
2509static int atmel_aes_register_algs(struct atmel_aes_dev *dd)
2510{
2511 int err, i, j;
2512
2513 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
bd3c7b5c
NR		 2514 err = crypto_register_alg(&aes_algs[i]);
2515 if (err)
2516 goto err_aes_algs;
2517 }
2518
cadc4ab8
NR		 2519 if (dd->caps.has_cfb64) {
2520 err = crypto_register_alg(&aes_cfb64_alg);
bd3c7b5c
NR		 2521 if (err)
2522 goto err_aes_cfb64_alg;
2523 }
2524
d4419548
CP		 2525 if (dd->caps.has_gcm) {
2526 err = crypto_register_aead(&aes_gcm_alg);
2527 if (err)
2528 goto err_aes_gcm_alg;
2529 }
2530
d52db518
CP		 2531 if (dd->caps.has_xts) {
2532 err = crypto_register_alg(&aes_xts_alg);
2533 if (err)
2534 goto err_aes_xts_alg;
2535 }
2536
89a82ef8
CP		 2537#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2538 if (dd->caps.has_authenc) {
2539 for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) {
2540 err = crypto_register_aead(&aes_authenc_algs[i]);
2541 if (err)
2542 goto err_aes_authenc_alg;
2543 }
2544 }
2545#endif
2546
bd3c7b5c
NR		 2547 return 0;
2548
89a82ef8
CP		 2549#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2550 /* i = ARRAY_SIZE(aes_authenc_algs); */
2551err_aes_authenc_alg:
2552 for (j = 0; j < i; j++)
2553 crypto_unregister_aead(&aes_authenc_algs[j]);
2554 crypto_unregister_alg(&aes_xts_alg);
2555#endif
d52db518
CP		 2556err_aes_xts_alg:
2557 crypto_unregister_aead(&aes_gcm_alg);
d4419548
CP		 2558err_aes_gcm_alg:
2559 crypto_unregister_alg(&aes_cfb64_alg);
bd3c7b5c
NR		 2560err_aes_cfb64_alg:
2561 i = ARRAY_SIZE(aes_algs);
2562err_aes_algs:
2563 for (j = 0; j < i; j++)
2564 crypto_unregister_alg(&aes_algs[j]);
2565
2566 return err;
2567}
2568
cadc4ab8
NR		 2569static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
2570{
2571 dd->caps.has_dualbuff = 0;
2572 dd->caps.has_cfb64 = 0;
fcac8365 2573 dd->caps.has_ctr32 = 0;
d4419548 2574 dd->caps.has_gcm = 0;
d52db518 2575 dd->caps.has_xts = 0;
89a82ef8 2576 dd->caps.has_authenc = 0;
cadc4ab8
NR		 2577 dd->caps.max_burst_size = 1;
2578
2579 /* keep only major version number */
2580 switch (dd->hw_version & 0xff0) {
973e209d
LZ		 2581 case 0x500:
2582 dd->caps.has_dualbuff = 1;
2583 dd->caps.has_cfb64 = 1;
fcac8365 2584 dd->caps.has_ctr32 = 1;
d4419548 2585 dd->caps.has_gcm = 1;
d52db518 2586 dd->caps.has_xts = 1;
89a82ef8 2587 dd->caps.has_authenc = 1;
973e209d
LZ		 2588 dd->caps.max_burst_size = 4;
2589 break;
cf1f0d12
LZ		 2590 case 0x200:
2591 dd->caps.has_dualbuff = 1;
2592 dd->caps.has_cfb64 = 1;
fcac8365 2593 dd->caps.has_ctr32 = 1;
d4419548 2594 dd->caps.has_gcm = 1;
cf1f0d12
LZ		 2595 dd->caps.max_burst_size = 4;
2596 break;
cadc4ab8
NR		 2597 case 0x130:
2598 dd->caps.has_dualbuff = 1;
2599 dd->caps.has_cfb64 = 1;
2600 dd->caps.max_burst_size = 4;
2601 break;
2602 case 0x120:
2603 break;
2604 default:
2605 dev_warn(dd->dev,
2606 "Unmanaged aes version, set minimum capabilities\n");
2607 break;
2608 }
2609}
2610
be943c7d
NF		 2611#if defined(CONFIG_OF)
2612static const struct of_device_id atmel_aes_dt_ids[] = {
2613 { .compatible = "atmel,at91sam9g46-aes" },
2614 { /* sentinel */ }
2615};
2616MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids);
2617
2618static struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev)
2619{
2620 struct device_node *np = pdev->dev.of_node;
2621 struct crypto_platform_data *pdata;
2622
2623 if (!np) {
2624 dev_err(&pdev->dev, "device node not found\n");
2625 return ERR_PTR(-EINVAL);
2626 }
2627
2628 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
2629 if (!pdata) {
2630 dev_err(&pdev->dev, "could not allocate memory for pdata\n");
2631 return ERR_PTR(-ENOMEM);
2632 }
2633
2634 pdata->dma_slave = devm_kzalloc(&pdev->dev,
2635 sizeof(*(pdata->dma_slave)),
2636 GFP_KERNEL);
2637 if (!pdata->dma_slave) {
2638 dev_err(&pdev->dev, "could not allocate memory for dma_slave\n");
2639 devm_kfree(&pdev->dev, pdata);
2640 return ERR_PTR(-ENOMEM);
2641 }
2642
2643 return pdata;
2644}
2645#else
2646static inline struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev)
2647{
2648 return ERR_PTR(-EINVAL);
2649}
2650#endif
2651
49cfe4db 2652static int atmel_aes_probe(struct platform_device *pdev)
bd3c7b5c
NR		 2653{
2654 struct atmel_aes_dev *aes_dd;
cadc4ab8 2655 struct crypto_platform_data *pdata;
bd3c7b5c
NR		 2656 struct device *dev = &pdev->dev;
2657 struct resource *aes_res;
bd3c7b5c
NR		 2658 int err;
2659
2660 pdata = pdev->dev.platform_data;
2661 if (!pdata) {
be943c7d
NF		 2662 pdata = atmel_aes_of_init(pdev);
2663 if (IS_ERR(pdata)) {
2664 err = PTR_ERR(pdata);
2665 goto aes_dd_err;
2666 }
2667 }
2668
2669 if (!pdata->dma_slave) {
bd3c7b5c
NR		 2670 err = -ENXIO;
2671 goto aes_dd_err;
2672 }
2673
b0e8b341 2674 aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL);
bd3c7b5c
NR		 2675 if (aes_dd == NULL) {
2676 dev_err(dev, "unable to alloc data struct.\n");
2677 err = -ENOMEM;
2678 goto aes_dd_err;
2679 }
2680
2681 aes_dd->dev = dev;
2682
2683 platform_set_drvdata(pdev, aes_dd);
2684
2685 INIT_LIST_HEAD(&aes_dd->list);
8a10eb8d 2686 spin_lock_init(&aes_dd->lock);
bd3c7b5c
NR		 2687
2688 tasklet_init(&aes_dd->done_task, atmel_aes_done_task,
2689 (unsigned long)aes_dd);
2690 tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task,
2691 (unsigned long)aes_dd);
2692
2693 crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH);
2694
bd3c7b5c
NR		 2695 /* Get the base address */
2696 aes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2697 if (!aes_res) {
2698 dev_err(dev, "no MEM resource info\n");
2699 err = -ENODEV;
2700 goto res_err;
2701 }
2702 aes_dd->phys_base = aes_res->start;
bd3c7b5c
NR		 2703
2704 /* Get the IRQ */
2705 aes_dd->irq = platform_get_irq(pdev, 0);
2706 if (aes_dd->irq < 0) {
2707 dev_err(dev, "no IRQ resource info\n");
2708 err = aes_dd->irq;
b0e8b341 2709 goto res_err;
bd3c7b5c
NR		 2710 }
2711
b0e8b341
LC		 2712 err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq,
2713 IRQF_SHARED, "atmel-aes", aes_dd);
bd3c7b5c
NR		 2714 if (err) {
2715 dev_err(dev, "unable to request aes irq.\n");
b0e8b341 2716 goto res_err;
bd3c7b5c
NR		 2717 }
2718
2719 /* Initializing the clock */
b0e8b341 2720 aes_dd->iclk = devm_clk_get(&pdev->dev, "aes_clk");
bd3c7b5c 2721 if (IS_ERR(aes_dd->iclk)) {
be208356 2722 dev_err(dev, "clock initialization failed.\n");
bd3c7b5c 2723 err = PTR_ERR(aes_dd->iclk);
b0e8b341 2724 goto res_err;
bd3c7b5c
NR		 2725 }
2726
b0e8b341 2727 aes_dd->io_base = devm_ioremap_resource(&pdev->dev, aes_res);
9b52d55f 2728 if (IS_ERR(aes_dd->io_base)) {
bd3c7b5c 2729 dev_err(dev, "can't ioremap\n");
9b52d55f 2730 err = PTR_ERR(aes_dd->io_base);
b0e8b341 2731 goto res_err;
bd3c7b5c
NR		 2732 }
2733
49a20454 2734 err = clk_prepare(aes_dd->iclk);
aab0a39b
CP		 2735 if (err)
2736 goto res_err;
cadc4ab8 2737
49a20454
CP		 2738 err = atmel_aes_hw_version_init(aes_dd);
2739 if (err)
2740 goto iclk_unprepare;
2741
cadc4ab8
NR		 2742 atmel_aes_get_cap(aes_dd);
2743
89a82ef8
CP		 2744#ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC
2745 if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) {
2746 err = -EPROBE_DEFER;
2747 goto iclk_unprepare;
2748 }
2749#endif
2750
cadc4ab8
NR		 2751 err = atmel_aes_buff_init(aes_dd);
2752 if (err)
2753 goto err_aes_buff;
2754
2755 err = atmel_aes_dma_init(aes_dd, pdata);
bd3c7b5c
NR		 2756 if (err)
2757 goto err_aes_dma;
2758
2759 spin_lock(&atmel_aes.lock);
2760 list_add_tail(&aes_dd->list, &atmel_aes.dev_list);
2761 spin_unlock(&atmel_aes.lock);
2762
2763 err = atmel_aes_register_algs(aes_dd);
2764 if (err)
2765 goto err_algs;
2766
be943c7d 2767 dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n",
bbe628ed
CP		 2768 dma_chan_name(aes_dd->src.chan),
2769 dma_chan_name(aes_dd->dst.chan));
bd3c7b5c
NR		 2770
2771 return 0;
2772
2773err_algs:
2774 spin_lock(&atmel_aes.lock);
2775 list_del(&aes_dd->list);
2776 spin_unlock(&atmel_aes.lock);
2777 atmel_aes_dma_cleanup(aes_dd);
2778err_aes_dma:
cadc4ab8
NR		 2779 atmel_aes_buff_cleanup(aes_dd);
2780err_aes_buff:
49a20454
CP		 2781iclk_unprepare:
2782 clk_unprepare(aes_dd->iclk);
bd3c7b5c
NR		 2783res_err:
2784 tasklet_kill(&aes_dd->done_task);
2785 tasklet_kill(&aes_dd->queue_task);
bd3c7b5c 2786aes_dd_err:
89a82ef8
CP		 2787 if (err != -EPROBE_DEFER)
2788 dev_err(dev, "initialization failed.\n");
bd3c7b5c
NR		 2789
2790 return err;
2791}
2792
49cfe4db 2793static int atmel_aes_remove(struct platform_device *pdev)
bd3c7b5c 2794{
fc783341 2795 struct atmel_aes_dev *aes_dd;
bd3c7b5c
NR		 2796
2797 aes_dd = platform_get_drvdata(pdev);
2798 if (!aes_dd)
2799 return -ENODEV;
2800 spin_lock(&atmel_aes.lock);
2801 list_del(&aes_dd->list);
2802 spin_unlock(&atmel_aes.lock);
2803
2804 atmel_aes_unregister_algs(aes_dd);
2805
2806 tasklet_kill(&aes_dd->done_task);
2807 tasklet_kill(&aes_dd->queue_task);
2808
2809 atmel_aes_dma_cleanup(aes_dd);
2a377828 2810 atmel_aes_buff_cleanup(aes_dd);
bd3c7b5c 2811
49a20454
CP		 2812 clk_unprepare(aes_dd->iclk);
2813
bd3c7b5c
NR		 2814 return 0;
2815}
2816
2817static struct platform_driver atmel_aes_driver = {
2818 .probe = atmel_aes_probe,
49cfe4db 2819 .remove = atmel_aes_remove,
bd3c7b5c
NR		 2820 .driver = {
2821 .name = "atmel_aes",
be943c7d 2822 .of_match_table = of_match_ptr(atmel_aes_dt_ids),
bd3c7b5c
NR		 2823 },
2824};
2825
2826module_platform_driver(atmel_aes_driver);
2827
2828MODULE_DESCRIPTION("Atmel AES hw acceleration support.");
2829MODULE_LICENSE("GPL v2");
2830MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");
Ubuntu Hirsute Linux kernel mirror