]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/crypto/talitos.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'sound-4.15-rc8' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai...
[mirror_ubuntu-bionic-kernel.git] / drivers / crypto / talitos.c
1 /*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_address.h>
36 #include <linux/of_irq.h>
37 #include <linux/of_platform.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/io.h>
40 #include <linux/spinlock.h>
41 #include <linux/rtnetlink.h>
42 #include <linux/slab.h>
43
44 #include <crypto/algapi.h>
45 #include <crypto/aes.h>
46 #include <crypto/des.h>
47 #include <crypto/sha.h>
48 #include <crypto/md5.h>
49 #include <crypto/internal/aead.h>
50 #include <crypto/authenc.h>
51 #include <crypto/skcipher.h>
52 #include <crypto/hash.h>
53 #include <crypto/internal/hash.h>
54 #include <crypto/scatterwalk.h>
55
56 #include "talitos.h"
57
58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
59 unsigned int len, bool is_sec1)
60 {
61 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
62 if (is_sec1) {
63 ptr->len1 = cpu_to_be16(len);
64 } else {
65 ptr->len = cpu_to_be16(len);
66 ptr->eptr = upper_32_bits(dma_addr);
67 }
68 }
69
70 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
71 struct talitos_ptr *src_ptr, bool is_sec1)
72 {
73 dst_ptr->ptr = src_ptr->ptr;
74 if (is_sec1) {
75 dst_ptr->len1 = src_ptr->len1;
76 } else {
77 dst_ptr->len = src_ptr->len;
78 dst_ptr->eptr = src_ptr->eptr;
79 }
80 }
81
82 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
83 bool is_sec1)
84 {
85 if (is_sec1)
86 return be16_to_cpu(ptr->len1);
87 else
88 return be16_to_cpu(ptr->len);
89 }
90
91 static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
92 bool is_sec1)
93 {
94 if (!is_sec1)
95 ptr->j_extent = val;
96 }
97
98 static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
99 {
100 if (!is_sec1)
101 ptr->j_extent |= val;
102 }
103
104 /*
105 * map virtual single (contiguous) pointer to h/w descriptor pointer
106 */
107 static void map_single_talitos_ptr(struct device *dev,
108 struct talitos_ptr *ptr,
109 unsigned int len, void *data,
110 enum dma_data_direction dir)
111 {
112 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
113 struct talitos_private *priv = dev_get_drvdata(dev);
114 bool is_sec1 = has_ftr_sec1(priv);
115
116 to_talitos_ptr(ptr, dma_addr, len, is_sec1);
117 }
118
119 /*
120 * unmap bus single (contiguous) h/w descriptor pointer
121 */
122 static void unmap_single_talitos_ptr(struct device *dev,
123 struct talitos_ptr *ptr,
124 enum dma_data_direction dir)
125 {
126 struct talitos_private *priv = dev_get_drvdata(dev);
127 bool is_sec1 = has_ftr_sec1(priv);
128
129 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
130 from_talitos_ptr_len(ptr, is_sec1), dir);
131 }
132
133 static int reset_channel(struct device *dev, int ch)
134 {
135 struct talitos_private *priv = dev_get_drvdata(dev);
136 unsigned int timeout = TALITOS_TIMEOUT;
137 bool is_sec1 = has_ftr_sec1(priv);
138
139 if (is_sec1) {
140 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
141 TALITOS1_CCCR_LO_RESET);
142
143 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
144 TALITOS1_CCCR_LO_RESET) && --timeout)
145 cpu_relax();
146 } else {
147 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
148 TALITOS2_CCCR_RESET);
149
150 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
151 TALITOS2_CCCR_RESET) && --timeout)
152 cpu_relax();
153 }
154
155 if (timeout == 0) {
156 dev_err(dev, "failed to reset channel %d\n", ch);
157 return -EIO;
158 }
159
160 /* set 36-bit addressing, done writeback enable and done IRQ enable */
161 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
162 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
163 /* enable chaining descriptors */
164 if (is_sec1)
165 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
166 TALITOS_CCCR_LO_NE);
167
168 /* and ICCR writeback, if available */
169 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
170 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
171 TALITOS_CCCR_LO_IWSE);
172
173 return 0;
174 }
175
176 static int reset_device(struct device *dev)
177 {
178 struct talitos_private *priv = dev_get_drvdata(dev);
179 unsigned int timeout = TALITOS_TIMEOUT;
180 bool is_sec1 = has_ftr_sec1(priv);
181 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
182
183 setbits32(priv->reg + TALITOS_MCR, mcr);
184
185 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
186 && --timeout)
187 cpu_relax();
188
189 if (priv->irq[1]) {
190 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
191 setbits32(priv->reg + TALITOS_MCR, mcr);
192 }
193
194 if (timeout == 0) {
195 dev_err(dev, "failed to reset device\n");
196 return -EIO;
197 }
198
199 return 0;
200 }
201
202 /*
203 * Reset and initialize the device
204 */
205 static int init_device(struct device *dev)
206 {
207 struct talitos_private *priv = dev_get_drvdata(dev);
208 int ch, err;
209 bool is_sec1 = has_ftr_sec1(priv);
210
211 /*
212 * Master reset
213 * errata documentation: warning: certain SEC interrupts
214 * are not fully cleared by writing the MCR:SWR bit,
215 * set bit twice to completely reset
216 */
217 err = reset_device(dev);
218 if (err)
219 return err;
220
221 err = reset_device(dev);
222 if (err)
223 return err;
224
225 /* reset channels */
226 for (ch = 0; ch < priv->num_channels; ch++) {
227 err = reset_channel(dev, ch);
228 if (err)
229 return err;
230 }
231
232 /* enable channel done and error interrupts */
233 if (is_sec1) {
234 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
235 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
236 /* disable parity error check in DEU (erroneous? test vect.) */
237 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
238 } else {
239 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
240 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
241 }
242
243 /* disable integrity check error interrupts (use writeback instead) */
244 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
245 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
246 TALITOS_MDEUICR_LO_ICE);
247
248 return 0;
249 }
250
251 /**
252 * talitos_submit - submits a descriptor to the device for processing
253 * @dev: the SEC device to be used
254 * @ch: the SEC device channel to be used
255 * @desc: the descriptor to be processed by the device
256 * @callback: whom to call when processing is complete
257 * @context: a handle for use by caller (optional)
258 *
259 * desc must contain valid dma-mapped (bus physical) address pointers.
260 * callback must check err and feedback in descriptor header
261 * for device processing status.
262 */
263 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
264 void (*callback)(struct device *dev,
265 struct talitos_desc *desc,
266 void *context, int error),
267 void *context)
268 {
269 struct talitos_private *priv = dev_get_drvdata(dev);
270 struct talitos_request *request;
271 unsigned long flags;
272 int head;
273 bool is_sec1 = has_ftr_sec1(priv);
274
275 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
276
277 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
278 /* h/w fifo is full */
279 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
280 return -EAGAIN;
281 }
282
283 head = priv->chan[ch].head;
284 request = &priv->chan[ch].fifo[head];
285
286 /* map descriptor and save caller data */
287 if (is_sec1) {
288 desc->hdr1 = desc->hdr;
289 request->dma_desc = dma_map_single(dev, &desc->hdr1,
290 TALITOS_DESC_SIZE,
291 DMA_BIDIRECTIONAL);
292 } else {
293 request->dma_desc = dma_map_single(dev, desc,
294 TALITOS_DESC_SIZE,
295 DMA_BIDIRECTIONAL);
296 }
297 request->callback = callback;
298 request->context = context;
299
300 /* increment fifo head */
301 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
302
303 smp_wmb();
304 request->desc = desc;
305
306 /* GO! */
307 wmb();
308 out_be32(priv->chan[ch].reg + TALITOS_FF,
309 upper_32_bits(request->dma_desc));
310 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
311 lower_32_bits(request->dma_desc));
312
313 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
314
315 return -EINPROGRESS;
316 }
317 EXPORT_SYMBOL(talitos_submit);
318
319 /*
320 * process what was done, notify callback of error if not
321 */
322 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
323 {
324 struct talitos_private *priv = dev_get_drvdata(dev);
325 struct talitos_request *request, saved_req;
326 unsigned long flags;
327 int tail, status;
328 bool is_sec1 = has_ftr_sec1(priv);
329
330 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
331
332 tail = priv->chan[ch].tail;
333 while (priv->chan[ch].fifo[tail].desc) {
334 __be32 hdr;
335
336 request = &priv->chan[ch].fifo[tail];
337
338 /* descriptors with their done bits set don't get the error */
339 rmb();
340 if (!is_sec1)
341 hdr = request->desc->hdr;
342 else if (request->desc->next_desc)
343 hdr = (request->desc + 1)->hdr1;
344 else
345 hdr = request->desc->hdr1;
346
347 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
348 status = 0;
349 else
350 if (!error)
351 break;
352 else
353 status = error;
354
355 dma_unmap_single(dev, request->dma_desc,
356 TALITOS_DESC_SIZE,
357 DMA_BIDIRECTIONAL);
358
359 /* copy entries so we can call callback outside lock */
360 saved_req.desc = request->desc;
361 saved_req.callback = request->callback;
362 saved_req.context = request->context;
363
364 /* release request entry in fifo */
365 smp_wmb();
366 request->desc = NULL;
367
368 /* increment fifo tail */
369 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
370
371 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
372
373 atomic_dec(&priv->chan[ch].submit_count);
374
375 saved_req.callback(dev, saved_req.desc, saved_req.context,
376 status);
377 /* channel may resume processing in single desc error case */
378 if (error && !reset_ch && status == error)
379 return;
380 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
381 tail = priv->chan[ch].tail;
382 }
383
384 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
385 }
386
387 /*
388 * process completed requests for channels that have done status
389 */
390 #define DEF_TALITOS1_DONE(name, ch_done_mask) \
391 static void talitos1_done_##name(unsigned long data) \
392 { \
393 struct device *dev = (struct device *)data; \
394 struct talitos_private *priv = dev_get_drvdata(dev); \
395 unsigned long flags; \
396 \
397 if (ch_done_mask & 0x10000000) \
398 flush_channel(dev, 0, 0, 0); \
399 if (ch_done_mask & 0x40000000) \
400 flush_channel(dev, 1, 0, 0); \
401 if (ch_done_mask & 0x00010000) \
402 flush_channel(dev, 2, 0, 0); \
403 if (ch_done_mask & 0x00040000) \
404 flush_channel(dev, 3, 0, 0); \
405 \
406 /* At this point, all completed channels have been processed */ \
407 /* Unmask done interrupts for channels completed later on. */ \
408 spin_lock_irqsave(&priv->reg_lock, flags); \
409 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
410 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
411 spin_unlock_irqrestore(&priv->reg_lock, flags); \
412 }
413
414 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
415 DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
416
417 #define DEF_TALITOS2_DONE(name, ch_done_mask) \
418 static void talitos2_done_##name(unsigned long data) \
419 { \
420 struct device *dev = (struct device *)data; \
421 struct talitos_private *priv = dev_get_drvdata(dev); \
422 unsigned long flags; \
423 \
424 if (ch_done_mask & 1) \
425 flush_channel(dev, 0, 0, 0); \
426 if (ch_done_mask & (1 << 2)) \
427 flush_channel(dev, 1, 0, 0); \
428 if (ch_done_mask & (1 << 4)) \
429 flush_channel(dev, 2, 0, 0); \
430 if (ch_done_mask & (1 << 6)) \
431 flush_channel(dev, 3, 0, 0); \
432 \
433 /* At this point, all completed channels have been processed */ \
434 /* Unmask done interrupts for channels completed later on. */ \
435 spin_lock_irqsave(&priv->reg_lock, flags); \
436 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
437 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
438 spin_unlock_irqrestore(&priv->reg_lock, flags); \
439 }
440
441 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
442 DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
443 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
444 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
445
446 /*
447 * locate current (offending) descriptor
448 */
449 static u32 current_desc_hdr(struct device *dev, int ch)
450 {
451 struct talitos_private *priv = dev_get_drvdata(dev);
452 int tail, iter;
453 dma_addr_t cur_desc;
454
455 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
456 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
457
458 if (!cur_desc) {
459 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
460 return 0;
461 }
462
463 tail = priv->chan[ch].tail;
464
465 iter = tail;
466 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
467 priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) {
468 iter = (iter + 1) & (priv->fifo_len - 1);
469 if (iter == tail) {
470 dev_err(dev, "couldn't locate current descriptor\n");
471 return 0;
472 }
473 }
474
475 if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc)
476 return (priv->chan[ch].fifo[iter].desc + 1)->hdr;
477
478 return priv->chan[ch].fifo[iter].desc->hdr;
479 }
480
481 /*
482 * user diagnostics; report root cause of error based on execution unit status
483 */
484 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
485 {
486 struct talitos_private *priv = dev_get_drvdata(dev);
487 int i;
488
489 if (!desc_hdr)
490 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
491
492 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
493 case DESC_HDR_SEL0_AFEU:
494 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
495 in_be32(priv->reg_afeu + TALITOS_EUISR),
496 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
497 break;
498 case DESC_HDR_SEL0_DEU:
499 dev_err(dev, "DEUISR 0x%08x_%08x\n",
500 in_be32(priv->reg_deu + TALITOS_EUISR),
501 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
502 break;
503 case DESC_HDR_SEL0_MDEUA:
504 case DESC_HDR_SEL0_MDEUB:
505 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
506 in_be32(priv->reg_mdeu + TALITOS_EUISR),
507 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
508 break;
509 case DESC_HDR_SEL0_RNG:
510 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
511 in_be32(priv->reg_rngu + TALITOS_ISR),
512 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
513 break;
514 case DESC_HDR_SEL0_PKEU:
515 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
516 in_be32(priv->reg_pkeu + TALITOS_EUISR),
517 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
518 break;
519 case DESC_HDR_SEL0_AESU:
520 dev_err(dev, "AESUISR 0x%08x_%08x\n",
521 in_be32(priv->reg_aesu + TALITOS_EUISR),
522 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
523 break;
524 case DESC_HDR_SEL0_CRCU:
525 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
526 in_be32(priv->reg_crcu + TALITOS_EUISR),
527 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
528 break;
529 case DESC_HDR_SEL0_KEU:
530 dev_err(dev, "KEUISR 0x%08x_%08x\n",
531 in_be32(priv->reg_pkeu + TALITOS_EUISR),
532 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
533 break;
534 }
535
536 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
537 case DESC_HDR_SEL1_MDEUA:
538 case DESC_HDR_SEL1_MDEUB:
539 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
540 in_be32(priv->reg_mdeu + TALITOS_EUISR),
541 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
542 break;
543 case DESC_HDR_SEL1_CRCU:
544 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
545 in_be32(priv->reg_crcu + TALITOS_EUISR),
546 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
547 break;
548 }
549
550 for (i = 0; i < 8; i++)
551 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
552 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
553 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
554 }
555
556 /*
557 * recover from error interrupts
558 */
559 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
560 {
561 struct talitos_private *priv = dev_get_drvdata(dev);
562 unsigned int timeout = TALITOS_TIMEOUT;
563 int ch, error, reset_dev = 0;
564 u32 v_lo;
565 bool is_sec1 = has_ftr_sec1(priv);
566 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
567
568 for (ch = 0; ch < priv->num_channels; ch++) {
569 /* skip channels without errors */
570 if (is_sec1) {
571 /* bits 29, 31, 17, 19 */
572 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
573 continue;
574 } else {
575 if (!(isr & (1 << (ch * 2 + 1))))
576 continue;
577 }
578
579 error = -EINVAL;
580
581 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
582
583 if (v_lo & TALITOS_CCPSR_LO_DOF) {
584 dev_err(dev, "double fetch fifo overflow error\n");
585 error = -EAGAIN;
586 reset_ch = 1;
587 }
588 if (v_lo & TALITOS_CCPSR_LO_SOF) {
589 /* h/w dropped descriptor */
590 dev_err(dev, "single fetch fifo overflow error\n");
591 error = -EAGAIN;
592 }
593 if (v_lo & TALITOS_CCPSR_LO_MDTE)
594 dev_err(dev, "master data transfer error\n");
595 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
596 dev_err(dev, is_sec1 ? "pointer not complete error\n"
597 : "s/g data length zero error\n");
598 if (v_lo & TALITOS_CCPSR_LO_FPZ)
599 dev_err(dev, is_sec1 ? "parity error\n"
600 : "fetch pointer zero error\n");
601 if (v_lo & TALITOS_CCPSR_LO_IDH)
602 dev_err(dev, "illegal descriptor header error\n");
603 if (v_lo & TALITOS_CCPSR_LO_IEU)
604 dev_err(dev, is_sec1 ? "static assignment error\n"
605 : "invalid exec unit error\n");
606 if (v_lo & TALITOS_CCPSR_LO_EU)
607 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
608 if (!is_sec1) {
609 if (v_lo & TALITOS_CCPSR_LO_GB)
610 dev_err(dev, "gather boundary error\n");
611 if (v_lo & TALITOS_CCPSR_LO_GRL)
612 dev_err(dev, "gather return/length error\n");
613 if (v_lo & TALITOS_CCPSR_LO_SB)
614 dev_err(dev, "scatter boundary error\n");
615 if (v_lo & TALITOS_CCPSR_LO_SRL)
616 dev_err(dev, "scatter return/length error\n");
617 }
618
619 flush_channel(dev, ch, error, reset_ch);
620
621 if (reset_ch) {
622 reset_channel(dev, ch);
623 } else {
624 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
625 TALITOS2_CCCR_CONT);
626 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
627 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
628 TALITOS2_CCCR_CONT) && --timeout)
629 cpu_relax();
630 if (timeout == 0) {
631 dev_err(dev, "failed to restart channel %d\n",
632 ch);
633 reset_dev = 1;
634 }
635 }
636 }
637 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
638 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
639 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
640 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
641 isr, isr_lo);
642 else
643 dev_err(dev, "done overflow, internal time out, or "
644 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
645
646 /* purge request queues */
647 for (ch = 0; ch < priv->num_channels; ch++)
648 flush_channel(dev, ch, -EIO, 1);
649
650 /* reset and reinitialize the device */
651 init_device(dev);
652 }
653 }
654
655 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
656 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
657 { \
658 struct device *dev = data; \
659 struct talitos_private *priv = dev_get_drvdata(dev); \
660 u32 isr, isr_lo; \
661 unsigned long flags; \
662 \
663 spin_lock_irqsave(&priv->reg_lock, flags); \
664 isr = in_be32(priv->reg + TALITOS_ISR); \
665 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
666 /* Acknowledge interrupt */ \
667 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
668 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
669 \
670 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
671 spin_unlock_irqrestore(&priv->reg_lock, flags); \
672 talitos_error(dev, isr & ch_err_mask, isr_lo); \
673 } \
674 else { \
675 if (likely(isr & ch_done_mask)) { \
676 /* mask further done interrupts. */ \
677 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
678 /* done_task will unmask done interrupts at exit */ \
679 tasklet_schedule(&priv->done_task[tlet]); \
680 } \
681 spin_unlock_irqrestore(&priv->reg_lock, flags); \
682 } \
683 \
684 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
685 IRQ_NONE; \
686 }
687
688 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
689
690 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
691 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
692 { \
693 struct device *dev = data; \
694 struct talitos_private *priv = dev_get_drvdata(dev); \
695 u32 isr, isr_lo; \
696 unsigned long flags; \
697 \
698 spin_lock_irqsave(&priv->reg_lock, flags); \
699 isr = in_be32(priv->reg + TALITOS_ISR); \
700 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
701 /* Acknowledge interrupt */ \
702 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
703 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
704 \
705 if (unlikely(isr & ch_err_mask || isr_lo)) { \
706 spin_unlock_irqrestore(&priv->reg_lock, flags); \
707 talitos_error(dev, isr & ch_err_mask, isr_lo); \
708 } \
709 else { \
710 if (likely(isr & ch_done_mask)) { \
711 /* mask further done interrupts. */ \
712 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
713 /* done_task will unmask done interrupts at exit */ \
714 tasklet_schedule(&priv->done_task[tlet]); \
715 } \
716 spin_unlock_irqrestore(&priv->reg_lock, flags); \
717 } \
718 \
719 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
720 IRQ_NONE; \
721 }
722
723 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
724 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
725 0)
726 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
727 1)
728
729 /*
730 * hwrng
731 */
732 static int talitos_rng_data_present(struct hwrng *rng, int wait)
733 {
734 struct device *dev = (struct device *)rng->priv;
735 struct talitos_private *priv = dev_get_drvdata(dev);
736 u32 ofl;
737 int i;
738
739 for (i = 0; i < 20; i++) {
740 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
741 TALITOS_RNGUSR_LO_OFL;
742 if (ofl || !wait)
743 break;
744 udelay(10);
745 }
746
747 return !!ofl;
748 }
749
750 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
751 {
752 struct device *dev = (struct device *)rng->priv;
753 struct talitos_private *priv = dev_get_drvdata(dev);
754
755 /* rng fifo requires 64-bit accesses */
756 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
757 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
758
759 return sizeof(u32);
760 }
761
762 static int talitos_rng_init(struct hwrng *rng)
763 {
764 struct device *dev = (struct device *)rng->priv;
765 struct talitos_private *priv = dev_get_drvdata(dev);
766 unsigned int timeout = TALITOS_TIMEOUT;
767
768 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
769 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
770 & TALITOS_RNGUSR_LO_RD)
771 && --timeout)
772 cpu_relax();
773 if (timeout == 0) {
774 dev_err(dev, "failed to reset rng hw\n");
775 return -ENODEV;
776 }
777
778 /* start generating */
779 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
780
781 return 0;
782 }
783
784 static int talitos_register_rng(struct device *dev)
785 {
786 struct talitos_private *priv = dev_get_drvdata(dev);
787 int err;
788
789 priv->rng.name = dev_driver_string(dev),
790 priv->rng.init = talitos_rng_init,
791 priv->rng.data_present = talitos_rng_data_present,
792 priv->rng.data_read = talitos_rng_data_read,
793 priv->rng.priv = (unsigned long)dev;
794
795 err = hwrng_register(&priv->rng);
796 if (!err)
797 priv->rng_registered = true;
798
799 return err;
800 }
801
802 static void talitos_unregister_rng(struct device *dev)
803 {
804 struct talitos_private *priv = dev_get_drvdata(dev);
805
806 if (!priv->rng_registered)
807 return;
808
809 hwrng_unregister(&priv->rng);
810 priv->rng_registered = false;
811 }
812
813 /*
814 * crypto alg
815 */
816 #define TALITOS_CRA_PRIORITY 3000
817 /*
818 * Defines a priority for doing AEAD with descriptors type
819 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
820 */
821 #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
822 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
823 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
824
825 struct talitos_ctx {
826 struct device *dev;
827 int ch;
828 __be32 desc_hdr_template;
829 u8 key[TALITOS_MAX_KEY_SIZE];
830 u8 iv[TALITOS_MAX_IV_LENGTH];
831 dma_addr_t dma_key;
832 unsigned int keylen;
833 unsigned int enckeylen;
834 unsigned int authkeylen;
835 dma_addr_t dma_buf;
836 dma_addr_t dma_hw_context;
837 };
838
839 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
840 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
841
842 struct talitos_ahash_req_ctx {
843 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
844 unsigned int hw_context_size;
845 u8 buf[2][HASH_MAX_BLOCK_SIZE];
846 int buf_idx;
847 unsigned int swinit;
848 unsigned int first;
849 unsigned int last;
850 unsigned int to_hash_later;
851 unsigned int nbuf;
852 struct scatterlist bufsl[2];
853 struct scatterlist *psrc;
854 };
855
856 struct talitos_export_state {
857 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
858 u8 buf[HASH_MAX_BLOCK_SIZE];
859 unsigned int swinit;
860 unsigned int first;
861 unsigned int last;
862 unsigned int to_hash_later;
863 unsigned int nbuf;
864 };
865
866 static int aead_setkey(struct crypto_aead *authenc,
867 const u8 *key, unsigned int keylen)
868 {
869 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
870 struct device *dev = ctx->dev;
871 struct crypto_authenc_keys keys;
872
873 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
874 goto badkey;
875
876 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
877 goto badkey;
878
879 if (ctx->keylen)
880 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
881
882 memcpy(ctx->key, keys.authkey, keys.authkeylen);
883 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
884
885 ctx->keylen = keys.authkeylen + keys.enckeylen;
886 ctx->enckeylen = keys.enckeylen;
887 ctx->authkeylen = keys.authkeylen;
888 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
889 DMA_TO_DEVICE);
890
891 return 0;
892
893 badkey:
894 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
895 return -EINVAL;
896 }
897
898 /*
899 * talitos_edesc - s/w-extended descriptor
900 * @src_nents: number of segments in input scatterlist
901 * @dst_nents: number of segments in output scatterlist
902 * @icv_ool: whether ICV is out-of-line
903 * @iv_dma: dma address of iv for checking continuity and link table
904 * @dma_len: length of dma mapped link_tbl space
905 * @dma_link_tbl: bus physical address of link_tbl/buf
906 * @desc: h/w descriptor
907 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
908 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
909 *
910 * if decrypting (with authcheck), or either one of src_nents or dst_nents
911 * is greater than 1, an integrity check value is concatenated to the end
912 * of link_tbl data
913 */
914 struct talitos_edesc {
915 int src_nents;
916 int dst_nents;
917 bool icv_ool;
918 dma_addr_t iv_dma;
919 int dma_len;
920 dma_addr_t dma_link_tbl;
921 struct talitos_desc desc;
922 union {
923 struct talitos_ptr link_tbl[0];
924 u8 buf[0];
925 };
926 };
927
928 static void talitos_sg_unmap(struct device *dev,
929 struct talitos_edesc *edesc,
930 struct scatterlist *src,
931 struct scatterlist *dst,
932 unsigned int len, unsigned int offset)
933 {
934 struct talitos_private *priv = dev_get_drvdata(dev);
935 bool is_sec1 = has_ftr_sec1(priv);
936 unsigned int src_nents = edesc->src_nents ? : 1;
937 unsigned int dst_nents = edesc->dst_nents ? : 1;
938
939 if (is_sec1 && dst && dst_nents > 1) {
940 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
941 len, DMA_FROM_DEVICE);
942 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
943 offset);
944 }
945 if (src != dst) {
946 if (src_nents == 1 || !is_sec1)
947 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
948
949 if (dst && (dst_nents == 1 || !is_sec1))
950 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
951 } else if (src_nents == 1 || !is_sec1) {
952 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
953 }
954 }
955
956 static void ipsec_esp_unmap(struct device *dev,
957 struct talitos_edesc *edesc,
958 struct aead_request *areq)
959 {
960 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
961 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
962 unsigned int ivsize = crypto_aead_ivsize(aead);
963 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
964 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
965
966 if (is_ipsec_esp)
967 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
968 DMA_FROM_DEVICE);
969 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
970
971 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
972 areq->assoclen);
973
974 if (edesc->dma_len)
975 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
976 DMA_BIDIRECTIONAL);
977
978 if (!is_ipsec_esp) {
979 unsigned int dst_nents = edesc->dst_nents ? : 1;
980
981 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
982 areq->assoclen + areq->cryptlen - ivsize);
983 }
984 }
985
986 /*
987 * ipsec_esp descriptor callbacks
988 */
989 static void ipsec_esp_encrypt_done(struct device *dev,
990 struct talitos_desc *desc, void *context,
991 int err)
992 {
993 struct talitos_private *priv = dev_get_drvdata(dev);
994 bool is_sec1 = has_ftr_sec1(priv);
995 struct aead_request *areq = context;
996 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
997 unsigned int authsize = crypto_aead_authsize(authenc);
998 unsigned int ivsize = crypto_aead_ivsize(authenc);
999 struct talitos_edesc *edesc;
1000 struct scatterlist *sg;
1001 void *icvdata;
1002
1003 edesc = container_of(desc, struct talitos_edesc, desc);
1004
1005 ipsec_esp_unmap(dev, edesc, areq);
1006
1007 /* copy the generated ICV to dst */
1008 if (edesc->icv_ool) {
1009 if (is_sec1)
1010 icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
1011 else
1012 icvdata = &edesc->link_tbl[edesc->src_nents +
1013 edesc->dst_nents + 2];
1014 sg = sg_last(areq->dst, edesc->dst_nents);
1015 memcpy((char *)sg_virt(sg) + sg->length - authsize,
1016 icvdata, authsize);
1017 }
1018
1019 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1020
1021 kfree(edesc);
1022
1023 aead_request_complete(areq, err);
1024 }
1025
1026 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1027 struct talitos_desc *desc,
1028 void *context, int err)
1029 {
1030 struct aead_request *req = context;
1031 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1032 unsigned int authsize = crypto_aead_authsize(authenc);
1033 struct talitos_edesc *edesc;
1034 struct scatterlist *sg;
1035 char *oicv, *icv;
1036 struct talitos_private *priv = dev_get_drvdata(dev);
1037 bool is_sec1 = has_ftr_sec1(priv);
1038
1039 edesc = container_of(desc, struct talitos_edesc, desc);
1040
1041 ipsec_esp_unmap(dev, edesc, req);
1042
1043 if (!err) {
1044 /* auth check */
1045 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
1046 icv = (char *)sg_virt(sg) + sg->length - authsize;
1047
1048 if (edesc->dma_len) {
1049 if (is_sec1)
1050 oicv = (char *)&edesc->dma_link_tbl +
1051 req->assoclen + req->cryptlen;
1052 else
1053 oicv = (char *)
1054 &edesc->link_tbl[edesc->src_nents +
1055 edesc->dst_nents + 2];
1056 if (edesc->icv_ool)
1057 icv = oicv + authsize;
1058 } else
1059 oicv = (char *)&edesc->link_tbl[0];
1060
1061 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
1062 }
1063
1064 kfree(edesc);
1065
1066 aead_request_complete(req, err);
1067 }
1068
1069 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1070 struct talitos_desc *desc,
1071 void *context, int err)
1072 {
1073 struct aead_request *req = context;
1074 struct talitos_edesc *edesc;
1075
1076 edesc = container_of(desc, struct talitos_edesc, desc);
1077
1078 ipsec_esp_unmap(dev, edesc, req);
1079
1080 /* check ICV auth status */
1081 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1082 DESC_HDR_LO_ICCR1_PASS))
1083 err = -EBADMSG;
1084
1085 kfree(edesc);
1086
1087 aead_request_complete(req, err);
1088 }
1089
1090 /*
1091 * convert scatterlist to SEC h/w link table format
1092 * stop at cryptlen bytes
1093 */
1094 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1095 unsigned int offset, int cryptlen,
1096 struct talitos_ptr *link_tbl_ptr)
1097 {
1098 int n_sg = sg_count;
1099 int count = 0;
1100
1101 while (cryptlen && sg && n_sg--) {
1102 unsigned int len = sg_dma_len(sg);
1103
1104 if (offset >= len) {
1105 offset -= len;
1106 goto next;
1107 }
1108
1109 len -= offset;
1110
1111 if (len > cryptlen)
1112 len = cryptlen;
1113
1114 to_talitos_ptr(link_tbl_ptr + count,
1115 sg_dma_address(sg) + offset, len, 0);
1116 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1117 count++;
1118 cryptlen -= len;
1119 offset = 0;
1120
1121 next:
1122 sg = sg_next(sg);
1123 }
1124
1125 /* tag end of link table */
1126 if (count > 0)
1127 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1128 DESC_PTR_LNKTBL_RETURN, 0);
1129
1130 return count;
1131 }
1132
1133 static int talitos_sg_map(struct device *dev, struct scatterlist *src,
1134 unsigned int len, struct talitos_edesc *edesc,
1135 struct talitos_ptr *ptr,
1136 int sg_count, unsigned int offset, int tbl_off)
1137 {
1138 struct talitos_private *priv = dev_get_drvdata(dev);
1139 bool is_sec1 = has_ftr_sec1(priv);
1140
1141 if (sg_count == 1) {
1142 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
1143 return sg_count;
1144 }
1145 if (is_sec1) {
1146 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
1147 return sg_count;
1148 }
1149 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len,
1150 &edesc->link_tbl[tbl_off]);
1151 if (sg_count == 1) {
1152 /* Only one segment now, so no link tbl needed*/
1153 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1154 return sg_count;
1155 }
1156 to_talitos_ptr(ptr, edesc->dma_link_tbl +
1157 tbl_off * sizeof(struct talitos_ptr), len, is_sec1);
1158 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1159
1160 return sg_count;
1161 }
1162
1163 /*
1164 * fill in and submit ipsec_esp descriptor
1165 */
1166 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1167 void (*callback)(struct device *dev,
1168 struct talitos_desc *desc,
1169 void *context, int error))
1170 {
1171 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1172 unsigned int authsize = crypto_aead_authsize(aead);
1173 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1174 struct device *dev = ctx->dev;
1175 struct talitos_desc *desc = &edesc->desc;
1176 unsigned int cryptlen = areq->cryptlen;
1177 unsigned int ivsize = crypto_aead_ivsize(aead);
1178 int tbl_off = 0;
1179 int sg_count, ret;
1180 int sg_link_tbl_len;
1181 bool sync_needed = false;
1182 struct talitos_private *priv = dev_get_drvdata(dev);
1183 bool is_sec1 = has_ftr_sec1(priv);
1184 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1185 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1186 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
1187
1188 /* hmac key */
1189 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
1190
1191 sg_count = edesc->src_nents ?: 1;
1192 if (is_sec1 && sg_count > 1)
1193 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1194 areq->assoclen + cryptlen);
1195 else
1196 sg_count = dma_map_sg(dev, areq->src, sg_count,
1197 (areq->src == areq->dst) ?
1198 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1199
1200 /* hmac data */
1201 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1202 &desc->ptr[1], sg_count, 0, tbl_off);
1203
1204 if (ret > 1) {
1205 tbl_off += ret;
1206 sync_needed = true;
1207 }
1208
1209 /* cipher iv */
1210 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);
1211
1212 /* cipher key */
1213 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen,
1214 ctx->enckeylen, is_sec1);
1215
1216 /*
1217 * cipher in
1218 * map and adjust cipher len to aead request cryptlen.
1219 * extent is bytes of HMAC postpended to ciphertext,
1220 * typically 12 for ipsec
1221 */
1222 sg_link_tbl_len = cryptlen;
1223
1224 if (is_ipsec_esp) {
1225 to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1);
1226
1227 if (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)
1228 sg_link_tbl_len += authsize;
1229 }
1230
1231 ret = talitos_sg_map(dev, areq->src, sg_link_tbl_len, edesc,
1232 &desc->ptr[4], sg_count, areq->assoclen, tbl_off);
1233
1234 if (ret > 1) {
1235 tbl_off += ret;
1236 sync_needed = true;
1237 }
1238
1239 /* cipher out */
1240 if (areq->src != areq->dst) {
1241 sg_count = edesc->dst_nents ? : 1;
1242 if (!is_sec1 || sg_count == 1)
1243 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1244 }
1245
1246 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1247 sg_count, areq->assoclen, tbl_off);
1248
1249 if (is_ipsec_esp)
1250 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
1251
1252 /* ICV data */
1253 if (ret > 1) {
1254 tbl_off += ret;
1255 edesc->icv_ool = true;
1256 sync_needed = true;
1257
1258 if (is_ipsec_esp) {
1259 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1260 int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1261 sizeof(struct talitos_ptr) + authsize;
1262
1263 /* Add an entry to the link table for ICV data */
1264 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
1265 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1266 is_sec1);
1267
1268 /* icv data follows link tables */
1269 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
1270 authsize, is_sec1);
1271 } else {
1272 dma_addr_t addr = edesc->dma_link_tbl;
1273
1274 if (is_sec1)
1275 addr += areq->assoclen + cryptlen;
1276 else
1277 addr += sizeof(struct talitos_ptr) * tbl_off;
1278
1279 to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1);
1280 }
1281 } else if (!is_ipsec_esp) {
1282 ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
1283 &desc->ptr[6], sg_count, areq->assoclen +
1284 cryptlen,
1285 tbl_off);
1286 if (ret > 1) {
1287 tbl_off += ret;
1288 edesc->icv_ool = true;
1289 sync_needed = true;
1290 } else {
1291 edesc->icv_ool = false;
1292 }
1293 } else {
1294 edesc->icv_ool = false;
1295 }
1296
1297 /* iv out */
1298 if (is_ipsec_esp)
1299 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1300 DMA_FROM_DEVICE);
1301
1302 if (sync_needed)
1303 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1304 edesc->dma_len,
1305 DMA_BIDIRECTIONAL);
1306
1307 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1308 if (ret != -EINPROGRESS) {
1309 ipsec_esp_unmap(dev, edesc, areq);
1310 kfree(edesc);
1311 }
1312 return ret;
1313 }
1314
1315 /*
1316 * allocate and map the extended descriptor
1317 */
1318 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1319 struct scatterlist *src,
1320 struct scatterlist *dst,
1321 u8 *iv,
1322 unsigned int assoclen,
1323 unsigned int cryptlen,
1324 unsigned int authsize,
1325 unsigned int ivsize,
1326 int icv_stashing,
1327 u32 cryptoflags,
1328 bool encrypt)
1329 {
1330 struct talitos_edesc *edesc;
1331 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1332 dma_addr_t iv_dma = 0;
1333 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1334 GFP_ATOMIC;
1335 struct talitos_private *priv = dev_get_drvdata(dev);
1336 bool is_sec1 = has_ftr_sec1(priv);
1337 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1338 void *err;
1339
1340 if (cryptlen + authsize > max_len) {
1341 dev_err(dev, "length exceeds h/w max limit\n");
1342 return ERR_PTR(-EINVAL);
1343 }
1344
1345 if (ivsize)
1346 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1347
1348 if (!dst || dst == src) {
1349 src_len = assoclen + cryptlen + authsize;
1350 src_nents = sg_nents_for_len(src, src_len);
1351 if (src_nents < 0) {
1352 dev_err(dev, "Invalid number of src SG.\n");
1353 err = ERR_PTR(-EINVAL);
1354 goto error_sg;
1355 }
1356 src_nents = (src_nents == 1) ? 0 : src_nents;
1357 dst_nents = dst ? src_nents : 0;
1358 dst_len = 0;
1359 } else { /* dst && dst != src*/
1360 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1361 src_nents = sg_nents_for_len(src, src_len);
1362 if (src_nents < 0) {
1363 dev_err(dev, "Invalid number of src SG.\n");
1364 err = ERR_PTR(-EINVAL);
1365 goto error_sg;
1366 }
1367 src_nents = (src_nents == 1) ? 0 : src_nents;
1368 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1369 dst_nents = sg_nents_for_len(dst, dst_len);
1370 if (dst_nents < 0) {
1371 dev_err(dev, "Invalid number of dst SG.\n");
1372 err = ERR_PTR(-EINVAL);
1373 goto error_sg;
1374 }
1375 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1376 }
1377
1378 /*
1379 * allocate space for base edesc plus the link tables,
1380 * allowing for two separate entries for AD and generated ICV (+ 2),
1381 * and space for two sets of ICVs (stashed and generated)
1382 */
1383 alloc_len = sizeof(struct talitos_edesc);
1384 if (src_nents || dst_nents) {
1385 if (is_sec1)
1386 dma_len = (src_nents ? src_len : 0) +
1387 (dst_nents ? dst_len : 0);
1388 else
1389 dma_len = (src_nents + dst_nents + 2) *
1390 sizeof(struct talitos_ptr) + authsize * 2;
1391 alloc_len += dma_len;
1392 } else {
1393 dma_len = 0;
1394 alloc_len += icv_stashing ? authsize : 0;
1395 }
1396
1397 /* if its a ahash, add space for a second desc next to the first one */
1398 if (is_sec1 && !dst)
1399 alloc_len += sizeof(struct talitos_desc);
1400
1401 edesc = kmalloc(alloc_len, GFP_DMA | flags);
1402 if (!edesc) {
1403 dev_err(dev, "could not allocate edescriptor\n");
1404 err = ERR_PTR(-ENOMEM);
1405 goto error_sg;
1406 }
1407 memset(&edesc->desc, 0, sizeof(edesc->desc));
1408
1409 edesc->src_nents = src_nents;
1410 edesc->dst_nents = dst_nents;
1411 edesc->iv_dma = iv_dma;
1412 edesc->dma_len = dma_len;
1413 if (dma_len) {
1414 void *addr = &edesc->link_tbl[0];
1415
1416 if (is_sec1 && !dst)
1417 addr += sizeof(struct talitos_desc);
1418 edesc->dma_link_tbl = dma_map_single(dev, addr,
1419 edesc->dma_len,
1420 DMA_BIDIRECTIONAL);
1421 }
1422 return edesc;
1423 error_sg:
1424 if (iv_dma)
1425 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1426 return err;
1427 }
1428
1429 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1430 int icv_stashing, bool encrypt)
1431 {
1432 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1433 unsigned int authsize = crypto_aead_authsize(authenc);
1434 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1435 unsigned int ivsize = crypto_aead_ivsize(authenc);
1436
1437 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1438 iv, areq->assoclen, areq->cryptlen,
1439 authsize, ivsize, icv_stashing,
1440 areq->base.flags, encrypt);
1441 }
1442
1443 static int aead_encrypt(struct aead_request *req)
1444 {
1445 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1446 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1447 struct talitos_edesc *edesc;
1448
1449 /* allocate extended descriptor */
1450 edesc = aead_edesc_alloc(req, req->iv, 0, true);
1451 if (IS_ERR(edesc))
1452 return PTR_ERR(edesc);
1453
1454 /* set encrypt */
1455 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1456
1457 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1458 }
1459
1460 static int aead_decrypt(struct aead_request *req)
1461 {
1462 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1463 unsigned int authsize = crypto_aead_authsize(authenc);
1464 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1465 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1466 struct talitos_edesc *edesc;
1467 struct scatterlist *sg;
1468 void *icvdata;
1469
1470 req->cryptlen -= authsize;
1471
1472 /* allocate extended descriptor */
1473 edesc = aead_edesc_alloc(req, req->iv, 1, false);
1474 if (IS_ERR(edesc))
1475 return PTR_ERR(edesc);
1476
1477 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1478 ((!edesc->src_nents && !edesc->dst_nents) ||
1479 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1480
1481 /* decrypt and check the ICV */
1482 edesc->desc.hdr = ctx->desc_hdr_template |
1483 DESC_HDR_DIR_INBOUND |
1484 DESC_HDR_MODE1_MDEU_CICV;
1485
1486 /* reset integrity check result bits */
1487
1488 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1489 }
1490
1491 /* Have to check the ICV with software */
1492 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1493
1494 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1495 if (edesc->dma_len)
1496 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1497 edesc->dst_nents + 2];
1498 else
1499 icvdata = &edesc->link_tbl[0];
1500
1501 sg = sg_last(req->src, edesc->src_nents ? : 1);
1502
1503 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1504
1505 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1506 }
1507
1508 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1509 const u8 *key, unsigned int keylen)
1510 {
1511 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1512 struct device *dev = ctx->dev;
1513 u32 tmp[DES_EXPKEY_WORDS];
1514
1515 if (keylen > TALITOS_MAX_KEY_SIZE) {
1516 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1517 return -EINVAL;
1518 }
1519
1520 if (unlikely(crypto_ablkcipher_get_flags(cipher) &
1521 CRYPTO_TFM_REQ_WEAK_KEY) &&
1522 !des_ekey(tmp, key)) {
1523 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY);
1524 return -EINVAL;
1525 }
1526
1527 if (ctx->keylen)
1528 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1529
1530 memcpy(&ctx->key, key, keylen);
1531 ctx->keylen = keylen;
1532
1533 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1534
1535 return 0;
1536 }
1537
1538 static void common_nonsnoop_unmap(struct device *dev,
1539 struct talitos_edesc *edesc,
1540 struct ablkcipher_request *areq)
1541 {
1542 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1543
1544 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
1545 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1546
1547 if (edesc->dma_len)
1548 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1549 DMA_BIDIRECTIONAL);
1550 }
1551
1552 static void ablkcipher_done(struct device *dev,
1553 struct talitos_desc *desc, void *context,
1554 int err)
1555 {
1556 struct ablkcipher_request *areq = context;
1557 struct talitos_edesc *edesc;
1558
1559 edesc = container_of(desc, struct talitos_edesc, desc);
1560
1561 common_nonsnoop_unmap(dev, edesc, areq);
1562
1563 kfree(edesc);
1564
1565 areq->base.complete(&areq->base, err);
1566 }
1567
1568 static int common_nonsnoop(struct talitos_edesc *edesc,
1569 struct ablkcipher_request *areq,
1570 void (*callback) (struct device *dev,
1571 struct talitos_desc *desc,
1572 void *context, int error))
1573 {
1574 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1575 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1576 struct device *dev = ctx->dev;
1577 struct talitos_desc *desc = &edesc->desc;
1578 unsigned int cryptlen = areq->nbytes;
1579 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1580 int sg_count, ret;
1581 bool sync_needed = false;
1582 struct talitos_private *priv = dev_get_drvdata(dev);
1583 bool is_sec1 = has_ftr_sec1(priv);
1584
1585 /* first DWORD empty */
1586
1587 /* cipher iv */
1588 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
1589
1590 /* cipher key */
1591 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1);
1592
1593 sg_count = edesc->src_nents ?: 1;
1594 if (is_sec1 && sg_count > 1)
1595 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1596 cryptlen);
1597 else
1598 sg_count = dma_map_sg(dev, areq->src, sg_count,
1599 (areq->src == areq->dst) ?
1600 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1601 /*
1602 * cipher in
1603 */
1604 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1605 &desc->ptr[3], sg_count, 0, 0);
1606 if (sg_count > 1)
1607 sync_needed = true;
1608
1609 /* cipher out */
1610 if (areq->src != areq->dst) {
1611 sg_count = edesc->dst_nents ? : 1;
1612 if (!is_sec1 || sg_count == 1)
1613 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1614 }
1615
1616 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1617 sg_count, 0, (edesc->src_nents + 1));
1618 if (ret > 1)
1619 sync_needed = true;
1620
1621 /* iv out */
1622 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1623 DMA_FROM_DEVICE);
1624
1625 /* last DWORD empty */
1626
1627 if (sync_needed)
1628 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1629 edesc->dma_len, DMA_BIDIRECTIONAL);
1630
1631 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1632 if (ret != -EINPROGRESS) {
1633 common_nonsnoop_unmap(dev, edesc, areq);
1634 kfree(edesc);
1635 }
1636 return ret;
1637 }
1638
1639 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1640 areq, bool encrypt)
1641 {
1642 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1643 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1644 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1645
1646 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1647 areq->info, 0, areq->nbytes, 0, ivsize, 0,
1648 areq->base.flags, encrypt);
1649 }
1650
1651 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1652 {
1653 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1654 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1655 struct talitos_edesc *edesc;
1656
1657 /* allocate extended descriptor */
1658 edesc = ablkcipher_edesc_alloc(areq, true);
1659 if (IS_ERR(edesc))
1660 return PTR_ERR(edesc);
1661
1662 /* set encrypt */
1663 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1664
1665 return common_nonsnoop(edesc, areq, ablkcipher_done);
1666 }
1667
1668 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1669 {
1670 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1671 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1672 struct talitos_edesc *edesc;
1673
1674 /* allocate extended descriptor */
1675 edesc = ablkcipher_edesc_alloc(areq, false);
1676 if (IS_ERR(edesc))
1677 return PTR_ERR(edesc);
1678
1679 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1680
1681 return common_nonsnoop(edesc, areq, ablkcipher_done);
1682 }
1683
1684 static void common_nonsnoop_hash_unmap(struct device *dev,
1685 struct talitos_edesc *edesc,
1686 struct ahash_request *areq)
1687 {
1688 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1689
1690 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
1691
1692 if (edesc->dma_len)
1693 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1694 DMA_BIDIRECTIONAL);
1695
1696 if (edesc->desc.next_desc)
1697 dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),
1698 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
1699 }
1700
1701 static void ahash_done(struct device *dev,
1702 struct talitos_desc *desc, void *context,
1703 int err)
1704 {
1705 struct ahash_request *areq = context;
1706 struct talitos_edesc *edesc =
1707 container_of(desc, struct talitos_edesc, desc);
1708 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1709
1710 if (!req_ctx->last && req_ctx->to_hash_later) {
1711 /* Position any partial block for next update/final/finup */
1712 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
1713 req_ctx->nbuf = req_ctx->to_hash_later;
1714 }
1715 common_nonsnoop_hash_unmap(dev, edesc, areq);
1716
1717 kfree(edesc);
1718
1719 areq->base.complete(&areq->base, err);
1720 }
1721
1722 /*
1723 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1724 * ourself and submit a padded block
1725 */
1726 static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1727 struct talitos_edesc *edesc,
1728 struct talitos_ptr *ptr)
1729 {
1730 static u8 padded_hash[64] = {
1731 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1732 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1733 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1734 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1735 };
1736
1737 pr_err_once("Bug in SEC1, padding ourself\n");
1738 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1739 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1740 (char *)padded_hash, DMA_TO_DEVICE);
1741 }
1742
1743 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1744 struct ahash_request *areq, unsigned int length,
1745 unsigned int offset,
1746 void (*callback) (struct device *dev,
1747 struct talitos_desc *desc,
1748 void *context, int error))
1749 {
1750 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1751 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1752 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1753 struct device *dev = ctx->dev;
1754 struct talitos_desc *desc = &edesc->desc;
1755 int ret;
1756 bool sync_needed = false;
1757 struct talitos_private *priv = dev_get_drvdata(dev);
1758 bool is_sec1 = has_ftr_sec1(priv);
1759 int sg_count;
1760
1761 /* first DWORD empty */
1762
1763 /* hash context in */
1764 if (!req_ctx->first || req_ctx->swinit) {
1765 to_talitos_ptr(&desc->ptr[1], ctx->dma_hw_context,
1766 req_ctx->hw_context_size, is_sec1);
1767 req_ctx->swinit = 0;
1768 }
1769 /* Indicate next op is not the first. */
1770 req_ctx->first = 0;
1771
1772 /* HMAC key */
1773 if (ctx->keylen)
1774 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
1775 is_sec1);
1776
1777 if (is_sec1 && req_ctx->nbuf)
1778 length -= req_ctx->nbuf;
1779
1780 sg_count = edesc->src_nents ?: 1;
1781 if (is_sec1 && sg_count > 1)
1782 sg_pcopy_to_buffer(req_ctx->psrc, sg_count,
1783 edesc->buf + sizeof(struct talitos_desc),
1784 length, req_ctx->nbuf);
1785 else if (length)
1786 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1787 DMA_TO_DEVICE);
1788 /*
1789 * data in
1790 */
1791 if (is_sec1 && req_ctx->nbuf) {
1792 dma_addr_t dma_buf = ctx->dma_buf + req_ctx->buf_idx *
1793 HASH_MAX_BLOCK_SIZE;
1794
1795 to_talitos_ptr(&desc->ptr[3], dma_buf, req_ctx->nbuf, is_sec1);
1796 } else {
1797 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1798 &desc->ptr[3], sg_count, offset, 0);
1799 if (sg_count > 1)
1800 sync_needed = true;
1801 }
1802
1803 /* fifth DWORD empty */
1804
1805 /* hash/HMAC out -or- hash context out */
1806 if (req_ctx->last)
1807 map_single_talitos_ptr(dev, &desc->ptr[5],
1808 crypto_ahash_digestsize(tfm),
1809 areq->result, DMA_FROM_DEVICE);
1810 else
1811 to_talitos_ptr(&desc->ptr[5], ctx->dma_hw_context,
1812 req_ctx->hw_context_size, is_sec1);
1813
1814 /* last DWORD empty */
1815
1816 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1817 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1818
1819 if (is_sec1 && req_ctx->nbuf && length) {
1820 struct talitos_desc *desc2 = desc + 1;
1821 dma_addr_t next_desc;
1822
1823 memset(desc2, 0, sizeof(*desc2));
1824 desc2->hdr = desc->hdr;
1825 desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1826 desc2->hdr1 = desc2->hdr;
1827 desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1828 desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
1829 desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
1830
1831 to_talitos_ptr(&desc2->ptr[1], ctx->dma_hw_context,
1832 req_ctx->hw_context_size, is_sec1);
1833
1834 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
1835 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1836 &desc2->ptr[3], sg_count, offset, 0);
1837 if (sg_count > 1)
1838 sync_needed = true;
1839 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
1840 if (req_ctx->last)
1841 to_talitos_ptr(&desc->ptr[5], ctx->dma_hw_context,
1842 req_ctx->hw_context_size, is_sec1);
1843
1844 next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1845 DMA_BIDIRECTIONAL);
1846 desc->next_desc = cpu_to_be32(next_desc);
1847 }
1848
1849 if (sync_needed)
1850 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1851 edesc->dma_len, DMA_BIDIRECTIONAL);
1852
1853 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1854 if (ret != -EINPROGRESS) {
1855 common_nonsnoop_hash_unmap(dev, edesc, areq);
1856 kfree(edesc);
1857 }
1858 return ret;
1859 }
1860
1861 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1862 unsigned int nbytes)
1863 {
1864 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1865 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1866 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1867 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1868 bool is_sec1 = has_ftr_sec1(priv);
1869
1870 if (is_sec1)
1871 nbytes -= req_ctx->nbuf;
1872
1873 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1874 nbytes, 0, 0, 0, areq->base.flags, false);
1875 }
1876
1877 static int ahash_init(struct ahash_request *areq)
1878 {
1879 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1880 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1881 struct device *dev = ctx->dev;
1882 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1883 unsigned int size;
1884 struct talitos_private *priv = dev_get_drvdata(dev);
1885 bool is_sec1 = has_ftr_sec1(priv);
1886
1887 /* Initialize the context */
1888 req_ctx->buf_idx = 0;
1889 req_ctx->nbuf = 0;
1890 req_ctx->first = 1; /* first indicates h/w must init its context */
1891 req_ctx->swinit = 0; /* assume h/w init of context */
1892 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1893 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1894 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1895 req_ctx->hw_context_size = size;
1896
1897 if (ctx->dma_hw_context)
1898 dma_unmap_single(dev, ctx->dma_hw_context, size,
1899 DMA_BIDIRECTIONAL);
1900 ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size,
1901 DMA_BIDIRECTIONAL);
1902 if (ctx->dma_buf)
1903 dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf),
1904 DMA_TO_DEVICE);
1905 if (is_sec1)
1906 ctx->dma_buf = dma_map_single(dev, req_ctx->buf,
1907 sizeof(req_ctx->buf),
1908 DMA_TO_DEVICE);
1909 return 0;
1910 }
1911
1912 /*
1913 * on h/w without explicit sha224 support, we initialize h/w context
1914 * manually with sha224 constants, and tell it to run sha256.
1915 */
1916 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1917 {
1918 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1919 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1920 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1921 struct device *dev = ctx->dev;
1922
1923 ahash_init(areq);
1924 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1925
1926 req_ctx->hw_context[0] = SHA224_H0;
1927 req_ctx->hw_context[1] = SHA224_H1;
1928 req_ctx->hw_context[2] = SHA224_H2;
1929 req_ctx->hw_context[3] = SHA224_H3;
1930 req_ctx->hw_context[4] = SHA224_H4;
1931 req_ctx->hw_context[5] = SHA224_H5;
1932 req_ctx->hw_context[6] = SHA224_H6;
1933 req_ctx->hw_context[7] = SHA224_H7;
1934
1935 /* init 64-bit count */
1936 req_ctx->hw_context[8] = 0;
1937 req_ctx->hw_context[9] = 0;
1938
1939 dma_sync_single_for_device(dev, ctx->dma_hw_context,
1940 req_ctx->hw_context_size, DMA_TO_DEVICE);
1941
1942 return 0;
1943 }
1944
1945 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1946 {
1947 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1948 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1949 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1950 struct talitos_edesc *edesc;
1951 unsigned int blocksize =
1952 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1953 unsigned int nbytes_to_hash;
1954 unsigned int to_hash_later;
1955 unsigned int nsg;
1956 int nents;
1957 struct device *dev = ctx->dev;
1958 struct talitos_private *priv = dev_get_drvdata(dev);
1959 bool is_sec1 = has_ftr_sec1(priv);
1960 int offset = 0;
1961 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
1962
1963 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1964 /* Buffer up to one whole block */
1965 nents = sg_nents_for_len(areq->src, nbytes);
1966 if (nents < 0) {
1967 dev_err(ctx->dev, "Invalid number of src SG.\n");
1968 return nents;
1969 }
1970 sg_copy_to_buffer(areq->src, nents,
1971 ctx_buf + req_ctx->nbuf, nbytes);
1972 req_ctx->nbuf += nbytes;
1973 return 0;
1974 }
1975
1976 /* At least (blocksize + 1) bytes are available to hash */
1977 nbytes_to_hash = nbytes + req_ctx->nbuf;
1978 to_hash_later = nbytes_to_hash & (blocksize - 1);
1979
1980 if (req_ctx->last)
1981 to_hash_later = 0;
1982 else if (to_hash_later)
1983 /* There is a partial block. Hash the full block(s) now */
1984 nbytes_to_hash -= to_hash_later;
1985 else {
1986 /* Keep one block buffered */
1987 nbytes_to_hash -= blocksize;
1988 to_hash_later = blocksize;
1989 }
1990
1991 /* Chain in any previously buffered data */
1992 if (!is_sec1 && req_ctx->nbuf) {
1993 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1994 sg_init_table(req_ctx->bufsl, nsg);
1995 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
1996 if (nsg > 1)
1997 sg_chain(req_ctx->bufsl, 2, areq->src);
1998 req_ctx->psrc = req_ctx->bufsl;
1999 } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
2000 if (nbytes_to_hash > blocksize)
2001 offset = blocksize - req_ctx->nbuf;
2002 else
2003 offset = nbytes_to_hash - req_ctx->nbuf;
2004 nents = sg_nents_for_len(areq->src, offset);
2005 if (nents < 0) {
2006 dev_err(ctx->dev, "Invalid number of src SG.\n");
2007 return nents;
2008 }
2009 sg_copy_to_buffer(areq->src, nents,
2010 ctx_buf + req_ctx->nbuf, offset);
2011 req_ctx->nbuf += offset;
2012 req_ctx->psrc = areq->src;
2013 } else
2014 req_ctx->psrc = areq->src;
2015
2016 if (to_hash_later) {
2017 nents = sg_nents_for_len(areq->src, nbytes);
2018 if (nents < 0) {
2019 dev_err(ctx->dev, "Invalid number of src SG.\n");
2020 return nents;
2021 }
2022 sg_pcopy_to_buffer(areq->src, nents,
2023 req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
2024 to_hash_later,
2025 nbytes - to_hash_later);
2026 }
2027 req_ctx->to_hash_later = to_hash_later;
2028
2029 /* Allocate extended descriptor */
2030 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2031 if (IS_ERR(edesc))
2032 return PTR_ERR(edesc);
2033
2034 edesc->desc.hdr = ctx->desc_hdr_template;
2035
2036 /* On last one, request SEC to pad; otherwise continue */
2037 if (req_ctx->last)
2038 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2039 else
2040 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2041
2042 /* request SEC to INIT hash. */
2043 if (req_ctx->first && !req_ctx->swinit)
2044 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2045 if (is_sec1) {
2046 dma_addr_t dma_buf = ctx->dma_buf + req_ctx->buf_idx *
2047 HASH_MAX_BLOCK_SIZE;
2048
2049 dma_sync_single_for_device(dev, dma_buf,
2050 req_ctx->nbuf, DMA_TO_DEVICE);
2051 }
2052
2053 /* When the tfm context has a keylen, it's an HMAC.
2054 * A first or last (ie. not middle) descriptor must request HMAC.
2055 */
2056 if (ctx->keylen && (req_ctx->first || req_ctx->last))
2057 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2058
2059 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset,
2060 ahash_done);
2061 }
2062
2063 static int ahash_update(struct ahash_request *areq)
2064 {
2065 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2066
2067 req_ctx->last = 0;
2068
2069 return ahash_process_req(areq, areq->nbytes);
2070 }
2071
2072 static int ahash_final(struct ahash_request *areq)
2073 {
2074 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2075
2076 req_ctx->last = 1;
2077
2078 return ahash_process_req(areq, 0);
2079 }
2080
2081 static int ahash_finup(struct ahash_request *areq)
2082 {
2083 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2084
2085 req_ctx->last = 1;
2086
2087 return ahash_process_req(areq, areq->nbytes);
2088 }
2089
2090 static int ahash_digest(struct ahash_request *areq)
2091 {
2092 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2093 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
2094
2095 ahash->init(areq);
2096 req_ctx->last = 1;
2097
2098 return ahash_process_req(areq, areq->nbytes);
2099 }
2100
2101 static int ahash_export(struct ahash_request *areq, void *out)
2102 {
2103 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2104 struct talitos_export_state *export = out;
2105 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
2106 struct talitos_ctx *ctx = crypto_ahash_ctx(ahash);
2107 struct device *dev = ctx->dev;
2108
2109 dma_sync_single_for_cpu(dev, ctx->dma_hw_context,
2110 req_ctx->hw_context_size, DMA_FROM_DEVICE);
2111 memcpy(export->hw_context, req_ctx->hw_context,
2112 req_ctx->hw_context_size);
2113 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
2114 export->swinit = req_ctx->swinit;
2115 export->first = req_ctx->first;
2116 export->last = req_ctx->last;
2117 export->to_hash_later = req_ctx->to_hash_later;
2118 export->nbuf = req_ctx->nbuf;
2119
2120 return 0;
2121 }
2122
2123 static int ahash_import(struct ahash_request *areq, const void *in)
2124 {
2125 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2126 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2127 const struct talitos_export_state *export = in;
2128 unsigned int size;
2129 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2130 struct device *dev = ctx->dev;
2131 struct talitos_private *priv = dev_get_drvdata(dev);
2132 bool is_sec1 = has_ftr_sec1(priv);
2133
2134 memset(req_ctx, 0, sizeof(*req_ctx));
2135 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2136 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2137 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2138 req_ctx->hw_context_size = size;
2139 if (ctx->dma_hw_context)
2140 dma_unmap_single(dev, ctx->dma_hw_context, size,
2141 DMA_BIDIRECTIONAL);
2142
2143 memcpy(req_ctx->hw_context, export->hw_context, size);
2144 ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size,
2145 DMA_BIDIRECTIONAL);
2146 if (ctx->dma_buf)
2147 dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf),
2148 DMA_TO_DEVICE);
2149 memcpy(req_ctx->buf[0], export->buf, export->nbuf);
2150 if (is_sec1)
2151 ctx->dma_buf = dma_map_single(dev, req_ctx->buf,
2152 sizeof(req_ctx->buf),
2153 DMA_TO_DEVICE);
2154 req_ctx->swinit = export->swinit;
2155 req_ctx->first = export->first;
2156 req_ctx->last = export->last;
2157 req_ctx->to_hash_later = export->to_hash_later;
2158 req_ctx->nbuf = export->nbuf;
2159
2160 return 0;
2161 }
2162
2163 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2164 u8 *hash)
2165 {
2166 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2167
2168 struct scatterlist sg[1];
2169 struct ahash_request *req;
2170 struct crypto_wait wait;
2171 int ret;
2172
2173 crypto_init_wait(&wait);
2174
2175 req = ahash_request_alloc(tfm, GFP_KERNEL);
2176 if (!req)
2177 return -ENOMEM;
2178
2179 /* Keep tfm keylen == 0 during hash of the long key */
2180 ctx->keylen = 0;
2181 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2182 crypto_req_done, &wait);
2183
2184 sg_init_one(&sg[0], key, keylen);
2185
2186 ahash_request_set_crypt(req, sg, hash, keylen);
2187 ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
2188
2189 ahash_request_free(req);
2190
2191 return ret;
2192 }
2193
2194 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2195 unsigned int keylen)
2196 {
2197 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2198 struct device *dev = ctx->dev;
2199 unsigned int blocksize =
2200 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2201 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2202 unsigned int keysize = keylen;
2203 u8 hash[SHA512_DIGEST_SIZE];
2204 int ret;
2205
2206 if (keylen <= blocksize)
2207 memcpy(ctx->key, key, keysize);
2208 else {
2209 /* Must get the hash of the long key */
2210 ret = keyhash(tfm, key, keylen, hash);
2211
2212 if (ret) {
2213 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2214 return -EINVAL;
2215 }
2216
2217 keysize = digestsize;
2218 memcpy(ctx->key, hash, digestsize);
2219 }
2220
2221 if (ctx->keylen)
2222 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2223
2224 ctx->keylen = keysize;
2225 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
2226
2227 return 0;
2228 }
2229
2230
2231 struct talitos_alg_template {
2232 u32 type;
2233 u32 priority;
2234 union {
2235 struct crypto_alg crypto;
2236 struct ahash_alg hash;
2237 struct aead_alg aead;
2238 } alg;
2239 __be32 desc_hdr_template;
2240 };
2241
2242 static struct talitos_alg_template driver_algs[] = {
2243 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
2244 { .type = CRYPTO_ALG_TYPE_AEAD,
2245 .alg.aead = {
2246 .base = {
2247 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2248 .cra_driver_name = "authenc-hmac-sha1-"
2249 "cbc-aes-talitos",
2250 .cra_blocksize = AES_BLOCK_SIZE,
2251 .cra_flags = CRYPTO_ALG_ASYNC,
2252 },
2253 .ivsize = AES_BLOCK_SIZE,
2254 .maxauthsize = SHA1_DIGEST_SIZE,
2255 },
2256 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2257 DESC_HDR_SEL0_AESU |
2258 DESC_HDR_MODE0_AESU_CBC |
2259 DESC_HDR_SEL1_MDEUA |
2260 DESC_HDR_MODE1_MDEU_INIT |
2261 DESC_HDR_MODE1_MDEU_PAD |
2262 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2263 },
2264 { .type = CRYPTO_ALG_TYPE_AEAD,
2265 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2266 .alg.aead = {
2267 .base = {
2268 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2269 .cra_driver_name = "authenc-hmac-sha1-"
2270 "cbc-aes-talitos",
2271 .cra_blocksize = AES_BLOCK_SIZE,
2272 .cra_flags = CRYPTO_ALG_ASYNC,
2273 },
2274 .ivsize = AES_BLOCK_SIZE,
2275 .maxauthsize = SHA1_DIGEST_SIZE,
2276 },
2277 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2278 DESC_HDR_SEL0_AESU |
2279 DESC_HDR_MODE0_AESU_CBC |
2280 DESC_HDR_SEL1_MDEUA |
2281 DESC_HDR_MODE1_MDEU_INIT |
2282 DESC_HDR_MODE1_MDEU_PAD |
2283 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2284 },
2285 { .type = CRYPTO_ALG_TYPE_AEAD,
2286 .alg.aead = {
2287 .base = {
2288 .cra_name = "authenc(hmac(sha1),"
2289 "cbc(des3_ede))",
2290 .cra_driver_name = "authenc-hmac-sha1-"
2291 "cbc-3des-talitos",
2292 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2293 .cra_flags = CRYPTO_ALG_ASYNC,
2294 },
2295 .ivsize = DES3_EDE_BLOCK_SIZE,
2296 .maxauthsize = SHA1_DIGEST_SIZE,
2297 },
2298 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2299 DESC_HDR_SEL0_DEU |
2300 DESC_HDR_MODE0_DEU_CBC |
2301 DESC_HDR_MODE0_DEU_3DES |
2302 DESC_HDR_SEL1_MDEUA |
2303 DESC_HDR_MODE1_MDEU_INIT |
2304 DESC_HDR_MODE1_MDEU_PAD |
2305 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2306 },
2307 { .type = CRYPTO_ALG_TYPE_AEAD,
2308 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2309 .alg.aead = {
2310 .base = {
2311 .cra_name = "authenc(hmac(sha1),"
2312 "cbc(des3_ede))",
2313 .cra_driver_name = "authenc-hmac-sha1-"
2314 "cbc-3des-talitos",
2315 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2316 .cra_flags = CRYPTO_ALG_ASYNC,
2317 },
2318 .ivsize = DES3_EDE_BLOCK_SIZE,
2319 .maxauthsize = SHA1_DIGEST_SIZE,
2320 },
2321 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2322 DESC_HDR_SEL0_DEU |
2323 DESC_HDR_MODE0_DEU_CBC |
2324 DESC_HDR_MODE0_DEU_3DES |
2325 DESC_HDR_SEL1_MDEUA |
2326 DESC_HDR_MODE1_MDEU_INIT |
2327 DESC_HDR_MODE1_MDEU_PAD |
2328 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2329 },
2330 { .type = CRYPTO_ALG_TYPE_AEAD,
2331 .alg.aead = {
2332 .base = {
2333 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2334 .cra_driver_name = "authenc-hmac-sha224-"
2335 "cbc-aes-talitos",
2336 .cra_blocksize = AES_BLOCK_SIZE,
2337 .cra_flags = CRYPTO_ALG_ASYNC,
2338 },
2339 .ivsize = AES_BLOCK_SIZE,
2340 .maxauthsize = SHA224_DIGEST_SIZE,
2341 },
2342 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2343 DESC_HDR_SEL0_AESU |
2344 DESC_HDR_MODE0_AESU_CBC |
2345 DESC_HDR_SEL1_MDEUA |
2346 DESC_HDR_MODE1_MDEU_INIT |
2347 DESC_HDR_MODE1_MDEU_PAD |
2348 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2349 },
2350 { .type = CRYPTO_ALG_TYPE_AEAD,
2351 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2352 .alg.aead = {
2353 .base = {
2354 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2355 .cra_driver_name = "authenc-hmac-sha224-"
2356 "cbc-aes-talitos",
2357 .cra_blocksize = AES_BLOCK_SIZE,
2358 .cra_flags = CRYPTO_ALG_ASYNC,
2359 },
2360 .ivsize = AES_BLOCK_SIZE,
2361 .maxauthsize = SHA224_DIGEST_SIZE,
2362 },
2363 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2364 DESC_HDR_SEL0_AESU |
2365 DESC_HDR_MODE0_AESU_CBC |
2366 DESC_HDR_SEL1_MDEUA |
2367 DESC_HDR_MODE1_MDEU_INIT |
2368 DESC_HDR_MODE1_MDEU_PAD |
2369 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2370 },
2371 { .type = CRYPTO_ALG_TYPE_AEAD,
2372 .alg.aead = {
2373 .base = {
2374 .cra_name = "authenc(hmac(sha224),"
2375 "cbc(des3_ede))",
2376 .cra_driver_name = "authenc-hmac-sha224-"
2377 "cbc-3des-talitos",
2378 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2379 .cra_flags = CRYPTO_ALG_ASYNC,
2380 },
2381 .ivsize = DES3_EDE_BLOCK_SIZE,
2382 .maxauthsize = SHA224_DIGEST_SIZE,
2383 },
2384 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2385 DESC_HDR_SEL0_DEU |
2386 DESC_HDR_MODE0_DEU_CBC |
2387 DESC_HDR_MODE0_DEU_3DES |
2388 DESC_HDR_SEL1_MDEUA |
2389 DESC_HDR_MODE1_MDEU_INIT |
2390 DESC_HDR_MODE1_MDEU_PAD |
2391 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2392 },
2393 { .type = CRYPTO_ALG_TYPE_AEAD,
2394 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2395 .alg.aead = {
2396 .base = {
2397 .cra_name = "authenc(hmac(sha224),"
2398 "cbc(des3_ede))",
2399 .cra_driver_name = "authenc-hmac-sha224-"
2400 "cbc-3des-talitos",
2401 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2402 .cra_flags = CRYPTO_ALG_ASYNC,
2403 },
2404 .ivsize = DES3_EDE_BLOCK_SIZE,
2405 .maxauthsize = SHA224_DIGEST_SIZE,
2406 },
2407 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2408 DESC_HDR_SEL0_DEU |
2409 DESC_HDR_MODE0_DEU_CBC |
2410 DESC_HDR_MODE0_DEU_3DES |
2411 DESC_HDR_SEL1_MDEUA |
2412 DESC_HDR_MODE1_MDEU_INIT |
2413 DESC_HDR_MODE1_MDEU_PAD |
2414 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2415 },
2416 { .type = CRYPTO_ALG_TYPE_AEAD,
2417 .alg.aead = {
2418 .base = {
2419 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2420 .cra_driver_name = "authenc-hmac-sha256-"
2421 "cbc-aes-talitos",
2422 .cra_blocksize = AES_BLOCK_SIZE,
2423 .cra_flags = CRYPTO_ALG_ASYNC,
2424 },
2425 .ivsize = AES_BLOCK_SIZE,
2426 .maxauthsize = SHA256_DIGEST_SIZE,
2427 },
2428 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2429 DESC_HDR_SEL0_AESU |
2430 DESC_HDR_MODE0_AESU_CBC |
2431 DESC_HDR_SEL1_MDEUA |
2432 DESC_HDR_MODE1_MDEU_INIT |
2433 DESC_HDR_MODE1_MDEU_PAD |
2434 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2435 },
2436 { .type = CRYPTO_ALG_TYPE_AEAD,
2437 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2438 .alg.aead = {
2439 .base = {
2440 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2441 .cra_driver_name = "authenc-hmac-sha256-"
2442 "cbc-aes-talitos",
2443 .cra_blocksize = AES_BLOCK_SIZE,
2444 .cra_flags = CRYPTO_ALG_ASYNC,
2445 },
2446 .ivsize = AES_BLOCK_SIZE,
2447 .maxauthsize = SHA256_DIGEST_SIZE,
2448 },
2449 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2450 DESC_HDR_SEL0_AESU |
2451 DESC_HDR_MODE0_AESU_CBC |
2452 DESC_HDR_SEL1_MDEUA |
2453 DESC_HDR_MODE1_MDEU_INIT |
2454 DESC_HDR_MODE1_MDEU_PAD |
2455 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2456 },
2457 { .type = CRYPTO_ALG_TYPE_AEAD,
2458 .alg.aead = {
2459 .base = {
2460 .cra_name = "authenc(hmac(sha256),"
2461 "cbc(des3_ede))",
2462 .cra_driver_name = "authenc-hmac-sha256-"
2463 "cbc-3des-talitos",
2464 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2465 .cra_flags = CRYPTO_ALG_ASYNC,
2466 },
2467 .ivsize = DES3_EDE_BLOCK_SIZE,
2468 .maxauthsize = SHA256_DIGEST_SIZE,
2469 },
2470 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2471 DESC_HDR_SEL0_DEU |
2472 DESC_HDR_MODE0_DEU_CBC |
2473 DESC_HDR_MODE0_DEU_3DES |
2474 DESC_HDR_SEL1_MDEUA |
2475 DESC_HDR_MODE1_MDEU_INIT |
2476 DESC_HDR_MODE1_MDEU_PAD |
2477 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2478 },
2479 { .type = CRYPTO_ALG_TYPE_AEAD,
2480 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2481 .alg.aead = {
2482 .base = {
2483 .cra_name = "authenc(hmac(sha256),"
2484 "cbc(des3_ede))",
2485 .cra_driver_name = "authenc-hmac-sha256-"
2486 "cbc-3des-talitos",
2487 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2488 .cra_flags = CRYPTO_ALG_ASYNC,
2489 },
2490 .ivsize = DES3_EDE_BLOCK_SIZE,
2491 .maxauthsize = SHA256_DIGEST_SIZE,
2492 },
2493 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2494 DESC_HDR_SEL0_DEU |
2495 DESC_HDR_MODE0_DEU_CBC |
2496 DESC_HDR_MODE0_DEU_3DES |
2497 DESC_HDR_SEL1_MDEUA |
2498 DESC_HDR_MODE1_MDEU_INIT |
2499 DESC_HDR_MODE1_MDEU_PAD |
2500 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2501 },
2502 { .type = CRYPTO_ALG_TYPE_AEAD,
2503 .alg.aead = {
2504 .base = {
2505 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2506 .cra_driver_name = "authenc-hmac-sha384-"
2507 "cbc-aes-talitos",
2508 .cra_blocksize = AES_BLOCK_SIZE,
2509 .cra_flags = CRYPTO_ALG_ASYNC,
2510 },
2511 .ivsize = AES_BLOCK_SIZE,
2512 .maxauthsize = SHA384_DIGEST_SIZE,
2513 },
2514 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2515 DESC_HDR_SEL0_AESU |
2516 DESC_HDR_MODE0_AESU_CBC |
2517 DESC_HDR_SEL1_MDEUB |
2518 DESC_HDR_MODE1_MDEU_INIT |
2519 DESC_HDR_MODE1_MDEU_PAD |
2520 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2521 },
2522 { .type = CRYPTO_ALG_TYPE_AEAD,
2523 .alg.aead = {
2524 .base = {
2525 .cra_name = "authenc(hmac(sha384),"
2526 "cbc(des3_ede))",
2527 .cra_driver_name = "authenc-hmac-sha384-"
2528 "cbc-3des-talitos",
2529 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2530 .cra_flags = CRYPTO_ALG_ASYNC,
2531 },
2532 .ivsize = DES3_EDE_BLOCK_SIZE,
2533 .maxauthsize = SHA384_DIGEST_SIZE,
2534 },
2535 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2536 DESC_HDR_SEL0_DEU |
2537 DESC_HDR_MODE0_DEU_CBC |
2538 DESC_HDR_MODE0_DEU_3DES |
2539 DESC_HDR_SEL1_MDEUB |
2540 DESC_HDR_MODE1_MDEU_INIT |
2541 DESC_HDR_MODE1_MDEU_PAD |
2542 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2543 },
2544 { .type = CRYPTO_ALG_TYPE_AEAD,
2545 .alg.aead = {
2546 .base = {
2547 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2548 .cra_driver_name = "authenc-hmac-sha512-"
2549 "cbc-aes-talitos",
2550 .cra_blocksize = AES_BLOCK_SIZE,
2551 .cra_flags = CRYPTO_ALG_ASYNC,
2552 },
2553 .ivsize = AES_BLOCK_SIZE,
2554 .maxauthsize = SHA512_DIGEST_SIZE,
2555 },
2556 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2557 DESC_HDR_SEL0_AESU |
2558 DESC_HDR_MODE0_AESU_CBC |
2559 DESC_HDR_SEL1_MDEUB |
2560 DESC_HDR_MODE1_MDEU_INIT |
2561 DESC_HDR_MODE1_MDEU_PAD |
2562 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2563 },
2564 { .type = CRYPTO_ALG_TYPE_AEAD,
2565 .alg.aead = {
2566 .base = {
2567 .cra_name = "authenc(hmac(sha512),"
2568 "cbc(des3_ede))",
2569 .cra_driver_name = "authenc-hmac-sha512-"
2570 "cbc-3des-talitos",
2571 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2572 .cra_flags = CRYPTO_ALG_ASYNC,
2573 },
2574 .ivsize = DES3_EDE_BLOCK_SIZE,
2575 .maxauthsize = SHA512_DIGEST_SIZE,
2576 },
2577 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2578 DESC_HDR_SEL0_DEU |
2579 DESC_HDR_MODE0_DEU_CBC |
2580 DESC_HDR_MODE0_DEU_3DES |
2581 DESC_HDR_SEL1_MDEUB |
2582 DESC_HDR_MODE1_MDEU_INIT |
2583 DESC_HDR_MODE1_MDEU_PAD |
2584 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2585 },
2586 { .type = CRYPTO_ALG_TYPE_AEAD,
2587 .alg.aead = {
2588 .base = {
2589 .cra_name = "authenc(hmac(md5),cbc(aes))",
2590 .cra_driver_name = "authenc-hmac-md5-"
2591 "cbc-aes-talitos",
2592 .cra_blocksize = AES_BLOCK_SIZE,
2593 .cra_flags = CRYPTO_ALG_ASYNC,
2594 },
2595 .ivsize = AES_BLOCK_SIZE,
2596 .maxauthsize = MD5_DIGEST_SIZE,
2597 },
2598 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2599 DESC_HDR_SEL0_AESU |
2600 DESC_HDR_MODE0_AESU_CBC |
2601 DESC_HDR_SEL1_MDEUA |
2602 DESC_HDR_MODE1_MDEU_INIT |
2603 DESC_HDR_MODE1_MDEU_PAD |
2604 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2605 },
2606 { .type = CRYPTO_ALG_TYPE_AEAD,
2607 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2608 .alg.aead = {
2609 .base = {
2610 .cra_name = "authenc(hmac(md5),cbc(aes))",
2611 .cra_driver_name = "authenc-hmac-md5-"
2612 "cbc-aes-talitos",
2613 .cra_blocksize = AES_BLOCK_SIZE,
2614 .cra_flags = CRYPTO_ALG_ASYNC,
2615 },
2616 .ivsize = AES_BLOCK_SIZE,
2617 .maxauthsize = MD5_DIGEST_SIZE,
2618 },
2619 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2620 DESC_HDR_SEL0_AESU |
2621 DESC_HDR_MODE0_AESU_CBC |
2622 DESC_HDR_SEL1_MDEUA |
2623 DESC_HDR_MODE1_MDEU_INIT |
2624 DESC_HDR_MODE1_MDEU_PAD |
2625 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2626 },
2627 { .type = CRYPTO_ALG_TYPE_AEAD,
2628 .alg.aead = {
2629 .base = {
2630 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2631 .cra_driver_name = "authenc-hmac-md5-"
2632 "cbc-3des-talitos",
2633 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2634 .cra_flags = CRYPTO_ALG_ASYNC,
2635 },
2636 .ivsize = DES3_EDE_BLOCK_SIZE,
2637 .maxauthsize = MD5_DIGEST_SIZE,
2638 },
2639 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2640 DESC_HDR_SEL0_DEU |
2641 DESC_HDR_MODE0_DEU_CBC |
2642 DESC_HDR_MODE0_DEU_3DES |
2643 DESC_HDR_SEL1_MDEUA |
2644 DESC_HDR_MODE1_MDEU_INIT |
2645 DESC_HDR_MODE1_MDEU_PAD |
2646 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2647 },
2648 { .type = CRYPTO_ALG_TYPE_AEAD,
2649 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2650 .alg.aead = {
2651 .base = {
2652 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2653 .cra_driver_name = "authenc-hmac-md5-"
2654 "cbc-3des-talitos",
2655 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2656 .cra_flags = CRYPTO_ALG_ASYNC,
2657 },
2658 .ivsize = DES3_EDE_BLOCK_SIZE,
2659 .maxauthsize = MD5_DIGEST_SIZE,
2660 },
2661 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2662 DESC_HDR_SEL0_DEU |
2663 DESC_HDR_MODE0_DEU_CBC |
2664 DESC_HDR_MODE0_DEU_3DES |
2665 DESC_HDR_SEL1_MDEUA |
2666 DESC_HDR_MODE1_MDEU_INIT |
2667 DESC_HDR_MODE1_MDEU_PAD |
2668 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2669 },
2670 /* ABLKCIPHER algorithms. */
2671 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2672 .alg.crypto = {
2673 .cra_name = "ecb(aes)",
2674 .cra_driver_name = "ecb-aes-talitos",
2675 .cra_blocksize = AES_BLOCK_SIZE,
2676 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2677 CRYPTO_ALG_ASYNC,
2678 .cra_ablkcipher = {
2679 .min_keysize = AES_MIN_KEY_SIZE,
2680 .max_keysize = AES_MAX_KEY_SIZE,
2681 .ivsize = AES_BLOCK_SIZE,
2682 }
2683 },
2684 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2685 DESC_HDR_SEL0_AESU,
2686 },
2687 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2688 .alg.crypto = {
2689 .cra_name = "cbc(aes)",
2690 .cra_driver_name = "cbc-aes-talitos",
2691 .cra_blocksize = AES_BLOCK_SIZE,
2692 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2693 CRYPTO_ALG_ASYNC,
2694 .cra_ablkcipher = {
2695 .min_keysize = AES_MIN_KEY_SIZE,
2696 .max_keysize = AES_MAX_KEY_SIZE,
2697 .ivsize = AES_BLOCK_SIZE,
2698 }
2699 },
2700 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2701 DESC_HDR_SEL0_AESU |
2702 DESC_HDR_MODE0_AESU_CBC,
2703 },
2704 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2705 .alg.crypto = {
2706 .cra_name = "ctr(aes)",
2707 .cra_driver_name = "ctr-aes-talitos",
2708 .cra_blocksize = AES_BLOCK_SIZE,
2709 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2710 CRYPTO_ALG_ASYNC,
2711 .cra_ablkcipher = {
2712 .min_keysize = AES_MIN_KEY_SIZE,
2713 .max_keysize = AES_MAX_KEY_SIZE,
2714 .ivsize = AES_BLOCK_SIZE,
2715 }
2716 },
2717 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
2718 DESC_HDR_SEL0_AESU |
2719 DESC_HDR_MODE0_AESU_CTR,
2720 },
2721 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2722 .alg.crypto = {
2723 .cra_name = "ecb(des)",
2724 .cra_driver_name = "ecb-des-talitos",
2725 .cra_blocksize = DES_BLOCK_SIZE,
2726 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2727 CRYPTO_ALG_ASYNC,
2728 .cra_ablkcipher = {
2729 .min_keysize = DES_KEY_SIZE,
2730 .max_keysize = DES_KEY_SIZE,
2731 .ivsize = DES_BLOCK_SIZE,
2732 }
2733 },
2734 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2735 DESC_HDR_SEL0_DEU,
2736 },
2737 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2738 .alg.crypto = {
2739 .cra_name = "cbc(des)",
2740 .cra_driver_name = "cbc-des-talitos",
2741 .cra_blocksize = DES_BLOCK_SIZE,
2742 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2743 CRYPTO_ALG_ASYNC,
2744 .cra_ablkcipher = {
2745 .min_keysize = DES_KEY_SIZE,
2746 .max_keysize = DES_KEY_SIZE,
2747 .ivsize = DES_BLOCK_SIZE,
2748 }
2749 },
2750 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2751 DESC_HDR_SEL0_DEU |
2752 DESC_HDR_MODE0_DEU_CBC,
2753 },
2754 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2755 .alg.crypto = {
2756 .cra_name = "ecb(des3_ede)",
2757 .cra_driver_name = "ecb-3des-talitos",
2758 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2759 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2760 CRYPTO_ALG_ASYNC,
2761 .cra_ablkcipher = {
2762 .min_keysize = DES3_EDE_KEY_SIZE,
2763 .max_keysize = DES3_EDE_KEY_SIZE,
2764 .ivsize = DES3_EDE_BLOCK_SIZE,
2765 }
2766 },
2767 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2768 DESC_HDR_SEL0_DEU |
2769 DESC_HDR_MODE0_DEU_3DES,
2770 },
2771 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2772 .alg.crypto = {
2773 .cra_name = "cbc(des3_ede)",
2774 .cra_driver_name = "cbc-3des-talitos",
2775 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2776 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2777 CRYPTO_ALG_ASYNC,
2778 .cra_ablkcipher = {
2779 .min_keysize = DES3_EDE_KEY_SIZE,
2780 .max_keysize = DES3_EDE_KEY_SIZE,
2781 .ivsize = DES3_EDE_BLOCK_SIZE,
2782 }
2783 },
2784 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2785 DESC_HDR_SEL0_DEU |
2786 DESC_HDR_MODE0_DEU_CBC |
2787 DESC_HDR_MODE0_DEU_3DES,
2788 },
2789 /* AHASH algorithms. */
2790 { .type = CRYPTO_ALG_TYPE_AHASH,
2791 .alg.hash = {
2792 .halg.digestsize = MD5_DIGEST_SIZE,
2793 .halg.statesize = sizeof(struct talitos_export_state),
2794 .halg.base = {
2795 .cra_name = "md5",
2796 .cra_driver_name = "md5-talitos",
2797 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2798 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2799 CRYPTO_ALG_ASYNC,
2800 }
2801 },
2802 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2803 DESC_HDR_SEL0_MDEUA |
2804 DESC_HDR_MODE0_MDEU_MD5,
2805 },
2806 { .type = CRYPTO_ALG_TYPE_AHASH,
2807 .alg.hash = {
2808 .halg.digestsize = SHA1_DIGEST_SIZE,
2809 .halg.statesize = sizeof(struct talitos_export_state),
2810 .halg.base = {
2811 .cra_name = "sha1",
2812 .cra_driver_name = "sha1-talitos",
2813 .cra_blocksize = SHA1_BLOCK_SIZE,
2814 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2815 CRYPTO_ALG_ASYNC,
2816 }
2817 },
2818 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2819 DESC_HDR_SEL0_MDEUA |
2820 DESC_HDR_MODE0_MDEU_SHA1,
2821 },
2822 { .type = CRYPTO_ALG_TYPE_AHASH,
2823 .alg.hash = {
2824 .halg.digestsize = SHA224_DIGEST_SIZE,
2825 .halg.statesize = sizeof(struct talitos_export_state),
2826 .halg.base = {
2827 .cra_name = "sha224",
2828 .cra_driver_name = "sha224-talitos",
2829 .cra_blocksize = SHA224_BLOCK_SIZE,
2830 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2831 CRYPTO_ALG_ASYNC,
2832 }
2833 },
2834 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2835 DESC_HDR_SEL0_MDEUA |
2836 DESC_HDR_MODE0_MDEU_SHA224,
2837 },
2838 { .type = CRYPTO_ALG_TYPE_AHASH,
2839 .alg.hash = {
2840 .halg.digestsize = SHA256_DIGEST_SIZE,
2841 .halg.statesize = sizeof(struct talitos_export_state),
2842 .halg.base = {
2843 .cra_name = "sha256",
2844 .cra_driver_name = "sha256-talitos",
2845 .cra_blocksize = SHA256_BLOCK_SIZE,
2846 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2847 CRYPTO_ALG_ASYNC,
2848 }
2849 },
2850 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2851 DESC_HDR_SEL0_MDEUA |
2852 DESC_HDR_MODE0_MDEU_SHA256,
2853 },
2854 { .type = CRYPTO_ALG_TYPE_AHASH,
2855 .alg.hash = {
2856 .halg.digestsize = SHA384_DIGEST_SIZE,
2857 .halg.statesize = sizeof(struct talitos_export_state),
2858 .halg.base = {
2859 .cra_name = "sha384",
2860 .cra_driver_name = "sha384-talitos",
2861 .cra_blocksize = SHA384_BLOCK_SIZE,
2862 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2863 CRYPTO_ALG_ASYNC,
2864 }
2865 },
2866 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2867 DESC_HDR_SEL0_MDEUB |
2868 DESC_HDR_MODE0_MDEUB_SHA384,
2869 },
2870 { .type = CRYPTO_ALG_TYPE_AHASH,
2871 .alg.hash = {
2872 .halg.digestsize = SHA512_DIGEST_SIZE,
2873 .halg.statesize = sizeof(struct talitos_export_state),
2874 .halg.base = {
2875 .cra_name = "sha512",
2876 .cra_driver_name = "sha512-talitos",
2877 .cra_blocksize = SHA512_BLOCK_SIZE,
2878 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2879 CRYPTO_ALG_ASYNC,
2880 }
2881 },
2882 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2883 DESC_HDR_SEL0_MDEUB |
2884 DESC_HDR_MODE0_MDEUB_SHA512,
2885 },
2886 { .type = CRYPTO_ALG_TYPE_AHASH,
2887 .alg.hash = {
2888 .halg.digestsize = MD5_DIGEST_SIZE,
2889 .halg.statesize = sizeof(struct talitos_export_state),
2890 .halg.base = {
2891 .cra_name = "hmac(md5)",
2892 .cra_driver_name = "hmac-md5-talitos",
2893 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2894 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2895 CRYPTO_ALG_ASYNC,
2896 }
2897 },
2898 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2899 DESC_HDR_SEL0_MDEUA |
2900 DESC_HDR_MODE0_MDEU_MD5,
2901 },
2902 { .type = CRYPTO_ALG_TYPE_AHASH,
2903 .alg.hash = {
2904 .halg.digestsize = SHA1_DIGEST_SIZE,
2905 .halg.statesize = sizeof(struct talitos_export_state),
2906 .halg.base = {
2907 .cra_name = "hmac(sha1)",
2908 .cra_driver_name = "hmac-sha1-talitos",
2909 .cra_blocksize = SHA1_BLOCK_SIZE,
2910 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2911 CRYPTO_ALG_ASYNC,
2912 }
2913 },
2914 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2915 DESC_HDR_SEL0_MDEUA |
2916 DESC_HDR_MODE0_MDEU_SHA1,
2917 },
2918 { .type = CRYPTO_ALG_TYPE_AHASH,
2919 .alg.hash = {
2920 .halg.digestsize = SHA224_DIGEST_SIZE,
2921 .halg.statesize = sizeof(struct talitos_export_state),
2922 .halg.base = {
2923 .cra_name = "hmac(sha224)",
2924 .cra_driver_name = "hmac-sha224-talitos",
2925 .cra_blocksize = SHA224_BLOCK_SIZE,
2926 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2927 CRYPTO_ALG_ASYNC,
2928 }
2929 },
2930 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2931 DESC_HDR_SEL0_MDEUA |
2932 DESC_HDR_MODE0_MDEU_SHA224,
2933 },
2934 { .type = CRYPTO_ALG_TYPE_AHASH,
2935 .alg.hash = {
2936 .halg.digestsize = SHA256_DIGEST_SIZE,
2937 .halg.statesize = sizeof(struct talitos_export_state),
2938 .halg.base = {
2939 .cra_name = "hmac(sha256)",
2940 .cra_driver_name = "hmac-sha256-talitos",
2941 .cra_blocksize = SHA256_BLOCK_SIZE,
2942 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2943 CRYPTO_ALG_ASYNC,
2944 }
2945 },
2946 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2947 DESC_HDR_SEL0_MDEUA |
2948 DESC_HDR_MODE0_MDEU_SHA256,
2949 },
2950 { .type = CRYPTO_ALG_TYPE_AHASH,
2951 .alg.hash = {
2952 .halg.digestsize = SHA384_DIGEST_SIZE,
2953 .halg.statesize = sizeof(struct talitos_export_state),
2954 .halg.base = {
2955 .cra_name = "hmac(sha384)",
2956 .cra_driver_name = "hmac-sha384-talitos",
2957 .cra_blocksize = SHA384_BLOCK_SIZE,
2958 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2959 CRYPTO_ALG_ASYNC,
2960 }
2961 },
2962 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2963 DESC_HDR_SEL0_MDEUB |
2964 DESC_HDR_MODE0_MDEUB_SHA384,
2965 },
2966 { .type = CRYPTO_ALG_TYPE_AHASH,
2967 .alg.hash = {
2968 .halg.digestsize = SHA512_DIGEST_SIZE,
2969 .halg.statesize = sizeof(struct talitos_export_state),
2970 .halg.base = {
2971 .cra_name = "hmac(sha512)",
2972 .cra_driver_name = "hmac-sha512-talitos",
2973 .cra_blocksize = SHA512_BLOCK_SIZE,
2974 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2975 CRYPTO_ALG_ASYNC,
2976 }
2977 },
2978 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2979 DESC_HDR_SEL0_MDEUB |
2980 DESC_HDR_MODE0_MDEUB_SHA512,
2981 }
2982 };
2983
2984 struct talitos_crypto_alg {
2985 struct list_head entry;
2986 struct device *dev;
2987 struct talitos_alg_template algt;
2988 };
2989
2990 static int talitos_init_common(struct talitos_ctx *ctx,
2991 struct talitos_crypto_alg *talitos_alg)
2992 {
2993 struct talitos_private *priv;
2994
2995 /* update context with ptr to dev */
2996 ctx->dev = talitos_alg->dev;
2997
2998 /* assign SEC channel to tfm in round-robin fashion */
2999 priv = dev_get_drvdata(ctx->dev);
3000 ctx->ch = atomic_inc_return(&priv->last_chan) &
3001 (priv->num_channels - 1);
3002
3003 /* copy descriptor header template value */
3004 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
3005
3006 /* select done notification */
3007 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3008
3009 return 0;
3010 }
3011
3012 static int talitos_cra_init(struct crypto_tfm *tfm)
3013 {
3014 struct crypto_alg *alg = tfm->__crt_alg;
3015 struct talitos_crypto_alg *talitos_alg;
3016 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3017
3018 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
3019 talitos_alg = container_of(__crypto_ahash_alg(alg),
3020 struct talitos_crypto_alg,
3021 algt.alg.hash);
3022 else
3023 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3024 algt.alg.crypto);
3025
3026 return talitos_init_common(ctx, talitos_alg);
3027 }
3028
3029 static int talitos_cra_init_aead(struct crypto_aead *tfm)
3030 {
3031 struct aead_alg *alg = crypto_aead_alg(tfm);
3032 struct talitos_crypto_alg *talitos_alg;
3033 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3034
3035 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3036 algt.alg.aead);
3037
3038 return talitos_init_common(ctx, talitos_alg);
3039 }
3040
3041 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3042 {
3043 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3044
3045 talitos_cra_init(tfm);
3046
3047 ctx->keylen = 0;
3048 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
3049 sizeof(struct talitos_ahash_req_ctx));
3050
3051 return 0;
3052 }
3053
3054 static void talitos_cra_exit(struct crypto_tfm *tfm)
3055 {
3056 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3057 struct device *dev = ctx->dev;
3058
3059 if (ctx->keylen)
3060 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3061 }
3062
3063 static void talitos_cra_exit_ahash(struct crypto_tfm *tfm)
3064 {
3065 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3066 struct device *dev = ctx->dev;
3067 unsigned int size;
3068
3069 talitos_cra_exit(tfm);
3070
3071 size = (crypto_ahash_digestsize(__crypto_ahash_cast(tfm)) <=
3072 SHA256_DIGEST_SIZE)
3073 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
3074 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
3075
3076 if (ctx->dma_hw_context)
3077 dma_unmap_single(dev, ctx->dma_hw_context, size,
3078 DMA_BIDIRECTIONAL);
3079 if (ctx->dma_buf)
3080 dma_unmap_single(dev, ctx->dma_buf, HASH_MAX_BLOCK_SIZE * 2,
3081 DMA_TO_DEVICE);
3082 }
3083
3084 /*
3085 * given the alg's descriptor header template, determine whether descriptor
3086 * type and primary/secondary execution units required match the hw
3087 * capabilities description provided in the device tree node.
3088 */
3089 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3090 {
3091 struct talitos_private *priv = dev_get_drvdata(dev);
3092 int ret;
3093
3094 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3095 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3096
3097 if (SECONDARY_EU(desc_hdr_template))
3098 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3099 & priv->exec_units);
3100
3101 return ret;
3102 }
3103
3104 static int talitos_remove(struct platform_device *ofdev)
3105 {
3106 struct device *dev = &ofdev->dev;
3107 struct talitos_private *priv = dev_get_drvdata(dev);
3108 struct talitos_crypto_alg *t_alg, *n;
3109 int i;
3110
3111 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
3112 switch (t_alg->algt.type) {
3113 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3114 break;
3115 case CRYPTO_ALG_TYPE_AEAD:
3116 crypto_unregister_aead(&t_alg->algt.alg.aead);
3117 case CRYPTO_ALG_TYPE_AHASH:
3118 crypto_unregister_ahash(&t_alg->algt.alg.hash);
3119 break;
3120 }
3121 list_del(&t_alg->entry);
3122 }
3123
3124 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3125 talitos_unregister_rng(dev);
3126
3127 for (i = 0; i < 2; i++)
3128 if (priv->irq[i]) {
3129 free_irq(priv->irq[i], dev);
3130 irq_dispose_mapping(priv->irq[i]);
3131 }
3132
3133 tasklet_kill(&priv->done_task[0]);
3134 if (priv->irq[1])
3135 tasklet_kill(&priv->done_task[1]);
3136
3137 return 0;
3138 }
3139
3140 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3141 struct talitos_alg_template
3142 *template)
3143 {
3144 struct talitos_private *priv = dev_get_drvdata(dev);
3145 struct talitos_crypto_alg *t_alg;
3146 struct crypto_alg *alg;
3147
3148 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3149 GFP_KERNEL);
3150 if (!t_alg)
3151 return ERR_PTR(-ENOMEM);
3152
3153 t_alg->algt = *template;
3154
3155 switch (t_alg->algt.type) {
3156 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3157 alg = &t_alg->algt.alg.crypto;
3158 alg->cra_init = talitos_cra_init;
3159 alg->cra_exit = talitos_cra_exit;
3160 alg->cra_type = &crypto_ablkcipher_type;
3161 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
3162 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3163 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3164 alg->cra_ablkcipher.geniv = "eseqiv";
3165 break;
3166 case CRYPTO_ALG_TYPE_AEAD:
3167 alg = &t_alg->algt.alg.aead.base;
3168 alg->cra_exit = talitos_cra_exit;
3169 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3170 t_alg->algt.alg.aead.setkey = aead_setkey;
3171 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3172 t_alg->algt.alg.aead.decrypt = aead_decrypt;
3173 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3174 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
3175 devm_kfree(dev, t_alg);
3176 return ERR_PTR(-ENOTSUPP);
3177 }
3178 break;
3179 case CRYPTO_ALG_TYPE_AHASH:
3180 alg = &t_alg->algt.alg.hash.halg.base;
3181 alg->cra_init = talitos_cra_init_ahash;
3182 alg->cra_exit = talitos_cra_exit_ahash;
3183 alg->cra_type = &crypto_ahash_type;
3184 t_alg->algt.alg.hash.init = ahash_init;
3185 t_alg->algt.alg.hash.update = ahash_update;
3186 t_alg->algt.alg.hash.final = ahash_final;
3187 t_alg->algt.alg.hash.finup = ahash_finup;
3188 t_alg->algt.alg.hash.digest = ahash_digest;
3189 if (!strncmp(alg->cra_name, "hmac", 4))
3190 t_alg->algt.alg.hash.setkey = ahash_setkey;
3191 t_alg->algt.alg.hash.import = ahash_import;
3192 t_alg->algt.alg.hash.export = ahash_export;
3193
3194 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3195 !strncmp(alg->cra_name, "hmac", 4)) {
3196 devm_kfree(dev, t_alg);
3197 return ERR_PTR(-ENOTSUPP);
3198 }
3199 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3200 (!strcmp(alg->cra_name, "sha224") ||
3201 !strcmp(alg->cra_name, "hmac(sha224)"))) {
3202 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3203 t_alg->algt.desc_hdr_template =
3204 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3205 DESC_HDR_SEL0_MDEUA |
3206 DESC_HDR_MODE0_MDEU_SHA256;
3207 }
3208 break;
3209 default:
3210 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3211 devm_kfree(dev, t_alg);
3212 return ERR_PTR(-EINVAL);
3213 }
3214
3215 alg->cra_module = THIS_MODULE;
3216 if (t_alg->algt.priority)
3217 alg->cra_priority = t_alg->algt.priority;
3218 else
3219 alg->cra_priority = TALITOS_CRA_PRIORITY;
3220 alg->cra_alignmask = 0;
3221 alg->cra_ctxsize = sizeof(struct talitos_ctx);
3222 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3223
3224 t_alg->dev = dev;
3225
3226 return t_alg;
3227 }
3228
3229 static int talitos_probe_irq(struct platform_device *ofdev)
3230 {
3231 struct device *dev = &ofdev->dev;
3232 struct device_node *np = ofdev->dev.of_node;
3233 struct talitos_private *priv = dev_get_drvdata(dev);
3234 int err;
3235 bool is_sec1 = has_ftr_sec1(priv);
3236
3237 priv->irq[0] = irq_of_parse_and_map(np, 0);
3238 if (!priv->irq[0]) {
3239 dev_err(dev, "failed to map irq\n");
3240 return -EINVAL;
3241 }
3242 if (is_sec1) {
3243 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3244 dev_driver_string(dev), dev);
3245 goto primary_out;
3246 }
3247
3248 priv->irq[1] = irq_of_parse_and_map(np, 1);
3249
3250 /* get the primary irq line */
3251 if (!priv->irq[1]) {
3252 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
3253 dev_driver_string(dev), dev);
3254 goto primary_out;
3255 }
3256
3257 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
3258 dev_driver_string(dev), dev);
3259 if (err)
3260 goto primary_out;
3261
3262 /* get the secondary irq line */
3263 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
3264 dev_driver_string(dev), dev);
3265 if (err) {
3266 dev_err(dev, "failed to request secondary irq\n");
3267 irq_dispose_mapping(priv->irq[1]);
3268 priv->irq[1] = 0;
3269 }
3270
3271 return err;
3272
3273 primary_out:
3274 if (err) {
3275 dev_err(dev, "failed to request primary irq\n");
3276 irq_dispose_mapping(priv->irq[0]);
3277 priv->irq[0] = 0;
3278 }
3279
3280 return err;
3281 }
3282
3283 static int talitos_probe(struct platform_device *ofdev)
3284 {
3285 struct device *dev = &ofdev->dev;
3286 struct device_node *np = ofdev->dev.of_node;
3287 struct talitos_private *priv;
3288 int i, err;
3289 int stride;
3290 struct resource *res;
3291
3292 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
3293 if (!priv)
3294 return -ENOMEM;
3295
3296 INIT_LIST_HEAD(&priv->alg_list);
3297
3298 dev_set_drvdata(dev, priv);
3299
3300 priv->ofdev = ofdev;
3301
3302 spin_lock_init(&priv->reg_lock);
3303
3304 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3305 if (!res)
3306 return -ENXIO;
3307 priv->reg = devm_ioremap(dev, res->start, resource_size(res));
3308 if (!priv->reg) {
3309 dev_err(dev, "failed to of_iomap\n");
3310 err = -ENOMEM;
3311 goto err_out;
3312 }
3313
3314 /* get SEC version capabilities from device tree */
3315 of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
3316 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3317 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3318 of_property_read_u32(np, "fsl,descriptor-types-mask",
3319 &priv->desc_types);
3320
3321 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3322 !priv->exec_units || !priv->desc_types) {
3323 dev_err(dev, "invalid property data in device tree node\n");
3324 err = -EINVAL;
3325 goto err_out;
3326 }
3327
3328 if (of_device_is_compatible(np, "fsl,sec3.0"))
3329 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3330
3331 if (of_device_is_compatible(np, "fsl,sec2.1"))
3332 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3333 TALITOS_FTR_SHA224_HWINIT |
3334 TALITOS_FTR_HMAC_OK;
3335
3336 if (of_device_is_compatible(np, "fsl,sec1.0"))
3337 priv->features |= TALITOS_FTR_SEC1;
3338
3339 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3340 priv->reg_deu = priv->reg + TALITOS12_DEU;
3341 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3342 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3343 stride = TALITOS1_CH_STRIDE;
3344 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3345 priv->reg_deu = priv->reg + TALITOS10_DEU;
3346 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3347 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3348 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3349 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3350 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3351 stride = TALITOS1_CH_STRIDE;
3352 } else {
3353 priv->reg_deu = priv->reg + TALITOS2_DEU;
3354 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3355 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3356 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3357 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3358 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3359 priv->reg_keu = priv->reg + TALITOS2_KEU;
3360 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3361 stride = TALITOS2_CH_STRIDE;
3362 }
3363
3364 err = talitos_probe_irq(ofdev);
3365 if (err)
3366 goto err_out;
3367
3368 if (of_device_is_compatible(np, "fsl,sec1.0")) {
3369 if (priv->num_channels == 1)
3370 tasklet_init(&priv->done_task[0], talitos1_done_ch0,
3371 (unsigned long)dev);
3372 else
3373 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3374 (unsigned long)dev);
3375 } else {
3376 if (priv->irq[1]) {
3377 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3378 (unsigned long)dev);
3379 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3380 (unsigned long)dev);
3381 } else if (priv->num_channels == 1) {
3382 tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3383 (unsigned long)dev);
3384 } else {
3385 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3386 (unsigned long)dev);
3387 }
3388 }
3389
3390 priv->chan = devm_kzalloc(dev, sizeof(struct talitos_channel) *
3391 priv->num_channels, GFP_KERNEL);
3392 if (!priv->chan) {
3393 dev_err(dev, "failed to allocate channel management space\n");
3394 err = -ENOMEM;
3395 goto err_out;
3396 }
3397
3398 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3399
3400 for (i = 0; i < priv->num_channels; i++) {
3401 priv->chan[i].reg = priv->reg + stride * (i + 1);
3402 if (!priv->irq[1] || !(i & 1))
3403 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3404
3405 spin_lock_init(&priv->chan[i].head_lock);
3406 spin_lock_init(&priv->chan[i].tail_lock);
3407
3408 priv->chan[i].fifo = devm_kzalloc(dev,
3409 sizeof(struct talitos_request) *
3410 priv->fifo_len, GFP_KERNEL);
3411 if (!priv->chan[i].fifo) {
3412 dev_err(dev, "failed to allocate request fifo %d\n", i);
3413 err = -ENOMEM;
3414 goto err_out;
3415 }
3416
3417 atomic_set(&priv->chan[i].submit_count,
3418 -(priv->chfifo_len - 1));
3419 }
3420
3421 dma_set_mask(dev, DMA_BIT_MASK(36));
3422
3423 /* reset and initialize the h/w */
3424 err = init_device(dev);
3425 if (err) {
3426 dev_err(dev, "failed to initialize device\n");
3427 goto err_out;
3428 }
3429
3430 /* register the RNG, if available */
3431 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3432 err = talitos_register_rng(dev);
3433 if (err) {
3434 dev_err(dev, "failed to register hwrng: %d\n", err);
3435 goto err_out;
3436 } else
3437 dev_info(dev, "hwrng\n");
3438 }
3439
3440 /* register crypto algorithms the device supports */
3441 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3442 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3443 struct talitos_crypto_alg *t_alg;
3444 struct crypto_alg *alg = NULL;
3445
3446 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3447 if (IS_ERR(t_alg)) {
3448 err = PTR_ERR(t_alg);
3449 if (err == -ENOTSUPP)
3450 continue;
3451 goto err_out;
3452 }
3453
3454 switch (t_alg->algt.type) {
3455 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3456 err = crypto_register_alg(
3457 &t_alg->algt.alg.crypto);
3458 alg = &t_alg->algt.alg.crypto;
3459 break;
3460
3461 case CRYPTO_ALG_TYPE_AEAD:
3462 err = crypto_register_aead(
3463 &t_alg->algt.alg.aead);
3464 alg = &t_alg->algt.alg.aead.base;
3465 break;
3466
3467 case CRYPTO_ALG_TYPE_AHASH:
3468 err = crypto_register_ahash(
3469 &t_alg->algt.alg.hash);
3470 alg = &t_alg->algt.alg.hash.halg.base;
3471 break;
3472 }
3473 if (err) {
3474 dev_err(dev, "%s alg registration failed\n",
3475 alg->cra_driver_name);
3476 devm_kfree(dev, t_alg);
3477 } else
3478 list_add_tail(&t_alg->entry, &priv->alg_list);
3479 }
3480 }
3481 if (!list_empty(&priv->alg_list))
3482 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3483 (char *)of_get_property(np, "compatible", NULL));
3484
3485 return 0;
3486
3487 err_out:
3488 talitos_remove(ofdev);
3489
3490 return err;
3491 }
3492
3493 static const struct of_device_id talitos_match[] = {
3494 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3495 {
3496 .compatible = "fsl,sec1.0",
3497 },
3498 #endif
3499 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3500 {
3501 .compatible = "fsl,sec2.0",
3502 },
3503 #endif
3504 {},
3505 };
3506 MODULE_DEVICE_TABLE(of, talitos_match);
3507
3508 static struct platform_driver talitos_driver = {
3509 .driver = {
3510 .name = "talitos",
3511 .of_match_table = talitos_match,
3512 },
3513 .probe = talitos_probe,
3514 .remove = talitos_remove,
3515 };
3516
3517 module_platform_driver(talitos_driver);
3518
3519 MODULE_LICENSE("GPL");
3520 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3521 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
ubuntu-bionic-kernel mirror