2 * talitos - Freescale Integrated Security Engine (SEC) device driver
4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_address.h>
36 #include <linux/of_irq.h>
37 #include <linux/of_platform.h>
38 #include <linux/dma-mapping.h>
40 #include <linux/spinlock.h>
41 #include <linux/rtnetlink.h>
42 #include <linux/slab.h>
44 #include <crypto/algapi.h>
45 #include <crypto/aes.h>
46 #include <crypto/des.h>
47 #include <crypto/sha.h>
48 #include <crypto/md5.h>
49 #include <crypto/internal/aead.h>
50 #include <crypto/authenc.h>
51 #include <crypto/skcipher.h>
52 #include <crypto/hash.h>
53 #include <crypto/internal/hash.h>
54 #include <crypto/scatterwalk.h>
58 static void to_talitos_ptr(struct talitos_ptr
*ptr
, dma_addr_t dma_addr
,
61 ptr
->ptr
= cpu_to_be32(lower_32_bits(dma_addr
));
63 ptr
->eptr
= upper_32_bits(dma_addr
);
66 static void to_talitos_ptr_len(struct talitos_ptr
*ptr
, unsigned int len
,
71 ptr
->len1
= cpu_to_be16(len
);
73 ptr
->len
= cpu_to_be16(len
);
77 static unsigned short from_talitos_ptr_len(struct talitos_ptr
*ptr
,
81 return be16_to_cpu(ptr
->len1
);
83 return be16_to_cpu(ptr
->len
);
86 static void to_talitos_ptr_extent_clear(struct talitos_ptr
*ptr
, bool is_sec1
)
93 * map virtual single (contiguous) pointer to h/w descriptor pointer
95 static void map_single_talitos_ptr(struct device
*dev
,
96 struct talitos_ptr
*ptr
,
97 unsigned int len
, void *data
,
98 enum dma_data_direction dir
)
100 dma_addr_t dma_addr
= dma_map_single(dev
, data
, len
, dir
);
101 struct talitos_private
*priv
= dev_get_drvdata(dev
);
102 bool is_sec1
= has_ftr_sec1(priv
);
104 to_talitos_ptr_len(ptr
, len
, is_sec1
);
105 to_talitos_ptr(ptr
, dma_addr
, is_sec1
);
106 to_talitos_ptr_extent_clear(ptr
, is_sec1
);
110 * unmap bus single (contiguous) h/w descriptor pointer
112 static void unmap_single_talitos_ptr(struct device
*dev
,
113 struct talitos_ptr
*ptr
,
114 enum dma_data_direction dir
)
116 struct talitos_private
*priv
= dev_get_drvdata(dev
);
117 bool is_sec1
= has_ftr_sec1(priv
);
119 dma_unmap_single(dev
, be32_to_cpu(ptr
->ptr
),
120 from_talitos_ptr_len(ptr
, is_sec1
), dir
);
123 static int reset_channel(struct device
*dev
, int ch
)
125 struct talitos_private
*priv
= dev_get_drvdata(dev
);
126 unsigned int timeout
= TALITOS_TIMEOUT
;
127 bool is_sec1
= has_ftr_sec1(priv
);
130 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR_LO
,
131 TALITOS1_CCCR_LO_RESET
);
133 while ((in_be32(priv
->chan
[ch
].reg
+ TALITOS_CCCR_LO
) &
134 TALITOS1_CCCR_LO_RESET
) && --timeout
)
137 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR
,
138 TALITOS2_CCCR_RESET
);
140 while ((in_be32(priv
->chan
[ch
].reg
+ TALITOS_CCCR
) &
141 TALITOS2_CCCR_RESET
) && --timeout
)
146 dev_err(dev
, "failed to reset channel %d\n", ch
);
150 /* set 36-bit addressing, done writeback enable and done IRQ enable */
151 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR_LO
, TALITOS_CCCR_LO_EAE
|
152 TALITOS_CCCR_LO_CDWE
| TALITOS_CCCR_LO_CDIE
);
154 /* and ICCR writeback, if available */
155 if (priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
)
156 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR_LO
,
157 TALITOS_CCCR_LO_IWSE
);
162 static int reset_device(struct device
*dev
)
164 struct talitos_private
*priv
= dev_get_drvdata(dev
);
165 unsigned int timeout
= TALITOS_TIMEOUT
;
166 bool is_sec1
= has_ftr_sec1(priv
);
167 u32 mcr
= is_sec1
? TALITOS1_MCR_SWR
: TALITOS2_MCR_SWR
;
169 setbits32(priv
->reg
+ TALITOS_MCR
, mcr
);
171 while ((in_be32(priv
->reg
+ TALITOS_MCR
) & mcr
)
176 mcr
= TALITOS_MCR_RCA1
| TALITOS_MCR_RCA3
;
177 setbits32(priv
->reg
+ TALITOS_MCR
, mcr
);
181 dev_err(dev
, "failed to reset device\n");
189 * Reset and initialize the device
191 static int init_device(struct device
*dev
)
193 struct talitos_private
*priv
= dev_get_drvdata(dev
);
195 bool is_sec1
= has_ftr_sec1(priv
);
199 * errata documentation: warning: certain SEC interrupts
200 * are not fully cleared by writing the MCR:SWR bit,
201 * set bit twice to completely reset
203 err
= reset_device(dev
);
207 err
= reset_device(dev
);
212 for (ch
= 0; ch
< priv
->num_channels
; ch
++) {
213 err
= reset_channel(dev
, ch
);
218 /* enable channel done and error interrupts */
220 clrbits32(priv
->reg
+ TALITOS_IMR
, TALITOS1_IMR_INIT
);
221 clrbits32(priv
->reg
+ TALITOS_IMR_LO
, TALITOS1_IMR_LO_INIT
);
222 /* disable parity error check in DEU (erroneous? test vect.) */
223 setbits32(priv
->reg_deu
+ TALITOS_EUICR
, TALITOS1_DEUICR_KPE
);
225 setbits32(priv
->reg
+ TALITOS_IMR
, TALITOS2_IMR_INIT
);
226 setbits32(priv
->reg
+ TALITOS_IMR_LO
, TALITOS2_IMR_LO_INIT
);
229 /* disable integrity check error interrupts (use writeback instead) */
230 if (priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
)
231 setbits32(priv
->reg_mdeu
+ TALITOS_EUICR_LO
,
232 TALITOS_MDEUICR_LO_ICE
);
238 * talitos_submit - submits a descriptor to the device for processing
239 * @dev: the SEC device to be used
240 * @ch: the SEC device channel to be used
241 * @desc: the descriptor to be processed by the device
242 * @callback: whom to call when processing is complete
243 * @context: a handle for use by caller (optional)
245 * desc must contain valid dma-mapped (bus physical) address pointers.
246 * callback must check err and feedback in descriptor header
247 * for device processing status.
249 int talitos_submit(struct device
*dev
, int ch
, struct talitos_desc
*desc
,
250 void (*callback
)(struct device
*dev
,
251 struct talitos_desc
*desc
,
252 void *context
, int error
),
255 struct talitos_private
*priv
= dev_get_drvdata(dev
);
256 struct talitos_request
*request
;
259 bool is_sec1
= has_ftr_sec1(priv
);
261 spin_lock_irqsave(&priv
->chan
[ch
].head_lock
, flags
);
263 if (!atomic_inc_not_zero(&priv
->chan
[ch
].submit_count
)) {
264 /* h/w fifo is full */
265 spin_unlock_irqrestore(&priv
->chan
[ch
].head_lock
, flags
);
269 head
= priv
->chan
[ch
].head
;
270 request
= &priv
->chan
[ch
].fifo
[head
];
272 /* map descriptor and save caller data */
274 desc
->hdr1
= desc
->hdr
;
276 request
->dma_desc
= dma_map_single(dev
, &desc
->hdr1
,
280 request
->dma_desc
= dma_map_single(dev
, desc
,
284 request
->callback
= callback
;
285 request
->context
= context
;
287 /* increment fifo head */
288 priv
->chan
[ch
].head
= (priv
->chan
[ch
].head
+ 1) & (priv
->fifo_len
- 1);
291 request
->desc
= desc
;
295 out_be32(priv
->chan
[ch
].reg
+ TALITOS_FF
,
296 upper_32_bits(request
->dma_desc
));
297 out_be32(priv
->chan
[ch
].reg
+ TALITOS_FF_LO
,
298 lower_32_bits(request
->dma_desc
));
300 spin_unlock_irqrestore(&priv
->chan
[ch
].head_lock
, flags
);
304 EXPORT_SYMBOL(talitos_submit
);
307 * process what was done, notify callback of error if not
309 static void flush_channel(struct device
*dev
, int ch
, int error
, int reset_ch
)
311 struct talitos_private
*priv
= dev_get_drvdata(dev
);
312 struct talitos_request
*request
, saved_req
;
315 bool is_sec1
= has_ftr_sec1(priv
);
317 spin_lock_irqsave(&priv
->chan
[ch
].tail_lock
, flags
);
319 tail
= priv
->chan
[ch
].tail
;
320 while (priv
->chan
[ch
].fifo
[tail
].desc
) {
323 request
= &priv
->chan
[ch
].fifo
[tail
];
325 /* descriptors with their done bits set don't get the error */
327 hdr
= is_sec1
? request
->desc
->hdr1
: request
->desc
->hdr
;
329 if ((hdr
& DESC_HDR_DONE
) == DESC_HDR_DONE
)
337 dma_unmap_single(dev
, request
->dma_desc
,
341 /* copy entries so we can call callback outside lock */
342 saved_req
.desc
= request
->desc
;
343 saved_req
.callback
= request
->callback
;
344 saved_req
.context
= request
->context
;
346 /* release request entry in fifo */
348 request
->desc
= NULL
;
350 /* increment fifo tail */
351 priv
->chan
[ch
].tail
= (tail
+ 1) & (priv
->fifo_len
- 1);
353 spin_unlock_irqrestore(&priv
->chan
[ch
].tail_lock
, flags
);
355 atomic_dec(&priv
->chan
[ch
].submit_count
);
357 saved_req
.callback(dev
, saved_req
.desc
, saved_req
.context
,
359 /* channel may resume processing in single desc error case */
360 if (error
&& !reset_ch
&& status
== error
)
362 spin_lock_irqsave(&priv
->chan
[ch
].tail_lock
, flags
);
363 tail
= priv
->chan
[ch
].tail
;
366 spin_unlock_irqrestore(&priv
->chan
[ch
].tail_lock
, flags
);
370 * process completed requests for channels that have done status
372 #define DEF_TALITOS1_DONE(name, ch_done_mask) \
373 static void talitos1_done_##name(unsigned long data) \
375 struct device *dev = (struct device *)data; \
376 struct talitos_private *priv = dev_get_drvdata(dev); \
377 unsigned long flags; \
379 if (ch_done_mask & 0x10000000) \
380 flush_channel(dev, 0, 0, 0); \
381 if (priv->num_channels == 1) \
383 if (ch_done_mask & 0x40000000) \
384 flush_channel(dev, 1, 0, 0); \
385 if (ch_done_mask & 0x00010000) \
386 flush_channel(dev, 2, 0, 0); \
387 if (ch_done_mask & 0x00040000) \
388 flush_channel(dev, 3, 0, 0); \
391 /* At this point, all completed channels have been processed */ \
392 /* Unmask done interrupts for channels completed later on. */ \
393 spin_lock_irqsave(&priv->reg_lock, flags); \
394 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
395 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
396 spin_unlock_irqrestore(&priv->reg_lock, flags); \
399 DEF_TALITOS1_DONE(4ch
, TALITOS1_ISR_4CHDONE
)
401 #define DEF_TALITOS2_DONE(name, ch_done_mask) \
402 static void talitos2_done_##name(unsigned long data) \
404 struct device *dev = (struct device *)data; \
405 struct talitos_private *priv = dev_get_drvdata(dev); \
406 unsigned long flags; \
408 if (ch_done_mask & 1) \
409 flush_channel(dev, 0, 0, 0); \
410 if (priv->num_channels == 1) \
412 if (ch_done_mask & (1 << 2)) \
413 flush_channel(dev, 1, 0, 0); \
414 if (ch_done_mask & (1 << 4)) \
415 flush_channel(dev, 2, 0, 0); \
416 if (ch_done_mask & (1 << 6)) \
417 flush_channel(dev, 3, 0, 0); \
420 /* At this point, all completed channels have been processed */ \
421 /* Unmask done interrupts for channels completed later on. */ \
422 spin_lock_irqsave(&priv->reg_lock, flags); \
423 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
424 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
425 spin_unlock_irqrestore(&priv->reg_lock, flags); \
428 DEF_TALITOS2_DONE(4ch
, TALITOS2_ISR_4CHDONE
)
429 DEF_TALITOS2_DONE(ch0_2
, TALITOS2_ISR_CH_0_2_DONE
)
430 DEF_TALITOS2_DONE(ch1_3
, TALITOS2_ISR_CH_1_3_DONE
)
433 * locate current (offending) descriptor
435 static u32
current_desc_hdr(struct device
*dev
, int ch
)
437 struct talitos_private
*priv
= dev_get_drvdata(dev
);
441 cur_desc
= ((u64
)in_be32(priv
->chan
[ch
].reg
+ TALITOS_CDPR
)) << 32;
442 cur_desc
|= in_be32(priv
->chan
[ch
].reg
+ TALITOS_CDPR_LO
);
445 dev_err(dev
, "CDPR is NULL, giving up search for offending descriptor\n");
449 tail
= priv
->chan
[ch
].tail
;
452 while (priv
->chan
[ch
].fifo
[iter
].dma_desc
!= cur_desc
) {
453 iter
= (iter
+ 1) & (priv
->fifo_len
- 1);
455 dev_err(dev
, "couldn't locate current descriptor\n");
460 return priv
->chan
[ch
].fifo
[iter
].desc
->hdr
;
464 * user diagnostics; report root cause of error based on execution unit status
466 static void report_eu_error(struct device
*dev
, int ch
, u32 desc_hdr
)
468 struct talitos_private
*priv
= dev_get_drvdata(dev
);
472 desc_hdr
= in_be32(priv
->chan
[ch
].reg
+ TALITOS_DESCBUF
);
474 switch (desc_hdr
& DESC_HDR_SEL0_MASK
) {
475 case DESC_HDR_SEL0_AFEU
:
476 dev_err(dev
, "AFEUISR 0x%08x_%08x\n",
477 in_be32(priv
->reg_afeu
+ TALITOS_EUISR
),
478 in_be32(priv
->reg_afeu
+ TALITOS_EUISR_LO
));
480 case DESC_HDR_SEL0_DEU
:
481 dev_err(dev
, "DEUISR 0x%08x_%08x\n",
482 in_be32(priv
->reg_deu
+ TALITOS_EUISR
),
483 in_be32(priv
->reg_deu
+ TALITOS_EUISR_LO
));
485 case DESC_HDR_SEL0_MDEUA
:
486 case DESC_HDR_SEL0_MDEUB
:
487 dev_err(dev
, "MDEUISR 0x%08x_%08x\n",
488 in_be32(priv
->reg_mdeu
+ TALITOS_EUISR
),
489 in_be32(priv
->reg_mdeu
+ TALITOS_EUISR_LO
));
491 case DESC_HDR_SEL0_RNG
:
492 dev_err(dev
, "RNGUISR 0x%08x_%08x\n",
493 in_be32(priv
->reg_rngu
+ TALITOS_ISR
),
494 in_be32(priv
->reg_rngu
+ TALITOS_ISR_LO
));
496 case DESC_HDR_SEL0_PKEU
:
497 dev_err(dev
, "PKEUISR 0x%08x_%08x\n",
498 in_be32(priv
->reg_pkeu
+ TALITOS_EUISR
),
499 in_be32(priv
->reg_pkeu
+ TALITOS_EUISR_LO
));
501 case DESC_HDR_SEL0_AESU
:
502 dev_err(dev
, "AESUISR 0x%08x_%08x\n",
503 in_be32(priv
->reg_aesu
+ TALITOS_EUISR
),
504 in_be32(priv
->reg_aesu
+ TALITOS_EUISR_LO
));
506 case DESC_HDR_SEL0_CRCU
:
507 dev_err(dev
, "CRCUISR 0x%08x_%08x\n",
508 in_be32(priv
->reg_crcu
+ TALITOS_EUISR
),
509 in_be32(priv
->reg_crcu
+ TALITOS_EUISR_LO
));
511 case DESC_HDR_SEL0_KEU
:
512 dev_err(dev
, "KEUISR 0x%08x_%08x\n",
513 in_be32(priv
->reg_pkeu
+ TALITOS_EUISR
),
514 in_be32(priv
->reg_pkeu
+ TALITOS_EUISR_LO
));
518 switch (desc_hdr
& DESC_HDR_SEL1_MASK
) {
519 case DESC_HDR_SEL1_MDEUA
:
520 case DESC_HDR_SEL1_MDEUB
:
521 dev_err(dev
, "MDEUISR 0x%08x_%08x\n",
522 in_be32(priv
->reg_mdeu
+ TALITOS_EUISR
),
523 in_be32(priv
->reg_mdeu
+ TALITOS_EUISR_LO
));
525 case DESC_HDR_SEL1_CRCU
:
526 dev_err(dev
, "CRCUISR 0x%08x_%08x\n",
527 in_be32(priv
->reg_crcu
+ TALITOS_EUISR
),
528 in_be32(priv
->reg_crcu
+ TALITOS_EUISR_LO
));
532 for (i
= 0; i
< 8; i
++)
533 dev_err(dev
, "DESCBUF 0x%08x_%08x\n",
534 in_be32(priv
->chan
[ch
].reg
+ TALITOS_DESCBUF
+ 8*i
),
535 in_be32(priv
->chan
[ch
].reg
+ TALITOS_DESCBUF_LO
+ 8*i
));
539 * recover from error interrupts
541 static void talitos_error(struct device
*dev
, u32 isr
, u32 isr_lo
)
543 struct talitos_private
*priv
= dev_get_drvdata(dev
);
544 unsigned int timeout
= TALITOS_TIMEOUT
;
545 int ch
, error
, reset_dev
= 0;
547 bool is_sec1
= has_ftr_sec1(priv
);
548 int reset_ch
= is_sec1
? 1 : 0; /* only SEC2 supports continuation */
550 for (ch
= 0; ch
< priv
->num_channels
; ch
++) {
551 /* skip channels without errors */
553 /* bits 29, 31, 17, 19 */
554 if (!(isr
& (1 << (29 + (ch
& 1) * 2 - (ch
& 2) * 6))))
557 if (!(isr
& (1 << (ch
* 2 + 1))))
563 v_lo
= in_be32(priv
->chan
[ch
].reg
+ TALITOS_CCPSR_LO
);
565 if (v_lo
& TALITOS_CCPSR_LO_DOF
) {
566 dev_err(dev
, "double fetch fifo overflow error\n");
570 if (v_lo
& TALITOS_CCPSR_LO_SOF
) {
571 /* h/w dropped descriptor */
572 dev_err(dev
, "single fetch fifo overflow error\n");
575 if (v_lo
& TALITOS_CCPSR_LO_MDTE
)
576 dev_err(dev
, "master data transfer error\n");
577 if (v_lo
& TALITOS_CCPSR_LO_SGDLZ
)
578 dev_err(dev
, is_sec1
? "pointeur not complete error\n"
579 : "s/g data length zero error\n");
580 if (v_lo
& TALITOS_CCPSR_LO_FPZ
)
581 dev_err(dev
, is_sec1
? "parity error\n"
582 : "fetch pointer zero error\n");
583 if (v_lo
& TALITOS_CCPSR_LO_IDH
)
584 dev_err(dev
, "illegal descriptor header error\n");
585 if (v_lo
& TALITOS_CCPSR_LO_IEU
)
586 dev_err(dev
, is_sec1
? "static assignment error\n"
587 : "invalid exec unit error\n");
588 if (v_lo
& TALITOS_CCPSR_LO_EU
)
589 report_eu_error(dev
, ch
, current_desc_hdr(dev
, ch
));
591 if (v_lo
& TALITOS_CCPSR_LO_GB
)
592 dev_err(dev
, "gather boundary error\n");
593 if (v_lo
& TALITOS_CCPSR_LO_GRL
)
594 dev_err(dev
, "gather return/length error\n");
595 if (v_lo
& TALITOS_CCPSR_LO_SB
)
596 dev_err(dev
, "scatter boundary error\n");
597 if (v_lo
& TALITOS_CCPSR_LO_SRL
)
598 dev_err(dev
, "scatter return/length error\n");
601 flush_channel(dev
, ch
, error
, reset_ch
);
604 reset_channel(dev
, ch
);
606 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR
,
608 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR_LO
, 0);
609 while ((in_be32(priv
->chan
[ch
].reg
+ TALITOS_CCCR
) &
610 TALITOS2_CCCR_CONT
) && --timeout
)
613 dev_err(dev
, "failed to restart channel %d\n",
619 if (reset_dev
|| (is_sec1
&& isr
& ~TALITOS1_ISR_4CHERR
) ||
620 (!is_sec1
&& isr
& ~TALITOS2_ISR_4CHERR
) || isr_lo
) {
621 if (is_sec1
&& (isr_lo
& TALITOS1_ISR_TEA_ERR
))
622 dev_err(dev
, "TEA error: ISR 0x%08x_%08x\n",
625 dev_err(dev
, "done overflow, internal time out, or "
626 "rngu error: ISR 0x%08x_%08x\n", isr
, isr_lo
);
628 /* purge request queues */
629 for (ch
= 0; ch
< priv
->num_channels
; ch
++)
630 flush_channel(dev
, ch
, -EIO
, 1);
632 /* reset and reinitialize the device */
637 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
638 static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
640 struct device *dev = data; \
641 struct talitos_private *priv = dev_get_drvdata(dev); \
643 unsigned long flags; \
645 spin_lock_irqsave(&priv->reg_lock, flags); \
646 isr = in_be32(priv->reg + TALITOS_ISR); \
647 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
648 /* Acknowledge interrupt */ \
649 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
650 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
652 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
653 spin_unlock_irqrestore(&priv->reg_lock, flags); \
654 talitos_error(dev, isr & ch_err_mask, isr_lo); \
657 if (likely(isr & ch_done_mask)) { \
658 /* mask further done interrupts. */ \
659 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
660 /* done_task will unmask done interrupts at exit */ \
661 tasklet_schedule(&priv->done_task[tlet]); \
663 spin_unlock_irqrestore(&priv->reg_lock, flags); \
666 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
670 DEF_TALITOS1_INTERRUPT(4ch
, TALITOS1_ISR_4CHDONE
, TALITOS1_ISR_4CHERR
, 0)
672 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
673 static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
675 struct device *dev = data; \
676 struct talitos_private *priv = dev_get_drvdata(dev); \
678 unsigned long flags; \
680 spin_lock_irqsave(&priv->reg_lock, flags); \
681 isr = in_be32(priv->reg + TALITOS_ISR); \
682 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
683 /* Acknowledge interrupt */ \
684 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
685 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
687 if (unlikely(isr & ch_err_mask || isr_lo)) { \
688 spin_unlock_irqrestore(&priv->reg_lock, flags); \
689 talitos_error(dev, isr & ch_err_mask, isr_lo); \
692 if (likely(isr & ch_done_mask)) { \
693 /* mask further done interrupts. */ \
694 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
695 /* done_task will unmask done interrupts at exit */ \
696 tasklet_schedule(&priv->done_task[tlet]); \
698 spin_unlock_irqrestore(&priv->reg_lock, flags); \
701 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
705 DEF_TALITOS2_INTERRUPT(4ch
, TALITOS2_ISR_4CHDONE
, TALITOS2_ISR_4CHERR
, 0)
706 DEF_TALITOS2_INTERRUPT(ch0_2
, TALITOS2_ISR_CH_0_2_DONE
, TALITOS2_ISR_CH_0_2_ERR
,
708 DEF_TALITOS2_INTERRUPT(ch1_3
, TALITOS2_ISR_CH_1_3_DONE
, TALITOS2_ISR_CH_1_3_ERR
,
714 static int talitos_rng_data_present(struct hwrng
*rng
, int wait
)
716 struct device
*dev
= (struct device
*)rng
->priv
;
717 struct talitos_private
*priv
= dev_get_drvdata(dev
);
721 for (i
= 0; i
< 20; i
++) {
722 ofl
= in_be32(priv
->reg_rngu
+ TALITOS_EUSR_LO
) &
723 TALITOS_RNGUSR_LO_OFL
;
732 static int talitos_rng_data_read(struct hwrng
*rng
, u32
*data
)
734 struct device
*dev
= (struct device
*)rng
->priv
;
735 struct talitos_private
*priv
= dev_get_drvdata(dev
);
737 /* rng fifo requires 64-bit accesses */
738 *data
= in_be32(priv
->reg_rngu
+ TALITOS_EU_FIFO
);
739 *data
= in_be32(priv
->reg_rngu
+ TALITOS_EU_FIFO_LO
);
744 static int talitos_rng_init(struct hwrng
*rng
)
746 struct device
*dev
= (struct device
*)rng
->priv
;
747 struct talitos_private
*priv
= dev_get_drvdata(dev
);
748 unsigned int timeout
= TALITOS_TIMEOUT
;
750 setbits32(priv
->reg_rngu
+ TALITOS_EURCR_LO
, TALITOS_RNGURCR_LO_SR
);
751 while (!(in_be32(priv
->reg_rngu
+ TALITOS_EUSR_LO
)
752 & TALITOS_RNGUSR_LO_RD
)
756 dev_err(dev
, "failed to reset rng hw\n");
760 /* start generating */
761 setbits32(priv
->reg_rngu
+ TALITOS_EUDSR_LO
, 0);
766 static int talitos_register_rng(struct device
*dev
)
768 struct talitos_private
*priv
= dev_get_drvdata(dev
);
771 priv
->rng
.name
= dev_driver_string(dev
),
772 priv
->rng
.init
= talitos_rng_init
,
773 priv
->rng
.data_present
= talitos_rng_data_present
,
774 priv
->rng
.data_read
= talitos_rng_data_read
,
775 priv
->rng
.priv
= (unsigned long)dev
;
777 err
= hwrng_register(&priv
->rng
);
779 priv
->rng_registered
= true;
784 static void talitos_unregister_rng(struct device
*dev
)
786 struct talitos_private
*priv
= dev_get_drvdata(dev
);
788 if (!priv
->rng_registered
)
791 hwrng_unregister(&priv
->rng
);
792 priv
->rng_registered
= false;
798 #define TALITOS_CRA_PRIORITY 3000
799 #define TALITOS_MAX_KEY_SIZE 96
800 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
805 __be32 desc_hdr_template
;
806 u8 key
[TALITOS_MAX_KEY_SIZE
];
807 u8 iv
[TALITOS_MAX_IV_LENGTH
];
809 unsigned int enckeylen
;
810 unsigned int authkeylen
;
813 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
814 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
816 struct talitos_ahash_req_ctx
{
817 u32 hw_context
[TALITOS_MDEU_MAX_CONTEXT_SIZE
/ sizeof(u32
)];
818 unsigned int hw_context_size
;
819 u8 buf
[HASH_MAX_BLOCK_SIZE
];
820 u8 bufnext
[HASH_MAX_BLOCK_SIZE
];
824 unsigned int to_hash_later
;
826 struct scatterlist bufsl
[2];
827 struct scatterlist
*psrc
;
830 static int aead_setkey(struct crypto_aead
*authenc
,
831 const u8
*key
, unsigned int keylen
)
833 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
834 struct crypto_authenc_keys keys
;
836 if (crypto_authenc_extractkeys(&keys
, key
, keylen
) != 0)
839 if (keys
.authkeylen
+ keys
.enckeylen
> TALITOS_MAX_KEY_SIZE
)
842 memcpy(ctx
->key
, keys
.authkey
, keys
.authkeylen
);
843 memcpy(&ctx
->key
[keys
.authkeylen
], keys
.enckey
, keys
.enckeylen
);
845 ctx
->keylen
= keys
.authkeylen
+ keys
.enckeylen
;
846 ctx
->enckeylen
= keys
.enckeylen
;
847 ctx
->authkeylen
= keys
.authkeylen
;
852 crypto_aead_set_flags(authenc
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
857 * talitos_edesc - s/w-extended descriptor
858 * @src_nents: number of segments in input scatterlist
859 * @dst_nents: number of segments in output scatterlist
860 * @src_chained: whether src is chained or not
861 * @dst_chained: whether dst is chained or not
862 * @icv_ool: whether ICV is out-of-line
863 * @iv_dma: dma address of iv for checking continuity and link table
864 * @dma_len: length of dma mapped link_tbl space
865 * @dma_link_tbl: bus physical address of link_tbl/buf
866 * @desc: h/w descriptor
867 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
868 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
870 * if decrypting (with authcheck), or either one of src_nents or dst_nents
871 * is greater than 1, an integrity check value is concatenated to the end
874 struct talitos_edesc
{
882 dma_addr_t dma_link_tbl
;
883 struct talitos_desc desc
;
885 struct talitos_ptr link_tbl
[0];
890 static int talitos_map_sg(struct device
*dev
, struct scatterlist
*sg
,
891 unsigned int nents
, enum dma_data_direction dir
,
894 if (unlikely(chained
))
896 dma_map_sg(dev
, sg
, 1, dir
);
900 dma_map_sg(dev
, sg
, nents
, dir
);
904 static void talitos_unmap_sg_chain(struct device
*dev
, struct scatterlist
*sg
,
905 enum dma_data_direction dir
)
908 dma_unmap_sg(dev
, sg
, 1, dir
);
913 static void talitos_sg_unmap(struct device
*dev
,
914 struct talitos_edesc
*edesc
,
915 struct scatterlist
*src
,
916 struct scatterlist
*dst
)
918 unsigned int src_nents
= edesc
->src_nents
? : 1;
919 unsigned int dst_nents
= edesc
->dst_nents
? : 1;
922 if (edesc
->src_chained
)
923 talitos_unmap_sg_chain(dev
, src
, DMA_TO_DEVICE
);
925 dma_unmap_sg(dev
, src
, src_nents
, DMA_TO_DEVICE
);
928 if (edesc
->dst_chained
)
929 talitos_unmap_sg_chain(dev
, dst
,
932 dma_unmap_sg(dev
, dst
, dst_nents
,
936 if (edesc
->src_chained
)
937 talitos_unmap_sg_chain(dev
, src
, DMA_BIDIRECTIONAL
);
939 dma_unmap_sg(dev
, src
, src_nents
, DMA_BIDIRECTIONAL
);
942 static void ipsec_esp_unmap(struct device
*dev
,
943 struct talitos_edesc
*edesc
,
944 struct aead_request
*areq
)
946 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[6], DMA_FROM_DEVICE
);
947 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[3], DMA_TO_DEVICE
);
948 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2], DMA_TO_DEVICE
);
949 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[0], DMA_TO_DEVICE
);
951 talitos_sg_unmap(dev
, edesc
, areq
->src
, areq
->dst
);
954 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
959 * ipsec_esp descriptor callbacks
961 static void ipsec_esp_encrypt_done(struct device
*dev
,
962 struct talitos_desc
*desc
, void *context
,
965 struct aead_request
*areq
= context
;
966 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
967 unsigned int authsize
= crypto_aead_authsize(authenc
);
968 struct talitos_edesc
*edesc
;
969 struct scatterlist
*sg
;
972 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
974 ipsec_esp_unmap(dev
, edesc
, areq
);
976 /* copy the generated ICV to dst */
977 if (edesc
->icv_ool
) {
978 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
979 edesc
->dst_nents
+ 2];
980 sg
= sg_last(areq
->dst
, edesc
->dst_nents
);
981 memcpy((char *)sg_virt(sg
) + sg
->length
- authsize
,
987 aead_request_complete(areq
, err
);
990 static void ipsec_esp_decrypt_swauth_done(struct device
*dev
,
991 struct talitos_desc
*desc
,
992 void *context
, int err
)
994 struct aead_request
*req
= context
;
995 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
996 unsigned int authsize
= crypto_aead_authsize(authenc
);
997 struct talitos_edesc
*edesc
;
998 struct scatterlist
*sg
;
1001 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
1003 ipsec_esp_unmap(dev
, edesc
, req
);
1007 sg
= sg_last(req
->dst
, edesc
->dst_nents
? : 1);
1008 icv
= (char *)sg_virt(sg
) + sg
->length
- authsize
;
1010 if (edesc
->dma_len
) {
1011 oicv
= (char *)&edesc
->link_tbl
[edesc
->src_nents
+
1012 edesc
->dst_nents
+ 2];
1014 icv
= oicv
+ authsize
;
1016 oicv
= (char *)&edesc
->link_tbl
[0];
1018 err
= memcmp(oicv
, icv
, authsize
) ? -EBADMSG
: 0;
1023 aead_request_complete(req
, err
);
1026 static void ipsec_esp_decrypt_hwauth_done(struct device
*dev
,
1027 struct talitos_desc
*desc
,
1028 void *context
, int err
)
1030 struct aead_request
*req
= context
;
1031 struct talitos_edesc
*edesc
;
1033 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
1035 ipsec_esp_unmap(dev
, edesc
, req
);
1037 /* check ICV auth status */
1038 if (!err
&& ((desc
->hdr_lo
& DESC_HDR_LO_ICCR1_MASK
) !=
1039 DESC_HDR_LO_ICCR1_PASS
))
1044 aead_request_complete(req
, err
);
1048 * convert scatterlist to SEC h/w link table format
1049 * stop at cryptlen bytes
1051 static int sg_to_link_tbl_offset(struct scatterlist
*sg
, int sg_count
,
1052 unsigned int offset
, int cryptlen
,
1053 struct talitos_ptr
*link_tbl_ptr
)
1055 int n_sg
= sg_count
;
1058 while (cryptlen
&& sg
&& n_sg
--) {
1059 unsigned int len
= sg_dma_len(sg
);
1061 if (offset
>= len
) {
1071 to_talitos_ptr(link_tbl_ptr
+ count
,
1072 sg_dma_address(sg
) + offset
, 0);
1073 link_tbl_ptr
[count
].len
= cpu_to_be16(len
);
1074 link_tbl_ptr
[count
].j_extent
= 0;
1083 /* tag end of link table */
1085 link_tbl_ptr
[count
- 1].j_extent
= DESC_PTR_LNKTBL_RETURN
;
1090 static inline int sg_to_link_tbl(struct scatterlist
*sg
, int sg_count
,
1092 struct talitos_ptr
*link_tbl_ptr
)
1094 return sg_to_link_tbl_offset(sg
, sg_count
, 0, cryptlen
,
1099 * fill in and submit ipsec_esp descriptor
1101 static int ipsec_esp(struct talitos_edesc
*edesc
, struct aead_request
*areq
,
1102 void (*callback
)(struct device
*dev
,
1103 struct talitos_desc
*desc
,
1104 void *context
, int error
))
1106 struct crypto_aead
*aead
= crypto_aead_reqtfm(areq
);
1107 unsigned int authsize
= crypto_aead_authsize(aead
);
1108 struct talitos_ctx
*ctx
= crypto_aead_ctx(aead
);
1109 struct device
*dev
= ctx
->dev
;
1110 struct talitos_desc
*desc
= &edesc
->desc
;
1111 unsigned int cryptlen
= areq
->cryptlen
;
1112 unsigned int ivsize
= crypto_aead_ivsize(aead
);
1115 int sg_link_tbl_len
;
1118 map_single_talitos_ptr(dev
, &desc
->ptr
[0], ctx
->authkeylen
, &ctx
->key
,
1121 sg_count
= talitos_map_sg(dev
, areq
->src
, edesc
->src_nents
?: 1,
1122 (areq
->src
== areq
->dst
) ? DMA_BIDIRECTIONAL
1124 edesc
->src_chained
);
1127 desc
->ptr
[1].len
= cpu_to_be16(areq
->assoclen
);
1129 (ret
= sg_to_link_tbl_offset(areq
->src
, sg_count
, 0,
1131 &edesc
->link_tbl
[tbl_off
])) > 1) {
1134 to_talitos_ptr(&desc
->ptr
[1], edesc
->dma_link_tbl
+ tbl_off
*
1135 sizeof(struct talitos_ptr
), 0);
1136 desc
->ptr
[1].j_extent
= DESC_PTR_LNKTBL_JUMP
;
1138 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1139 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1141 to_talitos_ptr(&desc
->ptr
[1], sg_dma_address(areq
->src
), 0);
1142 desc
->ptr
[1].j_extent
= 0;
1146 to_talitos_ptr(&desc
->ptr
[2], edesc
->iv_dma
, 0);
1147 desc
->ptr
[2].len
= cpu_to_be16(ivsize
);
1148 desc
->ptr
[2].j_extent
= 0;
1151 map_single_talitos_ptr(dev
, &desc
->ptr
[3], ctx
->enckeylen
,
1152 (char *)&ctx
->key
+ ctx
->authkeylen
,
1157 * map and adjust cipher len to aead request cryptlen.
1158 * extent is bytes of HMAC postpended to ciphertext,
1159 * typically 12 for ipsec
1161 desc
->ptr
[4].len
= cpu_to_be16(cryptlen
);
1162 desc
->ptr
[4].j_extent
= authsize
;
1164 sg_link_tbl_len
= cryptlen
;
1165 if (edesc
->desc
.hdr
& DESC_HDR_MODE1_MDEU_CICV
)
1166 sg_link_tbl_len
+= authsize
;
1169 (ret
= sg_to_link_tbl_offset(areq
->src
, sg_count
, areq
->assoclen
,
1171 &edesc
->link_tbl
[tbl_off
])) > 1) {
1173 desc
->ptr
[4].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1174 to_talitos_ptr(&desc
->ptr
[4], edesc
->dma_link_tbl
+
1176 sizeof(struct talitos_ptr
), 0);
1177 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1181 to_talitos_ptr(&desc
->ptr
[4], sg_dma_address(areq
->src
), 0);
1184 desc
->ptr
[5].len
= cpu_to_be16(cryptlen
);
1185 desc
->ptr
[5].j_extent
= authsize
;
1187 if (areq
->src
!= areq
->dst
)
1188 sg_count
= talitos_map_sg(dev
, areq
->dst
,
1189 edesc
->dst_nents
? : 1,
1190 DMA_FROM_DEVICE
, edesc
->dst_chained
);
1192 edesc
->icv_ool
= false;
1195 (sg_count
= sg_to_link_tbl_offset(areq
->dst
, sg_count
,
1196 areq
->assoclen
, cryptlen
,
1197 &edesc
->link_tbl
[tbl_off
])) >
1199 struct talitos_ptr
*tbl_ptr
= &edesc
->link_tbl
[tbl_off
];
1201 to_talitos_ptr(&desc
->ptr
[5], edesc
->dma_link_tbl
+
1202 tbl_off
* sizeof(struct talitos_ptr
), 0);
1204 /* Add an entry to the link table for ICV data */
1205 tbl_ptr
+= sg_count
- 1;
1206 tbl_ptr
->j_extent
= 0;
1208 tbl_ptr
->j_extent
= DESC_PTR_LNKTBL_RETURN
;
1209 tbl_ptr
->len
= cpu_to_be16(authsize
);
1211 /* icv data follows link tables */
1212 to_talitos_ptr(tbl_ptr
, edesc
->dma_link_tbl
+
1213 (edesc
->src_nents
+ edesc
->dst_nents
+
1214 2) * sizeof(struct talitos_ptr
) +
1216 desc
->ptr
[5].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1217 dma_sync_single_for_device(ctx
->dev
, edesc
->dma_link_tbl
,
1218 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1220 edesc
->icv_ool
= true;
1222 to_talitos_ptr(&desc
->ptr
[5], sg_dma_address(areq
->dst
), 0);
1225 map_single_talitos_ptr(dev
, &desc
->ptr
[6], ivsize
, ctx
->iv
,
1228 ret
= talitos_submit(dev
, ctx
->ch
, desc
, callback
, areq
);
1229 if (ret
!= -EINPROGRESS
) {
1230 ipsec_esp_unmap(dev
, edesc
, areq
);
1237 * derive number of elements in scatterlist
1239 static int sg_count(struct scatterlist
*sg_list
, int nbytes
, bool *chained
)
1241 struct scatterlist
*sg
= sg_list
;
1245 while (nbytes
> 0 && sg
) {
1247 nbytes
-= sg
->length
;
1248 if (!sg_is_last(sg
) && (sg
+ 1)->length
== 0)
1257 * allocate and map the extended descriptor
1259 static struct talitos_edesc
*talitos_edesc_alloc(struct device
*dev
,
1260 struct scatterlist
*src
,
1261 struct scatterlist
*dst
,
1263 unsigned int assoclen
,
1264 unsigned int cryptlen
,
1265 unsigned int authsize
,
1266 unsigned int ivsize
,
1271 struct talitos_edesc
*edesc
;
1272 int src_nents
, dst_nents
, alloc_len
, dma_len
;
1273 bool src_chained
= false, dst_chained
= false;
1274 dma_addr_t iv_dma
= 0;
1275 gfp_t flags
= cryptoflags
& CRYPTO_TFM_REQ_MAY_SLEEP
? GFP_KERNEL
:
1277 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1278 bool is_sec1
= has_ftr_sec1(priv
);
1279 int max_len
= is_sec1
? TALITOS1_MAX_DATA_LEN
: TALITOS2_MAX_DATA_LEN
;
1281 if (cryptlen
+ authsize
> max_len
) {
1282 dev_err(dev
, "length exceeds h/w max limit\n");
1283 return ERR_PTR(-EINVAL
);
1287 iv_dma
= dma_map_single(dev
, iv
, ivsize
, DMA_TO_DEVICE
);
1289 if (!dst
|| dst
== src
) {
1290 src_nents
= sg_count(src
, assoclen
+ cryptlen
+ authsize
,
1292 src_nents
= (src_nents
== 1) ? 0 : src_nents
;
1293 dst_nents
= dst
? src_nents
: 0;
1294 } else { /* dst && dst != src*/
1295 src_nents
= sg_count(src
, assoclen
+ cryptlen
+
1296 (encrypt
? 0 : authsize
),
1298 src_nents
= (src_nents
== 1) ? 0 : src_nents
;
1299 dst_nents
= sg_count(dst
, assoclen
+ cryptlen
+
1300 (encrypt
? authsize
: 0),
1302 dst_nents
= (dst_nents
== 1) ? 0 : dst_nents
;
1306 * allocate space for base edesc plus the link tables,
1307 * allowing for two separate entries for AD and generated ICV (+ 2),
1308 * and space for two sets of ICVs (stashed and generated)
1310 alloc_len
= sizeof(struct talitos_edesc
);
1311 if (src_nents
|| dst_nents
) {
1313 dma_len
= (src_nents
? cryptlen
: 0) +
1314 (dst_nents
? cryptlen
: 0);
1316 dma_len
= (src_nents
+ dst_nents
+ 2) *
1317 sizeof(struct talitos_ptr
) + authsize
* 2;
1318 alloc_len
+= dma_len
;
1321 alloc_len
+= icv_stashing
? authsize
: 0;
1324 edesc
= kmalloc(alloc_len
, GFP_DMA
| flags
);
1327 dma_unmap_single(dev
, iv_dma
, ivsize
, DMA_TO_DEVICE
);
1329 dev_err(dev
, "could not allocate edescriptor\n");
1330 return ERR_PTR(-ENOMEM
);
1333 edesc
->src_nents
= src_nents
;
1334 edesc
->dst_nents
= dst_nents
;
1335 edesc
->src_chained
= src_chained
;
1336 edesc
->dst_chained
= dst_chained
;
1337 edesc
->iv_dma
= iv_dma
;
1338 edesc
->dma_len
= dma_len
;
1340 edesc
->dma_link_tbl
= dma_map_single(dev
, &edesc
->link_tbl
[0],
1347 static struct talitos_edesc
*aead_edesc_alloc(struct aead_request
*areq
, u8
*iv
,
1348 int icv_stashing
, bool encrypt
)
1350 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
1351 unsigned int authsize
= crypto_aead_authsize(authenc
);
1352 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1353 unsigned int ivsize
= crypto_aead_ivsize(authenc
);
1355 return talitos_edesc_alloc(ctx
->dev
, areq
->src
, areq
->dst
,
1356 iv
, areq
->assoclen
, areq
->cryptlen
,
1357 authsize
, ivsize
, icv_stashing
,
1358 areq
->base
.flags
, encrypt
);
1361 static int aead_encrypt(struct aead_request
*req
)
1363 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
1364 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1365 struct talitos_edesc
*edesc
;
1367 /* allocate extended descriptor */
1368 edesc
= aead_edesc_alloc(req
, req
->iv
, 0, true);
1370 return PTR_ERR(edesc
);
1373 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1375 return ipsec_esp(edesc
, req
, ipsec_esp_encrypt_done
);
1378 static int aead_decrypt(struct aead_request
*req
)
1380 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
1381 unsigned int authsize
= crypto_aead_authsize(authenc
);
1382 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1383 struct talitos_private
*priv
= dev_get_drvdata(ctx
->dev
);
1384 struct talitos_edesc
*edesc
;
1385 struct scatterlist
*sg
;
1388 req
->cryptlen
-= authsize
;
1390 /* allocate extended descriptor */
1391 edesc
= aead_edesc_alloc(req
, req
->iv
, 1, false);
1393 return PTR_ERR(edesc
);
1395 if ((priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
) &&
1396 ((!edesc
->src_nents
&& !edesc
->dst_nents
) ||
1397 priv
->features
& TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT
)) {
1399 /* decrypt and check the ICV */
1400 edesc
->desc
.hdr
= ctx
->desc_hdr_template
|
1401 DESC_HDR_DIR_INBOUND
|
1402 DESC_HDR_MODE1_MDEU_CICV
;
1404 /* reset integrity check result bits */
1405 edesc
->desc
.hdr_lo
= 0;
1407 return ipsec_esp(edesc
, req
, ipsec_esp_decrypt_hwauth_done
);
1410 /* Have to check the ICV with software */
1411 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_DIR_INBOUND
;
1413 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1415 icvdata
= (char *)&edesc
->link_tbl
[edesc
->src_nents
+
1416 edesc
->dst_nents
+ 2];
1418 icvdata
= &edesc
->link_tbl
[0];
1420 sg
= sg_last(req
->src
, edesc
->src_nents
? : 1);
1422 memcpy(icvdata
, (char *)sg_virt(sg
) + sg
->length
- authsize
, authsize
);
1424 return ipsec_esp(edesc
, req
, ipsec_esp_decrypt_swauth_done
);
1427 static int ablkcipher_setkey(struct crypto_ablkcipher
*cipher
,
1428 const u8
*key
, unsigned int keylen
)
1430 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1432 memcpy(&ctx
->key
, key
, keylen
);
1433 ctx
->keylen
= keylen
;
1438 static void unmap_sg_talitos_ptr(struct device
*dev
, struct scatterlist
*src
,
1439 struct scatterlist
*dst
, unsigned int len
,
1440 struct talitos_edesc
*edesc
)
1442 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1443 bool is_sec1
= has_ftr_sec1(priv
);
1446 if (!edesc
->src_nents
) {
1447 dma_unmap_sg(dev
, src
, 1,
1448 dst
!= src
? DMA_TO_DEVICE
1449 : DMA_BIDIRECTIONAL
);
1451 if (dst
&& edesc
->dst_nents
) {
1452 dma_sync_single_for_device(dev
,
1453 edesc
->dma_link_tbl
+ len
,
1454 len
, DMA_FROM_DEVICE
);
1455 sg_copy_from_buffer(dst
, edesc
->dst_nents
? : 1,
1456 edesc
->buf
+ len
, len
);
1457 } else if (dst
&& dst
!= src
) {
1458 dma_unmap_sg(dev
, dst
, 1, DMA_FROM_DEVICE
);
1461 talitos_sg_unmap(dev
, edesc
, src
, dst
);
1465 static void common_nonsnoop_unmap(struct device
*dev
,
1466 struct talitos_edesc
*edesc
,
1467 struct ablkcipher_request
*areq
)
1469 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[5], DMA_FROM_DEVICE
);
1471 unmap_sg_talitos_ptr(dev
, areq
->src
, areq
->dst
, areq
->nbytes
, edesc
);
1472 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2], DMA_TO_DEVICE
);
1473 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[1], DMA_TO_DEVICE
);
1476 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
1480 static void ablkcipher_done(struct device
*dev
,
1481 struct talitos_desc
*desc
, void *context
,
1484 struct ablkcipher_request
*areq
= context
;
1485 struct talitos_edesc
*edesc
;
1487 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
1489 common_nonsnoop_unmap(dev
, edesc
, areq
);
1493 areq
->base
.complete(&areq
->base
, err
);
1496 int map_sg_in_talitos_ptr(struct device
*dev
, struct scatterlist
*src
,
1497 unsigned int len
, struct talitos_edesc
*edesc
,
1498 enum dma_data_direction dir
, struct talitos_ptr
*ptr
)
1501 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1502 bool is_sec1
= has_ftr_sec1(priv
);
1504 to_talitos_ptr_len(ptr
, len
, is_sec1
);
1507 sg_count
= edesc
->src_nents
? : 1;
1509 if (sg_count
== 1) {
1510 dma_map_sg(dev
, src
, 1, dir
);
1511 to_talitos_ptr(ptr
, sg_dma_address(src
), is_sec1
);
1513 sg_copy_to_buffer(src
, sg_count
, edesc
->buf
, len
);
1514 to_talitos_ptr(ptr
, edesc
->dma_link_tbl
, is_sec1
);
1515 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1516 len
, DMA_TO_DEVICE
);
1519 to_talitos_ptr_extent_clear(ptr
, is_sec1
);
1521 sg_count
= talitos_map_sg(dev
, src
, edesc
->src_nents
? : 1, dir
,
1522 edesc
->src_chained
);
1524 if (sg_count
== 1) {
1525 to_talitos_ptr(ptr
, sg_dma_address(src
), is_sec1
);
1527 sg_count
= sg_to_link_tbl(src
, sg_count
, len
,
1528 &edesc
->link_tbl
[0]);
1530 to_talitos_ptr(ptr
, edesc
->dma_link_tbl
, 0);
1531 ptr
->j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1532 dma_sync_single_for_device(dev
,
1533 edesc
->dma_link_tbl
,
1537 /* Only one segment now, so no link tbl needed*/
1538 to_talitos_ptr(ptr
, sg_dma_address(src
),
1546 void map_sg_out_talitos_ptr(struct device
*dev
, struct scatterlist
*dst
,
1547 unsigned int len
, struct talitos_edesc
*edesc
,
1548 enum dma_data_direction dir
,
1549 struct talitos_ptr
*ptr
, int sg_count
)
1551 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1552 bool is_sec1
= has_ftr_sec1(priv
);
1554 if (dir
!= DMA_NONE
)
1555 sg_count
= talitos_map_sg(dev
, dst
, edesc
->dst_nents
? : 1,
1556 dir
, edesc
->dst_chained
);
1558 to_talitos_ptr_len(ptr
, len
, is_sec1
);
1561 if (sg_count
== 1) {
1562 if (dir
!= DMA_NONE
)
1563 dma_map_sg(dev
, dst
, 1, dir
);
1564 to_talitos_ptr(ptr
, sg_dma_address(dst
), is_sec1
);
1566 to_talitos_ptr(ptr
, edesc
->dma_link_tbl
+ len
, is_sec1
);
1567 dma_sync_single_for_device(dev
,
1568 edesc
->dma_link_tbl
+ len
,
1569 len
, DMA_FROM_DEVICE
);
1572 to_talitos_ptr_extent_clear(ptr
, is_sec1
);
1574 if (sg_count
== 1) {
1575 to_talitos_ptr(ptr
, sg_dma_address(dst
), is_sec1
);
1577 struct talitos_ptr
*link_tbl_ptr
=
1578 &edesc
->link_tbl
[edesc
->src_nents
+ 1];
1580 to_talitos_ptr(ptr
, edesc
->dma_link_tbl
+
1581 (edesc
->src_nents
+ 1) *
1582 sizeof(struct talitos_ptr
), 0);
1583 ptr
->j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1584 sg_to_link_tbl(dst
, sg_count
, len
, link_tbl_ptr
);
1585 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1592 static int common_nonsnoop(struct talitos_edesc
*edesc
,
1593 struct ablkcipher_request
*areq
,
1594 void (*callback
) (struct device
*dev
,
1595 struct talitos_desc
*desc
,
1596 void *context
, int error
))
1598 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1599 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1600 struct device
*dev
= ctx
->dev
;
1601 struct talitos_desc
*desc
= &edesc
->desc
;
1602 unsigned int cryptlen
= areq
->nbytes
;
1603 unsigned int ivsize
= crypto_ablkcipher_ivsize(cipher
);
1605 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1606 bool is_sec1
= has_ftr_sec1(priv
);
1608 /* first DWORD empty */
1609 desc
->ptr
[0] = zero_entry
;
1612 to_talitos_ptr(&desc
->ptr
[1], edesc
->iv_dma
, is_sec1
);
1613 to_talitos_ptr_len(&desc
->ptr
[1], ivsize
, is_sec1
);
1614 to_talitos_ptr_extent_clear(&desc
->ptr
[1], is_sec1
);
1617 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ctx
->keylen
,
1618 (char *)&ctx
->key
, DMA_TO_DEVICE
);
1623 sg_count
= map_sg_in_talitos_ptr(dev
, areq
->src
, cryptlen
, edesc
,
1624 (areq
->src
== areq
->dst
) ?
1625 DMA_BIDIRECTIONAL
: DMA_TO_DEVICE
,
1629 map_sg_out_talitos_ptr(dev
, areq
->dst
, cryptlen
, edesc
,
1630 (areq
->src
== areq
->dst
) ? DMA_NONE
1632 &desc
->ptr
[4], sg_count
);
1635 map_single_talitos_ptr(dev
, &desc
->ptr
[5], ivsize
, ctx
->iv
,
1638 /* last DWORD empty */
1639 desc
->ptr
[6] = zero_entry
;
1641 ret
= talitos_submit(dev
, ctx
->ch
, desc
, callback
, areq
);
1642 if (ret
!= -EINPROGRESS
) {
1643 common_nonsnoop_unmap(dev
, edesc
, areq
);
1649 static struct talitos_edesc
*ablkcipher_edesc_alloc(struct ablkcipher_request
*
1652 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1653 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1654 unsigned int ivsize
= crypto_ablkcipher_ivsize(cipher
);
1656 return talitos_edesc_alloc(ctx
->dev
, areq
->src
, areq
->dst
,
1657 areq
->info
, 0, areq
->nbytes
, 0, ivsize
, 0,
1658 areq
->base
.flags
, encrypt
);
1661 static int ablkcipher_encrypt(struct ablkcipher_request
*areq
)
1663 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1664 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1665 struct talitos_edesc
*edesc
;
1667 /* allocate extended descriptor */
1668 edesc
= ablkcipher_edesc_alloc(areq
, true);
1670 return PTR_ERR(edesc
);
1673 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1675 return common_nonsnoop(edesc
, areq
, ablkcipher_done
);
1678 static int ablkcipher_decrypt(struct ablkcipher_request
*areq
)
1680 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1681 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1682 struct talitos_edesc
*edesc
;
1684 /* allocate extended descriptor */
1685 edesc
= ablkcipher_edesc_alloc(areq
, false);
1687 return PTR_ERR(edesc
);
1689 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_DIR_INBOUND
;
1691 return common_nonsnoop(edesc
, areq
, ablkcipher_done
);
1694 static void common_nonsnoop_hash_unmap(struct device
*dev
,
1695 struct talitos_edesc
*edesc
,
1696 struct ahash_request
*areq
)
1698 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1699 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1700 bool is_sec1
= has_ftr_sec1(priv
);
1702 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[5], DMA_FROM_DEVICE
);
1704 unmap_sg_talitos_ptr(dev
, req_ctx
->psrc
, NULL
, 0, edesc
);
1706 /* When using hashctx-in, must unmap it. */
1707 if (from_talitos_ptr_len(&edesc
->desc
.ptr
[1], is_sec1
))
1708 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[1],
1711 if (from_talitos_ptr_len(&edesc
->desc
.ptr
[2], is_sec1
))
1712 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2],
1716 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
1721 static void ahash_done(struct device
*dev
,
1722 struct talitos_desc
*desc
, void *context
,
1725 struct ahash_request
*areq
= context
;
1726 struct talitos_edesc
*edesc
=
1727 container_of(desc
, struct talitos_edesc
, desc
);
1728 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1730 if (!req_ctx
->last
&& req_ctx
->to_hash_later
) {
1731 /* Position any partial block for next update/final/finup */
1732 memcpy(req_ctx
->buf
, req_ctx
->bufnext
, req_ctx
->to_hash_later
);
1733 req_ctx
->nbuf
= req_ctx
->to_hash_later
;
1735 common_nonsnoop_hash_unmap(dev
, edesc
, areq
);
1739 areq
->base
.complete(&areq
->base
, err
);
1743 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1744 * ourself and submit a padded block
1746 void talitos_handle_buggy_hash(struct talitos_ctx
*ctx
,
1747 struct talitos_edesc
*edesc
,
1748 struct talitos_ptr
*ptr
)
1750 static u8 padded_hash
[64] = {
1751 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1752 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1753 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1754 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1757 pr_err_once("Bug in SEC1, padding ourself\n");
1758 edesc
->desc
.hdr
&= ~DESC_HDR_MODE0_MDEU_PAD
;
1759 map_single_talitos_ptr(ctx
->dev
, ptr
, sizeof(padded_hash
),
1760 (char *)padded_hash
, DMA_TO_DEVICE
);
1763 static int common_nonsnoop_hash(struct talitos_edesc
*edesc
,
1764 struct ahash_request
*areq
, unsigned int length
,
1765 void (*callback
) (struct device
*dev
,
1766 struct talitos_desc
*desc
,
1767 void *context
, int error
))
1769 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1770 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1771 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1772 struct device
*dev
= ctx
->dev
;
1773 struct talitos_desc
*desc
= &edesc
->desc
;
1775 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1776 bool is_sec1
= has_ftr_sec1(priv
);
1778 /* first DWORD empty */
1779 desc
->ptr
[0] = zero_entry
;
1781 /* hash context in */
1782 if (!req_ctx
->first
|| req_ctx
->swinit
) {
1783 map_single_talitos_ptr(dev
, &desc
->ptr
[1],
1784 req_ctx
->hw_context_size
,
1785 (char *)req_ctx
->hw_context
,
1787 req_ctx
->swinit
= 0;
1789 desc
->ptr
[1] = zero_entry
;
1790 /* Indicate next op is not the first. */
1796 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ctx
->keylen
,
1797 (char *)&ctx
->key
, DMA_TO_DEVICE
);
1799 desc
->ptr
[2] = zero_entry
;
1804 map_sg_in_talitos_ptr(dev
, req_ctx
->psrc
, length
, edesc
,
1805 DMA_TO_DEVICE
, &desc
->ptr
[3]);
1807 /* fifth DWORD empty */
1808 desc
->ptr
[4] = zero_entry
;
1810 /* hash/HMAC out -or- hash context out */
1812 map_single_talitos_ptr(dev
, &desc
->ptr
[5],
1813 crypto_ahash_digestsize(tfm
),
1814 areq
->result
, DMA_FROM_DEVICE
);
1816 map_single_talitos_ptr(dev
, &desc
->ptr
[5],
1817 req_ctx
->hw_context_size
,
1818 req_ctx
->hw_context
, DMA_FROM_DEVICE
);
1820 /* last DWORD empty */
1821 desc
->ptr
[6] = zero_entry
;
1823 if (is_sec1
&& from_talitos_ptr_len(&desc
->ptr
[3], true) == 0)
1824 talitos_handle_buggy_hash(ctx
, edesc
, &desc
->ptr
[3]);
1826 ret
= talitos_submit(dev
, ctx
->ch
, desc
, callback
, areq
);
1827 if (ret
!= -EINPROGRESS
) {
1828 common_nonsnoop_hash_unmap(dev
, edesc
, areq
);
1834 static struct talitos_edesc
*ahash_edesc_alloc(struct ahash_request
*areq
,
1835 unsigned int nbytes
)
1837 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1838 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1839 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1841 return talitos_edesc_alloc(ctx
->dev
, req_ctx
->psrc
, NULL
, NULL
, 0,
1842 nbytes
, 0, 0, 0, areq
->base
.flags
, false);
1845 static int ahash_init(struct ahash_request
*areq
)
1847 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1848 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1850 /* Initialize the context */
1852 req_ctx
->first
= 1; /* first indicates h/w must init its context */
1853 req_ctx
->swinit
= 0; /* assume h/w init of context */
1854 req_ctx
->hw_context_size
=
1855 (crypto_ahash_digestsize(tfm
) <= SHA256_DIGEST_SIZE
)
1856 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1857 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
;
1863 * on h/w without explicit sha224 support, we initialize h/w context
1864 * manually with sha224 constants, and tell it to run sha256.
1866 static int ahash_init_sha224_swinit(struct ahash_request
*areq
)
1868 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1871 req_ctx
->swinit
= 1;/* prevent h/w initting context with sha256 values*/
1873 req_ctx
->hw_context
[0] = SHA224_H0
;
1874 req_ctx
->hw_context
[1] = SHA224_H1
;
1875 req_ctx
->hw_context
[2] = SHA224_H2
;
1876 req_ctx
->hw_context
[3] = SHA224_H3
;
1877 req_ctx
->hw_context
[4] = SHA224_H4
;
1878 req_ctx
->hw_context
[5] = SHA224_H5
;
1879 req_ctx
->hw_context
[6] = SHA224_H6
;
1880 req_ctx
->hw_context
[7] = SHA224_H7
;
1882 /* init 64-bit count */
1883 req_ctx
->hw_context
[8] = 0;
1884 req_ctx
->hw_context
[9] = 0;
1889 static int ahash_process_req(struct ahash_request
*areq
, unsigned int nbytes
)
1891 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1892 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1893 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1894 struct talitos_edesc
*edesc
;
1895 unsigned int blocksize
=
1896 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
1897 unsigned int nbytes_to_hash
;
1898 unsigned int to_hash_later
;
1902 if (!req_ctx
->last
&& (nbytes
+ req_ctx
->nbuf
<= blocksize
)) {
1903 /* Buffer up to one whole block */
1904 sg_copy_to_buffer(areq
->src
,
1905 sg_count(areq
->src
, nbytes
, &chained
),
1906 req_ctx
->buf
+ req_ctx
->nbuf
, nbytes
);
1907 req_ctx
->nbuf
+= nbytes
;
1911 /* At least (blocksize + 1) bytes are available to hash */
1912 nbytes_to_hash
= nbytes
+ req_ctx
->nbuf
;
1913 to_hash_later
= nbytes_to_hash
& (blocksize
- 1);
1917 else if (to_hash_later
)
1918 /* There is a partial block. Hash the full block(s) now */
1919 nbytes_to_hash
-= to_hash_later
;
1921 /* Keep one block buffered */
1922 nbytes_to_hash
-= blocksize
;
1923 to_hash_later
= blocksize
;
1926 /* Chain in any previously buffered data */
1927 if (req_ctx
->nbuf
) {
1928 nsg
= (req_ctx
->nbuf
< nbytes_to_hash
) ? 2 : 1;
1929 sg_init_table(req_ctx
->bufsl
, nsg
);
1930 sg_set_buf(req_ctx
->bufsl
, req_ctx
->buf
, req_ctx
->nbuf
);
1932 scatterwalk_sg_chain(req_ctx
->bufsl
, 2, areq
->src
);
1933 req_ctx
->psrc
= req_ctx
->bufsl
;
1935 req_ctx
->psrc
= areq
->src
;
1937 if (to_hash_later
) {
1938 int nents
= sg_count(areq
->src
, nbytes
, &chained
);
1939 sg_pcopy_to_buffer(areq
->src
, nents
,
1942 nbytes
- to_hash_later
);
1944 req_ctx
->to_hash_later
= to_hash_later
;
1946 /* Allocate extended descriptor */
1947 edesc
= ahash_edesc_alloc(areq
, nbytes_to_hash
);
1949 return PTR_ERR(edesc
);
1951 edesc
->desc
.hdr
= ctx
->desc_hdr_template
;
1953 /* On last one, request SEC to pad; otherwise continue */
1955 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_PAD
;
1957 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_CONT
;
1959 /* request SEC to INIT hash. */
1960 if (req_ctx
->first
&& !req_ctx
->swinit
)
1961 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_INIT
;
1963 /* When the tfm context has a keylen, it's an HMAC.
1964 * A first or last (ie. not middle) descriptor must request HMAC.
1966 if (ctx
->keylen
&& (req_ctx
->first
|| req_ctx
->last
))
1967 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_HMAC
;
1969 return common_nonsnoop_hash(edesc
, areq
, nbytes_to_hash
,
1973 static int ahash_update(struct ahash_request
*areq
)
1975 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1979 return ahash_process_req(areq
, areq
->nbytes
);
1982 static int ahash_final(struct ahash_request
*areq
)
1984 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1988 return ahash_process_req(areq
, 0);
1991 static int ahash_finup(struct ahash_request
*areq
)
1993 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1997 return ahash_process_req(areq
, areq
->nbytes
);
2000 static int ahash_digest(struct ahash_request
*areq
)
2002 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
2003 struct crypto_ahash
*ahash
= crypto_ahash_reqtfm(areq
);
2008 return ahash_process_req(areq
, areq
->nbytes
);
2011 struct keyhash_result
{
2012 struct completion completion
;
2016 static void keyhash_complete(struct crypto_async_request
*req
, int err
)
2018 struct keyhash_result
*res
= req
->data
;
2020 if (err
== -EINPROGRESS
)
2024 complete(&res
->completion
);
2027 static int keyhash(struct crypto_ahash
*tfm
, const u8
*key
, unsigned int keylen
,
2030 struct talitos_ctx
*ctx
= crypto_tfm_ctx(crypto_ahash_tfm(tfm
));
2032 struct scatterlist sg
[1];
2033 struct ahash_request
*req
;
2034 struct keyhash_result hresult
;
2037 init_completion(&hresult
.completion
);
2039 req
= ahash_request_alloc(tfm
, GFP_KERNEL
);
2043 /* Keep tfm keylen == 0 during hash of the long key */
2045 ahash_request_set_callback(req
, CRYPTO_TFM_REQ_MAY_BACKLOG
,
2046 keyhash_complete
, &hresult
);
2048 sg_init_one(&sg
[0], key
, keylen
);
2050 ahash_request_set_crypt(req
, sg
, hash
, keylen
);
2051 ret
= crypto_ahash_digest(req
);
2057 ret
= wait_for_completion_interruptible(
2058 &hresult
.completion
);
2065 ahash_request_free(req
);
2070 static int ahash_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
2071 unsigned int keylen
)
2073 struct talitos_ctx
*ctx
= crypto_tfm_ctx(crypto_ahash_tfm(tfm
));
2074 unsigned int blocksize
=
2075 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
2076 unsigned int digestsize
= crypto_ahash_digestsize(tfm
);
2077 unsigned int keysize
= keylen
;
2078 u8 hash
[SHA512_DIGEST_SIZE
];
2081 if (keylen
<= blocksize
)
2082 memcpy(ctx
->key
, key
, keysize
);
2084 /* Must get the hash of the long key */
2085 ret
= keyhash(tfm
, key
, keylen
, hash
);
2088 crypto_ahash_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
2092 keysize
= digestsize
;
2093 memcpy(ctx
->key
, hash
, digestsize
);
2096 ctx
->keylen
= keysize
;
2102 struct talitos_alg_template
{
2105 struct crypto_alg crypto
;
2106 struct ahash_alg hash
;
2107 struct aead_alg aead
;
2109 __be32 desc_hdr_template
;
2112 static struct talitos_alg_template driver_algs
[] = {
2113 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
2114 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2117 .cra_name
= "authenc(hmac(sha1),cbc(aes))",
2118 .cra_driver_name
= "authenc-hmac-sha1-"
2120 .cra_blocksize
= AES_BLOCK_SIZE
,
2121 .cra_flags
= CRYPTO_ALG_ASYNC
,
2123 .ivsize
= AES_BLOCK_SIZE
,
2124 .maxauthsize
= SHA1_DIGEST_SIZE
,
2126 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2127 DESC_HDR_SEL0_AESU
|
2128 DESC_HDR_MODE0_AESU_CBC
|
2129 DESC_HDR_SEL1_MDEUA
|
2130 DESC_HDR_MODE1_MDEU_INIT
|
2131 DESC_HDR_MODE1_MDEU_PAD
|
2132 DESC_HDR_MODE1_MDEU_SHA1_HMAC
,
2134 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2137 .cra_name
= "authenc(hmac(sha1),"
2139 .cra_driver_name
= "authenc-hmac-sha1-"
2141 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2142 .cra_flags
= CRYPTO_ALG_ASYNC
,
2144 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2145 .maxauthsize
= SHA1_DIGEST_SIZE
,
2147 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2149 DESC_HDR_MODE0_DEU_CBC
|
2150 DESC_HDR_MODE0_DEU_3DES
|
2151 DESC_HDR_SEL1_MDEUA
|
2152 DESC_HDR_MODE1_MDEU_INIT
|
2153 DESC_HDR_MODE1_MDEU_PAD
|
2154 DESC_HDR_MODE1_MDEU_SHA1_HMAC
,
2156 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2159 .cra_name
= "authenc(hmac(sha224),cbc(aes))",
2160 .cra_driver_name
= "authenc-hmac-sha224-"
2162 .cra_blocksize
= AES_BLOCK_SIZE
,
2163 .cra_flags
= CRYPTO_ALG_ASYNC
,
2165 .ivsize
= AES_BLOCK_SIZE
,
2166 .maxauthsize
= SHA224_DIGEST_SIZE
,
2168 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2169 DESC_HDR_SEL0_AESU
|
2170 DESC_HDR_MODE0_AESU_CBC
|
2171 DESC_HDR_SEL1_MDEUA
|
2172 DESC_HDR_MODE1_MDEU_INIT
|
2173 DESC_HDR_MODE1_MDEU_PAD
|
2174 DESC_HDR_MODE1_MDEU_SHA224_HMAC
,
2176 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2179 .cra_name
= "authenc(hmac(sha224),"
2181 .cra_driver_name
= "authenc-hmac-sha224-"
2183 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2184 .cra_flags
= CRYPTO_ALG_ASYNC
,
2186 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2187 .maxauthsize
= SHA224_DIGEST_SIZE
,
2189 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2191 DESC_HDR_MODE0_DEU_CBC
|
2192 DESC_HDR_MODE0_DEU_3DES
|
2193 DESC_HDR_SEL1_MDEUA
|
2194 DESC_HDR_MODE1_MDEU_INIT
|
2195 DESC_HDR_MODE1_MDEU_PAD
|
2196 DESC_HDR_MODE1_MDEU_SHA224_HMAC
,
2198 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2201 .cra_name
= "authenc(hmac(sha256),cbc(aes))",
2202 .cra_driver_name
= "authenc-hmac-sha256-"
2204 .cra_blocksize
= AES_BLOCK_SIZE
,
2205 .cra_flags
= CRYPTO_ALG_ASYNC
,
2207 .ivsize
= AES_BLOCK_SIZE
,
2208 .maxauthsize
= SHA256_DIGEST_SIZE
,
2210 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2211 DESC_HDR_SEL0_AESU
|
2212 DESC_HDR_MODE0_AESU_CBC
|
2213 DESC_HDR_SEL1_MDEUA
|
2214 DESC_HDR_MODE1_MDEU_INIT
|
2215 DESC_HDR_MODE1_MDEU_PAD
|
2216 DESC_HDR_MODE1_MDEU_SHA256_HMAC
,
2218 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2221 .cra_name
= "authenc(hmac(sha256),"
2223 .cra_driver_name
= "authenc-hmac-sha256-"
2225 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2226 .cra_flags
= CRYPTO_ALG_ASYNC
,
2228 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2229 .maxauthsize
= SHA256_DIGEST_SIZE
,
2231 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2233 DESC_HDR_MODE0_DEU_CBC
|
2234 DESC_HDR_MODE0_DEU_3DES
|
2235 DESC_HDR_SEL1_MDEUA
|
2236 DESC_HDR_MODE1_MDEU_INIT
|
2237 DESC_HDR_MODE1_MDEU_PAD
|
2238 DESC_HDR_MODE1_MDEU_SHA256_HMAC
,
2240 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2243 .cra_name
= "authenc(hmac(sha384),cbc(aes))",
2244 .cra_driver_name
= "authenc-hmac-sha384-"
2246 .cra_blocksize
= AES_BLOCK_SIZE
,
2247 .cra_flags
= CRYPTO_ALG_ASYNC
,
2249 .ivsize
= AES_BLOCK_SIZE
,
2250 .maxauthsize
= SHA384_DIGEST_SIZE
,
2252 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2253 DESC_HDR_SEL0_AESU
|
2254 DESC_HDR_MODE0_AESU_CBC
|
2255 DESC_HDR_SEL1_MDEUB
|
2256 DESC_HDR_MODE1_MDEU_INIT
|
2257 DESC_HDR_MODE1_MDEU_PAD
|
2258 DESC_HDR_MODE1_MDEUB_SHA384_HMAC
,
2260 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2263 .cra_name
= "authenc(hmac(sha384),"
2265 .cra_driver_name
= "authenc-hmac-sha384-"
2267 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2268 .cra_flags
= CRYPTO_ALG_ASYNC
,
2270 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2271 .maxauthsize
= SHA384_DIGEST_SIZE
,
2273 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2275 DESC_HDR_MODE0_DEU_CBC
|
2276 DESC_HDR_MODE0_DEU_3DES
|
2277 DESC_HDR_SEL1_MDEUB
|
2278 DESC_HDR_MODE1_MDEU_INIT
|
2279 DESC_HDR_MODE1_MDEU_PAD
|
2280 DESC_HDR_MODE1_MDEUB_SHA384_HMAC
,
2282 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2285 .cra_name
= "authenc(hmac(sha512),cbc(aes))",
2286 .cra_driver_name
= "authenc-hmac-sha512-"
2288 .cra_blocksize
= AES_BLOCK_SIZE
,
2289 .cra_flags
= CRYPTO_ALG_ASYNC
,
2291 .ivsize
= AES_BLOCK_SIZE
,
2292 .maxauthsize
= SHA512_DIGEST_SIZE
,
2294 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2295 DESC_HDR_SEL0_AESU
|
2296 DESC_HDR_MODE0_AESU_CBC
|
2297 DESC_HDR_SEL1_MDEUB
|
2298 DESC_HDR_MODE1_MDEU_INIT
|
2299 DESC_HDR_MODE1_MDEU_PAD
|
2300 DESC_HDR_MODE1_MDEUB_SHA512_HMAC
,
2302 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2305 .cra_name
= "authenc(hmac(sha512),"
2307 .cra_driver_name
= "authenc-hmac-sha512-"
2309 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2310 .cra_flags
= CRYPTO_ALG_ASYNC
,
2312 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2313 .maxauthsize
= SHA512_DIGEST_SIZE
,
2315 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2317 DESC_HDR_MODE0_DEU_CBC
|
2318 DESC_HDR_MODE0_DEU_3DES
|
2319 DESC_HDR_SEL1_MDEUB
|
2320 DESC_HDR_MODE1_MDEU_INIT
|
2321 DESC_HDR_MODE1_MDEU_PAD
|
2322 DESC_HDR_MODE1_MDEUB_SHA512_HMAC
,
2324 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2327 .cra_name
= "authenc(hmac(md5),cbc(aes))",
2328 .cra_driver_name
= "authenc-hmac-md5-"
2330 .cra_blocksize
= AES_BLOCK_SIZE
,
2331 .cra_flags
= CRYPTO_ALG_ASYNC
,
2333 .ivsize
= AES_BLOCK_SIZE
,
2334 .maxauthsize
= MD5_DIGEST_SIZE
,
2336 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2337 DESC_HDR_SEL0_AESU
|
2338 DESC_HDR_MODE0_AESU_CBC
|
2339 DESC_HDR_SEL1_MDEUA
|
2340 DESC_HDR_MODE1_MDEU_INIT
|
2341 DESC_HDR_MODE1_MDEU_PAD
|
2342 DESC_HDR_MODE1_MDEU_MD5_HMAC
,
2344 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2347 .cra_name
= "authenc(hmac(md5),cbc(des3_ede))",
2348 .cra_driver_name
= "authenc-hmac-md5-"
2350 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2351 .cra_flags
= CRYPTO_ALG_ASYNC
,
2353 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2354 .maxauthsize
= MD5_DIGEST_SIZE
,
2356 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2358 DESC_HDR_MODE0_DEU_CBC
|
2359 DESC_HDR_MODE0_DEU_3DES
|
2360 DESC_HDR_SEL1_MDEUA
|
2361 DESC_HDR_MODE1_MDEU_INIT
|
2362 DESC_HDR_MODE1_MDEU_PAD
|
2363 DESC_HDR_MODE1_MDEU_MD5_HMAC
,
2365 /* ABLKCIPHER algorithms. */
2366 { .type
= CRYPTO_ALG_TYPE_ABLKCIPHER
,
2368 .cra_name
= "cbc(aes)",
2369 .cra_driver_name
= "cbc-aes-talitos",
2370 .cra_blocksize
= AES_BLOCK_SIZE
,
2371 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
2374 .min_keysize
= AES_MIN_KEY_SIZE
,
2375 .max_keysize
= AES_MAX_KEY_SIZE
,
2376 .ivsize
= AES_BLOCK_SIZE
,
2379 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2380 DESC_HDR_SEL0_AESU
|
2381 DESC_HDR_MODE0_AESU_CBC
,
2383 { .type
= CRYPTO_ALG_TYPE_ABLKCIPHER
,
2385 .cra_name
= "cbc(des3_ede)",
2386 .cra_driver_name
= "cbc-3des-talitos",
2387 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2388 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
2391 .min_keysize
= DES3_EDE_KEY_SIZE
,
2392 .max_keysize
= DES3_EDE_KEY_SIZE
,
2393 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2396 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2398 DESC_HDR_MODE0_DEU_CBC
|
2399 DESC_HDR_MODE0_DEU_3DES
,
2401 /* AHASH algorithms. */
2402 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2404 .halg
.digestsize
= MD5_DIGEST_SIZE
,
2407 .cra_driver_name
= "md5-talitos",
2408 .cra_blocksize
= MD5_HMAC_BLOCK_SIZE
,
2409 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2413 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2414 DESC_HDR_SEL0_MDEUA
|
2415 DESC_HDR_MODE0_MDEU_MD5
,
2417 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2419 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
2422 .cra_driver_name
= "sha1-talitos",
2423 .cra_blocksize
= SHA1_BLOCK_SIZE
,
2424 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2428 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2429 DESC_HDR_SEL0_MDEUA
|
2430 DESC_HDR_MODE0_MDEU_SHA1
,
2432 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2434 .halg
.digestsize
= SHA224_DIGEST_SIZE
,
2436 .cra_name
= "sha224",
2437 .cra_driver_name
= "sha224-talitos",
2438 .cra_blocksize
= SHA224_BLOCK_SIZE
,
2439 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2443 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2444 DESC_HDR_SEL0_MDEUA
|
2445 DESC_HDR_MODE0_MDEU_SHA224
,
2447 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2449 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
2451 .cra_name
= "sha256",
2452 .cra_driver_name
= "sha256-talitos",
2453 .cra_blocksize
= SHA256_BLOCK_SIZE
,
2454 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2458 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2459 DESC_HDR_SEL0_MDEUA
|
2460 DESC_HDR_MODE0_MDEU_SHA256
,
2462 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2464 .halg
.digestsize
= SHA384_DIGEST_SIZE
,
2466 .cra_name
= "sha384",
2467 .cra_driver_name
= "sha384-talitos",
2468 .cra_blocksize
= SHA384_BLOCK_SIZE
,
2469 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2473 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2474 DESC_HDR_SEL0_MDEUB
|
2475 DESC_HDR_MODE0_MDEUB_SHA384
,
2477 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2479 .halg
.digestsize
= SHA512_DIGEST_SIZE
,
2481 .cra_name
= "sha512",
2482 .cra_driver_name
= "sha512-talitos",
2483 .cra_blocksize
= SHA512_BLOCK_SIZE
,
2484 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2488 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2489 DESC_HDR_SEL0_MDEUB
|
2490 DESC_HDR_MODE0_MDEUB_SHA512
,
2492 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2494 .halg
.digestsize
= MD5_DIGEST_SIZE
,
2496 .cra_name
= "hmac(md5)",
2497 .cra_driver_name
= "hmac-md5-talitos",
2498 .cra_blocksize
= MD5_HMAC_BLOCK_SIZE
,
2499 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2503 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2504 DESC_HDR_SEL0_MDEUA
|
2505 DESC_HDR_MODE0_MDEU_MD5
,
2507 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2509 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
2511 .cra_name
= "hmac(sha1)",
2512 .cra_driver_name
= "hmac-sha1-talitos",
2513 .cra_blocksize
= SHA1_BLOCK_SIZE
,
2514 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2518 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2519 DESC_HDR_SEL0_MDEUA
|
2520 DESC_HDR_MODE0_MDEU_SHA1
,
2522 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2524 .halg
.digestsize
= SHA224_DIGEST_SIZE
,
2526 .cra_name
= "hmac(sha224)",
2527 .cra_driver_name
= "hmac-sha224-talitos",
2528 .cra_blocksize
= SHA224_BLOCK_SIZE
,
2529 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2533 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2534 DESC_HDR_SEL0_MDEUA
|
2535 DESC_HDR_MODE0_MDEU_SHA224
,
2537 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2539 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
2541 .cra_name
= "hmac(sha256)",
2542 .cra_driver_name
= "hmac-sha256-talitos",
2543 .cra_blocksize
= SHA256_BLOCK_SIZE
,
2544 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2548 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2549 DESC_HDR_SEL0_MDEUA
|
2550 DESC_HDR_MODE0_MDEU_SHA256
,
2552 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2554 .halg
.digestsize
= SHA384_DIGEST_SIZE
,
2556 .cra_name
= "hmac(sha384)",
2557 .cra_driver_name
= "hmac-sha384-talitos",
2558 .cra_blocksize
= SHA384_BLOCK_SIZE
,
2559 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2563 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2564 DESC_HDR_SEL0_MDEUB
|
2565 DESC_HDR_MODE0_MDEUB_SHA384
,
2567 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2569 .halg
.digestsize
= SHA512_DIGEST_SIZE
,
2571 .cra_name
= "hmac(sha512)",
2572 .cra_driver_name
= "hmac-sha512-talitos",
2573 .cra_blocksize
= SHA512_BLOCK_SIZE
,
2574 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2578 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2579 DESC_HDR_SEL0_MDEUB
|
2580 DESC_HDR_MODE0_MDEUB_SHA512
,
2584 struct talitos_crypto_alg
{
2585 struct list_head entry
;
2587 struct talitos_alg_template algt
;
2590 static int talitos_cra_init(struct crypto_tfm
*tfm
)
2592 struct crypto_alg
*alg
= tfm
->__crt_alg
;
2593 struct talitos_crypto_alg
*talitos_alg
;
2594 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
2595 struct talitos_private
*priv
;
2597 if ((alg
->cra_flags
& CRYPTO_ALG_TYPE_MASK
) == CRYPTO_ALG_TYPE_AHASH
)
2598 talitos_alg
= container_of(__crypto_ahash_alg(alg
),
2599 struct talitos_crypto_alg
,
2602 talitos_alg
= container_of(alg
, struct talitos_crypto_alg
,
2605 /* update context with ptr to dev */
2606 ctx
->dev
= talitos_alg
->dev
;
2608 /* assign SEC channel to tfm in round-robin fashion */
2609 priv
= dev_get_drvdata(ctx
->dev
);
2610 ctx
->ch
= atomic_inc_return(&priv
->last_chan
) &
2611 (priv
->num_channels
- 1);
2613 /* copy descriptor header template value */
2614 ctx
->desc_hdr_template
= talitos_alg
->algt
.desc_hdr_template
;
2616 /* select done notification */
2617 ctx
->desc_hdr_template
|= DESC_HDR_DONE_NOTIFY
;
2622 static int talitos_cra_init_aead(struct crypto_aead
*tfm
)
2624 talitos_cra_init(crypto_aead_tfm(tfm
));
2628 static int talitos_cra_init_ahash(struct crypto_tfm
*tfm
)
2630 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
2632 talitos_cra_init(tfm
);
2635 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
2636 sizeof(struct talitos_ahash_req_ctx
));
2642 * given the alg's descriptor header template, determine whether descriptor
2643 * type and primary/secondary execution units required match the hw
2644 * capabilities description provided in the device tree node.
2646 static int hw_supports(struct device
*dev
, __be32 desc_hdr_template
)
2648 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2651 ret
= (1 << DESC_TYPE(desc_hdr_template
) & priv
->desc_types
) &&
2652 (1 << PRIMARY_EU(desc_hdr_template
) & priv
->exec_units
);
2654 if (SECONDARY_EU(desc_hdr_template
))
2655 ret
= ret
&& (1 << SECONDARY_EU(desc_hdr_template
)
2656 & priv
->exec_units
);
2661 static int talitos_remove(struct platform_device
*ofdev
)
2663 struct device
*dev
= &ofdev
->dev
;
2664 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2665 struct talitos_crypto_alg
*t_alg
, *n
;
2668 list_for_each_entry_safe(t_alg
, n
, &priv
->alg_list
, entry
) {
2669 switch (t_alg
->algt
.type
) {
2670 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
2672 case CRYPTO_ALG_TYPE_AEAD
:
2673 crypto_unregister_aead(&t_alg
->algt
.alg
.aead
);
2674 case CRYPTO_ALG_TYPE_AHASH
:
2675 crypto_unregister_ahash(&t_alg
->algt
.alg
.hash
);
2678 list_del(&t_alg
->entry
);
2682 if (hw_supports(dev
, DESC_HDR_SEL0_RNG
))
2683 talitos_unregister_rng(dev
);
2685 for (i
= 0; priv
->chan
&& i
< priv
->num_channels
; i
++)
2686 kfree(priv
->chan
[i
].fifo
);
2690 for (i
= 0; i
< 2; i
++)
2692 free_irq(priv
->irq
[i
], dev
);
2693 irq_dispose_mapping(priv
->irq
[i
]);
2696 tasklet_kill(&priv
->done_task
[0]);
2698 tasklet_kill(&priv
->done_task
[1]);
2707 static struct talitos_crypto_alg
*talitos_alg_alloc(struct device
*dev
,
2708 struct talitos_alg_template
2711 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2712 struct talitos_crypto_alg
*t_alg
;
2713 struct crypto_alg
*alg
;
2715 t_alg
= kzalloc(sizeof(struct talitos_crypto_alg
), GFP_KERNEL
);
2717 return ERR_PTR(-ENOMEM
);
2719 t_alg
->algt
= *template;
2721 switch (t_alg
->algt
.type
) {
2722 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
2723 alg
= &t_alg
->algt
.alg
.crypto
;
2724 alg
->cra_init
= talitos_cra_init
;
2725 alg
->cra_type
= &crypto_ablkcipher_type
;
2726 alg
->cra_ablkcipher
.setkey
= ablkcipher_setkey
;
2727 alg
->cra_ablkcipher
.encrypt
= ablkcipher_encrypt
;
2728 alg
->cra_ablkcipher
.decrypt
= ablkcipher_decrypt
;
2729 alg
->cra_ablkcipher
.geniv
= "eseqiv";
2731 case CRYPTO_ALG_TYPE_AEAD
:
2732 alg
= &t_alg
->algt
.alg
.aead
.base
;
2733 t_alg
->algt
.alg
.aead
.init
= talitos_cra_init_aead
;
2734 t_alg
->algt
.alg
.aead
.setkey
= aead_setkey
;
2735 t_alg
->algt
.alg
.aead
.encrypt
= aead_encrypt
;
2736 t_alg
->algt
.alg
.aead
.decrypt
= aead_decrypt
;
2738 case CRYPTO_ALG_TYPE_AHASH
:
2739 alg
= &t_alg
->algt
.alg
.hash
.halg
.base
;
2740 alg
->cra_init
= talitos_cra_init_ahash
;
2741 alg
->cra_type
= &crypto_ahash_type
;
2742 t_alg
->algt
.alg
.hash
.init
= ahash_init
;
2743 t_alg
->algt
.alg
.hash
.update
= ahash_update
;
2744 t_alg
->algt
.alg
.hash
.final
= ahash_final
;
2745 t_alg
->algt
.alg
.hash
.finup
= ahash_finup
;
2746 t_alg
->algt
.alg
.hash
.digest
= ahash_digest
;
2747 t_alg
->algt
.alg
.hash
.setkey
= ahash_setkey
;
2749 if (!(priv
->features
& TALITOS_FTR_HMAC_OK
) &&
2750 !strncmp(alg
->cra_name
, "hmac", 4)) {
2752 return ERR_PTR(-ENOTSUPP
);
2754 if (!(priv
->features
& TALITOS_FTR_SHA224_HWINIT
) &&
2755 (!strcmp(alg
->cra_name
, "sha224") ||
2756 !strcmp(alg
->cra_name
, "hmac(sha224)"))) {
2757 t_alg
->algt
.alg
.hash
.init
= ahash_init_sha224_swinit
;
2758 t_alg
->algt
.desc_hdr_template
=
2759 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2760 DESC_HDR_SEL0_MDEUA
|
2761 DESC_HDR_MODE0_MDEU_SHA256
;
2765 dev_err(dev
, "unknown algorithm type %d\n", t_alg
->algt
.type
);
2767 return ERR_PTR(-EINVAL
);
2770 alg
->cra_module
= THIS_MODULE
;
2771 alg
->cra_priority
= TALITOS_CRA_PRIORITY
;
2772 alg
->cra_alignmask
= 0;
2773 alg
->cra_ctxsize
= sizeof(struct talitos_ctx
);
2774 alg
->cra_flags
|= CRYPTO_ALG_KERN_DRIVER_ONLY
;
2781 static int talitos_probe_irq(struct platform_device
*ofdev
)
2783 struct device
*dev
= &ofdev
->dev
;
2784 struct device_node
*np
= ofdev
->dev
.of_node
;
2785 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2787 bool is_sec1
= has_ftr_sec1(priv
);
2789 priv
->irq
[0] = irq_of_parse_and_map(np
, 0);
2790 if (!priv
->irq
[0]) {
2791 dev_err(dev
, "failed to map irq\n");
2795 err
= request_irq(priv
->irq
[0], talitos1_interrupt_4ch
, 0,
2796 dev_driver_string(dev
), dev
);
2800 priv
->irq
[1] = irq_of_parse_and_map(np
, 1);
2802 /* get the primary irq line */
2803 if (!priv
->irq
[1]) {
2804 err
= request_irq(priv
->irq
[0], talitos2_interrupt_4ch
, 0,
2805 dev_driver_string(dev
), dev
);
2809 err
= request_irq(priv
->irq
[0], talitos2_interrupt_ch0_2
, 0,
2810 dev_driver_string(dev
), dev
);
2814 /* get the secondary irq line */
2815 err
= request_irq(priv
->irq
[1], talitos2_interrupt_ch1_3
, 0,
2816 dev_driver_string(dev
), dev
);
2818 dev_err(dev
, "failed to request secondary irq\n");
2819 irq_dispose_mapping(priv
->irq
[1]);
2827 dev_err(dev
, "failed to request primary irq\n");
2828 irq_dispose_mapping(priv
->irq
[0]);
2835 static int talitos_probe(struct platform_device
*ofdev
)
2837 struct device
*dev
= &ofdev
->dev
;
2838 struct device_node
*np
= ofdev
->dev
.of_node
;
2839 struct talitos_private
*priv
;
2840 const unsigned int *prop
;
2844 priv
= kzalloc(sizeof(struct talitos_private
), GFP_KERNEL
);
2848 INIT_LIST_HEAD(&priv
->alg_list
);
2850 dev_set_drvdata(dev
, priv
);
2852 priv
->ofdev
= ofdev
;
2854 spin_lock_init(&priv
->reg_lock
);
2856 priv
->reg
= of_iomap(np
, 0);
2858 dev_err(dev
, "failed to of_iomap\n");
2863 /* get SEC version capabilities from device tree */
2864 prop
= of_get_property(np
, "fsl,num-channels", NULL
);
2866 priv
->num_channels
= *prop
;
2868 prop
= of_get_property(np
, "fsl,channel-fifo-len", NULL
);
2870 priv
->chfifo_len
= *prop
;
2872 prop
= of_get_property(np
, "fsl,exec-units-mask", NULL
);
2874 priv
->exec_units
= *prop
;
2876 prop
= of_get_property(np
, "fsl,descriptor-types-mask", NULL
);
2878 priv
->desc_types
= *prop
;
2880 if (!is_power_of_2(priv
->num_channels
) || !priv
->chfifo_len
||
2881 !priv
->exec_units
|| !priv
->desc_types
) {
2882 dev_err(dev
, "invalid property data in device tree node\n");
2887 if (of_device_is_compatible(np
, "fsl,sec3.0"))
2888 priv
->features
|= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT
;
2890 if (of_device_is_compatible(np
, "fsl,sec2.1"))
2891 priv
->features
|= TALITOS_FTR_HW_AUTH_CHECK
|
2892 TALITOS_FTR_SHA224_HWINIT
|
2893 TALITOS_FTR_HMAC_OK
;
2895 if (of_device_is_compatible(np
, "fsl,sec1.0"))
2896 priv
->features
|= TALITOS_FTR_SEC1
;
2898 if (of_device_is_compatible(np
, "fsl,sec1.2")) {
2899 priv
->reg_deu
= priv
->reg
+ TALITOS12_DEU
;
2900 priv
->reg_aesu
= priv
->reg
+ TALITOS12_AESU
;
2901 priv
->reg_mdeu
= priv
->reg
+ TALITOS12_MDEU
;
2902 stride
= TALITOS1_CH_STRIDE
;
2903 } else if (of_device_is_compatible(np
, "fsl,sec1.0")) {
2904 priv
->reg_deu
= priv
->reg
+ TALITOS10_DEU
;
2905 priv
->reg_aesu
= priv
->reg
+ TALITOS10_AESU
;
2906 priv
->reg_mdeu
= priv
->reg
+ TALITOS10_MDEU
;
2907 priv
->reg_afeu
= priv
->reg
+ TALITOS10_AFEU
;
2908 priv
->reg_rngu
= priv
->reg
+ TALITOS10_RNGU
;
2909 priv
->reg_pkeu
= priv
->reg
+ TALITOS10_PKEU
;
2910 stride
= TALITOS1_CH_STRIDE
;
2912 priv
->reg_deu
= priv
->reg
+ TALITOS2_DEU
;
2913 priv
->reg_aesu
= priv
->reg
+ TALITOS2_AESU
;
2914 priv
->reg_mdeu
= priv
->reg
+ TALITOS2_MDEU
;
2915 priv
->reg_afeu
= priv
->reg
+ TALITOS2_AFEU
;
2916 priv
->reg_rngu
= priv
->reg
+ TALITOS2_RNGU
;
2917 priv
->reg_pkeu
= priv
->reg
+ TALITOS2_PKEU
;
2918 priv
->reg_keu
= priv
->reg
+ TALITOS2_KEU
;
2919 priv
->reg_crcu
= priv
->reg
+ TALITOS2_CRCU
;
2920 stride
= TALITOS2_CH_STRIDE
;
2923 err
= talitos_probe_irq(ofdev
);
2927 if (of_device_is_compatible(np
, "fsl,sec1.0")) {
2928 tasklet_init(&priv
->done_task
[0], talitos1_done_4ch
,
2929 (unsigned long)dev
);
2931 if (!priv
->irq
[1]) {
2932 tasklet_init(&priv
->done_task
[0], talitos2_done_4ch
,
2933 (unsigned long)dev
);
2935 tasklet_init(&priv
->done_task
[0], talitos2_done_ch0_2
,
2936 (unsigned long)dev
);
2937 tasklet_init(&priv
->done_task
[1], talitos2_done_ch1_3
,
2938 (unsigned long)dev
);
2942 priv
->chan
= kzalloc(sizeof(struct talitos_channel
) *
2943 priv
->num_channels
, GFP_KERNEL
);
2945 dev_err(dev
, "failed to allocate channel management space\n");
2950 priv
->fifo_len
= roundup_pow_of_two(priv
->chfifo_len
);
2952 for (i
= 0; i
< priv
->num_channels
; i
++) {
2953 priv
->chan
[i
].reg
= priv
->reg
+ stride
* (i
+ 1);
2954 if (!priv
->irq
[1] || !(i
& 1))
2955 priv
->chan
[i
].reg
+= TALITOS_CH_BASE_OFFSET
;
2957 spin_lock_init(&priv
->chan
[i
].head_lock
);
2958 spin_lock_init(&priv
->chan
[i
].tail_lock
);
2960 priv
->chan
[i
].fifo
= kzalloc(sizeof(struct talitos_request
) *
2961 priv
->fifo_len
, GFP_KERNEL
);
2962 if (!priv
->chan
[i
].fifo
) {
2963 dev_err(dev
, "failed to allocate request fifo %d\n", i
);
2968 atomic_set(&priv
->chan
[i
].submit_count
,
2969 -(priv
->chfifo_len
- 1));
2972 dma_set_mask(dev
, DMA_BIT_MASK(36));
2974 /* reset and initialize the h/w */
2975 err
= init_device(dev
);
2977 dev_err(dev
, "failed to initialize device\n");
2981 /* register the RNG, if available */
2982 if (hw_supports(dev
, DESC_HDR_SEL0_RNG
)) {
2983 err
= talitos_register_rng(dev
);
2985 dev_err(dev
, "failed to register hwrng: %d\n", err
);
2988 dev_info(dev
, "hwrng\n");
2991 /* register crypto algorithms the device supports */
2992 for (i
= 0; i
< ARRAY_SIZE(driver_algs
); i
++) {
2993 if (hw_supports(dev
, driver_algs
[i
].desc_hdr_template
)) {
2994 struct talitos_crypto_alg
*t_alg
;
2995 struct crypto_alg
*alg
= NULL
;
2997 t_alg
= talitos_alg_alloc(dev
, &driver_algs
[i
]);
2998 if (IS_ERR(t_alg
)) {
2999 err
= PTR_ERR(t_alg
);
3000 if (err
== -ENOTSUPP
)
3005 switch (t_alg
->algt
.type
) {
3006 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
3007 err
= crypto_register_alg(
3008 &t_alg
->algt
.alg
.crypto
);
3009 alg
= &t_alg
->algt
.alg
.crypto
;
3012 case CRYPTO_ALG_TYPE_AEAD
:
3013 err
= crypto_register_aead(
3014 &t_alg
->algt
.alg
.aead
);
3015 alg
= &t_alg
->algt
.alg
.aead
.base
;
3018 case CRYPTO_ALG_TYPE_AHASH
:
3019 err
= crypto_register_ahash(
3020 &t_alg
->algt
.alg
.hash
);
3021 alg
= &t_alg
->algt
.alg
.hash
.halg
.base
;
3025 dev_err(dev
, "%s alg registration failed\n",
3026 alg
->cra_driver_name
);
3029 list_add_tail(&t_alg
->entry
, &priv
->alg_list
);
3032 if (!list_empty(&priv
->alg_list
))
3033 dev_info(dev
, "%s algorithms registered in /proc/crypto\n",
3034 (char *)of_get_property(np
, "compatible", NULL
));
3039 talitos_remove(ofdev
);
3044 static const struct of_device_id talitos_match
[] = {
3045 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3047 .compatible
= "fsl,sec1.0",
3050 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3052 .compatible
= "fsl,sec2.0",
3057 MODULE_DEVICE_TABLE(of
, talitos_match
);
3059 static struct platform_driver talitos_driver
= {
3062 .of_match_table
= talitos_match
,
3064 .probe
= talitos_probe
,
3065 .remove
= talitos_remove
,
3068 module_platform_driver(talitos_driver
);
3070 MODULE_LICENSE("GPL");
3071 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3072 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");