2 * Block chaining cipher operations.
4 * Generic encrypt/decrypt wrapper for ciphers, handles operations across
5 * multiple page boundaries by using temporary blocks. In user context,
6 * the kernel is given a chance to schedule us once per page.
8 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free
12 * Software Foundation; either version 2 of the License, or (at your option)
17 #include <crypto/aead.h>
18 #include <crypto/internal/skcipher.h>
19 #include <crypto/scatterwalk.h>
20 #include <linux/errno.h>
21 #include <linux/hardirq.h>
22 #include <linux/kernel.h>
23 #include <linux/module.h>
24 #include <linux/seq_file.h>
25 #include <linux/slab.h>
26 #include <linux/string.h>
27 #include <linux/cryptouser.h>
28 #include <net/netlink.h>
33 BLKCIPHER_WALK_PHYS
= 1 << 0,
34 BLKCIPHER_WALK_SLOW
= 1 << 1,
35 BLKCIPHER_WALK_COPY
= 1 << 2,
36 BLKCIPHER_WALK_DIFF
= 1 << 3,
39 static int blkcipher_walk_next(struct blkcipher_desc
*desc
,
40 struct blkcipher_walk
*walk
);
41 static int blkcipher_walk_first(struct blkcipher_desc
*desc
,
42 struct blkcipher_walk
*walk
);
44 static inline void blkcipher_map_src(struct blkcipher_walk
*walk
)
46 walk
->src
.virt
.addr
= scatterwalk_map(&walk
->in
);
49 static inline void blkcipher_map_dst(struct blkcipher_walk
*walk
)
51 walk
->dst
.virt
.addr
= scatterwalk_map(&walk
->out
);
54 static inline void blkcipher_unmap_src(struct blkcipher_walk
*walk
)
56 scatterwalk_unmap(walk
->src
.virt
.addr
);
59 static inline void blkcipher_unmap_dst(struct blkcipher_walk
*walk
)
61 scatterwalk_unmap(walk
->dst
.virt
.addr
);
64 /* Get a spot of the specified length that does not straddle a page.
65 * The caller needs to ensure that there is enough space for this operation.
67 static inline u8
*blkcipher_get_spot(u8
*start
, unsigned int len
)
69 u8
*end_page
= (u8
*)(((unsigned long)(start
+ len
- 1)) & PAGE_MASK
);
70 return max(start
, end_page
);
73 static inline unsigned int blkcipher_done_slow(struct blkcipher_walk
*walk
,
78 addr
= (u8
*)ALIGN((unsigned long)walk
->buffer
, walk
->alignmask
+ 1);
79 addr
= blkcipher_get_spot(addr
, bsize
);
80 scatterwalk_copychunks(addr
, &walk
->out
, bsize
, 1);
84 static inline unsigned int blkcipher_done_fast(struct blkcipher_walk
*walk
,
87 if (walk
->flags
& BLKCIPHER_WALK_COPY
) {
88 blkcipher_map_dst(walk
);
89 memcpy(walk
->dst
.virt
.addr
, walk
->page
, n
);
90 blkcipher_unmap_dst(walk
);
91 } else if (!(walk
->flags
& BLKCIPHER_WALK_PHYS
)) {
92 if (walk
->flags
& BLKCIPHER_WALK_DIFF
)
93 blkcipher_unmap_dst(walk
);
94 blkcipher_unmap_src(walk
);
97 scatterwalk_advance(&walk
->in
, n
);
98 scatterwalk_advance(&walk
->out
, n
);
103 int blkcipher_walk_done(struct blkcipher_desc
*desc
,
104 struct blkcipher_walk
*walk
, int err
)
106 unsigned int nbytes
= 0;
108 if (likely(err
>= 0)) {
109 unsigned int n
= walk
->nbytes
- err
;
111 if (likely(!(walk
->flags
& BLKCIPHER_WALK_SLOW
)))
112 n
= blkcipher_done_fast(walk
, n
);
113 else if (WARN_ON(err
)) {
117 n
= blkcipher_done_slow(walk
, n
);
119 nbytes
= walk
->total
- n
;
123 scatterwalk_done(&walk
->in
, 0, nbytes
);
124 scatterwalk_done(&walk
->out
, 1, nbytes
);
127 walk
->total
= nbytes
;
128 walk
->nbytes
= nbytes
;
131 crypto_yield(desc
->flags
);
132 return blkcipher_walk_next(desc
, walk
);
135 if (walk
->iv
!= desc
->info
)
136 memcpy(desc
->info
, walk
->iv
, walk
->ivsize
);
137 if (walk
->buffer
!= walk
->page
)
140 free_page((unsigned long)walk
->page
);
144 EXPORT_SYMBOL_GPL(blkcipher_walk_done
);
146 static inline int blkcipher_next_slow(struct blkcipher_desc
*desc
,
147 struct blkcipher_walk
*walk
,
149 unsigned int alignmask
)
152 unsigned aligned_bsize
= ALIGN(bsize
, alignmask
+ 1);
157 walk
->buffer
= walk
->page
;
161 n
= aligned_bsize
* 3 - (alignmask
+ 1) +
162 (alignmask
& ~(crypto_tfm_ctx_alignment() - 1));
163 walk
->buffer
= kmalloc(n
, GFP_ATOMIC
);
165 return blkcipher_walk_done(desc
, walk
, -ENOMEM
);
168 walk
->dst
.virt
.addr
= (u8
*)ALIGN((unsigned long)walk
->buffer
,
170 walk
->dst
.virt
.addr
= blkcipher_get_spot(walk
->dst
.virt
.addr
, bsize
);
171 walk
->src
.virt
.addr
= blkcipher_get_spot(walk
->dst
.virt
.addr
+
172 aligned_bsize
, bsize
);
174 scatterwalk_copychunks(walk
->src
.virt
.addr
, &walk
->in
, bsize
, 0);
176 walk
->nbytes
= bsize
;
177 walk
->flags
|= BLKCIPHER_WALK_SLOW
;
182 static inline int blkcipher_next_copy(struct blkcipher_walk
*walk
)
184 u8
*tmp
= walk
->page
;
186 blkcipher_map_src(walk
);
187 memcpy(tmp
, walk
->src
.virt
.addr
, walk
->nbytes
);
188 blkcipher_unmap_src(walk
);
190 walk
->src
.virt
.addr
= tmp
;
191 walk
->dst
.virt
.addr
= tmp
;
196 static inline int blkcipher_next_fast(struct blkcipher_desc
*desc
,
197 struct blkcipher_walk
*walk
)
201 walk
->src
.phys
.page
= scatterwalk_page(&walk
->in
);
202 walk
->src
.phys
.offset
= offset_in_page(walk
->in
.offset
);
203 walk
->dst
.phys
.page
= scatterwalk_page(&walk
->out
);
204 walk
->dst
.phys
.offset
= offset_in_page(walk
->out
.offset
);
206 if (walk
->flags
& BLKCIPHER_WALK_PHYS
)
209 diff
= walk
->src
.phys
.offset
- walk
->dst
.phys
.offset
;
210 diff
|= walk
->src
.virt
.page
- walk
->dst
.virt
.page
;
212 blkcipher_map_src(walk
);
213 walk
->dst
.virt
.addr
= walk
->src
.virt
.addr
;
216 walk
->flags
|= BLKCIPHER_WALK_DIFF
;
217 blkcipher_map_dst(walk
);
223 static int blkcipher_walk_next(struct blkcipher_desc
*desc
,
224 struct blkcipher_walk
*walk
)
231 if (unlikely(n
< walk
->cipher_blocksize
)) {
232 desc
->flags
|= CRYPTO_TFM_RES_BAD_BLOCK_LEN
;
233 return blkcipher_walk_done(desc
, walk
, -EINVAL
);
236 bsize
= min(walk
->walk_blocksize
, n
);
238 walk
->flags
&= ~(BLKCIPHER_WALK_SLOW
| BLKCIPHER_WALK_COPY
|
239 BLKCIPHER_WALK_DIFF
);
240 if (!scatterwalk_aligned(&walk
->in
, walk
->alignmask
) ||
241 !scatterwalk_aligned(&walk
->out
, walk
->alignmask
)) {
242 walk
->flags
|= BLKCIPHER_WALK_COPY
;
244 walk
->page
= (void *)__get_free_page(GFP_ATOMIC
);
250 n
= scatterwalk_clamp(&walk
->in
, n
);
251 n
= scatterwalk_clamp(&walk
->out
, n
);
253 if (unlikely(n
< bsize
)) {
254 err
= blkcipher_next_slow(desc
, walk
, bsize
, walk
->alignmask
);
255 goto set_phys_lowmem
;
259 if (walk
->flags
& BLKCIPHER_WALK_COPY
) {
260 err
= blkcipher_next_copy(walk
);
261 goto set_phys_lowmem
;
264 return blkcipher_next_fast(desc
, walk
);
267 if (walk
->flags
& BLKCIPHER_WALK_PHYS
) {
268 walk
->src
.phys
.page
= virt_to_page(walk
->src
.virt
.addr
);
269 walk
->dst
.phys
.page
= virt_to_page(walk
->dst
.virt
.addr
);
270 walk
->src
.phys
.offset
&= PAGE_SIZE
- 1;
271 walk
->dst
.phys
.offset
&= PAGE_SIZE
- 1;
276 static inline int blkcipher_copy_iv(struct blkcipher_walk
*walk
)
278 unsigned bs
= walk
->walk_blocksize
;
279 unsigned aligned_bs
= ALIGN(bs
, walk
->alignmask
+ 1);
280 unsigned int size
= aligned_bs
* 2 +
281 walk
->ivsize
+ max(aligned_bs
, walk
->ivsize
) -
282 (walk
->alignmask
+ 1);
285 size
+= walk
->alignmask
& ~(crypto_tfm_ctx_alignment() - 1);
286 walk
->buffer
= kmalloc(size
, GFP_ATOMIC
);
290 iv
= (u8
*)ALIGN((unsigned long)walk
->buffer
, walk
->alignmask
+ 1);
291 iv
= blkcipher_get_spot(iv
, bs
) + aligned_bs
;
292 iv
= blkcipher_get_spot(iv
, bs
) + aligned_bs
;
293 iv
= blkcipher_get_spot(iv
, walk
->ivsize
);
295 walk
->iv
= memcpy(iv
, walk
->iv
, walk
->ivsize
);
299 int blkcipher_walk_virt(struct blkcipher_desc
*desc
,
300 struct blkcipher_walk
*walk
)
302 walk
->flags
&= ~BLKCIPHER_WALK_PHYS
;
303 walk
->walk_blocksize
= crypto_blkcipher_blocksize(desc
->tfm
);
304 walk
->cipher_blocksize
= walk
->walk_blocksize
;
305 walk
->ivsize
= crypto_blkcipher_ivsize(desc
->tfm
);
306 walk
->alignmask
= crypto_blkcipher_alignmask(desc
->tfm
);
307 return blkcipher_walk_first(desc
, walk
);
309 EXPORT_SYMBOL_GPL(blkcipher_walk_virt
);
311 int blkcipher_walk_phys(struct blkcipher_desc
*desc
,
312 struct blkcipher_walk
*walk
)
314 walk
->flags
|= BLKCIPHER_WALK_PHYS
;
315 walk
->walk_blocksize
= crypto_blkcipher_blocksize(desc
->tfm
);
316 walk
->cipher_blocksize
= walk
->walk_blocksize
;
317 walk
->ivsize
= crypto_blkcipher_ivsize(desc
->tfm
);
318 walk
->alignmask
= crypto_blkcipher_alignmask(desc
->tfm
);
319 return blkcipher_walk_first(desc
, walk
);
321 EXPORT_SYMBOL_GPL(blkcipher_walk_phys
);
323 static int blkcipher_walk_first(struct blkcipher_desc
*desc
,
324 struct blkcipher_walk
*walk
)
326 if (WARN_ON_ONCE(in_irq()))
329 walk
->iv
= desc
->info
;
330 walk
->nbytes
= walk
->total
;
331 if (unlikely(!walk
->total
))
335 if (unlikely(((unsigned long)walk
->iv
& walk
->alignmask
))) {
336 int err
= blkcipher_copy_iv(walk
);
341 scatterwalk_start(&walk
->in
, walk
->in
.sg
);
342 scatterwalk_start(&walk
->out
, walk
->out
.sg
);
345 return blkcipher_walk_next(desc
, walk
);
348 int blkcipher_walk_virt_block(struct blkcipher_desc
*desc
,
349 struct blkcipher_walk
*walk
,
350 unsigned int blocksize
)
352 walk
->flags
&= ~BLKCIPHER_WALK_PHYS
;
353 walk
->walk_blocksize
= blocksize
;
354 walk
->cipher_blocksize
= crypto_blkcipher_blocksize(desc
->tfm
);
355 walk
->ivsize
= crypto_blkcipher_ivsize(desc
->tfm
);
356 walk
->alignmask
= crypto_blkcipher_alignmask(desc
->tfm
);
357 return blkcipher_walk_first(desc
, walk
);
359 EXPORT_SYMBOL_GPL(blkcipher_walk_virt_block
);
361 int blkcipher_aead_walk_virt_block(struct blkcipher_desc
*desc
,
362 struct blkcipher_walk
*walk
,
363 struct crypto_aead
*tfm
,
364 unsigned int blocksize
)
366 walk
->flags
&= ~BLKCIPHER_WALK_PHYS
;
367 walk
->walk_blocksize
= blocksize
;
368 walk
->cipher_blocksize
= crypto_aead_blocksize(tfm
);
369 walk
->ivsize
= crypto_aead_ivsize(tfm
);
370 walk
->alignmask
= crypto_aead_alignmask(tfm
);
371 return blkcipher_walk_first(desc
, walk
);
373 EXPORT_SYMBOL_GPL(blkcipher_aead_walk_virt_block
);
375 static int setkey_unaligned(struct crypto_tfm
*tfm
, const u8
*key
,
378 struct blkcipher_alg
*cipher
= &tfm
->__crt_alg
->cra_blkcipher
;
379 unsigned long alignmask
= crypto_tfm_alg_alignmask(tfm
);
381 u8
*buffer
, *alignbuffer
;
382 unsigned long absize
;
384 absize
= keylen
+ alignmask
;
385 buffer
= kmalloc(absize
, GFP_ATOMIC
);
389 alignbuffer
= (u8
*)ALIGN((unsigned long)buffer
, alignmask
+ 1);
390 memcpy(alignbuffer
, key
, keylen
);
391 ret
= cipher
->setkey(tfm
, alignbuffer
, keylen
);
392 memset(alignbuffer
, 0, keylen
);
397 static int setkey(struct crypto_tfm
*tfm
, const u8
*key
, unsigned int keylen
)
399 struct blkcipher_alg
*cipher
= &tfm
->__crt_alg
->cra_blkcipher
;
400 unsigned long alignmask
= crypto_tfm_alg_alignmask(tfm
);
402 if (keylen
< cipher
->min_keysize
|| keylen
> cipher
->max_keysize
) {
403 tfm
->crt_flags
|= CRYPTO_TFM_RES_BAD_KEY_LEN
;
407 if ((unsigned long)key
& alignmask
)
408 return setkey_unaligned(tfm
, key
, keylen
);
410 return cipher
->setkey(tfm
, key
, keylen
);
413 static int async_setkey(struct crypto_ablkcipher
*tfm
, const u8
*key
,
416 return setkey(crypto_ablkcipher_tfm(tfm
), key
, keylen
);
419 static int async_encrypt(struct ablkcipher_request
*req
)
421 struct crypto_tfm
*tfm
= req
->base
.tfm
;
422 struct blkcipher_alg
*alg
= &tfm
->__crt_alg
->cra_blkcipher
;
423 struct blkcipher_desc desc
= {
424 .tfm
= __crypto_blkcipher_cast(tfm
),
426 .flags
= req
->base
.flags
,
430 return alg
->encrypt(&desc
, req
->dst
, req
->src
, req
->nbytes
);
433 static int async_decrypt(struct ablkcipher_request
*req
)
435 struct crypto_tfm
*tfm
= req
->base
.tfm
;
436 struct blkcipher_alg
*alg
= &tfm
->__crt_alg
->cra_blkcipher
;
437 struct blkcipher_desc desc
= {
438 .tfm
= __crypto_blkcipher_cast(tfm
),
440 .flags
= req
->base
.flags
,
443 return alg
->decrypt(&desc
, req
->dst
, req
->src
, req
->nbytes
);
446 static unsigned int crypto_blkcipher_ctxsize(struct crypto_alg
*alg
, u32 type
,
449 struct blkcipher_alg
*cipher
= &alg
->cra_blkcipher
;
450 unsigned int len
= alg
->cra_ctxsize
;
452 if ((mask
& CRYPTO_ALG_TYPE_MASK
) == CRYPTO_ALG_TYPE_MASK
&&
454 len
= ALIGN(len
, (unsigned long)alg
->cra_alignmask
+ 1);
455 len
+= cipher
->ivsize
;
461 static int crypto_init_blkcipher_ops_async(struct crypto_tfm
*tfm
)
463 struct ablkcipher_tfm
*crt
= &tfm
->crt_ablkcipher
;
464 struct blkcipher_alg
*alg
= &tfm
->__crt_alg
->cra_blkcipher
;
466 crt
->setkey
= async_setkey
;
467 crt
->encrypt
= async_encrypt
;
468 crt
->decrypt
= async_decrypt
;
469 crt
->base
= __crypto_ablkcipher_cast(tfm
);
470 crt
->ivsize
= alg
->ivsize
;
475 static int crypto_init_blkcipher_ops_sync(struct crypto_tfm
*tfm
)
477 struct blkcipher_tfm
*crt
= &tfm
->crt_blkcipher
;
478 struct blkcipher_alg
*alg
= &tfm
->__crt_alg
->cra_blkcipher
;
479 unsigned long align
= crypto_tfm_alg_alignmask(tfm
) + 1;
482 crt
->setkey
= setkey
;
483 crt
->encrypt
= alg
->encrypt
;
484 crt
->decrypt
= alg
->decrypt
;
486 addr
= (unsigned long)crypto_tfm_ctx(tfm
);
487 addr
= ALIGN(addr
, align
);
488 addr
+= ALIGN(tfm
->__crt_alg
->cra_ctxsize
, align
);
489 crt
->iv
= (void *)addr
;
494 static int crypto_init_blkcipher_ops(struct crypto_tfm
*tfm
, u32 type
, u32 mask
)
496 struct blkcipher_alg
*alg
= &tfm
->__crt_alg
->cra_blkcipher
;
498 if (alg
->ivsize
> PAGE_SIZE
/ 8)
501 if ((mask
& CRYPTO_ALG_TYPE_MASK
) == CRYPTO_ALG_TYPE_MASK
)
502 return crypto_init_blkcipher_ops_sync(tfm
);
504 return crypto_init_blkcipher_ops_async(tfm
);
508 static int crypto_blkcipher_report(struct sk_buff
*skb
, struct crypto_alg
*alg
)
510 struct crypto_report_blkcipher rblkcipher
;
512 strncpy(rblkcipher
.type
, "blkcipher", sizeof(rblkcipher
.type
));
513 strncpy(rblkcipher
.geniv
, alg
->cra_blkcipher
.geniv
?: "<default>",
514 sizeof(rblkcipher
.geniv
));
516 rblkcipher
.blocksize
= alg
->cra_blocksize
;
517 rblkcipher
.min_keysize
= alg
->cra_blkcipher
.min_keysize
;
518 rblkcipher
.max_keysize
= alg
->cra_blkcipher
.max_keysize
;
519 rblkcipher
.ivsize
= alg
->cra_blkcipher
.ivsize
;
521 if (nla_put(skb
, CRYPTOCFGA_REPORT_BLKCIPHER
,
522 sizeof(struct crypto_report_blkcipher
), &rblkcipher
))
523 goto nla_put_failure
;
530 static int crypto_blkcipher_report(struct sk_buff
*skb
, struct crypto_alg
*alg
)
536 static void crypto_blkcipher_show(struct seq_file
*m
, struct crypto_alg
*alg
)
537 __attribute__ ((unused
));
538 static void crypto_blkcipher_show(struct seq_file
*m
, struct crypto_alg
*alg
)
540 seq_printf(m
, "type : blkcipher\n");
541 seq_printf(m
, "blocksize : %u\n", alg
->cra_blocksize
);
542 seq_printf(m
, "min keysize : %u\n", alg
->cra_blkcipher
.min_keysize
);
543 seq_printf(m
, "max keysize : %u\n", alg
->cra_blkcipher
.max_keysize
);
544 seq_printf(m
, "ivsize : %u\n", alg
->cra_blkcipher
.ivsize
);
545 seq_printf(m
, "geniv : %s\n", alg
->cra_blkcipher
.geniv
?:
549 const struct crypto_type crypto_blkcipher_type
= {
550 .ctxsize
= crypto_blkcipher_ctxsize
,
551 .init
= crypto_init_blkcipher_ops
,
552 #ifdef CONFIG_PROC_FS
553 .show
= crypto_blkcipher_show
,
555 .report
= crypto_blkcipher_report
,
557 EXPORT_SYMBOL_GPL(crypto_blkcipher_type
);
559 MODULE_LICENSE("GPL");
560 MODULE_DESCRIPTION("Generic block chaining cipher type");