2 * Copyright (C) 2003 Jana Saout <jana@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
4 * Copyright (C) 2006-2017 Red Hat, Inc. All rights reserved.
5 * Copyright (C) 2013-2017 Milan Broz <gmazyland@gmail.com>
7 * This file is released under the GPL.
10 #include <linux/completion.h>
11 #include <linux/err.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/kernel.h>
15 #include <linux/key.h>
16 #include <linux/bio.h>
17 #include <linux/blkdev.h>
18 #include <linux/mempool.h>
19 #include <linux/slab.h>
20 #include <linux/crypto.h>
21 #include <linux/workqueue.h>
22 #include <linux/kthread.h>
23 #include <linux/backing-dev.h>
24 #include <linux/atomic.h>
25 #include <linux/scatterlist.h>
26 #include <linux/rbtree.h>
27 #include <linux/ctype.h>
29 #include <asm/unaligned.h>
30 #include <crypto/hash.h>
31 #include <crypto/md5.h>
32 #include <crypto/algapi.h>
33 #include <crypto/skcipher.h>
34 #include <crypto/aead.h>
35 #include <crypto/authenc.h>
36 #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */
37 #include <keys/user-type.h>
39 #include <linux/device-mapper.h>
41 #define DM_MSG_PREFIX "crypt"
44 * context holding the current state of a multi-part conversion
46 struct convert_context
{
47 struct completion restart
;
50 struct bvec_iter iter_in
;
51 struct bvec_iter iter_out
;
55 struct skcipher_request
*req
;
56 struct aead_request
*req_aead
;
62 * per bio private data
65 struct crypt_config
*cc
;
67 u8
*integrity_metadata
;
68 bool integrity_metadata_from_pool
;
69 struct work_struct work
;
71 struct convert_context ctx
;
77 struct rb_node rb_node
;
78 } CRYPTO_MINALIGN_ATTR
;
80 struct dm_crypt_request
{
81 struct convert_context
*ctx
;
82 struct scatterlist sg_in
[4];
83 struct scatterlist sg_out
[4];
89 struct crypt_iv_operations
{
90 int (*ctr
)(struct crypt_config
*cc
, struct dm_target
*ti
,
92 void (*dtr
)(struct crypt_config
*cc
);
93 int (*init
)(struct crypt_config
*cc
);
94 int (*wipe
)(struct crypt_config
*cc
);
95 int (*generator
)(struct crypt_config
*cc
, u8
*iv
,
96 struct dm_crypt_request
*dmreq
);
97 int (*post
)(struct crypt_config
*cc
, u8
*iv
,
98 struct dm_crypt_request
*dmreq
);
101 struct iv_essiv_private
{
102 struct crypto_ahash
*hash_tfm
;
106 struct iv_benbi_private
{
110 #define LMK_SEED_SIZE 64 /* hash + 0 */
111 struct iv_lmk_private
{
112 struct crypto_shash
*hash_tfm
;
116 #define TCW_WHITENING_SIZE 16
117 struct iv_tcw_private
{
118 struct crypto_shash
*crc32_tfm
;
124 * Crypt: maps a linear range of a block device
125 * and encrypts / decrypts at the same time.
127 enum flags
{ DM_CRYPT_SUSPENDED
, DM_CRYPT_KEY_VALID
,
128 DM_CRYPT_SAME_CPU
, DM_CRYPT_NO_OFFLOAD
};
131 CRYPT_MODE_INTEGRITY_AEAD
, /* Use authenticated mode for cihper */
132 CRYPT_IV_LARGE_SECTORS
, /* Calculate IV from sector_size, not 512B sectors */
136 * The fields in here must be read only after initialization.
138 struct crypt_config
{
143 * pool for per bio private data, crypto requests,
144 * encryption requeusts/buffer pages and integrity tags
147 mempool_t
*page_pool
;
149 unsigned tag_pool_max_sectors
;
152 struct mutex bio_alloc_lock
;
154 struct workqueue_struct
*io_queue
;
155 struct workqueue_struct
*crypt_queue
;
157 struct task_struct
*write_thread
;
158 wait_queue_head_t write_thread_wait
;
159 struct rb_root write_tree
;
166 const struct crypt_iv_operations
*iv_gen_ops
;
168 struct iv_essiv_private essiv
;
169 struct iv_benbi_private benbi
;
170 struct iv_lmk_private lmk
;
171 struct iv_tcw_private tcw
;
174 unsigned int iv_size
;
175 unsigned short int sector_size
;
176 unsigned char sector_shift
;
178 /* ESSIV: struct crypto_cipher *essiv_tfm */
181 struct crypto_skcipher
**tfms
;
182 struct crypto_aead
**tfms_aead
;
185 unsigned long cipher_flags
;
188 * Layout of each crypto request:
190 * struct skcipher_request
193 * struct dm_crypt_request
197 * The padding is added so that dm_crypt_request and the IV are
200 unsigned int dmreq_start
;
202 unsigned int per_bio_data_size
;
205 unsigned int key_size
;
206 unsigned int key_parts
; /* independent parts in key buffer */
207 unsigned int key_extra_size
; /* additional keys length */
208 unsigned int key_mac_size
; /* MAC key size for authenc(...) */
210 unsigned int integrity_tag_size
;
211 unsigned int integrity_iv_size
;
212 unsigned int on_disk_tag_size
;
214 u8
*authenc_key
; /* space for keys in authenc() format (if used) */
219 #define MAX_TAG_SIZE 480
220 #define POOL_ENTRY_SIZE 512
222 static void clone_init(struct dm_crypt_io
*, struct bio
*);
223 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
);
224 static struct scatterlist
*crypt_get_sg_data(struct crypt_config
*cc
,
225 struct scatterlist
*sg
);
228 * Use this to access cipher attributes that are independent of the key.
230 static struct crypto_skcipher
*any_tfm(struct crypt_config
*cc
)
232 return cc
->cipher_tfm
.tfms
[0];
235 static struct crypto_aead
*any_tfm_aead(struct crypt_config
*cc
)
237 return cc
->cipher_tfm
.tfms_aead
[0];
241 * Different IV generation algorithms:
243 * plain: the initial vector is the 32-bit little-endian version of the sector
244 * number, padded with zeros if necessary.
246 * plain64: the initial vector is the 64-bit little-endian version of the sector
247 * number, padded with zeros if necessary.
249 * essiv: "encrypted sector|salt initial vector", the sector number is
250 * encrypted with the bulk cipher using a salt as key. The salt
251 * should be derived from the bulk cipher's key via hashing.
253 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
254 * (needed for LRW-32-AES and possible other narrow block modes)
256 * null: the initial vector is always zero. Provides compatibility with
257 * obsolete loop_fish2 devices. Do not use for new devices.
259 * lmk: Compatible implementation of the block chaining mode used
260 * by the Loop-AES block device encryption system
261 * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
262 * It operates on full 512 byte sectors and uses CBC
263 * with an IV derived from the sector number, the data and
264 * optionally extra IV seed.
265 * This means that after decryption the first block
266 * of sector must be tweaked according to decrypted data.
267 * Loop-AES can use three encryption schemes:
268 * version 1: is plain aes-cbc mode
269 * version 2: uses 64 multikey scheme with lmk IV generator
270 * version 3: the same as version 2 with additional IV seed
271 * (it uses 65 keys, last key is used as IV seed)
273 * tcw: Compatible implementation of the block chaining mode used
274 * by the TrueCrypt device encryption system (prior to version 4.1).
275 * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat
276 * It operates on full 512 byte sectors and uses CBC
277 * with an IV derived from initial key and the sector number.
278 * In addition, whitening value is applied on every sector, whitening
279 * is calculated from initial key, sector number and mixed using CRC32.
280 * Note that this encryption scheme is vulnerable to watermarking attacks
281 * and should be used for old compatible containers access only.
283 * plumb: unimplemented, see:
284 * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
287 static int crypt_iv_plain_gen(struct crypt_config
*cc
, u8
*iv
,
288 struct dm_crypt_request
*dmreq
)
290 memset(iv
, 0, cc
->iv_size
);
291 *(__le32
*)iv
= cpu_to_le32(dmreq
->iv_sector
& 0xffffffff);
296 static int crypt_iv_plain64_gen(struct crypt_config
*cc
, u8
*iv
,
297 struct dm_crypt_request
*dmreq
)
299 memset(iv
, 0, cc
->iv_size
);
300 *(__le64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
305 /* Initialise ESSIV - compute salt but no local memory allocations */
306 static int crypt_iv_essiv_init(struct crypt_config
*cc
)
308 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
309 AHASH_REQUEST_ON_STACK(req
, essiv
->hash_tfm
);
310 struct scatterlist sg
;
311 struct crypto_cipher
*essiv_tfm
;
314 sg_init_one(&sg
, cc
->key
, cc
->key_size
);
315 ahash_request_set_tfm(req
, essiv
->hash_tfm
);
316 ahash_request_set_callback(req
, CRYPTO_TFM_REQ_MAY_SLEEP
, NULL
, NULL
);
317 ahash_request_set_crypt(req
, &sg
, essiv
->salt
, cc
->key_size
);
319 err
= crypto_ahash_digest(req
);
320 ahash_request_zero(req
);
324 essiv_tfm
= cc
->iv_private
;
326 err
= crypto_cipher_setkey(essiv_tfm
, essiv
->salt
,
327 crypto_ahash_digestsize(essiv
->hash_tfm
));
334 /* Wipe salt and reset key derived from volume key */
335 static int crypt_iv_essiv_wipe(struct crypt_config
*cc
)
337 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
338 unsigned salt_size
= crypto_ahash_digestsize(essiv
->hash_tfm
);
339 struct crypto_cipher
*essiv_tfm
;
342 memset(essiv
->salt
, 0, salt_size
);
344 essiv_tfm
= cc
->iv_private
;
345 r
= crypto_cipher_setkey(essiv_tfm
, essiv
->salt
, salt_size
);
352 /* Allocate the cipher for ESSIV */
353 static struct crypto_cipher
*alloc_essiv_cipher(struct crypt_config
*cc
,
354 struct dm_target
*ti
,
356 unsigned int saltsize
)
358 struct crypto_cipher
*essiv_tfm
;
361 /* Setup the essiv_tfm with the given salt */
362 essiv_tfm
= crypto_alloc_cipher(cc
->cipher
, 0, CRYPTO_ALG_ASYNC
);
363 if (IS_ERR(essiv_tfm
)) {
364 ti
->error
= "Error allocating crypto tfm for ESSIV";
368 if (crypto_cipher_blocksize(essiv_tfm
) != cc
->iv_size
) {
369 ti
->error
= "Block size of ESSIV cipher does "
370 "not match IV size of block cipher";
371 crypto_free_cipher(essiv_tfm
);
372 return ERR_PTR(-EINVAL
);
375 err
= crypto_cipher_setkey(essiv_tfm
, salt
, saltsize
);
377 ti
->error
= "Failed to set key for ESSIV cipher";
378 crypto_free_cipher(essiv_tfm
);
385 static void crypt_iv_essiv_dtr(struct crypt_config
*cc
)
387 struct crypto_cipher
*essiv_tfm
;
388 struct iv_essiv_private
*essiv
= &cc
->iv_gen_private
.essiv
;
390 crypto_free_ahash(essiv
->hash_tfm
);
391 essiv
->hash_tfm
= NULL
;
396 essiv_tfm
= cc
->iv_private
;
399 crypto_free_cipher(essiv_tfm
);
401 cc
->iv_private
= NULL
;
404 static int crypt_iv_essiv_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
407 struct crypto_cipher
*essiv_tfm
= NULL
;
408 struct crypto_ahash
*hash_tfm
= NULL
;
413 ti
->error
= "Digest algorithm missing for ESSIV mode";
417 /* Allocate hash algorithm */
418 hash_tfm
= crypto_alloc_ahash(opts
, 0, CRYPTO_ALG_ASYNC
);
419 if (IS_ERR(hash_tfm
)) {
420 ti
->error
= "Error initializing ESSIV hash";
421 err
= PTR_ERR(hash_tfm
);
425 salt
= kzalloc(crypto_ahash_digestsize(hash_tfm
), GFP_KERNEL
);
427 ti
->error
= "Error kmallocing salt storage in ESSIV";
432 cc
->iv_gen_private
.essiv
.salt
= salt
;
433 cc
->iv_gen_private
.essiv
.hash_tfm
= hash_tfm
;
435 essiv_tfm
= alloc_essiv_cipher(cc
, ti
, salt
,
436 crypto_ahash_digestsize(hash_tfm
));
437 if (IS_ERR(essiv_tfm
)) {
438 crypt_iv_essiv_dtr(cc
);
439 return PTR_ERR(essiv_tfm
);
441 cc
->iv_private
= essiv_tfm
;
446 if (hash_tfm
&& !IS_ERR(hash_tfm
))
447 crypto_free_ahash(hash_tfm
);
452 static int crypt_iv_essiv_gen(struct crypt_config
*cc
, u8
*iv
,
453 struct dm_crypt_request
*dmreq
)
455 struct crypto_cipher
*essiv_tfm
= cc
->iv_private
;
457 memset(iv
, 0, cc
->iv_size
);
458 *(__le64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
459 crypto_cipher_encrypt_one(essiv_tfm
, iv
, iv
);
464 static int crypt_iv_benbi_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
467 unsigned bs
= crypto_skcipher_blocksize(any_tfm(cc
));
470 /* we need to calculate how far we must shift the sector count
471 * to get the cipher block count, we use this shift in _gen */
473 if (1 << log
!= bs
) {
474 ti
->error
= "cypher blocksize is not a power of 2";
479 ti
->error
= "cypher blocksize is > 512";
483 cc
->iv_gen_private
.benbi
.shift
= 9 - log
;
488 static void crypt_iv_benbi_dtr(struct crypt_config
*cc
)
492 static int crypt_iv_benbi_gen(struct crypt_config
*cc
, u8
*iv
,
493 struct dm_crypt_request
*dmreq
)
497 memset(iv
, 0, cc
->iv_size
- sizeof(u64
)); /* rest is cleared below */
499 val
= cpu_to_be64(((u64
)dmreq
->iv_sector
<< cc
->iv_gen_private
.benbi
.shift
) + 1);
500 put_unaligned(val
, (__be64
*)(iv
+ cc
->iv_size
- sizeof(u64
)));
505 static int crypt_iv_null_gen(struct crypt_config
*cc
, u8
*iv
,
506 struct dm_crypt_request
*dmreq
)
508 memset(iv
, 0, cc
->iv_size
);
513 static void crypt_iv_lmk_dtr(struct crypt_config
*cc
)
515 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
517 if (lmk
->hash_tfm
&& !IS_ERR(lmk
->hash_tfm
))
518 crypto_free_shash(lmk
->hash_tfm
);
519 lmk
->hash_tfm
= NULL
;
525 static int crypt_iv_lmk_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
528 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
530 if (cc
->sector_size
!= (1 << SECTOR_SHIFT
)) {
531 ti
->error
= "Unsupported sector size for LMK";
535 lmk
->hash_tfm
= crypto_alloc_shash("md5", 0, 0);
536 if (IS_ERR(lmk
->hash_tfm
)) {
537 ti
->error
= "Error initializing LMK hash";
538 return PTR_ERR(lmk
->hash_tfm
);
541 /* No seed in LMK version 2 */
542 if (cc
->key_parts
== cc
->tfms_count
) {
547 lmk
->seed
= kzalloc(LMK_SEED_SIZE
, GFP_KERNEL
);
549 crypt_iv_lmk_dtr(cc
);
550 ti
->error
= "Error kmallocing seed storage in LMK";
557 static int crypt_iv_lmk_init(struct crypt_config
*cc
)
559 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
560 int subkey_size
= cc
->key_size
/ cc
->key_parts
;
562 /* LMK seed is on the position of LMK_KEYS + 1 key */
564 memcpy(lmk
->seed
, cc
->key
+ (cc
->tfms_count
* subkey_size
),
565 crypto_shash_digestsize(lmk
->hash_tfm
));
570 static int crypt_iv_lmk_wipe(struct crypt_config
*cc
)
572 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
575 memset(lmk
->seed
, 0, LMK_SEED_SIZE
);
580 static int crypt_iv_lmk_one(struct crypt_config
*cc
, u8
*iv
,
581 struct dm_crypt_request
*dmreq
,
584 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
585 SHASH_DESC_ON_STACK(desc
, lmk
->hash_tfm
);
586 struct md5_state md5state
;
590 desc
->tfm
= lmk
->hash_tfm
;
591 desc
->flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
593 r
= crypto_shash_init(desc
);
598 r
= crypto_shash_update(desc
, lmk
->seed
, LMK_SEED_SIZE
);
603 /* Sector is always 512B, block size 16, add data of blocks 1-31 */
604 r
= crypto_shash_update(desc
, data
+ 16, 16 * 31);
608 /* Sector is cropped to 56 bits here */
609 buf
[0] = cpu_to_le32(dmreq
->iv_sector
& 0xFFFFFFFF);
610 buf
[1] = cpu_to_le32((((u64
)dmreq
->iv_sector
>> 32) & 0x00FFFFFF) | 0x80000000);
611 buf
[2] = cpu_to_le32(4024);
613 r
= crypto_shash_update(desc
, (u8
*)buf
, sizeof(buf
));
617 /* No MD5 padding here */
618 r
= crypto_shash_export(desc
, &md5state
);
622 for (i
= 0; i
< MD5_HASH_WORDS
; i
++)
623 __cpu_to_le32s(&md5state
.hash
[i
]);
624 memcpy(iv
, &md5state
.hash
, cc
->iv_size
);
629 static int crypt_iv_lmk_gen(struct crypt_config
*cc
, u8
*iv
,
630 struct dm_crypt_request
*dmreq
)
632 struct scatterlist
*sg
;
636 if (bio_data_dir(dmreq
->ctx
->bio_in
) == WRITE
) {
637 sg
= crypt_get_sg_data(cc
, dmreq
->sg_in
);
638 src
= kmap_atomic(sg_page(sg
));
639 r
= crypt_iv_lmk_one(cc
, iv
, dmreq
, src
+ sg
->offset
);
642 memset(iv
, 0, cc
->iv_size
);
647 static int crypt_iv_lmk_post(struct crypt_config
*cc
, u8
*iv
,
648 struct dm_crypt_request
*dmreq
)
650 struct scatterlist
*sg
;
654 if (bio_data_dir(dmreq
->ctx
->bio_in
) == WRITE
)
657 sg
= crypt_get_sg_data(cc
, dmreq
->sg_out
);
658 dst
= kmap_atomic(sg_page(sg
));
659 r
= crypt_iv_lmk_one(cc
, iv
, dmreq
, dst
+ sg
->offset
);
661 /* Tweak the first block of plaintext sector */
663 crypto_xor(dst
+ sg
->offset
, iv
, cc
->iv_size
);
669 static void crypt_iv_tcw_dtr(struct crypt_config
*cc
)
671 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
673 kzfree(tcw
->iv_seed
);
675 kzfree(tcw
->whitening
);
676 tcw
->whitening
= NULL
;
678 if (tcw
->crc32_tfm
&& !IS_ERR(tcw
->crc32_tfm
))
679 crypto_free_shash(tcw
->crc32_tfm
);
680 tcw
->crc32_tfm
= NULL
;
683 static int crypt_iv_tcw_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
686 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
688 if (cc
->sector_size
!= (1 << SECTOR_SHIFT
)) {
689 ti
->error
= "Unsupported sector size for TCW";
693 if (cc
->key_size
<= (cc
->iv_size
+ TCW_WHITENING_SIZE
)) {
694 ti
->error
= "Wrong key size for TCW";
698 tcw
->crc32_tfm
= crypto_alloc_shash("crc32", 0, 0);
699 if (IS_ERR(tcw
->crc32_tfm
)) {
700 ti
->error
= "Error initializing CRC32 in TCW";
701 return PTR_ERR(tcw
->crc32_tfm
);
704 tcw
->iv_seed
= kzalloc(cc
->iv_size
, GFP_KERNEL
);
705 tcw
->whitening
= kzalloc(TCW_WHITENING_SIZE
, GFP_KERNEL
);
706 if (!tcw
->iv_seed
|| !tcw
->whitening
) {
707 crypt_iv_tcw_dtr(cc
);
708 ti
->error
= "Error allocating seed storage in TCW";
715 static int crypt_iv_tcw_init(struct crypt_config
*cc
)
717 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
718 int key_offset
= cc
->key_size
- cc
->iv_size
- TCW_WHITENING_SIZE
;
720 memcpy(tcw
->iv_seed
, &cc
->key
[key_offset
], cc
->iv_size
);
721 memcpy(tcw
->whitening
, &cc
->key
[key_offset
+ cc
->iv_size
],
727 static int crypt_iv_tcw_wipe(struct crypt_config
*cc
)
729 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
731 memset(tcw
->iv_seed
, 0, cc
->iv_size
);
732 memset(tcw
->whitening
, 0, TCW_WHITENING_SIZE
);
737 static int crypt_iv_tcw_whitening(struct crypt_config
*cc
,
738 struct dm_crypt_request
*dmreq
,
741 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
742 __le64 sector
= cpu_to_le64(dmreq
->iv_sector
);
743 u8 buf
[TCW_WHITENING_SIZE
];
744 SHASH_DESC_ON_STACK(desc
, tcw
->crc32_tfm
);
747 /* xor whitening with sector number */
748 memcpy(buf
, tcw
->whitening
, TCW_WHITENING_SIZE
);
749 crypto_xor(buf
, (u8
*)§or
, 8);
750 crypto_xor(&buf
[8], (u8
*)§or
, 8);
752 /* calculate crc32 for every 32bit part and xor it */
753 desc
->tfm
= tcw
->crc32_tfm
;
754 desc
->flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
755 for (i
= 0; i
< 4; i
++) {
756 r
= crypto_shash_init(desc
);
759 r
= crypto_shash_update(desc
, &buf
[i
* 4], 4);
762 r
= crypto_shash_final(desc
, &buf
[i
* 4]);
766 crypto_xor(&buf
[0], &buf
[12], 4);
767 crypto_xor(&buf
[4], &buf
[8], 4);
769 /* apply whitening (8 bytes) to whole sector */
770 for (i
= 0; i
< ((1 << SECTOR_SHIFT
) / 8); i
++)
771 crypto_xor(data
+ i
* 8, buf
, 8);
773 memzero_explicit(buf
, sizeof(buf
));
777 static int crypt_iv_tcw_gen(struct crypt_config
*cc
, u8
*iv
,
778 struct dm_crypt_request
*dmreq
)
780 struct scatterlist
*sg
;
781 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
782 __le64 sector
= cpu_to_le64(dmreq
->iv_sector
);
786 /* Remove whitening from ciphertext */
787 if (bio_data_dir(dmreq
->ctx
->bio_in
) != WRITE
) {
788 sg
= crypt_get_sg_data(cc
, dmreq
->sg_in
);
789 src
= kmap_atomic(sg_page(sg
));
790 r
= crypt_iv_tcw_whitening(cc
, dmreq
, src
+ sg
->offset
);
795 memcpy(iv
, tcw
->iv_seed
, cc
->iv_size
);
796 crypto_xor(iv
, (u8
*)§or
, 8);
798 crypto_xor(&iv
[8], (u8
*)§or
, cc
->iv_size
- 8);
803 static int crypt_iv_tcw_post(struct crypt_config
*cc
, u8
*iv
,
804 struct dm_crypt_request
*dmreq
)
806 struct scatterlist
*sg
;
810 if (bio_data_dir(dmreq
->ctx
->bio_in
) != WRITE
)
813 /* Apply whitening on ciphertext */
814 sg
= crypt_get_sg_data(cc
, dmreq
->sg_out
);
815 dst
= kmap_atomic(sg_page(sg
));
816 r
= crypt_iv_tcw_whitening(cc
, dmreq
, dst
+ sg
->offset
);
822 static int crypt_iv_random_gen(struct crypt_config
*cc
, u8
*iv
,
823 struct dm_crypt_request
*dmreq
)
825 /* Used only for writes, there must be an additional space to store IV */
826 get_random_bytes(iv
, cc
->iv_size
);
830 static const struct crypt_iv_operations crypt_iv_plain_ops
= {
831 .generator
= crypt_iv_plain_gen
834 static const struct crypt_iv_operations crypt_iv_plain64_ops
= {
835 .generator
= crypt_iv_plain64_gen
838 static const struct crypt_iv_operations crypt_iv_essiv_ops
= {
839 .ctr
= crypt_iv_essiv_ctr
,
840 .dtr
= crypt_iv_essiv_dtr
,
841 .init
= crypt_iv_essiv_init
,
842 .wipe
= crypt_iv_essiv_wipe
,
843 .generator
= crypt_iv_essiv_gen
846 static const struct crypt_iv_operations crypt_iv_benbi_ops
= {
847 .ctr
= crypt_iv_benbi_ctr
,
848 .dtr
= crypt_iv_benbi_dtr
,
849 .generator
= crypt_iv_benbi_gen
852 static const struct crypt_iv_operations crypt_iv_null_ops
= {
853 .generator
= crypt_iv_null_gen
856 static const struct crypt_iv_operations crypt_iv_lmk_ops
= {
857 .ctr
= crypt_iv_lmk_ctr
,
858 .dtr
= crypt_iv_lmk_dtr
,
859 .init
= crypt_iv_lmk_init
,
860 .wipe
= crypt_iv_lmk_wipe
,
861 .generator
= crypt_iv_lmk_gen
,
862 .post
= crypt_iv_lmk_post
865 static const struct crypt_iv_operations crypt_iv_tcw_ops
= {
866 .ctr
= crypt_iv_tcw_ctr
,
867 .dtr
= crypt_iv_tcw_dtr
,
868 .init
= crypt_iv_tcw_init
,
869 .wipe
= crypt_iv_tcw_wipe
,
870 .generator
= crypt_iv_tcw_gen
,
871 .post
= crypt_iv_tcw_post
874 static struct crypt_iv_operations crypt_iv_random_ops
= {
875 .generator
= crypt_iv_random_gen
879 * Integrity extensions
881 static bool crypt_integrity_aead(struct crypt_config
*cc
)
883 return test_bit(CRYPT_MODE_INTEGRITY_AEAD
, &cc
->cipher_flags
);
886 static bool crypt_integrity_hmac(struct crypt_config
*cc
)
888 return crypt_integrity_aead(cc
) && cc
->key_mac_size
;
891 /* Get sg containing data */
892 static struct scatterlist
*crypt_get_sg_data(struct crypt_config
*cc
,
893 struct scatterlist
*sg
)
895 if (unlikely(crypt_integrity_aead(cc
)))
901 static int dm_crypt_integrity_io_alloc(struct dm_crypt_io
*io
, struct bio
*bio
)
903 struct bio_integrity_payload
*bip
;
904 unsigned int tag_len
;
907 if (!bio_sectors(bio
) || !io
->cc
->on_disk_tag_size
)
910 bip
= bio_integrity_alloc(bio
, GFP_NOIO
, 1);
914 tag_len
= io
->cc
->on_disk_tag_size
* bio_sectors(bio
);
916 bip
->bip_iter
.bi_size
= tag_len
;
917 bip
->bip_iter
.bi_sector
= io
->cc
->start
+ io
->sector
;
919 /* We own the metadata, do not let bio_free to release it */
920 bip
->bip_flags
&= ~BIP_BLOCK_INTEGRITY
;
922 ret
= bio_integrity_add_page(bio
, virt_to_page(io
->integrity_metadata
),
923 tag_len
, offset_in_page(io
->integrity_metadata
));
924 if (unlikely(ret
!= tag_len
))
930 static int crypt_integrity_ctr(struct crypt_config
*cc
, struct dm_target
*ti
)
932 #ifdef CONFIG_BLK_DEV_INTEGRITY
933 struct blk_integrity
*bi
= blk_get_integrity(cc
->dev
->bdev
->bd_disk
);
935 /* From now we require underlying device with our integrity profile */
936 if (!bi
|| strcasecmp(bi
->profile
->name
, "DM-DIF-EXT-TAG")) {
937 ti
->error
= "Integrity profile not supported.";
941 if (bi
->tag_size
!= cc
->on_disk_tag_size
||
942 bi
->tuple_size
!= cc
->on_disk_tag_size
) {
943 ti
->error
= "Integrity profile tag size mismatch.";
946 if (1 << bi
->interval_exp
!= cc
->sector_size
) {
947 ti
->error
= "Integrity profile sector size mismatch.";
951 if (crypt_integrity_aead(cc
)) {
952 cc
->integrity_tag_size
= cc
->on_disk_tag_size
- cc
->integrity_iv_size
;
953 DMINFO("Integrity AEAD, tag size %u, IV size %u.",
954 cc
->integrity_tag_size
, cc
->integrity_iv_size
);
956 if (crypto_aead_setauthsize(any_tfm_aead(cc
), cc
->integrity_tag_size
)) {
957 ti
->error
= "Integrity AEAD auth tag size is not supported.";
960 } else if (cc
->integrity_iv_size
)
961 DMINFO("Additional per-sector space %u bytes for IV.",
962 cc
->integrity_iv_size
);
964 if ((cc
->integrity_tag_size
+ cc
->integrity_iv_size
) != bi
->tag_size
) {
965 ti
->error
= "Not enough space for integrity tag in the profile.";
971 ti
->error
= "Integrity profile not supported.";
976 static void crypt_convert_init(struct crypt_config
*cc
,
977 struct convert_context
*ctx
,
978 struct bio
*bio_out
, struct bio
*bio_in
,
981 ctx
->bio_in
= bio_in
;
982 ctx
->bio_out
= bio_out
;
984 ctx
->iter_in
= bio_in
->bi_iter
;
986 ctx
->iter_out
= bio_out
->bi_iter
;
987 ctx
->cc_sector
= sector
+ cc
->iv_offset
;
988 init_completion(&ctx
->restart
);
991 static struct dm_crypt_request
*dmreq_of_req(struct crypt_config
*cc
,
994 return (struct dm_crypt_request
*)((char *)req
+ cc
->dmreq_start
);
997 static void *req_of_dmreq(struct crypt_config
*cc
, struct dm_crypt_request
*dmreq
)
999 return (void *)((char *)dmreq
- cc
->dmreq_start
);
1002 static u8
*iv_of_dmreq(struct crypt_config
*cc
,
1003 struct dm_crypt_request
*dmreq
)
1005 if (crypt_integrity_aead(cc
))
1006 return (u8
*)ALIGN((unsigned long)(dmreq
+ 1),
1007 crypto_aead_alignmask(any_tfm_aead(cc
)) + 1);
1009 return (u8
*)ALIGN((unsigned long)(dmreq
+ 1),
1010 crypto_skcipher_alignmask(any_tfm(cc
)) + 1);
1013 static u8
*org_iv_of_dmreq(struct crypt_config
*cc
,
1014 struct dm_crypt_request
*dmreq
)
1016 return iv_of_dmreq(cc
, dmreq
) + cc
->iv_size
;
1019 static uint64_t *org_sector_of_dmreq(struct crypt_config
*cc
,
1020 struct dm_crypt_request
*dmreq
)
1022 u8
*ptr
= iv_of_dmreq(cc
, dmreq
) + cc
->iv_size
+ cc
->iv_size
;
1023 return (uint64_t*) ptr
;
1026 static unsigned int *org_tag_of_dmreq(struct crypt_config
*cc
,
1027 struct dm_crypt_request
*dmreq
)
1029 u8
*ptr
= iv_of_dmreq(cc
, dmreq
) + cc
->iv_size
+
1030 cc
->iv_size
+ sizeof(uint64_t);
1031 return (unsigned int*)ptr
;
1034 static void *tag_from_dmreq(struct crypt_config
*cc
,
1035 struct dm_crypt_request
*dmreq
)
1037 struct convert_context
*ctx
= dmreq
->ctx
;
1038 struct dm_crypt_io
*io
= container_of(ctx
, struct dm_crypt_io
, ctx
);
1040 return &io
->integrity_metadata
[*org_tag_of_dmreq(cc
, dmreq
) *
1041 cc
->on_disk_tag_size
];
1044 static void *iv_tag_from_dmreq(struct crypt_config
*cc
,
1045 struct dm_crypt_request
*dmreq
)
1047 return tag_from_dmreq(cc
, dmreq
) + cc
->integrity_tag_size
;
1050 static int crypt_convert_block_aead(struct crypt_config
*cc
,
1051 struct convert_context
*ctx
,
1052 struct aead_request
*req
,
1053 unsigned int tag_offset
)
1055 struct bio_vec bv_in
= bio_iter_iovec(ctx
->bio_in
, ctx
->iter_in
);
1056 struct bio_vec bv_out
= bio_iter_iovec(ctx
->bio_out
, ctx
->iter_out
);
1057 struct dm_crypt_request
*dmreq
;
1058 u8
*iv
, *org_iv
, *tag_iv
, *tag
;
1062 BUG_ON(cc
->integrity_iv_size
&& cc
->integrity_iv_size
!= cc
->iv_size
);
1064 /* Reject unexpected unaligned bio. */
1065 if (unlikely(bv_in
.bv_offset
& (cc
->sector_size
- 1)))
1068 dmreq
= dmreq_of_req(cc
, req
);
1069 dmreq
->iv_sector
= ctx
->cc_sector
;
1070 if (test_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
))
1071 dmreq
->iv_sector
>>= cc
->sector_shift
;
1074 *org_tag_of_dmreq(cc
, dmreq
) = tag_offset
;
1076 sector
= org_sector_of_dmreq(cc
, dmreq
);
1077 *sector
= cpu_to_le64(ctx
->cc_sector
- cc
->iv_offset
);
1079 iv
= iv_of_dmreq(cc
, dmreq
);
1080 org_iv
= org_iv_of_dmreq(cc
, dmreq
);
1081 tag
= tag_from_dmreq(cc
, dmreq
);
1082 tag_iv
= iv_tag_from_dmreq(cc
, dmreq
);
1085 * |----- AAD -------|------ DATA -------|-- AUTH TAG --|
1086 * | (authenticated) | (auth+encryption) | |
1087 * | sector_LE | IV | sector in/out | tag in/out |
1089 sg_init_table(dmreq
->sg_in
, 4);
1090 sg_set_buf(&dmreq
->sg_in
[0], sector
, sizeof(uint64_t));
1091 sg_set_buf(&dmreq
->sg_in
[1], org_iv
, cc
->iv_size
);
1092 sg_set_page(&dmreq
->sg_in
[2], bv_in
.bv_page
, cc
->sector_size
, bv_in
.bv_offset
);
1093 sg_set_buf(&dmreq
->sg_in
[3], tag
, cc
->integrity_tag_size
);
1095 sg_init_table(dmreq
->sg_out
, 4);
1096 sg_set_buf(&dmreq
->sg_out
[0], sector
, sizeof(uint64_t));
1097 sg_set_buf(&dmreq
->sg_out
[1], org_iv
, cc
->iv_size
);
1098 sg_set_page(&dmreq
->sg_out
[2], bv_out
.bv_page
, cc
->sector_size
, bv_out
.bv_offset
);
1099 sg_set_buf(&dmreq
->sg_out
[3], tag
, cc
->integrity_tag_size
);
1101 if (cc
->iv_gen_ops
) {
1102 /* For READs use IV stored in integrity metadata */
1103 if (cc
->integrity_iv_size
&& bio_data_dir(ctx
->bio_in
) != WRITE
) {
1104 memcpy(org_iv
, tag_iv
, cc
->iv_size
);
1106 r
= cc
->iv_gen_ops
->generator(cc
, org_iv
, dmreq
);
1109 /* Store generated IV in integrity metadata */
1110 if (cc
->integrity_iv_size
)
1111 memcpy(tag_iv
, org_iv
, cc
->iv_size
);
1113 /* Working copy of IV, to be modified in crypto API */
1114 memcpy(iv
, org_iv
, cc
->iv_size
);
1117 aead_request_set_ad(req
, sizeof(uint64_t) + cc
->iv_size
);
1118 if (bio_data_dir(ctx
->bio_in
) == WRITE
) {
1119 aead_request_set_crypt(req
, dmreq
->sg_in
, dmreq
->sg_out
,
1120 cc
->sector_size
, iv
);
1121 r
= crypto_aead_encrypt(req
);
1122 if (cc
->integrity_tag_size
+ cc
->integrity_iv_size
!= cc
->on_disk_tag_size
)
1123 memset(tag
+ cc
->integrity_tag_size
+ cc
->integrity_iv_size
, 0,
1124 cc
->on_disk_tag_size
- (cc
->integrity_tag_size
+ cc
->integrity_iv_size
));
1126 aead_request_set_crypt(req
, dmreq
->sg_in
, dmreq
->sg_out
,
1127 cc
->sector_size
+ cc
->integrity_tag_size
, iv
);
1128 r
= crypto_aead_decrypt(req
);
1132 DMERR_LIMIT("INTEGRITY AEAD ERROR, sector %llu",
1133 (unsigned long long)le64_to_cpu(*sector
));
1135 if (!r
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1136 r
= cc
->iv_gen_ops
->post(cc
, org_iv
, dmreq
);
1138 bio_advance_iter(ctx
->bio_in
, &ctx
->iter_in
, cc
->sector_size
);
1139 bio_advance_iter(ctx
->bio_out
, &ctx
->iter_out
, cc
->sector_size
);
1144 static int crypt_convert_block_skcipher(struct crypt_config
*cc
,
1145 struct convert_context
*ctx
,
1146 struct skcipher_request
*req
,
1147 unsigned int tag_offset
)
1149 struct bio_vec bv_in
= bio_iter_iovec(ctx
->bio_in
, ctx
->iter_in
);
1150 struct bio_vec bv_out
= bio_iter_iovec(ctx
->bio_out
, ctx
->iter_out
);
1151 struct scatterlist
*sg_in
, *sg_out
;
1152 struct dm_crypt_request
*dmreq
;
1153 u8
*iv
, *org_iv
, *tag_iv
;
1157 /* Reject unexpected unaligned bio. */
1158 if (unlikely(bv_in
.bv_offset
& (cc
->sector_size
- 1)))
1161 dmreq
= dmreq_of_req(cc
, req
);
1162 dmreq
->iv_sector
= ctx
->cc_sector
;
1163 if (test_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
))
1164 dmreq
->iv_sector
>>= cc
->sector_shift
;
1167 *org_tag_of_dmreq(cc
, dmreq
) = tag_offset
;
1169 iv
= iv_of_dmreq(cc
, dmreq
);
1170 org_iv
= org_iv_of_dmreq(cc
, dmreq
);
1171 tag_iv
= iv_tag_from_dmreq(cc
, dmreq
);
1173 sector
= org_sector_of_dmreq(cc
, dmreq
);
1174 *sector
= cpu_to_le64(ctx
->cc_sector
- cc
->iv_offset
);
1176 /* For skcipher we use only the first sg item */
1177 sg_in
= &dmreq
->sg_in
[0];
1178 sg_out
= &dmreq
->sg_out
[0];
1180 sg_init_table(sg_in
, 1);
1181 sg_set_page(sg_in
, bv_in
.bv_page
, cc
->sector_size
, bv_in
.bv_offset
);
1183 sg_init_table(sg_out
, 1);
1184 sg_set_page(sg_out
, bv_out
.bv_page
, cc
->sector_size
, bv_out
.bv_offset
);
1186 if (cc
->iv_gen_ops
) {
1187 /* For READs use IV stored in integrity metadata */
1188 if (cc
->integrity_iv_size
&& bio_data_dir(ctx
->bio_in
) != WRITE
) {
1189 memcpy(org_iv
, tag_iv
, cc
->integrity_iv_size
);
1191 r
= cc
->iv_gen_ops
->generator(cc
, org_iv
, dmreq
);
1194 /* Store generated IV in integrity metadata */
1195 if (cc
->integrity_iv_size
)
1196 memcpy(tag_iv
, org_iv
, cc
->integrity_iv_size
);
1198 /* Working copy of IV, to be modified in crypto API */
1199 memcpy(iv
, org_iv
, cc
->iv_size
);
1202 skcipher_request_set_crypt(req
, sg_in
, sg_out
, cc
->sector_size
, iv
);
1204 if (bio_data_dir(ctx
->bio_in
) == WRITE
)
1205 r
= crypto_skcipher_encrypt(req
);
1207 r
= crypto_skcipher_decrypt(req
);
1209 if (!r
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1210 r
= cc
->iv_gen_ops
->post(cc
, org_iv
, dmreq
);
1212 bio_advance_iter(ctx
->bio_in
, &ctx
->iter_in
, cc
->sector_size
);
1213 bio_advance_iter(ctx
->bio_out
, &ctx
->iter_out
, cc
->sector_size
);
1218 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
1221 static void crypt_alloc_req_skcipher(struct crypt_config
*cc
,
1222 struct convert_context
*ctx
)
1224 unsigned key_index
= ctx
->cc_sector
& (cc
->tfms_count
- 1);
1227 ctx
->r
.req
= mempool_alloc(cc
->req_pool
, GFP_NOIO
);
1229 skcipher_request_set_tfm(ctx
->r
.req
, cc
->cipher_tfm
.tfms
[key_index
]);
1232 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1233 * requests if driver request queue is full.
1235 skcipher_request_set_callback(ctx
->r
.req
,
1236 CRYPTO_TFM_REQ_MAY_BACKLOG
| CRYPTO_TFM_REQ_MAY_SLEEP
,
1237 kcryptd_async_done
, dmreq_of_req(cc
, ctx
->r
.req
));
1240 static void crypt_alloc_req_aead(struct crypt_config
*cc
,
1241 struct convert_context
*ctx
)
1243 if (!ctx
->r
.req_aead
)
1244 ctx
->r
.req_aead
= mempool_alloc(cc
->req_pool
, GFP_NOIO
);
1246 aead_request_set_tfm(ctx
->r
.req_aead
, cc
->cipher_tfm
.tfms_aead
[0]);
1249 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1250 * requests if driver request queue is full.
1252 aead_request_set_callback(ctx
->r
.req_aead
,
1253 CRYPTO_TFM_REQ_MAY_BACKLOG
| CRYPTO_TFM_REQ_MAY_SLEEP
,
1254 kcryptd_async_done
, dmreq_of_req(cc
, ctx
->r
.req_aead
));
1257 static void crypt_alloc_req(struct crypt_config
*cc
,
1258 struct convert_context
*ctx
)
1260 if (crypt_integrity_aead(cc
))
1261 crypt_alloc_req_aead(cc
, ctx
);
1263 crypt_alloc_req_skcipher(cc
, ctx
);
1266 static void crypt_free_req_skcipher(struct crypt_config
*cc
,
1267 struct skcipher_request
*req
, struct bio
*base_bio
)
1269 struct dm_crypt_io
*io
= dm_per_bio_data(base_bio
, cc
->per_bio_data_size
);
1271 if ((struct skcipher_request
*)(io
+ 1) != req
)
1272 mempool_free(req
, cc
->req_pool
);
1275 static void crypt_free_req_aead(struct crypt_config
*cc
,
1276 struct aead_request
*req
, struct bio
*base_bio
)
1278 struct dm_crypt_io
*io
= dm_per_bio_data(base_bio
, cc
->per_bio_data_size
);
1280 if ((struct aead_request
*)(io
+ 1) != req
)
1281 mempool_free(req
, cc
->req_pool
);
1284 static void crypt_free_req(struct crypt_config
*cc
, void *req
, struct bio
*base_bio
)
1286 if (crypt_integrity_aead(cc
))
1287 crypt_free_req_aead(cc
, req
, base_bio
);
1289 crypt_free_req_skcipher(cc
, req
, base_bio
);
1293 * Encrypt / decrypt data from one bio to another one (can be the same one)
1295 static int crypt_convert(struct crypt_config
*cc
,
1296 struct convert_context
*ctx
)
1298 unsigned int tag_offset
= 0;
1299 unsigned int sector_step
= cc
->sector_size
>> SECTOR_SHIFT
;
1302 atomic_set(&ctx
->cc_pending
, 1);
1304 while (ctx
->iter_in
.bi_size
&& ctx
->iter_out
.bi_size
) {
1306 crypt_alloc_req(cc
, ctx
);
1307 atomic_inc(&ctx
->cc_pending
);
1309 if (crypt_integrity_aead(cc
))
1310 r
= crypt_convert_block_aead(cc
, ctx
, ctx
->r
.req_aead
, tag_offset
);
1312 r
= crypt_convert_block_skcipher(cc
, ctx
, ctx
->r
.req
, tag_offset
);
1316 * The request was queued by a crypto driver
1317 * but the driver request queue is full, let's wait.
1320 wait_for_completion(&ctx
->restart
);
1321 reinit_completion(&ctx
->restart
);
1324 * The request is queued and processed asynchronously,
1325 * completion function kcryptd_async_done() will be called.
1329 ctx
->cc_sector
+= sector_step
;
1333 * The request was already processed (synchronously).
1336 atomic_dec(&ctx
->cc_pending
);
1337 ctx
->cc_sector
+= sector_step
;
1342 * There was a data integrity error.
1345 atomic_dec(&ctx
->cc_pending
);
1348 * There was an error while processing the request.
1351 atomic_dec(&ctx
->cc_pending
);
1359 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
);
1362 * Generate a new unfragmented bio with the given size
1363 * This should never violate the device limitations (but only because
1364 * max_segment_size is being constrained to PAGE_SIZE).
1366 * This function may be called concurrently. If we allocate from the mempool
1367 * concurrently, there is a possibility of deadlock. For example, if we have
1368 * mempool of 256 pages, two processes, each wanting 256, pages allocate from
1369 * the mempool concurrently, it may deadlock in a situation where both processes
1370 * have allocated 128 pages and the mempool is exhausted.
1372 * In order to avoid this scenario we allocate the pages under a mutex.
1374 * In order to not degrade performance with excessive locking, we try
1375 * non-blocking allocations without a mutex first but on failure we fallback
1376 * to blocking allocations with a mutex.
1378 static struct bio
*crypt_alloc_buffer(struct dm_crypt_io
*io
, unsigned size
)
1380 struct crypt_config
*cc
= io
->cc
;
1382 unsigned int nr_iovecs
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
1383 gfp_t gfp_mask
= GFP_NOWAIT
| __GFP_HIGHMEM
;
1384 unsigned i
, len
, remaining_size
;
1388 if (unlikely(gfp_mask
& __GFP_DIRECT_RECLAIM
))
1389 mutex_lock(&cc
->bio_alloc_lock
);
1391 clone
= bio_alloc_bioset(GFP_NOIO
, nr_iovecs
, cc
->bs
);
1395 clone_init(io
, clone
);
1397 remaining_size
= size
;
1399 for (i
= 0; i
< nr_iovecs
; i
++) {
1400 page
= mempool_alloc(cc
->page_pool
, gfp_mask
);
1402 crypt_free_buffer_pages(cc
, clone
);
1404 gfp_mask
|= __GFP_DIRECT_RECLAIM
;
1408 len
= (remaining_size
> PAGE_SIZE
) ? PAGE_SIZE
: remaining_size
;
1410 bio_add_page(clone
, page
, len
, 0);
1412 remaining_size
-= len
;
1415 /* Allocate space for integrity tags */
1416 if (dm_crypt_integrity_io_alloc(io
, clone
)) {
1417 crypt_free_buffer_pages(cc
, clone
);
1422 if (unlikely(gfp_mask
& __GFP_DIRECT_RECLAIM
))
1423 mutex_unlock(&cc
->bio_alloc_lock
);
1428 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
)
1433 bio_for_each_segment_all(bv
, clone
, i
) {
1434 BUG_ON(!bv
->bv_page
);
1435 mempool_free(bv
->bv_page
, cc
->page_pool
);
1440 static void crypt_io_init(struct dm_crypt_io
*io
, struct crypt_config
*cc
,
1441 struct bio
*bio
, sector_t sector
)
1445 io
->sector
= sector
;
1447 io
->ctx
.r
.req
= NULL
;
1448 io
->integrity_metadata
= NULL
;
1449 io
->integrity_metadata_from_pool
= false;
1450 atomic_set(&io
->io_pending
, 0);
1453 static void crypt_inc_pending(struct dm_crypt_io
*io
)
1455 atomic_inc(&io
->io_pending
);
1459 * One of the bios was finished. Check for completion of
1460 * the whole request and correctly clean up the buffer.
1462 static void crypt_dec_pending(struct dm_crypt_io
*io
)
1464 struct crypt_config
*cc
= io
->cc
;
1465 struct bio
*base_bio
= io
->base_bio
;
1466 int error
= io
->error
;
1468 if (!atomic_dec_and_test(&io
->io_pending
))
1472 crypt_free_req(cc
, io
->ctx
.r
.req
, base_bio
);
1474 if (unlikely(io
->integrity_metadata_from_pool
))
1475 mempool_free(io
->integrity_metadata
, io
->cc
->tag_pool
);
1477 kfree(io
->integrity_metadata
);
1479 base_bio
->bi_error
= error
;
1480 bio_endio(base_bio
);
1484 * kcryptd/kcryptd_io:
1486 * Needed because it would be very unwise to do decryption in an
1487 * interrupt context.
1489 * kcryptd performs the actual encryption or decryption.
1491 * kcryptd_io performs the IO submission.
1493 * They must be separated as otherwise the final stages could be
1494 * starved by new requests which can block in the first stages due
1495 * to memory allocation.
1497 * The work is done per CPU global for all dm-crypt instances.
1498 * They should not depend on each other and do not block.
1500 static void crypt_endio(struct bio
*clone
)
1502 struct dm_crypt_io
*io
= clone
->bi_private
;
1503 struct crypt_config
*cc
= io
->cc
;
1504 unsigned rw
= bio_data_dir(clone
);
1508 * free the processed pages
1511 crypt_free_buffer_pages(cc
, clone
);
1513 error
= clone
->bi_error
;
1516 if (rw
== READ
&& !error
) {
1517 kcryptd_queue_crypt(io
);
1521 if (unlikely(error
))
1524 crypt_dec_pending(io
);
1527 static void clone_init(struct dm_crypt_io
*io
, struct bio
*clone
)
1529 struct crypt_config
*cc
= io
->cc
;
1531 clone
->bi_private
= io
;
1532 clone
->bi_end_io
= crypt_endio
;
1533 clone
->bi_bdev
= cc
->dev
->bdev
;
1534 clone
->bi_opf
= io
->base_bio
->bi_opf
;
1537 static int kcryptd_io_read(struct dm_crypt_io
*io
, gfp_t gfp
)
1539 struct crypt_config
*cc
= io
->cc
;
1543 * We need the original biovec array in order to decrypt
1544 * the whole bio data *afterwards* -- thanks to immutable
1545 * biovecs we don't need to worry about the block layer
1546 * modifying the biovec array; so leverage bio_clone_fast().
1548 clone
= bio_clone_fast(io
->base_bio
, gfp
, cc
->bs
);
1552 crypt_inc_pending(io
);
1554 clone_init(io
, clone
);
1555 clone
->bi_iter
.bi_sector
= cc
->start
+ io
->sector
;
1557 if (dm_crypt_integrity_io_alloc(io
, clone
)) {
1558 crypt_dec_pending(io
);
1563 generic_make_request(clone
);
1567 static void kcryptd_io_read_work(struct work_struct
*work
)
1569 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1571 crypt_inc_pending(io
);
1572 if (kcryptd_io_read(io
, GFP_NOIO
))
1573 io
->error
= -ENOMEM
;
1574 crypt_dec_pending(io
);
1577 static void kcryptd_queue_read(struct dm_crypt_io
*io
)
1579 struct crypt_config
*cc
= io
->cc
;
1581 INIT_WORK(&io
->work
, kcryptd_io_read_work
);
1582 queue_work(cc
->io_queue
, &io
->work
);
1585 static void kcryptd_io_write(struct dm_crypt_io
*io
)
1587 struct bio
*clone
= io
->ctx
.bio_out
;
1589 generic_make_request(clone
);
1592 #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
1594 static int dmcrypt_write(void *data
)
1596 struct crypt_config
*cc
= data
;
1597 struct dm_crypt_io
*io
;
1600 struct rb_root write_tree
;
1601 struct blk_plug plug
;
1603 DECLARE_WAITQUEUE(wait
, current
);
1605 spin_lock_irq(&cc
->write_thread_wait
.lock
);
1608 if (!RB_EMPTY_ROOT(&cc
->write_tree
))
1611 set_current_state(TASK_INTERRUPTIBLE
);
1612 __add_wait_queue(&cc
->write_thread_wait
, &wait
);
1614 spin_unlock_irq(&cc
->write_thread_wait
.lock
);
1616 if (unlikely(kthread_should_stop())) {
1617 set_current_state(TASK_RUNNING
);
1618 remove_wait_queue(&cc
->write_thread_wait
, &wait
);
1624 set_current_state(TASK_RUNNING
);
1625 spin_lock_irq(&cc
->write_thread_wait
.lock
);
1626 __remove_wait_queue(&cc
->write_thread_wait
, &wait
);
1627 goto continue_locked
;
1630 write_tree
= cc
->write_tree
;
1631 cc
->write_tree
= RB_ROOT
;
1632 spin_unlock_irq(&cc
->write_thread_wait
.lock
);
1634 BUG_ON(rb_parent(write_tree
.rb_node
));
1637 * Note: we cannot walk the tree here with rb_next because
1638 * the structures may be freed when kcryptd_io_write is called.
1640 blk_start_plug(&plug
);
1642 io
= crypt_io_from_node(rb_first(&write_tree
));
1643 rb_erase(&io
->rb_node
, &write_tree
);
1644 kcryptd_io_write(io
);
1645 } while (!RB_EMPTY_ROOT(&write_tree
));
1646 blk_finish_plug(&plug
);
1651 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io
*io
, int async
)
1653 struct bio
*clone
= io
->ctx
.bio_out
;
1654 struct crypt_config
*cc
= io
->cc
;
1655 unsigned long flags
;
1657 struct rb_node
**rbp
, *parent
;
1659 if (unlikely(io
->error
< 0)) {
1660 crypt_free_buffer_pages(cc
, clone
);
1662 crypt_dec_pending(io
);
1666 /* crypt_convert should have filled the clone bio */
1667 BUG_ON(io
->ctx
.iter_out
.bi_size
);
1669 clone
->bi_iter
.bi_sector
= cc
->start
+ io
->sector
;
1671 if (likely(!async
) && test_bit(DM_CRYPT_NO_OFFLOAD
, &cc
->flags
)) {
1672 generic_make_request(clone
);
1676 spin_lock_irqsave(&cc
->write_thread_wait
.lock
, flags
);
1677 rbp
= &cc
->write_tree
.rb_node
;
1679 sector
= io
->sector
;
1682 if (sector
< crypt_io_from_node(parent
)->sector
)
1683 rbp
= &(*rbp
)->rb_left
;
1685 rbp
= &(*rbp
)->rb_right
;
1687 rb_link_node(&io
->rb_node
, parent
, rbp
);
1688 rb_insert_color(&io
->rb_node
, &cc
->write_tree
);
1690 wake_up_locked(&cc
->write_thread_wait
);
1691 spin_unlock_irqrestore(&cc
->write_thread_wait
.lock
, flags
);
1694 static void kcryptd_crypt_write_convert(struct dm_crypt_io
*io
)
1696 struct crypt_config
*cc
= io
->cc
;
1699 sector_t sector
= io
->sector
;
1703 * Prevent io from disappearing until this function completes.
1705 crypt_inc_pending(io
);
1706 crypt_convert_init(cc
, &io
->ctx
, NULL
, io
->base_bio
, sector
);
1708 clone
= crypt_alloc_buffer(io
, io
->base_bio
->bi_iter
.bi_size
);
1709 if (unlikely(!clone
)) {
1714 io
->ctx
.bio_out
= clone
;
1715 io
->ctx
.iter_out
= clone
->bi_iter
;
1717 sector
+= bio_sectors(clone
);
1719 crypt_inc_pending(io
);
1720 r
= crypt_convert(cc
, &io
->ctx
);
1723 crypt_finished
= atomic_dec_and_test(&io
->ctx
.cc_pending
);
1725 /* Encryption was already finished, submit io now */
1726 if (crypt_finished
) {
1727 kcryptd_crypt_write_io_submit(io
, 0);
1728 io
->sector
= sector
;
1732 crypt_dec_pending(io
);
1735 static void kcryptd_crypt_read_done(struct dm_crypt_io
*io
)
1737 crypt_dec_pending(io
);
1740 static void kcryptd_crypt_read_convert(struct dm_crypt_io
*io
)
1742 struct crypt_config
*cc
= io
->cc
;
1745 crypt_inc_pending(io
);
1747 crypt_convert_init(cc
, &io
->ctx
, io
->base_bio
, io
->base_bio
,
1750 r
= crypt_convert(cc
, &io
->ctx
);
1754 if (atomic_dec_and_test(&io
->ctx
.cc_pending
))
1755 kcryptd_crypt_read_done(io
);
1757 crypt_dec_pending(io
);
1760 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
1763 struct dm_crypt_request
*dmreq
= async_req
->data
;
1764 struct convert_context
*ctx
= dmreq
->ctx
;
1765 struct dm_crypt_io
*io
= container_of(ctx
, struct dm_crypt_io
, ctx
);
1766 struct crypt_config
*cc
= io
->cc
;
1769 * A request from crypto driver backlog is going to be processed now,
1770 * finish the completion and continue in crypt_convert().
1771 * (Callback will be called for the second time for this request.)
1773 if (error
== -EINPROGRESS
) {
1774 complete(&ctx
->restart
);
1778 if (!error
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1779 error
= cc
->iv_gen_ops
->post(cc
, org_iv_of_dmreq(cc
, dmreq
), dmreq
);
1781 if (error
== -EBADMSG
) {
1782 DMERR_LIMIT("INTEGRITY AEAD ERROR, sector %llu",
1783 (unsigned long long)le64_to_cpu(*org_sector_of_dmreq(cc
, dmreq
)));
1784 io
->error
= -EILSEQ
;
1785 } else if (error
< 0)
1788 crypt_free_req(cc
, req_of_dmreq(cc
, dmreq
), io
->base_bio
);
1790 if (!atomic_dec_and_test(&ctx
->cc_pending
))
1793 if (bio_data_dir(io
->base_bio
) == READ
)
1794 kcryptd_crypt_read_done(io
);
1796 kcryptd_crypt_write_io_submit(io
, 1);
1799 static void kcryptd_crypt(struct work_struct
*work
)
1801 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1803 if (bio_data_dir(io
->base_bio
) == READ
)
1804 kcryptd_crypt_read_convert(io
);
1806 kcryptd_crypt_write_convert(io
);
1809 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
)
1811 struct crypt_config
*cc
= io
->cc
;
1813 INIT_WORK(&io
->work
, kcryptd_crypt
);
1814 queue_work(cc
->crypt_queue
, &io
->work
);
1817 static void crypt_free_tfms_aead(struct crypt_config
*cc
)
1819 if (!cc
->cipher_tfm
.tfms_aead
)
1822 if (cc
->cipher_tfm
.tfms_aead
[0] && !IS_ERR(cc
->cipher_tfm
.tfms_aead
[0])) {
1823 crypto_free_aead(cc
->cipher_tfm
.tfms_aead
[0]);
1824 cc
->cipher_tfm
.tfms_aead
[0] = NULL
;
1827 kfree(cc
->cipher_tfm
.tfms_aead
);
1828 cc
->cipher_tfm
.tfms_aead
= NULL
;
1831 static void crypt_free_tfms_skcipher(struct crypt_config
*cc
)
1835 if (!cc
->cipher_tfm
.tfms
)
1838 for (i
= 0; i
< cc
->tfms_count
; i
++)
1839 if (cc
->cipher_tfm
.tfms
[i
] && !IS_ERR(cc
->cipher_tfm
.tfms
[i
])) {
1840 crypto_free_skcipher(cc
->cipher_tfm
.tfms
[i
]);
1841 cc
->cipher_tfm
.tfms
[i
] = NULL
;
1844 kfree(cc
->cipher_tfm
.tfms
);
1845 cc
->cipher_tfm
.tfms
= NULL
;
1848 static void crypt_free_tfms(struct crypt_config
*cc
)
1850 if (crypt_integrity_aead(cc
))
1851 crypt_free_tfms_aead(cc
);
1853 crypt_free_tfms_skcipher(cc
);
1856 static int crypt_alloc_tfms_skcipher(struct crypt_config
*cc
, char *ciphermode
)
1861 cc
->cipher_tfm
.tfms
= kzalloc(cc
->tfms_count
*
1862 sizeof(struct crypto_skcipher
*), GFP_KERNEL
);
1863 if (!cc
->cipher_tfm
.tfms
)
1866 for (i
= 0; i
< cc
->tfms_count
; i
++) {
1867 cc
->cipher_tfm
.tfms
[i
] = crypto_alloc_skcipher(ciphermode
, 0, 0);
1868 if (IS_ERR(cc
->cipher_tfm
.tfms
[i
])) {
1869 err
= PTR_ERR(cc
->cipher_tfm
.tfms
[i
]);
1870 crypt_free_tfms(cc
);
1878 static int crypt_alloc_tfms_aead(struct crypt_config
*cc
, char *ciphermode
)
1882 cc
->cipher_tfm
.tfms
= kmalloc(sizeof(struct crypto_aead
*), GFP_KERNEL
);
1883 if (!cc
->cipher_tfm
.tfms
)
1886 cc
->cipher_tfm
.tfms_aead
[0] = crypto_alloc_aead(ciphermode
, 0, 0);
1887 if (IS_ERR(cc
->cipher_tfm
.tfms_aead
[0])) {
1888 err
= PTR_ERR(cc
->cipher_tfm
.tfms_aead
[0]);
1889 crypt_free_tfms(cc
);
1896 static int crypt_alloc_tfms(struct crypt_config
*cc
, char *ciphermode
)
1898 if (crypt_integrity_aead(cc
))
1899 return crypt_alloc_tfms_aead(cc
, ciphermode
);
1901 return crypt_alloc_tfms_skcipher(cc
, ciphermode
);
1904 static unsigned crypt_subkey_size(struct crypt_config
*cc
)
1906 return (cc
->key_size
- cc
->key_extra_size
) >> ilog2(cc
->tfms_count
);
1909 static unsigned crypt_authenckey_size(struct crypt_config
*cc
)
1911 return crypt_subkey_size(cc
) + RTA_SPACE(sizeof(struct crypto_authenc_key_param
));
1915 * If AEAD is composed like authenc(hmac(sha256),xts(aes)),
1916 * the key must be for some reason in special format.
1917 * This funcion converts cc->key to this special format.
1919 static void crypt_copy_authenckey(char *p
, const void *key
,
1920 unsigned enckeylen
, unsigned authkeylen
)
1922 struct crypto_authenc_key_param
*param
;
1925 rta
= (struct rtattr
*)p
;
1926 param
= RTA_DATA(rta
);
1927 param
->enckeylen
= cpu_to_be32(enckeylen
);
1928 rta
->rta_len
= RTA_LENGTH(sizeof(*param
));
1929 rta
->rta_type
= CRYPTO_AUTHENC_KEYA_PARAM
;
1930 p
+= RTA_SPACE(sizeof(*param
));
1931 memcpy(p
, key
+ enckeylen
, authkeylen
);
1933 memcpy(p
, key
, enckeylen
);
1936 static int crypt_setkey(struct crypt_config
*cc
)
1938 unsigned subkey_size
;
1941 /* Ignore extra keys (which are used for IV etc) */
1942 subkey_size
= crypt_subkey_size(cc
);
1944 if (crypt_integrity_hmac(cc
))
1945 crypt_copy_authenckey(cc
->authenc_key
, cc
->key
,
1946 subkey_size
- cc
->key_mac_size
,
1948 for (i
= 0; i
< cc
->tfms_count
; i
++) {
1949 if (crypt_integrity_hmac(cc
))
1950 r
= crypto_aead_setkey(cc
->cipher_tfm
.tfms_aead
[i
],
1951 cc
->authenc_key
, crypt_authenckey_size(cc
));
1952 else if (crypt_integrity_aead(cc
))
1953 r
= crypto_aead_setkey(cc
->cipher_tfm
.tfms_aead
[i
],
1954 cc
->key
+ (i
* subkey_size
),
1957 r
= crypto_skcipher_setkey(cc
->cipher_tfm
.tfms
[i
],
1958 cc
->key
+ (i
* subkey_size
),
1964 if (crypt_integrity_hmac(cc
))
1965 memzero_explicit(cc
->authenc_key
, crypt_authenckey_size(cc
));
1972 static bool contains_whitespace(const char *str
)
1975 if (isspace(*str
++))
1980 static int crypt_set_keyring_key(struct crypt_config
*cc
, const char *key_string
)
1982 char *new_key_string
, *key_desc
;
1985 const struct user_key_payload
*ukp
;
1988 * Reject key_string with whitespace. dm core currently lacks code for
1989 * proper whitespace escaping in arguments on DM_TABLE_STATUS path.
1991 if (contains_whitespace(key_string
)) {
1992 DMERR("whitespace chars not allowed in key string");
1996 /* look for next ':' separating key_type from key_description */
1997 key_desc
= strpbrk(key_string
, ":");
1998 if (!key_desc
|| key_desc
== key_string
|| !strlen(key_desc
+ 1))
2001 if (strncmp(key_string
, "logon:", key_desc
- key_string
+ 1) &&
2002 strncmp(key_string
, "user:", key_desc
- key_string
+ 1))
2005 new_key_string
= kstrdup(key_string
, GFP_KERNEL
);
2006 if (!new_key_string
)
2009 key
= request_key(key_string
[0] == 'l' ? &key_type_logon
: &key_type_user
,
2010 key_desc
+ 1, NULL
);
2012 kzfree(new_key_string
);
2013 return PTR_ERR(key
);
2016 down_read(&key
->sem
);
2018 ukp
= user_key_payload_locked(key
);
2022 kzfree(new_key_string
);
2023 return -EKEYREVOKED
;
2026 if (cc
->key_size
!= ukp
->datalen
) {
2029 kzfree(new_key_string
);
2033 memcpy(cc
->key
, ukp
->data
, cc
->key_size
);
2038 /* clear the flag since following operations may invalidate previously valid key */
2039 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
2041 ret
= crypt_setkey(cc
);
2043 /* wipe the kernel key payload copy in each case */
2044 memset(cc
->key
, 0, cc
->key_size
* sizeof(u8
));
2047 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
2048 kzfree(cc
->key_string
);
2049 cc
->key_string
= new_key_string
;
2051 kzfree(new_key_string
);
2056 static int get_key_size(char **key_string
)
2061 if (*key_string
[0] != ':')
2062 return strlen(*key_string
) >> 1;
2064 /* look for next ':' in key string */
2065 colon
= strpbrk(*key_string
+ 1, ":");
2069 if (sscanf(*key_string
+ 1, "%u%c", &ret
, &dummy
) != 2 || dummy
!= ':')
2072 *key_string
= colon
;
2074 /* remaining key string should be :<logon|user>:<key_desc> */
2081 static int crypt_set_keyring_key(struct crypt_config
*cc
, const char *key_string
)
2086 static int get_key_size(char **key_string
)
2088 return (*key_string
[0] == ':') ? -EINVAL
: strlen(*key_string
) >> 1;
2093 static int crypt_set_key(struct crypt_config
*cc
, char *key
)
2096 int key_string_len
= strlen(key
);
2098 /* Hyphen (which gives a key_size of zero) means there is no key. */
2099 if (!cc
->key_size
&& strcmp(key
, "-"))
2102 /* ':' means the key is in kernel keyring, short-circuit normal key processing */
2103 if (key
[0] == ':') {
2104 r
= crypt_set_keyring_key(cc
, key
+ 1);
2108 /* clear the flag since following operations may invalidate previously valid key */
2109 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
2111 /* wipe references to any kernel keyring key */
2112 kzfree(cc
->key_string
);
2113 cc
->key_string
= NULL
;
2115 /* Decode key from its hex representation. */
2116 if (cc
->key_size
&& hex2bin(cc
->key
, key
, cc
->key_size
) < 0)
2119 r
= crypt_setkey(cc
);
2121 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
2124 /* Hex key string not needed after here, so wipe it. */
2125 memset(key
, '0', key_string_len
);
2130 static int crypt_wipe_key(struct crypt_config
*cc
)
2134 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
2135 get_random_bytes(&cc
->key
, cc
->key_size
);
2136 kzfree(cc
->key_string
);
2137 cc
->key_string
= NULL
;
2138 r
= crypt_setkey(cc
);
2139 memset(&cc
->key
, 0, cc
->key_size
* sizeof(u8
));
2144 static void crypt_dtr(struct dm_target
*ti
)
2146 struct crypt_config
*cc
= ti
->private;
2153 if (cc
->write_thread
)
2154 kthread_stop(cc
->write_thread
);
2157 destroy_workqueue(cc
->io_queue
);
2158 if (cc
->crypt_queue
)
2159 destroy_workqueue(cc
->crypt_queue
);
2161 crypt_free_tfms(cc
);
2164 bioset_free(cc
->bs
);
2166 mempool_destroy(cc
->page_pool
);
2167 mempool_destroy(cc
->req_pool
);
2168 mempool_destroy(cc
->tag_pool
);
2170 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
2171 cc
->iv_gen_ops
->dtr(cc
);
2174 dm_put_device(ti
, cc
->dev
);
2177 kzfree(cc
->cipher_string
);
2178 kzfree(cc
->key_string
);
2179 kzfree(cc
->cipher_auth
);
2180 kzfree(cc
->authenc_key
);
2182 /* Must zero key material before freeing */
2186 static int crypt_ctr_ivmode(struct dm_target
*ti
, const char *ivmode
)
2188 struct crypt_config
*cc
= ti
->private;
2190 if (crypt_integrity_aead(cc
))
2191 cc
->iv_size
= crypto_aead_ivsize(any_tfm_aead(cc
));
2193 cc
->iv_size
= crypto_skcipher_ivsize(any_tfm(cc
));
2196 /* at least a 64 bit sector number should fit in our buffer */
2197 cc
->iv_size
= max(cc
->iv_size
,
2198 (unsigned int)(sizeof(u64
) / sizeof(u8
)));
2200 DMWARN("Selected cipher does not support IVs");
2204 /* Choose ivmode, see comments at iv code. */
2206 cc
->iv_gen_ops
= NULL
;
2207 else if (strcmp(ivmode
, "plain") == 0)
2208 cc
->iv_gen_ops
= &crypt_iv_plain_ops
;
2209 else if (strcmp(ivmode
, "plain64") == 0)
2210 cc
->iv_gen_ops
= &crypt_iv_plain64_ops
;
2211 else if (strcmp(ivmode
, "essiv") == 0)
2212 cc
->iv_gen_ops
= &crypt_iv_essiv_ops
;
2213 else if (strcmp(ivmode
, "benbi") == 0)
2214 cc
->iv_gen_ops
= &crypt_iv_benbi_ops
;
2215 else if (strcmp(ivmode
, "null") == 0)
2216 cc
->iv_gen_ops
= &crypt_iv_null_ops
;
2217 else if (strcmp(ivmode
, "lmk") == 0) {
2218 cc
->iv_gen_ops
= &crypt_iv_lmk_ops
;
2220 * Version 2 and 3 is recognised according
2221 * to length of provided multi-key string.
2222 * If present (version 3), last key is used as IV seed.
2223 * All keys (including IV seed) are always the same size.
2225 if (cc
->key_size
% cc
->key_parts
) {
2227 cc
->key_extra_size
= cc
->key_size
/ cc
->key_parts
;
2229 } else if (strcmp(ivmode
, "tcw") == 0) {
2230 cc
->iv_gen_ops
= &crypt_iv_tcw_ops
;
2231 cc
->key_parts
+= 2; /* IV + whitening */
2232 cc
->key_extra_size
= cc
->iv_size
+ TCW_WHITENING_SIZE
;
2233 } else if (strcmp(ivmode
, "random") == 0) {
2234 cc
->iv_gen_ops
= &crypt_iv_random_ops
;
2235 /* Need storage space in integrity fields. */
2236 cc
->integrity_iv_size
= cc
->iv_size
;
2238 ti
->error
= "Invalid IV mode";
2246 * Workaround to parse cipher algorithm from crypto API spec.
2247 * The cc->cipher is currently used only in ESSIV.
2248 * This should be probably done by crypto-api calls (once available...)
2250 static int crypt_ctr_blkdev_cipher(struct crypt_config
*cc
)
2252 const char *alg_name
= NULL
;
2255 if (crypt_integrity_aead(cc
)) {
2256 alg_name
= crypto_tfm_alg_name(crypto_aead_tfm(any_tfm_aead(cc
)));
2259 if (crypt_integrity_hmac(cc
)) {
2260 alg_name
= strchr(alg_name
, ',');
2266 alg_name
= crypto_tfm_alg_name(crypto_skcipher_tfm(any_tfm(cc
)));
2271 start
= strchr(alg_name
, '(');
2272 end
= strchr(alg_name
, ')');
2274 if (!start
&& !end
) {
2275 cc
->cipher
= kstrdup(alg_name
, GFP_KERNEL
);
2276 return cc
->cipher
? 0 : -ENOMEM
;
2279 if (!start
|| !end
|| ++start
>= end
)
2282 cc
->cipher
= kzalloc(end
- start
+ 1, GFP_KERNEL
);
2286 strncpy(cc
->cipher
, start
, end
- start
);
2292 * Workaround to parse HMAC algorithm from AEAD crypto API spec.
2293 * The HMAC is needed to calculate tag size (HMAC digest size).
2294 * This should be probably done by crypto-api calls (once available...)
2296 static int crypt_ctr_auth_cipher(struct crypt_config
*cc
, char *cipher_api
)
2298 char *start
, *end
, *mac_alg
= NULL
;
2299 struct crypto_ahash
*mac
;
2301 if (!strstarts(cipher_api
, "authenc("))
2304 start
= strchr(cipher_api
, '(');
2305 end
= strchr(cipher_api
, ',');
2306 if (!start
|| !end
|| ++start
> end
)
2309 mac_alg
= kzalloc(end
- start
+ 1, GFP_KERNEL
);
2312 strncpy(mac_alg
, start
, end
- start
);
2314 mac
= crypto_alloc_ahash(mac_alg
, 0, 0);
2318 return PTR_ERR(mac
);
2320 cc
->key_mac_size
= crypto_ahash_digestsize(mac
);
2321 crypto_free_ahash(mac
);
2323 cc
->authenc_key
= kmalloc(crypt_authenckey_size(cc
), GFP_KERNEL
);
2324 if (!cc
->authenc_key
)
2330 static int crypt_ctr_cipher_new(struct dm_target
*ti
, char *cipher_in
, char *key
,
2331 char **ivmode
, char **ivopts
)
2333 struct crypt_config
*cc
= ti
->private;
2334 char *tmp
, *cipher_api
;
2340 * New format (capi: prefix)
2341 * capi:cipher_api_spec-iv:ivopts
2343 tmp
= &cipher_in
[strlen("capi:")];
2344 cipher_api
= strsep(&tmp
, "-");
2345 *ivmode
= strsep(&tmp
, ":");
2348 if (*ivmode
&& !strcmp(*ivmode
, "lmk"))
2349 cc
->tfms_count
= 64;
2351 cc
->key_parts
= cc
->tfms_count
;
2353 /* Allocate cipher */
2354 ret
= crypt_alloc_tfms(cc
, cipher_api
);
2356 ti
->error
= "Error allocating crypto tfm";
2360 /* Alloc AEAD, can be used only in new format. */
2361 if (crypt_integrity_aead(cc
)) {
2362 ret
= crypt_ctr_auth_cipher(cc
, cipher_api
);
2364 ti
->error
= "Invalid AEAD cipher spec";
2367 cc
->iv_size
= crypto_aead_ivsize(any_tfm_aead(cc
));
2369 cc
->iv_size
= crypto_skcipher_ivsize(any_tfm(cc
));
2371 ret
= crypt_ctr_blkdev_cipher(cc
);
2373 ti
->error
= "Cannot allocate cipher string";
2380 static int crypt_ctr_cipher_old(struct dm_target
*ti
, char *cipher_in
, char *key
,
2381 char **ivmode
, char **ivopts
)
2383 struct crypt_config
*cc
= ti
->private;
2384 char *tmp
, *cipher
, *chainmode
, *keycount
;
2385 char *cipher_api
= NULL
;
2389 if (strchr(cipher_in
, '(') || crypt_integrity_aead(cc
)) {
2390 ti
->error
= "Bad cipher specification";
2395 * Legacy dm-crypt cipher specification
2396 * cipher[:keycount]-mode-iv:ivopts
2399 keycount
= strsep(&tmp
, "-");
2400 cipher
= strsep(&keycount
, ":");
2404 else if (sscanf(keycount
, "%u%c", &cc
->tfms_count
, &dummy
) != 1 ||
2405 !is_power_of_2(cc
->tfms_count
)) {
2406 ti
->error
= "Bad cipher key count specification";
2409 cc
->key_parts
= cc
->tfms_count
;
2411 cc
->cipher
= kstrdup(cipher
, GFP_KERNEL
);
2415 chainmode
= strsep(&tmp
, "-");
2416 *ivopts
= strsep(&tmp
, "-");
2417 *ivmode
= strsep(&*ivopts
, ":");
2420 DMWARN("Ignoring unexpected additional cipher options");
2423 * For compatibility with the original dm-crypt mapping format, if
2424 * only the cipher name is supplied, use cbc-plain.
2426 if (!chainmode
|| (!strcmp(chainmode
, "plain") && !*ivmode
)) {
2431 if (strcmp(chainmode
, "ecb") && !*ivmode
) {
2432 ti
->error
= "IV mechanism required";
2436 cipher_api
= kmalloc(CRYPTO_MAX_ALG_NAME
, GFP_KERNEL
);
2440 ret
= snprintf(cipher_api
, CRYPTO_MAX_ALG_NAME
,
2441 "%s(%s)", chainmode
, cipher
);
2447 /* Allocate cipher */
2448 ret
= crypt_alloc_tfms(cc
, cipher_api
);
2450 ti
->error
= "Error allocating crypto tfm";
2457 ti
->error
= "Cannot allocate cipher strings";
2461 static int crypt_ctr_cipher(struct dm_target
*ti
, char *cipher_in
, char *key
)
2463 struct crypt_config
*cc
= ti
->private;
2464 char *ivmode
= NULL
, *ivopts
= NULL
;
2467 cc
->cipher_string
= kstrdup(cipher_in
, GFP_KERNEL
);
2468 if (!cc
->cipher_string
) {
2469 ti
->error
= "Cannot allocate cipher strings";
2473 if (strstarts(cipher_in
, "capi:"))
2474 ret
= crypt_ctr_cipher_new(ti
, cipher_in
, key
, &ivmode
, &ivopts
);
2476 ret
= crypt_ctr_cipher_old(ti
, cipher_in
, key
, &ivmode
, &ivopts
);
2481 ret
= crypt_ctr_ivmode(ti
, ivmode
);
2485 /* Initialize and set key */
2486 ret
= crypt_set_key(cc
, key
);
2488 ti
->error
= "Error decoding and setting key";
2493 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->ctr
) {
2494 ret
= cc
->iv_gen_ops
->ctr(cc
, ti
, ivopts
);
2496 ti
->error
= "Error creating IV";
2501 /* Initialize IV (set keys for ESSIV etc) */
2502 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
) {
2503 ret
= cc
->iv_gen_ops
->init(cc
);
2505 ti
->error
= "Error initialising IV";
2513 static int crypt_ctr_optional(struct dm_target
*ti
, unsigned int argc
, char **argv
)
2515 struct crypt_config
*cc
= ti
->private;
2516 struct dm_arg_set as
;
2517 static struct dm_arg _args
[] = {
2518 {0, 6, "Invalid number of feature args"},
2520 unsigned int opt_params
, val
;
2521 const char *opt_string
, *sval
;
2525 /* Optional parameters */
2529 ret
= dm_read_arg_group(_args
, &as
, &opt_params
, &ti
->error
);
2533 while (opt_params
--) {
2534 opt_string
= dm_shift_arg(&as
);
2536 ti
->error
= "Not enough feature arguments";
2540 if (!strcasecmp(opt_string
, "allow_discards"))
2541 ti
->num_discard_bios
= 1;
2543 else if (!strcasecmp(opt_string
, "same_cpu_crypt"))
2544 set_bit(DM_CRYPT_SAME_CPU
, &cc
->flags
);
2546 else if (!strcasecmp(opt_string
, "submit_from_crypt_cpus"))
2547 set_bit(DM_CRYPT_NO_OFFLOAD
, &cc
->flags
);
2548 else if (sscanf(opt_string
, "integrity:%u:", &val
) == 1) {
2549 if (val
== 0 || val
> MAX_TAG_SIZE
) {
2550 ti
->error
= "Invalid integrity arguments";
2553 cc
->on_disk_tag_size
= val
;
2554 sval
= strchr(opt_string
+ strlen("integrity:"), ':') + 1;
2555 if (!strcasecmp(sval
, "aead")) {
2556 set_bit(CRYPT_MODE_INTEGRITY_AEAD
, &cc
->cipher_flags
);
2557 } else if (strcasecmp(sval
, "none")) {
2558 ti
->error
= "Unknown integrity profile";
2562 cc
->cipher_auth
= kstrdup(sval
, GFP_KERNEL
);
2563 if (!cc
->cipher_auth
)
2565 } else if (sscanf(opt_string
, "sector_size:%hu%c", &cc
->sector_size
, &dummy
) == 1) {
2566 if (cc
->sector_size
< (1 << SECTOR_SHIFT
) ||
2567 cc
->sector_size
> 4096 ||
2568 (cc
->sector_size
& (cc
->sector_size
- 1))) {
2569 ti
->error
= "Invalid feature value for sector_size";
2572 cc
->sector_shift
= __ffs(cc
->sector_size
) - SECTOR_SHIFT
;
2573 } else if (!strcasecmp(opt_string
, "iv_large_sectors"))
2574 set_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
);
2576 ti
->error
= "Invalid feature arguments";
2585 * Construct an encryption mapping:
2586 * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>
2588 static int crypt_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
2590 struct crypt_config
*cc
;
2592 unsigned int align_mask
;
2593 unsigned long long tmpll
;
2595 size_t iv_size_padding
, additional_req_size
;
2599 ti
->error
= "Not enough arguments";
2603 key_size
= get_key_size(&argv
[1]);
2605 ti
->error
= "Cannot parse key size";
2609 cc
= kzalloc(sizeof(*cc
) + key_size
* sizeof(u8
), GFP_KERNEL
);
2611 ti
->error
= "Cannot allocate encryption context";
2614 cc
->key_size
= key_size
;
2615 cc
->sector_size
= (1 << SECTOR_SHIFT
);
2616 cc
->sector_shift
= 0;
2620 /* Optional parameters need to be read before cipher constructor */
2622 ret
= crypt_ctr_optional(ti
, argc
- 5, &argv
[5]);
2627 ret
= crypt_ctr_cipher(ti
, argv
[0], argv
[1]);
2631 if (crypt_integrity_aead(cc
)) {
2632 cc
->dmreq_start
= sizeof(struct aead_request
);
2633 cc
->dmreq_start
+= crypto_aead_reqsize(any_tfm_aead(cc
));
2634 align_mask
= crypto_aead_alignmask(any_tfm_aead(cc
));
2636 cc
->dmreq_start
= sizeof(struct skcipher_request
);
2637 cc
->dmreq_start
+= crypto_skcipher_reqsize(any_tfm(cc
));
2638 align_mask
= crypto_skcipher_alignmask(any_tfm(cc
));
2640 cc
->dmreq_start
= ALIGN(cc
->dmreq_start
, __alignof__(struct dm_crypt_request
));
2642 if (align_mask
< CRYPTO_MINALIGN
) {
2643 /* Allocate the padding exactly */
2644 iv_size_padding
= -(cc
->dmreq_start
+ sizeof(struct dm_crypt_request
))
2648 * If the cipher requires greater alignment than kmalloc
2649 * alignment, we don't know the exact position of the
2650 * initialization vector. We must assume worst case.
2652 iv_size_padding
= align_mask
;
2657 /* ...| IV + padding | original IV | original sec. number | bio tag offset | */
2658 additional_req_size
= sizeof(struct dm_crypt_request
) +
2659 iv_size_padding
+ cc
->iv_size
+
2662 sizeof(unsigned int);
2664 cc
->req_pool
= mempool_create_kmalloc_pool(MIN_IOS
, cc
->dmreq_start
+ additional_req_size
);
2665 if (!cc
->req_pool
) {
2666 ti
->error
= "Cannot allocate crypt request mempool";
2670 cc
->per_bio_data_size
= ti
->per_io_data_size
=
2671 ALIGN(sizeof(struct dm_crypt_io
) + cc
->dmreq_start
+ additional_req_size
,
2672 ARCH_KMALLOC_MINALIGN
);
2674 cc
->page_pool
= mempool_create_page_pool(BIO_MAX_PAGES
, 0);
2675 if (!cc
->page_pool
) {
2676 ti
->error
= "Cannot allocate page mempool";
2680 cc
->bs
= bioset_create(MIN_IOS
, 0);
2682 ti
->error
= "Cannot allocate crypt bioset";
2686 mutex_init(&cc
->bio_alloc_lock
);
2689 if ((sscanf(argv
[2], "%llu%c", &tmpll
, &dummy
) != 1) ||
2690 (tmpll
& ((cc
->sector_size
>> SECTOR_SHIFT
) - 1))) {
2691 ti
->error
= "Invalid iv_offset sector";
2694 cc
->iv_offset
= tmpll
;
2696 ret
= dm_get_device(ti
, argv
[3], dm_table_get_mode(ti
->table
), &cc
->dev
);
2698 ti
->error
= "Device lookup failed";
2703 if (sscanf(argv
[4], "%llu%c", &tmpll
, &dummy
) != 1) {
2704 ti
->error
= "Invalid device sector";
2709 if (crypt_integrity_aead(cc
) || cc
->integrity_iv_size
) {
2710 ret
= crypt_integrity_ctr(cc
, ti
);
2714 cc
->tag_pool_max_sectors
= POOL_ENTRY_SIZE
/ cc
->on_disk_tag_size
;
2715 if (!cc
->tag_pool_max_sectors
)
2716 cc
->tag_pool_max_sectors
= 1;
2718 cc
->tag_pool
= mempool_create_kmalloc_pool(MIN_IOS
,
2719 cc
->tag_pool_max_sectors
* cc
->on_disk_tag_size
);
2720 if (!cc
->tag_pool
) {
2721 ti
->error
= "Cannot allocate integrity tags mempool";
2725 cc
->tag_pool_max_sectors
<<= cc
->sector_shift
;
2729 cc
->io_queue
= alloc_workqueue("kcryptd_io", WQ_HIGHPRI
| WQ_CPU_INTENSIVE
| WQ_MEM_RECLAIM
, 1);
2730 if (!cc
->io_queue
) {
2731 ti
->error
= "Couldn't create kcryptd io queue";
2735 if (test_bit(DM_CRYPT_SAME_CPU
, &cc
->flags
))
2736 cc
->crypt_queue
= alloc_workqueue("kcryptd", WQ_HIGHPRI
| WQ_CPU_INTENSIVE
| WQ_MEM_RECLAIM
, 1);
2738 cc
->crypt_queue
= alloc_workqueue("kcryptd",
2739 WQ_HIGHPRI
| WQ_CPU_INTENSIVE
| WQ_MEM_RECLAIM
| WQ_UNBOUND
,
2741 if (!cc
->crypt_queue
) {
2742 ti
->error
= "Couldn't create kcryptd queue";
2746 init_waitqueue_head(&cc
->write_thread_wait
);
2747 cc
->write_tree
= RB_ROOT
;
2749 cc
->write_thread
= kthread_create(dmcrypt_write
, cc
, "dmcrypt_write");
2750 if (IS_ERR(cc
->write_thread
)) {
2751 ret
= PTR_ERR(cc
->write_thread
);
2752 cc
->write_thread
= NULL
;
2753 ti
->error
= "Couldn't spawn write thread";
2756 wake_up_process(cc
->write_thread
);
2758 ti
->num_flush_bios
= 1;
2767 static int crypt_map(struct dm_target
*ti
, struct bio
*bio
)
2769 struct dm_crypt_io
*io
;
2770 struct crypt_config
*cc
= ti
->private;
2773 * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
2774 * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight
2775 * - for REQ_OP_DISCARD caller must use flush if IO ordering matters
2777 if (unlikely(bio
->bi_opf
& REQ_PREFLUSH
||
2778 bio_op(bio
) == REQ_OP_DISCARD
)) {
2779 bio
->bi_bdev
= cc
->dev
->bdev
;
2780 if (bio_sectors(bio
))
2781 bio
->bi_iter
.bi_sector
= cc
->start
+
2782 dm_target_offset(ti
, bio
->bi_iter
.bi_sector
);
2783 return DM_MAPIO_REMAPPED
;
2787 * Check if bio is too large, split as needed.
2789 if (unlikely(bio
->bi_iter
.bi_size
> (BIO_MAX_PAGES
<< PAGE_SHIFT
)) &&
2790 (bio_data_dir(bio
) == WRITE
|| cc
->on_disk_tag_size
))
2791 dm_accept_partial_bio(bio
, ((BIO_MAX_PAGES
<< PAGE_SHIFT
) >> SECTOR_SHIFT
));
2794 * Ensure that bio is a multiple of internal sector encryption size
2795 * and is aligned to this size as defined in IO hints.
2797 if (unlikely((bio
->bi_iter
.bi_sector
& ((cc
->sector_size
>> SECTOR_SHIFT
) - 1)) != 0))
2800 if (unlikely(bio
->bi_iter
.bi_size
& (cc
->sector_size
- 1)))
2803 io
= dm_per_bio_data(bio
, cc
->per_bio_data_size
);
2804 crypt_io_init(io
, cc
, bio
, dm_target_offset(ti
, bio
->bi_iter
.bi_sector
));
2806 if (cc
->on_disk_tag_size
) {
2807 unsigned tag_len
= cc
->on_disk_tag_size
* (bio_sectors(bio
) >> cc
->sector_shift
);
2809 if (unlikely(tag_len
> KMALLOC_MAX_SIZE
) ||
2810 unlikely(!(io
->integrity_metadata
= kmalloc(tag_len
,
2811 GFP_NOIO
| __GFP_NORETRY
| __GFP_NOMEMALLOC
| __GFP_NOWARN
)))) {
2812 if (bio_sectors(bio
) > cc
->tag_pool_max_sectors
)
2813 dm_accept_partial_bio(bio
, cc
->tag_pool_max_sectors
);
2814 io
->integrity_metadata
= mempool_alloc(cc
->tag_pool
, GFP_NOIO
);
2815 io
->integrity_metadata_from_pool
= true;
2819 if (crypt_integrity_aead(cc
))
2820 io
->ctx
.r
.req_aead
= (struct aead_request
*)(io
+ 1);
2822 io
->ctx
.r
.req
= (struct skcipher_request
*)(io
+ 1);
2824 if (bio_data_dir(io
->base_bio
) == READ
) {
2825 if (kcryptd_io_read(io
, GFP_NOWAIT
))
2826 kcryptd_queue_read(io
);
2828 kcryptd_queue_crypt(io
);
2830 return DM_MAPIO_SUBMITTED
;
2833 static void crypt_status(struct dm_target
*ti
, status_type_t type
,
2834 unsigned status_flags
, char *result
, unsigned maxlen
)
2836 struct crypt_config
*cc
= ti
->private;
2838 int num_feature_args
= 0;
2841 case STATUSTYPE_INFO
:
2845 case STATUSTYPE_TABLE
:
2846 DMEMIT("%s ", cc
->cipher_string
);
2848 if (cc
->key_size
> 0) {
2850 DMEMIT(":%u:%s", cc
->key_size
, cc
->key_string
);
2852 for (i
= 0; i
< cc
->key_size
; i
++)
2853 DMEMIT("%02x", cc
->key
[i
]);
2857 DMEMIT(" %llu %s %llu", (unsigned long long)cc
->iv_offset
,
2858 cc
->dev
->name
, (unsigned long long)cc
->start
);
2860 num_feature_args
+= !!ti
->num_discard_bios
;
2861 num_feature_args
+= test_bit(DM_CRYPT_SAME_CPU
, &cc
->flags
);
2862 num_feature_args
+= test_bit(DM_CRYPT_NO_OFFLOAD
, &cc
->flags
);
2863 num_feature_args
+= cc
->sector_size
!= (1 << SECTOR_SHIFT
);
2864 num_feature_args
+= test_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
);
2865 if (cc
->on_disk_tag_size
)
2867 if (num_feature_args
) {
2868 DMEMIT(" %d", num_feature_args
);
2869 if (ti
->num_discard_bios
)
2870 DMEMIT(" allow_discards");
2871 if (test_bit(DM_CRYPT_SAME_CPU
, &cc
->flags
))
2872 DMEMIT(" same_cpu_crypt");
2873 if (test_bit(DM_CRYPT_NO_OFFLOAD
, &cc
->flags
))
2874 DMEMIT(" submit_from_crypt_cpus");
2875 if (cc
->on_disk_tag_size
)
2876 DMEMIT(" integrity:%u:%s", cc
->on_disk_tag_size
, cc
->cipher_auth
);
2877 if (cc
->sector_size
!= (1 << SECTOR_SHIFT
))
2878 DMEMIT(" sector_size:%d", cc
->sector_size
);
2879 if (test_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
))
2880 DMEMIT(" iv_large_sectors");
2887 static void crypt_postsuspend(struct dm_target
*ti
)
2889 struct crypt_config
*cc
= ti
->private;
2891 set_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
2894 static int crypt_preresume(struct dm_target
*ti
)
2896 struct crypt_config
*cc
= ti
->private;
2898 if (!test_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
)) {
2899 DMERR("aborting resume - crypt key is not set.");
2906 static void crypt_resume(struct dm_target
*ti
)
2908 struct crypt_config
*cc
= ti
->private;
2910 clear_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
2913 /* Message interface
2917 static int crypt_message(struct dm_target
*ti
, unsigned argc
, char **argv
)
2919 struct crypt_config
*cc
= ti
->private;
2920 int key_size
, ret
= -EINVAL
;
2925 if (!strcasecmp(argv
[0], "key")) {
2926 if (!test_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
)) {
2927 DMWARN("not suspended during key manipulation.");
2930 if (argc
== 3 && !strcasecmp(argv
[1], "set")) {
2931 /* The key size may not be changed. */
2932 key_size
= get_key_size(&argv
[2]);
2933 if (key_size
< 0 || cc
->key_size
!= key_size
) {
2934 memset(argv
[2], '0', strlen(argv
[2]));
2938 ret
= crypt_set_key(cc
, argv
[2]);
2941 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
)
2942 ret
= cc
->iv_gen_ops
->init(cc
);
2945 if (argc
== 2 && !strcasecmp(argv
[1], "wipe")) {
2946 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->wipe
) {
2947 ret
= cc
->iv_gen_ops
->wipe(cc
);
2951 return crypt_wipe_key(cc
);
2956 DMWARN("unrecognised message received.");
2960 static int crypt_iterate_devices(struct dm_target
*ti
,
2961 iterate_devices_callout_fn fn
, void *data
)
2963 struct crypt_config
*cc
= ti
->private;
2965 return fn(ti
, cc
->dev
, cc
->start
, ti
->len
, data
);
2968 static void crypt_io_hints(struct dm_target
*ti
, struct queue_limits
*limits
)
2970 struct crypt_config
*cc
= ti
->private;
2973 * Unfortunate constraint that is required to avoid the potential
2974 * for exceeding underlying device's max_segments limits -- due to
2975 * crypt_alloc_buffer() possibly allocating pages for the encryption
2976 * bio that are not as physically contiguous as the original bio.
2978 limits
->max_segment_size
= PAGE_SIZE
;
2980 if (cc
->sector_size
!= (1 << SECTOR_SHIFT
)) {
2981 limits
->logical_block_size
= cc
->sector_size
;
2982 limits
->physical_block_size
= cc
->sector_size
;
2983 blk_limits_io_min(limits
, cc
->sector_size
);
2987 static struct target_type crypt_target
= {
2989 .version
= {1, 17, 0},
2990 .module
= THIS_MODULE
,
2994 .status
= crypt_status
,
2995 .postsuspend
= crypt_postsuspend
,
2996 .preresume
= crypt_preresume
,
2997 .resume
= crypt_resume
,
2998 .message
= crypt_message
,
2999 .iterate_devices
= crypt_iterate_devices
,
3000 .io_hints
= crypt_io_hints
,
3003 static int __init
dm_crypt_init(void)
3007 r
= dm_register_target(&crypt_target
);
3009 DMERR("register failed %d", r
);
3014 static void __exit
dm_crypt_exit(void)
3016 dm_unregister_target(&crypt_target
);
3019 module_init(dm_crypt_init
);
3020 module_exit(dm_crypt_exit
);
3022 MODULE_AUTHOR("Jana Saout <jana@saout.de>");
3023 MODULE_DESCRIPTION(DM_NAME
" target for transparent encryption / decryption");
3024 MODULE_LICENSE("GPL");