1 // SPDX-License-Identifier: GPL-2.0
3 * Inline encryption support for fscrypt
5 * Copyright 2019 Google LLC
9 * With "inline encryption", the block layer handles the decryption/encryption
10 * as part of the bio, instead of the filesystem doing the crypto itself via
11 * crypto API. See Documentation/block/inline-encryption.rst. fscrypt still
12 * provides the key and IV to use.
15 #include <linux/blk-crypto.h>
16 #include <linux/blkdev.h>
17 #include <linux/buffer_head.h>
18 #include <linux/sched/mm.h>
19 #include <linux/slab.h>
21 #include "fscrypt_private.h"
23 struct fscrypt_blk_crypto_key
{
24 struct blk_crypto_key base
;
26 struct request_queue
*devs
[];
29 static int fscrypt_get_num_devices(struct super_block
*sb
)
31 if (sb
->s_cop
->get_num_devices
)
32 return sb
->s_cop
->get_num_devices(sb
);
36 static void fscrypt_get_devices(struct super_block
*sb
, int num_devs
,
37 struct request_queue
**devs
)
40 devs
[0] = bdev_get_queue(sb
->s_bdev
);
42 sb
->s_cop
->get_devices(sb
, devs
);
45 static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info
*ci
)
47 struct super_block
*sb
= ci
->ci_inode
->i_sb
;
48 unsigned int flags
= fscrypt_policy_flags(&ci
->ci_policy
);
49 int ino_bits
= 64, lblk_bits
= 64;
51 if (flags
& FSCRYPT_POLICY_FLAG_DIRECT_KEY
)
52 return offsetofend(union fscrypt_iv
, nonce
);
54 if (flags
& FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64
)
55 return sizeof(__le64
);
57 if (flags
& FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32
)
58 return sizeof(__le32
);
60 /* Default case: IVs are just the file logical block number */
61 if (sb
->s_cop
->get_ino_and_lblk_bits
)
62 sb
->s_cop
->get_ino_and_lblk_bits(sb
, &ino_bits
, &lblk_bits
);
63 return DIV_ROUND_UP(lblk_bits
, 8);
66 /* Enable inline encryption for this file if supported. */
67 int fscrypt_select_encryption_impl(struct fscrypt_info
*ci
)
69 const struct inode
*inode
= ci
->ci_inode
;
70 struct super_block
*sb
= inode
->i_sb
;
71 struct blk_crypto_config crypto_cfg
;
73 struct request_queue
**devs
;
76 /* The file must need contents encryption, not filenames encryption */
77 if (!S_ISREG(inode
->i_mode
))
80 /* The crypto mode must have a blk-crypto counterpart */
81 if (ci
->ci_mode
->blk_crypto_mode
== BLK_ENCRYPTION_MODE_INVALID
)
84 /* The filesystem must be mounted with -o inlinecrypt */
85 if (!(sb
->s_flags
& SB_INLINECRYPT
))
89 * When a page contains multiple logically contiguous filesystem blocks,
90 * some filesystem code only calls fscrypt_mergeable_bio() for the first
91 * block in the page. This is fine for most of fscrypt's IV generation
92 * strategies, where contiguous blocks imply contiguous IVs. But it
93 * doesn't work with IV_INO_LBLK_32. For now, simply exclude
94 * IV_INO_LBLK_32 with blocksize != PAGE_SIZE from inline encryption.
96 if ((fscrypt_policy_flags(&ci
->ci_policy
) &
97 FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32
) &&
98 sb
->s_blocksize
!= PAGE_SIZE
)
102 * On all the filesystem's devices, blk-crypto must support the crypto
103 * configuration that the file would use.
105 crypto_cfg
.crypto_mode
= ci
->ci_mode
->blk_crypto_mode
;
106 crypto_cfg
.data_unit_size
= sb
->s_blocksize
;
107 crypto_cfg
.dun_bytes
= fscrypt_get_dun_bytes(ci
);
108 num_devs
= fscrypt_get_num_devices(sb
);
109 devs
= kmalloc_array(num_devs
, sizeof(*devs
), GFP_KERNEL
);
112 fscrypt_get_devices(sb
, num_devs
, devs
);
114 for (i
= 0; i
< num_devs
; i
++) {
115 if (!blk_crypto_config_supported(devs
[i
], &crypto_cfg
))
119 ci
->ci_inlinecrypt
= true;
126 int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key
*prep_key
,
128 const struct fscrypt_info
*ci
)
130 const struct inode
*inode
= ci
->ci_inode
;
131 struct super_block
*sb
= inode
->i_sb
;
132 enum blk_crypto_mode_num crypto_mode
= ci
->ci_mode
->blk_crypto_mode
;
133 int num_devs
= fscrypt_get_num_devices(sb
);
135 struct fscrypt_blk_crypto_key
*blk_key
;
139 blk_key
= kzalloc(struct_size(blk_key
, devs
, num_devs
), GFP_KERNEL
);
143 blk_key
->num_devs
= num_devs
;
144 fscrypt_get_devices(sb
, num_devs
, blk_key
->devs
);
146 err
= blk_crypto_init_key(&blk_key
->base
, raw_key
, crypto_mode
,
147 fscrypt_get_dun_bytes(ci
), sb
->s_blocksize
);
149 fscrypt_err(inode
, "error %d initializing blk-crypto key", err
);
154 * We have to start using blk-crypto on all the filesystem's devices.
155 * We also have to save all the request_queue's for later so that the
156 * key can be evicted from them. This is needed because some keys
157 * aren't destroyed until after the filesystem was already unmounted
158 * (namely, the per-mode keys in struct fscrypt_master_key).
160 for (i
= 0; i
< num_devs
; i
++) {
161 if (!blk_get_queue(blk_key
->devs
[i
])) {
162 fscrypt_err(inode
, "couldn't get request_queue");
168 err
= blk_crypto_start_using_key(&blk_key
->base
,
172 "error %d starting to use blk-crypto", err
);
177 * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared().
178 * I.e., here we publish ->blk_key with a RELEASE barrier so that
179 * concurrent tasks can ACQUIRE it. Note that this concurrency is only
180 * possible for per-mode keys, not for per-file keys.
182 smp_store_release(&prep_key
->blk_key
, blk_key
);
186 for (i
= 0; i
< queue_refs
; i
++)
187 blk_put_queue(blk_key
->devs
[i
]);
188 kfree_sensitive(blk_key
);
192 void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key
*prep_key
)
194 struct fscrypt_blk_crypto_key
*blk_key
= prep_key
->blk_key
;
198 for (i
= 0; i
< blk_key
->num_devs
; i
++) {
199 blk_crypto_evict_key(blk_key
->devs
[i
], &blk_key
->base
);
200 blk_put_queue(blk_key
->devs
[i
]);
202 kfree_sensitive(blk_key
);
206 bool __fscrypt_inode_uses_inline_crypto(const struct inode
*inode
)
208 return inode
->i_crypt_info
->ci_inlinecrypt
;
210 EXPORT_SYMBOL_GPL(__fscrypt_inode_uses_inline_crypto
);
212 static void fscrypt_generate_dun(const struct fscrypt_info
*ci
, u64 lblk_num
,
213 u64 dun
[BLK_CRYPTO_DUN_ARRAY_SIZE
])
218 fscrypt_generate_iv(&iv
, lblk_num
, ci
);
220 BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE
> BLK_CRYPTO_MAX_IV_SIZE
);
221 memset(dun
, 0, BLK_CRYPTO_MAX_IV_SIZE
);
222 for (i
= 0; i
< ci
->ci_mode
->ivsize
/sizeof(dun
[0]); i
++)
223 dun
[i
] = le64_to_cpu(iv
.dun
[i
]);
227 * fscrypt_set_bio_crypt_ctx() - prepare a file contents bio for inline crypto
228 * @bio: a bio which will eventually be submitted to the file
229 * @inode: the file's inode
230 * @first_lblk: the first file logical block number in the I/O
231 * @gfp_mask: memory allocation flags - these must be a waiting mask so that
232 * bio_crypt_set_ctx can't fail.
234 * If the contents of the file should be encrypted (or decrypted) with inline
235 * encryption, then assign the appropriate encryption context to the bio.
237 * Normally the bio should be newly allocated (i.e. no pages added yet), as
238 * otherwise fscrypt_mergeable_bio() won't work as intended.
240 * The encryption context will be freed automatically when the bio is freed.
242 void fscrypt_set_bio_crypt_ctx(struct bio
*bio
, const struct inode
*inode
,
243 u64 first_lblk
, gfp_t gfp_mask
)
245 const struct fscrypt_info
*ci
;
246 u64 dun
[BLK_CRYPTO_DUN_ARRAY_SIZE
];
248 if (!fscrypt_inode_uses_inline_crypto(inode
))
250 ci
= inode
->i_crypt_info
;
252 fscrypt_generate_dun(ci
, first_lblk
, dun
);
253 bio_crypt_set_ctx(bio
, &ci
->ci_enc_key
.blk_key
->base
, dun
, gfp_mask
);
255 EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx
);
257 /* Extract the inode and logical block number from a buffer_head. */
258 static bool bh_get_inode_and_lblk_num(const struct buffer_head
*bh
,
259 const struct inode
**inode_ret
,
262 struct page
*page
= bh
->b_page
;
263 const struct address_space
*mapping
;
264 const struct inode
*inode
;
267 * The ext4 journal (jbd2) can submit a buffer_head it directly created
268 * for a non-pagecache page. fscrypt doesn't care about these.
270 mapping
= page_mapping(page
);
273 inode
= mapping
->host
;
276 *lblk_num_ret
= ((u64
)page
->index
<< (PAGE_SHIFT
- inode
->i_blkbits
)) +
277 (bh_offset(bh
) >> inode
->i_blkbits
);
282 * fscrypt_set_bio_crypt_ctx_bh() - prepare a file contents bio for inline
284 * @bio: a bio which will eventually be submitted to the file
285 * @first_bh: the first buffer_head for which I/O will be submitted
286 * @gfp_mask: memory allocation flags
288 * Same as fscrypt_set_bio_crypt_ctx(), except this takes a buffer_head instead
289 * of an inode and block number directly.
291 void fscrypt_set_bio_crypt_ctx_bh(struct bio
*bio
,
292 const struct buffer_head
*first_bh
,
295 const struct inode
*inode
;
298 if (bh_get_inode_and_lblk_num(first_bh
, &inode
, &first_lblk
))
299 fscrypt_set_bio_crypt_ctx(bio
, inode
, first_lblk
, gfp_mask
);
301 EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx_bh
);
304 * fscrypt_mergeable_bio() - test whether data can be added to a bio
305 * @bio: the bio being built up
306 * @inode: the inode for the next part of the I/O
307 * @next_lblk: the next file logical block number in the I/O
309 * When building a bio which may contain data which should undergo inline
310 * encryption (or decryption) via fscrypt, filesystems should call this function
311 * to ensure that the resulting bio contains only contiguous data unit numbers.
312 * This will return false if the next part of the I/O cannot be merged with the
313 * bio because either the encryption key would be different or the encryption
314 * data unit numbers would be discontiguous.
316 * fscrypt_set_bio_crypt_ctx() must have already been called on the bio.
318 * Return: true iff the I/O is mergeable
320 bool fscrypt_mergeable_bio(struct bio
*bio
, const struct inode
*inode
,
323 const struct bio_crypt_ctx
*bc
= bio
->bi_crypt_context
;
324 u64 next_dun
[BLK_CRYPTO_DUN_ARRAY_SIZE
];
326 if (!!bc
!= fscrypt_inode_uses_inline_crypto(inode
))
332 * Comparing the key pointers is good enough, as all I/O for each key
333 * uses the same pointer. I.e., there's currently no need to support
334 * merging requests where the keys are the same but the pointers differ.
336 if (bc
->bc_key
!= &inode
->i_crypt_info
->ci_enc_key
.blk_key
->base
)
339 fscrypt_generate_dun(inode
->i_crypt_info
, next_lblk
, next_dun
);
340 return bio_crypt_dun_is_contiguous(bc
, bio
->bi_iter
.bi_size
, next_dun
);
342 EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio
);
345 * fscrypt_mergeable_bio_bh() - test whether data can be added to a bio
346 * @bio: the bio being built up
347 * @next_bh: the next buffer_head for which I/O will be submitted
349 * Same as fscrypt_mergeable_bio(), except this takes a buffer_head instead of
350 * an inode and block number directly.
352 * Return: true iff the I/O is mergeable
354 bool fscrypt_mergeable_bio_bh(struct bio
*bio
,
355 const struct buffer_head
*next_bh
)
357 const struct inode
*inode
;
360 if (!bh_get_inode_and_lblk_num(next_bh
, &inode
, &next_lblk
))
361 return !bio
->bi_crypt_context
;
363 return fscrypt_mergeable_bio(bio
, inode
, next_lblk
);
365 EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio_bh
);