1 // SPDX-License-Identifier: GPL-2.0
3 * bio-integrity.c - bio data integrity extensions
5 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
6 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
9 #include <linux/blkdev.h>
10 #include <linux/mempool.h>
11 #include <linux/export.h>
12 #include <linux/bio.h>
13 #include <linux/workqueue.h>
14 #include <linux/slab.h>
17 static struct kmem_cache
*bip_slab
;
18 static struct workqueue_struct
*kintegrityd_wq
;
20 void blk_flush_integrity(void)
22 flush_workqueue(kintegrityd_wq
);
25 static void __bio_integrity_free(struct bio_set
*bs
,
26 struct bio_integrity_payload
*bip
)
28 if (bs
&& mempool_initialized(&bs
->bio_integrity_pool
)) {
30 bvec_free(&bs
->bvec_integrity_pool
, bip
->bip_vec
,
32 mempool_free(bip
, &bs
->bio_integrity_pool
);
39 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
40 * @bio: bio to attach integrity metadata to
41 * @gfp_mask: Memory allocation mask
42 * @nr_vecs: Number of integrity metadata scatter-gather elements
44 * Description: This function prepares a bio for attaching integrity
45 * metadata. nr_vecs specifies the maximum number of pages containing
46 * integrity metadata that can be attached.
48 struct bio_integrity_payload
*bio_integrity_alloc(struct bio
*bio
,
52 struct bio_integrity_payload
*bip
;
53 struct bio_set
*bs
= bio
->bi_pool
;
56 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio
)))
57 return ERR_PTR(-EOPNOTSUPP
);
59 if (!bs
|| !mempool_initialized(&bs
->bio_integrity_pool
)) {
60 bip
= kmalloc(struct_size(bip
, bip_inline_vecs
, nr_vecs
), gfp_mask
);
61 inline_vecs
= nr_vecs
;
63 bip
= mempool_alloc(&bs
->bio_integrity_pool
, gfp_mask
);
64 inline_vecs
= BIO_INLINE_VECS
;
68 return ERR_PTR(-ENOMEM
);
70 memset(bip
, 0, sizeof(*bip
));
72 if (nr_vecs
> inline_vecs
) {
73 bip
->bip_max_vcnt
= nr_vecs
;
74 bip
->bip_vec
= bvec_alloc(&bs
->bvec_integrity_pool
,
75 &bip
->bip_max_vcnt
, gfp_mask
);
79 bip
->bip_vec
= bip
->bip_inline_vecs
;
80 bip
->bip_max_vcnt
= inline_vecs
;
84 bio
->bi_integrity
= bip
;
85 bio
->bi_opf
|= REQ_INTEGRITY
;
89 __bio_integrity_free(bs
, bip
);
90 return ERR_PTR(-ENOMEM
);
92 EXPORT_SYMBOL(bio_integrity_alloc
);
95 * bio_integrity_free - Free bio integrity payload
96 * @bio: bio containing bip to be freed
98 * Description: Used to free the integrity portion of a bio. Usually
99 * called from bio_free().
101 void bio_integrity_free(struct bio
*bio
)
103 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
104 struct bio_set
*bs
= bio
->bi_pool
;
106 if (bip
->bip_flags
& BIP_BLOCK_INTEGRITY
)
107 kfree(page_address(bip
->bip_vec
->bv_page
) +
108 bip
->bip_vec
->bv_offset
);
110 __bio_integrity_free(bs
, bip
);
111 bio
->bi_integrity
= NULL
;
112 bio
->bi_opf
&= ~REQ_INTEGRITY
;
116 * bio_integrity_add_page - Attach integrity metadata
117 * @bio: bio to update
118 * @page: page containing integrity metadata
119 * @len: number of bytes of integrity metadata in page
120 * @offset: start offset within page
122 * Description: Attach a page containing integrity metadata to bio.
124 int bio_integrity_add_page(struct bio
*bio
, struct page
*page
,
125 unsigned int len
, unsigned int offset
)
127 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
130 if (bip
->bip_vcnt
>= bip
->bip_max_vcnt
) {
131 printk(KERN_ERR
"%s: bip_vec full\n", __func__
);
135 iv
= bip
->bip_vec
+ bip
->bip_vcnt
;
138 bvec_gap_to_prev(bio
->bi_bdev
->bd_disk
->queue
,
139 &bip
->bip_vec
[bip
->bip_vcnt
- 1], offset
))
144 iv
->bv_offset
= offset
;
149 EXPORT_SYMBOL(bio_integrity_add_page
);
152 * bio_integrity_process - Process integrity metadata for a bio
153 * @bio: bio to generate/verify integrity metadata for
154 * @proc_iter: iterator to process
155 * @proc_fn: Pointer to the relevant processing function
157 static blk_status_t
bio_integrity_process(struct bio
*bio
,
158 struct bvec_iter
*proc_iter
, integrity_processing_fn
*proc_fn
)
160 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_bdev
->bd_disk
);
161 struct blk_integrity_iter iter
;
162 struct bvec_iter bviter
;
164 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
165 blk_status_t ret
= BLK_STS_OK
;
166 void *prot_buf
= page_address(bip
->bip_vec
->bv_page
) +
167 bip
->bip_vec
->bv_offset
;
169 iter
.disk_name
= bio
->bi_bdev
->bd_disk
->disk_name
;
170 iter
.interval
= 1 << bi
->interval_exp
;
171 iter
.seed
= proc_iter
->bi_sector
;
172 iter
.prot_buf
= prot_buf
;
174 __bio_for_each_segment(bv
, bio
, bviter
, *proc_iter
) {
175 void *kaddr
= kmap_atomic(bv
.bv_page
);
177 iter
.data_buf
= kaddr
+ bv
.bv_offset
;
178 iter
.data_size
= bv
.bv_len
;
180 ret
= proc_fn(&iter
);
182 kunmap_atomic(kaddr
);
186 kunmap_atomic(kaddr
);
192 * bio_integrity_prep - Prepare bio for integrity I/O
193 * @bio: bio to prepare
195 * Description: Checks if the bio already has an integrity payload attached.
196 * If it does, the payload has been generated by another kernel subsystem,
197 * and we just pass it through. Otherwise allocates integrity payload.
198 * The bio must have data direction, target device and start sector set priot
199 * to calling. In the WRITE case, integrity metadata will be generated using
200 * the block device's integrity function. In the READ case, the buffer
201 * will be prepared for DMA and a suitable end_io handler set up.
203 bool bio_integrity_prep(struct bio
*bio
)
205 struct bio_integrity_payload
*bip
;
206 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_bdev
->bd_disk
);
207 struct request_queue
*q
= bio
->bi_bdev
->bd_disk
->queue
;
209 unsigned long start
, end
;
210 unsigned int len
, nr_pages
;
211 unsigned int bytes
, offset
, i
;
212 unsigned int intervals
;
218 if (bio_op(bio
) != REQ_OP_READ
&& bio_op(bio
) != REQ_OP_WRITE
)
221 if (!bio_sectors(bio
))
224 /* Already protected? */
225 if (bio_integrity(bio
))
228 if (bio_data_dir(bio
) == READ
) {
229 if (!bi
->profile
->verify_fn
||
230 !(bi
->flags
& BLK_INTEGRITY_VERIFY
))
233 if (!bi
->profile
->generate_fn
||
234 !(bi
->flags
& BLK_INTEGRITY_GENERATE
))
237 intervals
= bio_integrity_intervals(bi
, bio_sectors(bio
));
239 /* Allocate kernel buffer for protection data */
240 len
= intervals
* bi
->tuple_size
;
241 buf
= kmalloc(len
, GFP_NOIO
| q
->bounce_gfp
);
242 status
= BLK_STS_RESOURCE
;
243 if (unlikely(buf
== NULL
)) {
244 printk(KERN_ERR
"could not allocate integrity buffer\n");
248 end
= (((unsigned long) buf
) + len
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
249 start
= ((unsigned long) buf
) >> PAGE_SHIFT
;
250 nr_pages
= end
- start
;
252 /* Allocate bio integrity payload and integrity vectors */
253 bip
= bio_integrity_alloc(bio
, GFP_NOIO
, nr_pages
);
255 printk(KERN_ERR
"could not allocate data integrity bioset\n");
257 status
= BLK_STS_RESOURCE
;
261 bip
->bip_flags
|= BIP_BLOCK_INTEGRITY
;
262 bip
->bip_iter
.bi_size
= len
;
263 bip_set_seed(bip
, bio
->bi_iter
.bi_sector
);
265 if (bi
->flags
& BLK_INTEGRITY_IP_CHECKSUM
)
266 bip
->bip_flags
|= BIP_IP_CHECKSUM
;
269 offset
= offset_in_page(buf
);
270 for (i
= 0 ; i
< nr_pages
; i
++) {
272 bytes
= PAGE_SIZE
- offset
;
280 ret
= bio_integrity_add_page(bio
, virt_to_page(buf
),
284 printk(KERN_ERR
"could not attach integrity payload\n");
285 status
= BLK_STS_RESOURCE
;
297 /* Auto-generate integrity metadata if this is a write */
298 if (bio_data_dir(bio
) == WRITE
) {
299 bio_integrity_process(bio
, &bio
->bi_iter
,
300 bi
->profile
->generate_fn
);
302 bip
->bio_iter
= bio
->bi_iter
;
307 bio
->bi_status
= status
;
312 EXPORT_SYMBOL(bio_integrity_prep
);
315 * bio_integrity_verify_fn - Integrity I/O completion worker
316 * @work: Work struct stored in bio to be verified
318 * Description: This workqueue function is called to complete a READ
319 * request. The function verifies the transferred integrity metadata
320 * and then calls the original bio end_io function.
322 static void bio_integrity_verify_fn(struct work_struct
*work
)
324 struct bio_integrity_payload
*bip
=
325 container_of(work
, struct bio_integrity_payload
, bip_work
);
326 struct bio
*bio
= bip
->bip_bio
;
327 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_bdev
->bd_disk
);
330 * At the moment verify is called bio's iterator was advanced
331 * during split and completion, we need to rewind iterator to
332 * it's original position.
334 bio
->bi_status
= bio_integrity_process(bio
, &bip
->bio_iter
,
335 bi
->profile
->verify_fn
);
336 bio_integrity_free(bio
);
341 * __bio_integrity_endio - Integrity I/O completion function
342 * @bio: Protected bio
344 * Description: Completion for integrity I/O
346 * Normally I/O completion is done in interrupt context. However,
347 * verifying I/O integrity is a time-consuming task which must be run
348 * in process context. This function postpones completion
351 bool __bio_integrity_endio(struct bio
*bio
)
353 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_bdev
->bd_disk
);
354 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
356 if (bio_op(bio
) == REQ_OP_READ
&& !bio
->bi_status
&&
357 (bip
->bip_flags
& BIP_BLOCK_INTEGRITY
) && bi
->profile
->verify_fn
) {
358 INIT_WORK(&bip
->bip_work
, bio_integrity_verify_fn
);
359 queue_work(kintegrityd_wq
, &bip
->bip_work
);
363 bio_integrity_free(bio
);
368 * bio_integrity_advance - Advance integrity vector
369 * @bio: bio whose integrity vector to update
370 * @bytes_done: number of data bytes that have been completed
372 * Description: This function calculates how many integrity bytes the
373 * number of completed data bytes correspond to and advances the
374 * integrity vector accordingly.
376 void bio_integrity_advance(struct bio
*bio
, unsigned int bytes_done
)
378 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
379 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_bdev
->bd_disk
);
380 unsigned bytes
= bio_integrity_bytes(bi
, bytes_done
>> 9);
382 bip
->bip_iter
.bi_sector
+= bytes_done
>> 9;
383 bvec_iter_advance(bip
->bip_vec
, &bip
->bip_iter
, bytes
);
387 * bio_integrity_trim - Trim integrity vector
388 * @bio: bio whose integrity vector to update
390 * Description: Used to trim the integrity vector in a cloned bio.
392 void bio_integrity_trim(struct bio
*bio
)
394 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
395 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_bdev
->bd_disk
);
397 bip
->bip_iter
.bi_size
= bio_integrity_bytes(bi
, bio_sectors(bio
));
399 EXPORT_SYMBOL(bio_integrity_trim
);
402 * bio_integrity_clone - Callback for cloning bios with integrity metadata
404 * @bio_src: Original bio
405 * @gfp_mask: Memory allocation mask
407 * Description: Called to allocate a bip when cloning a bio
409 int bio_integrity_clone(struct bio
*bio
, struct bio
*bio_src
,
412 struct bio_integrity_payload
*bip_src
= bio_integrity(bio_src
);
413 struct bio_integrity_payload
*bip
;
415 BUG_ON(bip_src
== NULL
);
417 bip
= bio_integrity_alloc(bio
, gfp_mask
, bip_src
->bip_vcnt
);
421 memcpy(bip
->bip_vec
, bip_src
->bip_vec
,
422 bip_src
->bip_vcnt
* sizeof(struct bio_vec
));
424 bip
->bip_vcnt
= bip_src
->bip_vcnt
;
425 bip
->bip_iter
= bip_src
->bip_iter
;
429 EXPORT_SYMBOL(bio_integrity_clone
);
431 int bioset_integrity_create(struct bio_set
*bs
, int pool_size
)
433 if (mempool_initialized(&bs
->bio_integrity_pool
))
436 if (mempool_init_slab_pool(&bs
->bio_integrity_pool
,
437 pool_size
, bip_slab
))
440 if (biovec_init_pool(&bs
->bvec_integrity_pool
, pool_size
)) {
441 mempool_exit(&bs
->bio_integrity_pool
);
447 EXPORT_SYMBOL(bioset_integrity_create
);
449 void bioset_integrity_free(struct bio_set
*bs
)
451 mempool_exit(&bs
->bio_integrity_pool
);
452 mempool_exit(&bs
->bvec_integrity_pool
);
455 void __init
bio_integrity_init(void)
458 * kintegrityd won't block much but may burn a lot of CPU cycles.
459 * Make it highpri CPU intensive wq with max concurrency of 1.
461 kintegrityd_wq
= alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM
|
462 WQ_HIGHPRI
| WQ_CPU_INTENSIVE
, 1);
464 panic("Failed to create kintegrityd\n");
466 bip_slab
= kmem_cache_create("bio_integrity_payload",
467 sizeof(struct bio_integrity_payload
) +
468 sizeof(struct bio_vec
) * BIO_INLINE_VECS
,
469 0, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
, NULL
);