1 // SPDX-License-Identifier: GPL-2.0
3 * bio-integrity.c - bio data integrity extensions
5 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
6 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
9 #include <linux/blkdev.h>
10 #include <linux/mempool.h>
11 #include <linux/export.h>
12 #include <linux/bio.h>
13 #include <linux/workqueue.h>
14 #include <linux/slab.h>
17 #define BIP_INLINE_VECS 4
19 static struct kmem_cache
*bip_slab
;
20 static struct workqueue_struct
*kintegrityd_wq
;
22 void blk_flush_integrity(void)
24 flush_workqueue(kintegrityd_wq
);
27 void __bio_integrity_free(struct bio_set
*bs
, struct bio_integrity_payload
*bip
)
29 if (bs
&& mempool_initialized(&bs
->bio_integrity_pool
)) {
31 bvec_free(&bs
->bvec_integrity_pool
, bip
->bip_vec
,
33 mempool_free(bip
, &bs
->bio_integrity_pool
);
40 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
41 * @bio: bio to attach integrity metadata to
42 * @gfp_mask: Memory allocation mask
43 * @nr_vecs: Number of integrity metadata scatter-gather elements
45 * Description: This function prepares a bio for attaching integrity
46 * metadata. nr_vecs specifies the maximum number of pages containing
47 * integrity metadata that can be attached.
49 struct bio_integrity_payload
*bio_integrity_alloc(struct bio
*bio
,
53 struct bio_integrity_payload
*bip
;
54 struct bio_set
*bs
= bio
->bi_pool
;
57 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio
)))
58 return ERR_PTR(-EOPNOTSUPP
);
60 if (!bs
|| !mempool_initialized(&bs
->bio_integrity_pool
)) {
61 bip
= kmalloc(struct_size(bip
, bip_inline_vecs
, nr_vecs
), gfp_mask
);
62 inline_vecs
= nr_vecs
;
64 bip
= mempool_alloc(&bs
->bio_integrity_pool
, gfp_mask
);
65 inline_vecs
= BIP_INLINE_VECS
;
69 return ERR_PTR(-ENOMEM
);
71 memset(bip
, 0, sizeof(*bip
));
73 if (nr_vecs
> inline_vecs
) {
74 unsigned long idx
= 0;
76 bip
->bip_vec
= bvec_alloc(gfp_mask
, nr_vecs
, &idx
,
77 &bs
->bvec_integrity_pool
);
80 bip
->bip_max_vcnt
= bvec_nr_vecs(idx
);
83 bip
->bip_vec
= bip
->bip_inline_vecs
;
84 bip
->bip_max_vcnt
= inline_vecs
;
88 bio
->bi_integrity
= bip
;
89 bio
->bi_opf
|= REQ_INTEGRITY
;
93 __bio_integrity_free(bs
, bip
);
94 return ERR_PTR(-ENOMEM
);
96 EXPORT_SYMBOL(bio_integrity_alloc
);
99 * bio_integrity_free - Free bio integrity payload
100 * @bio: bio containing bip to be freed
102 * Description: Used to free the integrity portion of a bio. Usually
103 * called from bio_free().
105 void bio_integrity_free(struct bio
*bio
)
107 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
108 struct bio_set
*bs
= bio
->bi_pool
;
110 if (bip
->bip_flags
& BIP_BLOCK_INTEGRITY
)
111 kfree(page_address(bip
->bip_vec
->bv_page
) +
112 bip
->bip_vec
->bv_offset
);
114 __bio_integrity_free(bs
, bip
);
115 bio
->bi_integrity
= NULL
;
116 bio
->bi_opf
&= ~REQ_INTEGRITY
;
120 * bio_integrity_add_page - Attach integrity metadata
121 * @bio: bio to update
122 * @page: page containing integrity metadata
123 * @len: number of bytes of integrity metadata in page
124 * @offset: start offset within page
126 * Description: Attach a page containing integrity metadata to bio.
128 int bio_integrity_add_page(struct bio
*bio
, struct page
*page
,
129 unsigned int len
, unsigned int offset
)
131 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
134 if (bip
->bip_vcnt
>= bip
->bip_max_vcnt
) {
135 printk(KERN_ERR
"%s: bip_vec full\n", __func__
);
139 iv
= bip
->bip_vec
+ bip
->bip_vcnt
;
142 bvec_gap_to_prev(bio
->bi_disk
->queue
,
143 &bip
->bip_vec
[bip
->bip_vcnt
- 1], offset
))
148 iv
->bv_offset
= offset
;
153 EXPORT_SYMBOL(bio_integrity_add_page
);
156 * bio_integrity_process - Process integrity metadata for a bio
157 * @bio: bio to generate/verify integrity metadata for
158 * @proc_iter: iterator to process
159 * @proc_fn: Pointer to the relevant processing function
161 static blk_status_t
bio_integrity_process(struct bio
*bio
,
162 struct bvec_iter
*proc_iter
, integrity_processing_fn
*proc_fn
)
164 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
165 struct blk_integrity_iter iter
;
166 struct bvec_iter bviter
;
168 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
169 blk_status_t ret
= BLK_STS_OK
;
170 void *prot_buf
= page_address(bip
->bip_vec
->bv_page
) +
171 bip
->bip_vec
->bv_offset
;
173 iter
.disk_name
= bio
->bi_disk
->disk_name
;
174 iter
.interval
= 1 << bi
->interval_exp
;
175 iter
.seed
= proc_iter
->bi_sector
;
176 iter
.prot_buf
= prot_buf
;
178 __bio_for_each_segment(bv
, bio
, bviter
, *proc_iter
) {
179 void *kaddr
= kmap_atomic(bv
.bv_page
);
181 iter
.data_buf
= kaddr
+ bv
.bv_offset
;
182 iter
.data_size
= bv
.bv_len
;
184 ret
= proc_fn(&iter
);
186 kunmap_atomic(kaddr
);
190 kunmap_atomic(kaddr
);
196 * bio_integrity_prep - Prepare bio for integrity I/O
197 * @bio: bio to prepare
199 * Description: Checks if the bio already has an integrity payload attached.
200 * If it does, the payload has been generated by another kernel subsystem,
201 * and we just pass it through. Otherwise allocates integrity payload.
202 * The bio must have data direction, target device and start sector set priot
203 * to calling. In the WRITE case, integrity metadata will be generated using
204 * the block device's integrity function. In the READ case, the buffer
205 * will be prepared for DMA and a suitable end_io handler set up.
207 bool bio_integrity_prep(struct bio
*bio
)
209 struct bio_integrity_payload
*bip
;
210 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
211 struct request_queue
*q
= bio
->bi_disk
->queue
;
213 unsigned long start
, end
;
214 unsigned int len
, nr_pages
;
215 unsigned int bytes
, offset
, i
;
216 unsigned int intervals
;
222 if (bio_op(bio
) != REQ_OP_READ
&& bio_op(bio
) != REQ_OP_WRITE
)
225 if (!bio_sectors(bio
))
228 /* Already protected? */
229 if (bio_integrity(bio
))
232 if (bio_data_dir(bio
) == READ
) {
233 if (!bi
->profile
->verify_fn
||
234 !(bi
->flags
& BLK_INTEGRITY_VERIFY
))
237 if (!bi
->profile
->generate_fn
||
238 !(bi
->flags
& BLK_INTEGRITY_GENERATE
))
241 intervals
= bio_integrity_intervals(bi
, bio_sectors(bio
));
243 /* Allocate kernel buffer for protection data */
244 len
= intervals
* bi
->tuple_size
;
245 buf
= kmalloc(len
, GFP_NOIO
| q
->bounce_gfp
);
246 status
= BLK_STS_RESOURCE
;
247 if (unlikely(buf
== NULL
)) {
248 printk(KERN_ERR
"could not allocate integrity buffer\n");
252 end
= (((unsigned long) buf
) + len
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
253 start
= ((unsigned long) buf
) >> PAGE_SHIFT
;
254 nr_pages
= end
- start
;
256 /* Allocate bio integrity payload and integrity vectors */
257 bip
= bio_integrity_alloc(bio
, GFP_NOIO
, nr_pages
);
259 printk(KERN_ERR
"could not allocate data integrity bioset\n");
261 status
= BLK_STS_RESOURCE
;
265 bip
->bip_flags
|= BIP_BLOCK_INTEGRITY
;
266 bip
->bip_iter
.bi_size
= len
;
267 bip_set_seed(bip
, bio
->bi_iter
.bi_sector
);
269 if (bi
->flags
& BLK_INTEGRITY_IP_CHECKSUM
)
270 bip
->bip_flags
|= BIP_IP_CHECKSUM
;
273 offset
= offset_in_page(buf
);
274 for (i
= 0 ; i
< nr_pages
; i
++) {
276 bytes
= PAGE_SIZE
- offset
;
284 ret
= bio_integrity_add_page(bio
, virt_to_page(buf
),
288 printk(KERN_ERR
"could not attach integrity payload\n");
289 status
= BLK_STS_RESOURCE
;
301 /* Auto-generate integrity metadata if this is a write */
302 if (bio_data_dir(bio
) == WRITE
) {
303 bio_integrity_process(bio
, &bio
->bi_iter
,
304 bi
->profile
->generate_fn
);
306 bip
->bio_iter
= bio
->bi_iter
;
311 bio
->bi_status
= status
;
316 EXPORT_SYMBOL(bio_integrity_prep
);
319 * bio_integrity_verify_fn - Integrity I/O completion worker
320 * @work: Work struct stored in bio to be verified
322 * Description: This workqueue function is called to complete a READ
323 * request. The function verifies the transferred integrity metadata
324 * and then calls the original bio end_io function.
326 static void bio_integrity_verify_fn(struct work_struct
*work
)
328 struct bio_integrity_payload
*bip
=
329 container_of(work
, struct bio_integrity_payload
, bip_work
);
330 struct bio
*bio
= bip
->bip_bio
;
331 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
334 * At the moment verify is called bio's iterator was advanced
335 * during split and completion, we need to rewind iterator to
336 * it's original position.
338 bio
->bi_status
= bio_integrity_process(bio
, &bip
->bio_iter
,
339 bi
->profile
->verify_fn
);
340 bio_integrity_free(bio
);
345 * __bio_integrity_endio - Integrity I/O completion function
346 * @bio: Protected bio
348 * Description: Completion for integrity I/O
350 * Normally I/O completion is done in interrupt context. However,
351 * verifying I/O integrity is a time-consuming task which must be run
352 * in process context. This function postpones completion
355 bool __bio_integrity_endio(struct bio
*bio
)
357 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
358 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
360 if (bio_op(bio
) == REQ_OP_READ
&& !bio
->bi_status
&&
361 (bip
->bip_flags
& BIP_BLOCK_INTEGRITY
) && bi
->profile
->verify_fn
) {
362 INIT_WORK(&bip
->bip_work
, bio_integrity_verify_fn
);
363 queue_work(kintegrityd_wq
, &bip
->bip_work
);
367 bio_integrity_free(bio
);
372 * bio_integrity_advance - Advance integrity vector
373 * @bio: bio whose integrity vector to update
374 * @bytes_done: number of data bytes that have been completed
376 * Description: This function calculates how many integrity bytes the
377 * number of completed data bytes correspond to and advances the
378 * integrity vector accordingly.
380 void bio_integrity_advance(struct bio
*bio
, unsigned int bytes_done
)
382 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
383 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
384 unsigned bytes
= bio_integrity_bytes(bi
, bytes_done
>> 9);
386 bip
->bip_iter
.bi_sector
+= bytes_done
>> 9;
387 bvec_iter_advance(bip
->bip_vec
, &bip
->bip_iter
, bytes
);
391 * bio_integrity_trim - Trim integrity vector
392 * @bio: bio whose integrity vector to update
394 * Description: Used to trim the integrity vector in a cloned bio.
396 void bio_integrity_trim(struct bio
*bio
)
398 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
399 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
401 bip
->bip_iter
.bi_size
= bio_integrity_bytes(bi
, bio_sectors(bio
));
403 EXPORT_SYMBOL(bio_integrity_trim
);
406 * bio_integrity_clone - Callback for cloning bios with integrity metadata
408 * @bio_src: Original bio
409 * @gfp_mask: Memory allocation mask
411 * Description: Called to allocate a bip when cloning a bio
413 int bio_integrity_clone(struct bio
*bio
, struct bio
*bio_src
,
416 struct bio_integrity_payload
*bip_src
= bio_integrity(bio_src
);
417 struct bio_integrity_payload
*bip
;
419 BUG_ON(bip_src
== NULL
);
421 bip
= bio_integrity_alloc(bio
, gfp_mask
, bip_src
->bip_vcnt
);
425 memcpy(bip
->bip_vec
, bip_src
->bip_vec
,
426 bip_src
->bip_vcnt
* sizeof(struct bio_vec
));
428 bip
->bip_vcnt
= bip_src
->bip_vcnt
;
429 bip
->bip_iter
= bip_src
->bip_iter
;
433 EXPORT_SYMBOL(bio_integrity_clone
);
435 int bioset_integrity_create(struct bio_set
*bs
, int pool_size
)
437 if (mempool_initialized(&bs
->bio_integrity_pool
))
440 if (mempool_init_slab_pool(&bs
->bio_integrity_pool
,
441 pool_size
, bip_slab
))
444 if (biovec_init_pool(&bs
->bvec_integrity_pool
, pool_size
)) {
445 mempool_exit(&bs
->bio_integrity_pool
);
451 EXPORT_SYMBOL(bioset_integrity_create
);
453 void bioset_integrity_free(struct bio_set
*bs
)
455 mempool_exit(&bs
->bio_integrity_pool
);
456 mempool_exit(&bs
->bvec_integrity_pool
);
459 void __init
bio_integrity_init(void)
462 * kintegrityd won't block much but may burn a lot of CPU cycles.
463 * Make it highpri CPU intensive wq with max concurrency of 1.
465 kintegrityd_wq
= alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM
|
466 WQ_HIGHPRI
| WQ_CPU_INTENSIVE
, 1);
468 panic("Failed to create kintegrityd\n");
470 bip_slab
= kmem_cache_create("bio_integrity_payload",
471 sizeof(struct bio_integrity_payload
) +
472 sizeof(struct bio_vec
) * BIP_INLINE_VECS
,
473 0, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
, NULL
);