1 // SPDX-License-Identifier: GPL-2.0
3 * bio-integrity.c - bio data integrity extensions
5 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
6 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
9 #include <linux/blkdev.h>
10 #include <linux/mempool.h>
11 #include <linux/export.h>
12 #include <linux/bio.h>
13 #include <linux/workqueue.h>
14 #include <linux/slab.h>
17 #define BIP_INLINE_VECS 4
19 static struct kmem_cache
*bip_slab
;
20 static struct workqueue_struct
*kintegrityd_wq
;
22 void blk_flush_integrity(void)
24 flush_workqueue(kintegrityd_wq
);
28 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
29 * @bio: bio to attach integrity metadata to
30 * @gfp_mask: Memory allocation mask
31 * @nr_vecs: Number of integrity metadata scatter-gather elements
33 * Description: This function prepares a bio for attaching integrity
34 * metadata. nr_vecs specifies the maximum number of pages containing
35 * integrity metadata that can be attached.
37 struct bio_integrity_payload
*bio_integrity_alloc(struct bio
*bio
,
41 struct bio_integrity_payload
*bip
;
42 struct bio_set
*bs
= bio
->bi_pool
;
45 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio
)))
46 return ERR_PTR(-EOPNOTSUPP
);
48 if (!bs
|| !mempool_initialized(&bs
->bio_integrity_pool
)) {
49 bip
= kmalloc(struct_size(bip
, bip_inline_vecs
, nr_vecs
), gfp_mask
);
50 inline_vecs
= nr_vecs
;
52 bip
= mempool_alloc(&bs
->bio_integrity_pool
, gfp_mask
);
53 inline_vecs
= BIP_INLINE_VECS
;
57 return ERR_PTR(-ENOMEM
);
59 memset(bip
, 0, sizeof(*bip
));
61 if (nr_vecs
> inline_vecs
) {
62 unsigned long idx
= 0;
64 bip
->bip_vec
= bvec_alloc(gfp_mask
, nr_vecs
, &idx
,
65 &bs
->bvec_integrity_pool
);
68 bip
->bip_max_vcnt
= bvec_nr_vecs(idx
);
71 bip
->bip_vec
= bip
->bip_inline_vecs
;
72 bip
->bip_max_vcnt
= inline_vecs
;
76 bio
->bi_integrity
= bip
;
77 bio
->bi_opf
|= REQ_INTEGRITY
;
81 mempool_free(bip
, &bs
->bio_integrity_pool
);
82 return ERR_PTR(-ENOMEM
);
84 EXPORT_SYMBOL(bio_integrity_alloc
);
87 * bio_integrity_free - Free bio integrity payload
88 * @bio: bio containing bip to be freed
90 * Description: Used to free the integrity portion of a bio. Usually
91 * called from bio_free().
93 void bio_integrity_free(struct bio
*bio
)
95 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
96 struct bio_set
*bs
= bio
->bi_pool
;
98 if (bip
->bip_flags
& BIP_BLOCK_INTEGRITY
)
99 kfree(page_address(bip
->bip_vec
->bv_page
) +
100 bip
->bip_vec
->bv_offset
);
102 if (bs
&& mempool_initialized(&bs
->bio_integrity_pool
)) {
103 bvec_free(&bs
->bvec_integrity_pool
, bip
->bip_vec
, bip
->bip_slab
);
105 mempool_free(bip
, &bs
->bio_integrity_pool
);
110 bio
->bi_integrity
= NULL
;
111 bio
->bi_opf
&= ~REQ_INTEGRITY
;
115 * bio_integrity_add_page - Attach integrity metadata
116 * @bio: bio to update
117 * @page: page containing integrity metadata
118 * @len: number of bytes of integrity metadata in page
119 * @offset: start offset within page
121 * Description: Attach a page containing integrity metadata to bio.
123 int bio_integrity_add_page(struct bio
*bio
, struct page
*page
,
124 unsigned int len
, unsigned int offset
)
126 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
129 if (bip
->bip_vcnt
>= bip
->bip_max_vcnt
) {
130 printk(KERN_ERR
"%s: bip_vec full\n", __func__
);
134 iv
= bip
->bip_vec
+ bip
->bip_vcnt
;
137 bvec_gap_to_prev(bio
->bi_disk
->queue
,
138 &bip
->bip_vec
[bip
->bip_vcnt
- 1], offset
))
143 iv
->bv_offset
= offset
;
148 EXPORT_SYMBOL(bio_integrity_add_page
);
151 * bio_integrity_process - Process integrity metadata for a bio
152 * @bio: bio to generate/verify integrity metadata for
153 * @proc_iter: iterator to process
154 * @proc_fn: Pointer to the relevant processing function
156 static blk_status_t
bio_integrity_process(struct bio
*bio
,
157 struct bvec_iter
*proc_iter
, integrity_processing_fn
*proc_fn
)
159 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
160 struct blk_integrity_iter iter
;
161 struct bvec_iter bviter
;
163 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
164 blk_status_t ret
= BLK_STS_OK
;
165 void *prot_buf
= page_address(bip
->bip_vec
->bv_page
) +
166 bip
->bip_vec
->bv_offset
;
168 iter
.disk_name
= bio
->bi_disk
->disk_name
;
169 iter
.interval
= 1 << bi
->interval_exp
;
170 iter
.seed
= proc_iter
->bi_sector
;
171 iter
.prot_buf
= prot_buf
;
173 __bio_for_each_segment(bv
, bio
, bviter
, *proc_iter
) {
174 void *kaddr
= kmap_atomic(bv
.bv_page
);
176 iter
.data_buf
= kaddr
+ bv
.bv_offset
;
177 iter
.data_size
= bv
.bv_len
;
179 ret
= proc_fn(&iter
);
181 kunmap_atomic(kaddr
);
185 kunmap_atomic(kaddr
);
191 * bio_integrity_prep - Prepare bio for integrity I/O
192 * @bio: bio to prepare
194 * Description: Checks if the bio already has an integrity payload attached.
195 * If it does, the payload has been generated by another kernel subsystem,
196 * and we just pass it through. Otherwise allocates integrity payload.
197 * The bio must have data direction, target device and start sector set priot
198 * to calling. In the WRITE case, integrity metadata will be generated using
199 * the block device's integrity function. In the READ case, the buffer
200 * will be prepared for DMA and a suitable end_io handler set up.
202 bool bio_integrity_prep(struct bio
*bio
)
204 struct bio_integrity_payload
*bip
;
205 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
206 struct request_queue
*q
= bio
->bi_disk
->queue
;
208 unsigned long start
, end
;
209 unsigned int len
, nr_pages
;
210 unsigned int bytes
, offset
, i
;
211 unsigned int intervals
;
217 if (bio_op(bio
) != REQ_OP_READ
&& bio_op(bio
) != REQ_OP_WRITE
)
220 if (!bio_sectors(bio
))
223 /* Already protected? */
224 if (bio_integrity(bio
))
227 if (bio_data_dir(bio
) == READ
) {
228 if (!bi
->profile
->verify_fn
||
229 !(bi
->flags
& BLK_INTEGRITY_VERIFY
))
232 if (!bi
->profile
->generate_fn
||
233 !(bi
->flags
& BLK_INTEGRITY_GENERATE
))
236 intervals
= bio_integrity_intervals(bi
, bio_sectors(bio
));
238 /* Allocate kernel buffer for protection data */
239 len
= intervals
* bi
->tuple_size
;
240 buf
= kmalloc(len
, GFP_NOIO
| q
->bounce_gfp
);
241 status
= BLK_STS_RESOURCE
;
242 if (unlikely(buf
== NULL
)) {
243 printk(KERN_ERR
"could not allocate integrity buffer\n");
247 end
= (((unsigned long) buf
) + len
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
248 start
= ((unsigned long) buf
) >> PAGE_SHIFT
;
249 nr_pages
= end
- start
;
251 /* Allocate bio integrity payload and integrity vectors */
252 bip
= bio_integrity_alloc(bio
, GFP_NOIO
, nr_pages
);
254 printk(KERN_ERR
"could not allocate data integrity bioset\n");
256 status
= BLK_STS_RESOURCE
;
260 bip
->bip_flags
|= BIP_BLOCK_INTEGRITY
;
261 bip
->bip_iter
.bi_size
= len
;
262 bip_set_seed(bip
, bio
->bi_iter
.bi_sector
);
264 if (bi
->flags
& BLK_INTEGRITY_IP_CHECKSUM
)
265 bip
->bip_flags
|= BIP_IP_CHECKSUM
;
268 offset
= offset_in_page(buf
);
269 for (i
= 0 ; i
< nr_pages
; i
++) {
271 bytes
= PAGE_SIZE
- offset
;
279 ret
= bio_integrity_add_page(bio
, virt_to_page(buf
),
283 printk(KERN_ERR
"could not attach integrity payload\n");
284 status
= BLK_STS_RESOURCE
;
296 /* Auto-generate integrity metadata if this is a write */
297 if (bio_data_dir(bio
) == WRITE
) {
298 bio_integrity_process(bio
, &bio
->bi_iter
,
299 bi
->profile
->generate_fn
);
301 bip
->bio_iter
= bio
->bi_iter
;
306 bio
->bi_status
= status
;
311 EXPORT_SYMBOL(bio_integrity_prep
);
314 * bio_integrity_verify_fn - Integrity I/O completion worker
315 * @work: Work struct stored in bio to be verified
317 * Description: This workqueue function is called to complete a READ
318 * request. The function verifies the transferred integrity metadata
319 * and then calls the original bio end_io function.
321 static void bio_integrity_verify_fn(struct work_struct
*work
)
323 struct bio_integrity_payload
*bip
=
324 container_of(work
, struct bio_integrity_payload
, bip_work
);
325 struct bio
*bio
= bip
->bip_bio
;
326 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
329 * At the moment verify is called bio's iterator was advanced
330 * during split and completion, we need to rewind iterator to
331 * it's original position.
333 bio
->bi_status
= bio_integrity_process(bio
, &bip
->bio_iter
,
334 bi
->profile
->verify_fn
);
335 bio_integrity_free(bio
);
340 * __bio_integrity_endio - Integrity I/O completion function
341 * @bio: Protected bio
343 * Description: Completion for integrity I/O
345 * Normally I/O completion is done in interrupt context. However,
346 * verifying I/O integrity is a time-consuming task which must be run
347 * in process context. This function postpones completion
350 bool __bio_integrity_endio(struct bio
*bio
)
352 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
353 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
355 if (bio_op(bio
) == REQ_OP_READ
&& !bio
->bi_status
&&
356 (bip
->bip_flags
& BIP_BLOCK_INTEGRITY
) && bi
->profile
->verify_fn
) {
357 INIT_WORK(&bip
->bip_work
, bio_integrity_verify_fn
);
358 queue_work(kintegrityd_wq
, &bip
->bip_work
);
362 bio_integrity_free(bio
);
367 * bio_integrity_advance - Advance integrity vector
368 * @bio: bio whose integrity vector to update
369 * @bytes_done: number of data bytes that have been completed
371 * Description: This function calculates how many integrity bytes the
372 * number of completed data bytes correspond to and advances the
373 * integrity vector accordingly.
375 void bio_integrity_advance(struct bio
*bio
, unsigned int bytes_done
)
377 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
378 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
379 unsigned bytes
= bio_integrity_bytes(bi
, bytes_done
>> 9);
381 bip
->bip_iter
.bi_sector
+= bytes_done
>> 9;
382 bvec_iter_advance(bip
->bip_vec
, &bip
->bip_iter
, bytes
);
386 * bio_integrity_trim - Trim integrity vector
387 * @bio: bio whose integrity vector to update
389 * Description: Used to trim the integrity vector in a cloned bio.
391 void bio_integrity_trim(struct bio
*bio
)
393 struct bio_integrity_payload
*bip
= bio_integrity(bio
);
394 struct blk_integrity
*bi
= blk_get_integrity(bio
->bi_disk
);
396 bip
->bip_iter
.bi_size
= bio_integrity_bytes(bi
, bio_sectors(bio
));
398 EXPORT_SYMBOL(bio_integrity_trim
);
401 * bio_integrity_clone - Callback for cloning bios with integrity metadata
403 * @bio_src: Original bio
404 * @gfp_mask: Memory allocation mask
406 * Description: Called to allocate a bip when cloning a bio
408 int bio_integrity_clone(struct bio
*bio
, struct bio
*bio_src
,
411 struct bio_integrity_payload
*bip_src
= bio_integrity(bio_src
);
412 struct bio_integrity_payload
*bip
;
414 BUG_ON(bip_src
== NULL
);
416 bip
= bio_integrity_alloc(bio
, gfp_mask
, bip_src
->bip_vcnt
);
420 memcpy(bip
->bip_vec
, bip_src
->bip_vec
,
421 bip_src
->bip_vcnt
* sizeof(struct bio_vec
));
423 bip
->bip_vcnt
= bip_src
->bip_vcnt
;
424 bip
->bip_iter
= bip_src
->bip_iter
;
428 EXPORT_SYMBOL(bio_integrity_clone
);
430 int bioset_integrity_create(struct bio_set
*bs
, int pool_size
)
432 if (mempool_initialized(&bs
->bio_integrity_pool
))
435 if (mempool_init_slab_pool(&bs
->bio_integrity_pool
,
436 pool_size
, bip_slab
))
439 if (biovec_init_pool(&bs
->bvec_integrity_pool
, pool_size
)) {
440 mempool_exit(&bs
->bio_integrity_pool
);
446 EXPORT_SYMBOL(bioset_integrity_create
);
448 void bioset_integrity_free(struct bio_set
*bs
)
450 mempool_exit(&bs
->bio_integrity_pool
);
451 mempool_exit(&bs
->bvec_integrity_pool
);
454 void __init
bio_integrity_init(void)
457 * kintegrityd won't block much but may burn a lot of CPU cycles.
458 * Make it highpri CPU intensive wq with max concurrency of 1.
460 kintegrityd_wq
= alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM
|
461 WQ_HIGHPRI
| WQ_CPU_INTENSIVE
, 1);
463 panic("Failed to create kintegrityd\n");
465 bip_slab
= kmem_cache_create("bio_integrity_payload",
466 sizeof(struct bio_integrity_payload
) +
467 sizeof(struct bio_vec
) * BIP_INLINE_VECS
,
468 0, SLAB_HWCACHE_ALIGN
|SLAB_PANIC
, NULL
);