]>
git.proxmox.com Git - mirror_ubuntu-kernels.git/blob - block/bounce.c
1 // SPDX-License-Identifier: GPL-2.0
2 /* bounce buffer handling for block devices
4 * - Split from highmem.c
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10 #include <linux/export.h>
11 #include <linux/swap.h>
12 #include <linux/gfp.h>
13 #include <linux/bio.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempool.h>
16 #include <linux/blkdev.h>
17 #include <linux/backing-dev.h>
18 #include <linux/init.h>
19 #include <linux/hash.h>
20 #include <linux/highmem.h>
21 #include <linux/printk.h>
22 #include <asm/tlbflush.h>
24 #include <trace/events/block.h>
28 #define ISA_POOL_SIZE 16
30 static struct bio_set bounce_bio_set
, bounce_bio_split
;
31 static mempool_t page_pool
;
33 static void init_bounce_bioset(void)
35 static bool bounce_bs_setup
;
41 ret
= bioset_init(&bounce_bio_set
, BIO_POOL_SIZE
, 0, BIOSET_NEED_BVECS
);
43 if (bioset_integrity_create(&bounce_bio_set
, BIO_POOL_SIZE
))
46 ret
= bioset_init(&bounce_bio_split
, BIO_POOL_SIZE
, 0, 0);
48 bounce_bs_setup
= true;
51 static __init
int init_emergency_pool(void)
55 #ifndef CONFIG_MEMORY_HOTPLUG
56 if (max_pfn
<= max_low_pfn
)
60 ret
= mempool_init_page_pool(&page_pool
, POOL_SIZE
, 0);
62 pr_info("pool size: %d pages\n", POOL_SIZE
);
68 __initcall(init_emergency_pool
);
71 * highmem version, map in to vec
73 static void bounce_copy_vec(struct bio_vec
*to
, unsigned char *vfrom
)
77 vto
= kmap_atomic(to
->bv_page
);
78 memcpy(vto
+ to
->bv_offset
, vfrom
, to
->bv_len
);
83 * Simple bounce buffer support for highmem pages. Depending on the
84 * queue gfp mask set, *to may or may not be a highmem page. kmap it
85 * always, it will do the Right Thing
87 static void copy_to_high_bio_irq(struct bio
*to
, struct bio
*from
)
90 struct bio_vec tovec
, fromvec
;
91 struct bvec_iter iter
;
93 * The bio of @from is created by bounce, so we can iterate
94 * its bvec from start to end, but the @from->bi_iter can't be
95 * trusted because it might be changed by splitting.
97 struct bvec_iter from_iter
= BVEC_ITER_ALL_INIT
;
99 bio_for_each_segment(tovec
, to
, iter
) {
100 fromvec
= bio_iter_iovec(from
, from_iter
);
101 if (tovec
.bv_page
!= fromvec
.bv_page
) {
103 * fromvec->bv_offset and fromvec->bv_len might have
104 * been modified by the block layer, so use the original
105 * copy, bounce_copy_vec already uses tovec->bv_len
107 vfrom
= page_address(fromvec
.bv_page
) +
110 bounce_copy_vec(&tovec
, vfrom
);
111 flush_dcache_page(tovec
.bv_page
);
113 bio_advance_iter(from
, &from_iter
, tovec
.bv_len
);
117 static void bounce_end_io(struct bio
*bio
)
119 struct bio
*bio_orig
= bio
->bi_private
;
120 struct bio_vec
*bvec
, orig_vec
;
121 struct bvec_iter orig_iter
= bio_orig
->bi_iter
;
122 struct bvec_iter_all iter_all
;
125 * free up bounce indirect pages used
127 bio_for_each_segment_all(bvec
, bio
, iter_all
) {
128 orig_vec
= bio_iter_iovec(bio_orig
, orig_iter
);
129 if (bvec
->bv_page
!= orig_vec
.bv_page
) {
130 dec_zone_page_state(bvec
->bv_page
, NR_BOUNCE
);
131 mempool_free(bvec
->bv_page
, &page_pool
);
133 bio_advance_iter(bio_orig
, &orig_iter
, orig_vec
.bv_len
);
136 bio_orig
->bi_status
= bio
->bi_status
;
141 static void bounce_end_io_write(struct bio
*bio
)
146 static void bounce_end_io_read(struct bio
*bio
)
148 struct bio
*bio_orig
= bio
->bi_private
;
151 copy_to_high_bio_irq(bio_orig
, bio
);
156 static struct bio
*bounce_clone_bio(struct bio
*bio_src
)
158 struct bvec_iter iter
;
163 * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
164 * bio_src->bi_io_vec to bio->bi_io_vec.
166 * We can't do that anymore, because:
168 * - The point of cloning the biovec is to produce a bio with a biovec
169 * the caller can modify: bi_idx and bi_bvec_done should be 0.
171 * - The original bio could've had more than BIO_MAX_VECS biovecs; if
172 * we tried to clone the whole thing bio_alloc_bioset() would fail.
173 * But the clone should succeed as long as the number of biovecs we
174 * actually need to allocate is fewer than BIO_MAX_VECS.
176 * - Lastly, bi_vcnt should not be looked at or relied upon by code
177 * that does not own the bio - reason being drivers don't use it for
178 * iterating over the biovec anymore, so expecting it to be kept up
179 * to date (i.e. for clones that share the parent biovec) is just
180 * asking for trouble and would force extra work on
181 * __bio_clone_fast() anyways.
183 bio
= bio_alloc_bioset(GFP_NOIO
, bio_segments(bio_src
),
185 bio
->bi_bdev
= bio_src
->bi_bdev
;
186 if (bio_flagged(bio_src
, BIO_REMAPPED
))
187 bio_set_flag(bio
, BIO_REMAPPED
);
188 bio
->bi_opf
= bio_src
->bi_opf
;
189 bio
->bi_ioprio
= bio_src
->bi_ioprio
;
190 bio
->bi_write_hint
= bio_src
->bi_write_hint
;
191 bio
->bi_iter
.bi_sector
= bio_src
->bi_iter
.bi_sector
;
192 bio
->bi_iter
.bi_size
= bio_src
->bi_iter
.bi_size
;
194 switch (bio_op(bio
)) {
196 case REQ_OP_SECURE_ERASE
:
197 case REQ_OP_WRITE_ZEROES
:
199 case REQ_OP_WRITE_SAME
:
200 bio
->bi_io_vec
[bio
->bi_vcnt
++] = bio_src
->bi_io_vec
[0];
203 bio_for_each_segment(bv
, bio_src
, iter
)
204 bio
->bi_io_vec
[bio
->bi_vcnt
++] = bv
;
208 if (bio_crypt_clone(bio
, bio_src
, GFP_NOIO
) < 0)
211 if (bio_integrity(bio_src
) &&
212 bio_integrity_clone(bio
, bio_src
, GFP_NOIO
) < 0)
215 bio_clone_blkg_association(bio
, bio_src
);
216 blkcg_bio_issue_init(bio
);
225 void __blk_queue_bounce(struct request_queue
*q
, struct bio
**bio_orig
)
228 int rw
= bio_data_dir(*bio_orig
);
229 struct bio_vec
*to
, from
;
230 struct bvec_iter iter
;
235 bio_for_each_segment(from
, *bio_orig
, iter
) {
236 if (i
++ < BIO_MAX_VECS
)
237 sectors
+= from
.bv_len
>> 9;
238 if (PageHighMem(from
.bv_page
))
244 if (sectors
< bio_sectors(*bio_orig
)) {
245 bio
= bio_split(*bio_orig
, sectors
, GFP_NOIO
, &bounce_bio_split
);
246 bio_chain(bio
, *bio_orig
);
247 submit_bio_noacct(*bio_orig
);
250 bio
= bounce_clone_bio(*bio_orig
);
253 * Bvec table can't be updated by bio_for_each_segment_all(),
254 * so retrieve bvec from the table directly. This way is safe
255 * because the 'bio' is single-page bvec.
257 for (i
= 0, to
= bio
->bi_io_vec
; i
< bio
->bi_vcnt
; to
++, i
++) {
258 struct page
*page
= to
->bv_page
;
260 if (!PageHighMem(page
))
263 to
->bv_page
= mempool_alloc(&page_pool
, GFP_NOIO
);
264 inc_zone_page_state(to
->bv_page
, NR_BOUNCE
);
269 flush_dcache_page(page
);
271 vto
= page_address(to
->bv_page
) + to
->bv_offset
;
272 vfrom
= kmap_atomic(page
) + to
->bv_offset
;
273 memcpy(vto
, vfrom
, to
->bv_len
);
274 kunmap_atomic(vfrom
);
278 trace_block_bio_bounce(*bio_orig
);
280 bio
->bi_flags
|= (1 << BIO_BOUNCED
);
283 bio
->bi_end_io
= bounce_end_io_read
;
285 bio
->bi_end_io
= bounce_end_io_write
;
287 bio
->bi_private
= *bio_orig
;