block: add a lower-level bio_add_page interface

[mirror_ubuntu-bionic-kernel.git] / block / bio.c

diff --git a/block/bio.c b/block/bio.c
index 9ef6cf3addb38cae822d0e5c5ef18ba9e98cd2d7..2636d15af97960fdd73cd6023bf57cc7061b2932 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -43,9 +43,9 @@
  * break badly! cannot be bigger than what you can fit into an
  * unsigned short
  */
-#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
+#define BV(x, n) { .nr_vecs = x, .name = "biovec-"#n }
 static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
-       BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
+       BV(1, 1), BV(4, 4), BV(16, 16), BV(64, 64), BV(128, 128), BV(BIO_MAX_PAGES, max),
 };
 #undef BV
 
@@ -773,7 +773,7 @@ int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page
                        return 0;
        }
 
-       if (bio->bi_vcnt >= bio->bi_max_vecs)
+       if (bio_full(bio))
                return 0;
 
        /*
@@ -821,52 +821,82 @@ int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page
 EXPORT_SYMBOL(bio_add_pc_page);
 
 /**
- *     bio_add_page    -       attempt to add page to bio
- *     @bio: destination bio
- *     @page: page to add
- *     @len: vec entry length
- *     @offset: vec entry offset
+ * __bio_try_merge_page - try appending data to an existing bvec.
+ * @bio: destination bio
+ * @page: page to add
+ * @len: length of the data to add
+ * @off: offset of the data in @page
  *
- *     Attempt to add a page to the bio_vec maplist. This will only fail
- *     if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
+ * Try to add the data at @page + @off to the last bvec of @bio.  This is a
+ * a useful optimisation for file systems with a block size smaller than the
+ * page size.
+ *
+ * Return %true on success or %false on failure.
  */
-int bio_add_page(struct bio *bio, struct page *page,
-                unsigned int len, unsigned int offset)
+bool __bio_try_merge_page(struct bio *bio, struct page *page,
+               unsigned int len, unsigned int off)
 {
-       struct bio_vec *bv;
-
-       /*
-        * cloned bio must not modify vec list
-        */
        if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
-               return 0;
+               return false;
 
-       /*
-        * For filesystems with a blocksize smaller than the pagesize
-        * we will often be called with the same page as last time and
-        * a consecutive offset.  Optimize this special case.
-        */
        if (bio->bi_vcnt > 0) {
-               bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
+               struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
 
-               if (page == bv->bv_page &&
-                   offset == bv->bv_offset + bv->bv_len) {
+               if (page == bv->bv_page && off == bv->bv_offset + bv->bv_len) {
                        bv->bv_len += len;
-                       goto done;
+                       bio->bi_iter.bi_size += len;
+                       return true;
                }
        }
+       return false;
+}
+EXPORT_SYMBOL_GPL(__bio_try_merge_page);
 
-       if (bio->bi_vcnt >= bio->bi_max_vecs)
-               return 0;
+/**
+ * __bio_add_page - add page to a bio in a new segment
+ * @bio: destination bio
+ * @page: page to add
+ * @len: length of the data to add
+ * @off: offset of the data in @page
+ *
+ * Add the data at @page + @off to @bio as a new bvec.  The caller must ensure
+ * that @bio has space for another bvec.
+ */
+void __bio_add_page(struct bio *bio, struct page *page,
+               unsigned int len, unsigned int off)
+{
+       struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
 
-       bv              = &bio->bi_io_vec[bio->bi_vcnt];
-       bv->bv_page     = page;
-       bv->bv_len      = len;
-       bv->bv_offset   = offset;
+       WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
+       WARN_ON_ONCE(bio_full(bio));
+
+       bv->bv_page = page;
+       bv->bv_offset = off;
+       bv->bv_len = len;
 
-       bio->bi_vcnt++;
-done:
        bio->bi_iter.bi_size += len;
+       bio->bi_vcnt++;
+}
+EXPORT_SYMBOL_GPL(__bio_add_page);
+
+/**
+ *     bio_add_page    -       attempt to add page to bio
+ *     @bio: destination bio
+ *     @page: page to add
+ *     @len: vec entry length
+ *     @offset: vec entry offset
+ *
+ *     Attempt to add a page to the bio_vec maplist. This will only fail
+ *     if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
+ */
+int bio_add_page(struct bio *bio, struct page *page,
+                unsigned int len, unsigned int offset)
+{
+       if (!__bio_try_merge_page(bio, page, len, offset)) {
+               if (bio_full(bio))
+                       return 0;
+               __bio_add_page(bio, page, len, offset);
+       }
        return len;
 }
 EXPORT_SYMBOL(bio_add_page);
@@ -1838,7 +1868,7 @@ struct bio *bio_split(struct bio *bio, int sectors,
        bio_advance(bio, split->bi_iter.bi_size);
 
        if (bio_flagged(bio, BIO_TRACE_COMPLETION))
-               bio_set_flag(bio, BIO_TRACE_COMPLETION);
+               bio_set_flag(split, BIO_TRACE_COMPLETION);
 
        return split;
 }