1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/bitops.h>
4 #include <linux/slab.h>
7 #include <linux/pagemap.h>
8 #include <linux/page-flags.h>
9 #include <linux/spinlock.h>
10 #include <linux/blkdev.h>
11 #include <linux/swap.h>
12 #include <linux/writeback.h>
13 #include <linux/pagevec.h>
14 #include <linux/prefetch.h>
15 #include <linux/cleancache.h>
16 #include "extent_io.h"
17 #include "extent_map.h"
19 #include "btrfs_inode.h"
21 #include "check-integrity.h"
23 #include "rcu-string.h"
27 static struct kmem_cache
*extent_state_cache
;
28 static struct kmem_cache
*extent_buffer_cache
;
29 static struct bio_set btrfs_bioset
;
31 static inline bool extent_state_in_tree(const struct extent_state
*state
)
33 return !RB_EMPTY_NODE(&state
->rb_node
);
36 #ifdef CONFIG_BTRFS_DEBUG
37 static LIST_HEAD(buffers
);
38 static LIST_HEAD(states
);
40 static DEFINE_SPINLOCK(leak_lock
);
43 void btrfs_leak_debug_add(struct list_head
*new, struct list_head
*head
)
47 spin_lock_irqsave(&leak_lock
, flags
);
49 spin_unlock_irqrestore(&leak_lock
, flags
);
53 void btrfs_leak_debug_del(struct list_head
*entry
)
57 spin_lock_irqsave(&leak_lock
, flags
);
59 spin_unlock_irqrestore(&leak_lock
, flags
);
63 void btrfs_leak_debug_check(void)
65 struct extent_state
*state
;
66 struct extent_buffer
*eb
;
68 while (!list_empty(&states
)) {
69 state
= list_entry(states
.next
, struct extent_state
, leak_list
);
70 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
71 state
->start
, state
->end
, state
->state
,
72 extent_state_in_tree(state
),
73 refcount_read(&state
->refs
));
74 list_del(&state
->leak_list
);
75 kmem_cache_free(extent_state_cache
, state
);
78 while (!list_empty(&buffers
)) {
79 eb
= list_entry(buffers
.next
, struct extent_buffer
, leak_list
);
80 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
81 eb
->start
, eb
->len
, atomic_read(&eb
->refs
), eb
->bflags
);
82 list_del(&eb
->leak_list
);
83 kmem_cache_free(extent_buffer_cache
, eb
);
87 #define btrfs_debug_check_extent_io_range(tree, start, end) \
88 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
89 static inline void __btrfs_debug_check_extent_io_range(const char *caller
,
90 struct extent_io_tree
*tree
, u64 start
, u64 end
)
92 struct inode
*inode
= tree
->private_data
;
95 if (!inode
|| !is_data_inode(inode
))
98 isize
= i_size_read(inode
);
99 if (end
>= PAGE_SIZE
&& (end
% 2) == 0 && end
!= isize
- 1) {
100 btrfs_debug_rl(BTRFS_I(inode
)->root
->fs_info
,
101 "%s: ino %llu isize %llu odd range [%llu,%llu]",
102 caller
, btrfs_ino(BTRFS_I(inode
)), isize
, start
, end
);
106 #define btrfs_leak_debug_add(new, head) do {} while (0)
107 #define btrfs_leak_debug_del(entry) do {} while (0)
108 #define btrfs_leak_debug_check() do {} while (0)
109 #define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
115 struct rb_node rb_node
;
118 struct extent_page_data
{
120 struct extent_io_tree
*tree
;
121 /* tells writepage not to lock the state bits for this range
122 * it still does the unlocking
124 unsigned int extent_locked
:1;
126 /* tells the submit_bio code to use REQ_SYNC */
127 unsigned int sync_io
:1;
130 static int add_extent_changeset(struct extent_state
*state
, unsigned bits
,
131 struct extent_changeset
*changeset
,
138 if (set
&& (state
->state
& bits
) == bits
)
140 if (!set
&& (state
->state
& bits
) == 0)
142 changeset
->bytes_changed
+= state
->end
- state
->start
+ 1;
143 ret
= ulist_add(&changeset
->range_changed
, state
->start
, state
->end
,
148 static int __must_check
submit_one_bio(struct bio
*bio
, int mirror_num
,
149 unsigned long bio_flags
)
151 blk_status_t ret
= 0;
152 struct bio_vec
*bvec
= bio_last_bvec_all(bio
);
154 struct extent_io_tree
*tree
= bio
->bi_private
;
157 mp_bvec_last_segment(bvec
, &bv
);
158 start
= page_offset(bv
.bv_page
) + bv
.bv_offset
;
160 bio
->bi_private
= NULL
;
163 ret
= tree
->ops
->submit_bio_hook(tree
->private_data
, bio
,
164 mirror_num
, bio_flags
, start
);
166 btrfsic_submit_bio(bio
);
168 return blk_status_to_errno(ret
);
171 /* Cleanup unsubmitted bios */
172 static void end_write_bio(struct extent_page_data
*epd
, int ret
)
175 epd
->bio
->bi_status
= errno_to_blk_status(ret
);
182 * Submit bio from extent page data via submit_one_bio
184 * Return 0 if everything is OK.
185 * Return <0 for error.
187 static int __must_check
flush_write_bio(struct extent_page_data
*epd
)
192 ret
= submit_one_bio(epd
->bio
, 0, 0);
194 * Clean up of epd->bio is handled by its endio function.
195 * And endio is either triggered by successful bio execution
196 * or the error handler of submit bio hook.
197 * So at this point, no matter what happened, we don't need
198 * to clean up epd->bio.
205 int __init
extent_io_init(void)
207 extent_state_cache
= kmem_cache_create("btrfs_extent_state",
208 sizeof(struct extent_state
), 0,
209 SLAB_MEM_SPREAD
, NULL
);
210 if (!extent_state_cache
)
213 extent_buffer_cache
= kmem_cache_create("btrfs_extent_buffer",
214 sizeof(struct extent_buffer
), 0,
215 SLAB_MEM_SPREAD
, NULL
);
216 if (!extent_buffer_cache
)
217 goto free_state_cache
;
219 if (bioset_init(&btrfs_bioset
, BIO_POOL_SIZE
,
220 offsetof(struct btrfs_io_bio
, bio
),
222 goto free_buffer_cache
;
224 if (bioset_integrity_create(&btrfs_bioset
, BIO_POOL_SIZE
))
230 bioset_exit(&btrfs_bioset
);
233 kmem_cache_destroy(extent_buffer_cache
);
234 extent_buffer_cache
= NULL
;
237 kmem_cache_destroy(extent_state_cache
);
238 extent_state_cache
= NULL
;
242 void __cold
extent_io_exit(void)
244 btrfs_leak_debug_check();
247 * Make sure all delayed rcu free are flushed before we
251 kmem_cache_destroy(extent_state_cache
);
252 kmem_cache_destroy(extent_buffer_cache
);
253 bioset_exit(&btrfs_bioset
);
256 void extent_io_tree_init(struct btrfs_fs_info
*fs_info
,
257 struct extent_io_tree
*tree
, unsigned int owner
,
260 tree
->fs_info
= fs_info
;
261 tree
->state
= RB_ROOT
;
263 tree
->dirty_bytes
= 0;
264 spin_lock_init(&tree
->lock
);
265 tree
->private_data
= private_data
;
269 static struct extent_state
*alloc_extent_state(gfp_t mask
)
271 struct extent_state
*state
;
274 * The given mask might be not appropriate for the slab allocator,
275 * drop the unsupported bits
277 mask
&= ~(__GFP_DMA32
|__GFP_HIGHMEM
);
278 state
= kmem_cache_alloc(extent_state_cache
, mask
);
282 state
->failrec
= NULL
;
283 RB_CLEAR_NODE(&state
->rb_node
);
284 btrfs_leak_debug_add(&state
->leak_list
, &states
);
285 refcount_set(&state
->refs
, 1);
286 init_waitqueue_head(&state
->wq
);
287 trace_alloc_extent_state(state
, mask
, _RET_IP_
);
291 void free_extent_state(struct extent_state
*state
)
295 if (refcount_dec_and_test(&state
->refs
)) {
296 WARN_ON(extent_state_in_tree(state
));
297 btrfs_leak_debug_del(&state
->leak_list
);
298 trace_free_extent_state(state
, _RET_IP_
);
299 kmem_cache_free(extent_state_cache
, state
);
303 static struct rb_node
*tree_insert(struct rb_root
*root
,
304 struct rb_node
*search_start
,
306 struct rb_node
*node
,
307 struct rb_node
***p_in
,
308 struct rb_node
**parent_in
)
311 struct rb_node
*parent
= NULL
;
312 struct tree_entry
*entry
;
314 if (p_in
&& parent_in
) {
320 p
= search_start
? &search_start
: &root
->rb_node
;
323 entry
= rb_entry(parent
, struct tree_entry
, rb_node
);
325 if (offset
< entry
->start
)
327 else if (offset
> entry
->end
)
334 rb_link_node(node
, parent
, p
);
335 rb_insert_color(node
, root
);
339 static struct rb_node
*__etree_search(struct extent_io_tree
*tree
, u64 offset
,
340 struct rb_node
**next_ret
,
341 struct rb_node
**prev_ret
,
342 struct rb_node
***p_ret
,
343 struct rb_node
**parent_ret
)
345 struct rb_root
*root
= &tree
->state
;
346 struct rb_node
**n
= &root
->rb_node
;
347 struct rb_node
*prev
= NULL
;
348 struct rb_node
*orig_prev
= NULL
;
349 struct tree_entry
*entry
;
350 struct tree_entry
*prev_entry
= NULL
;
354 entry
= rb_entry(prev
, struct tree_entry
, rb_node
);
357 if (offset
< entry
->start
)
359 else if (offset
> entry
->end
)
372 while (prev
&& offset
> prev_entry
->end
) {
373 prev
= rb_next(prev
);
374 prev_entry
= rb_entry(prev
, struct tree_entry
, rb_node
);
381 prev_entry
= rb_entry(prev
, struct tree_entry
, rb_node
);
382 while (prev
&& offset
< prev_entry
->start
) {
383 prev
= rb_prev(prev
);
384 prev_entry
= rb_entry(prev
, struct tree_entry
, rb_node
);
391 static inline struct rb_node
*
392 tree_search_for_insert(struct extent_io_tree
*tree
,
394 struct rb_node
***p_ret
,
395 struct rb_node
**parent_ret
)
397 struct rb_node
*next
= NULL
;
400 ret
= __etree_search(tree
, offset
, &next
, NULL
, p_ret
, parent_ret
);
406 static inline struct rb_node
*tree_search(struct extent_io_tree
*tree
,
409 return tree_search_for_insert(tree
, offset
, NULL
, NULL
);
413 * utility function to look for merge candidates inside a given range.
414 * Any extents with matching state are merged together into a single
415 * extent in the tree. Extents with EXTENT_IO in their state field
416 * are not merged because the end_io handlers need to be able to do
417 * operations on them without sleeping (or doing allocations/splits).
419 * This should be called with the tree lock held.
421 static void merge_state(struct extent_io_tree
*tree
,
422 struct extent_state
*state
)
424 struct extent_state
*other
;
425 struct rb_node
*other_node
;
427 if (state
->state
& (EXTENT_LOCKED
| EXTENT_BOUNDARY
))
430 other_node
= rb_prev(&state
->rb_node
);
432 other
= rb_entry(other_node
, struct extent_state
, rb_node
);
433 if (other
->end
== state
->start
- 1 &&
434 other
->state
== state
->state
) {
435 if (tree
->private_data
&&
436 is_data_inode(tree
->private_data
))
437 btrfs_merge_delalloc_extent(tree
->private_data
,
439 state
->start
= other
->start
;
440 rb_erase(&other
->rb_node
, &tree
->state
);
441 RB_CLEAR_NODE(&other
->rb_node
);
442 free_extent_state(other
);
445 other_node
= rb_next(&state
->rb_node
);
447 other
= rb_entry(other_node
, struct extent_state
, rb_node
);
448 if (other
->start
== state
->end
+ 1 &&
449 other
->state
== state
->state
) {
450 if (tree
->private_data
&&
451 is_data_inode(tree
->private_data
))
452 btrfs_merge_delalloc_extent(tree
->private_data
,
454 state
->end
= other
->end
;
455 rb_erase(&other
->rb_node
, &tree
->state
);
456 RB_CLEAR_NODE(&other
->rb_node
);
457 free_extent_state(other
);
462 static void set_state_bits(struct extent_io_tree
*tree
,
463 struct extent_state
*state
, unsigned *bits
,
464 struct extent_changeset
*changeset
);
467 * insert an extent_state struct into the tree. 'bits' are set on the
468 * struct before it is inserted.
470 * This may return -EEXIST if the extent is already there, in which case the
471 * state struct is freed.
473 * The tree lock is not taken internally. This is a utility function and
474 * probably isn't what you want to call (see set/clear_extent_bit).
476 static int insert_state(struct extent_io_tree
*tree
,
477 struct extent_state
*state
, u64 start
, u64 end
,
479 struct rb_node
**parent
,
480 unsigned *bits
, struct extent_changeset
*changeset
)
482 struct rb_node
*node
;
485 WARN(1, KERN_ERR
"BTRFS: end < start %llu %llu\n",
487 state
->start
= start
;
490 set_state_bits(tree
, state
, bits
, changeset
);
492 node
= tree_insert(&tree
->state
, NULL
, end
, &state
->rb_node
, p
, parent
);
494 struct extent_state
*found
;
495 found
= rb_entry(node
, struct extent_state
, rb_node
);
496 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
497 found
->start
, found
->end
, start
, end
);
500 merge_state(tree
, state
);
505 * split a given extent state struct in two, inserting the preallocated
506 * struct 'prealloc' as the newly created second half. 'split' indicates an
507 * offset inside 'orig' where it should be split.
510 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
511 * are two extent state structs in the tree:
512 * prealloc: [orig->start, split - 1]
513 * orig: [ split, orig->end ]
515 * The tree locks are not taken by this function. They need to be held
518 static int split_state(struct extent_io_tree
*tree
, struct extent_state
*orig
,
519 struct extent_state
*prealloc
, u64 split
)
521 struct rb_node
*node
;
523 if (tree
->private_data
&& is_data_inode(tree
->private_data
))
524 btrfs_split_delalloc_extent(tree
->private_data
, orig
, split
);
526 prealloc
->start
= orig
->start
;
527 prealloc
->end
= split
- 1;
528 prealloc
->state
= orig
->state
;
531 node
= tree_insert(&tree
->state
, &orig
->rb_node
, prealloc
->end
,
532 &prealloc
->rb_node
, NULL
, NULL
);
534 free_extent_state(prealloc
);
540 static struct extent_state
*next_state(struct extent_state
*state
)
542 struct rb_node
*next
= rb_next(&state
->rb_node
);
544 return rb_entry(next
, struct extent_state
, rb_node
);
550 * utility function to clear some bits in an extent state struct.
551 * it will optionally wake up anyone waiting on this state (wake == 1).
553 * If no bits are set on the state struct after clearing things, the
554 * struct is freed and removed from the tree
556 static struct extent_state
*clear_state_bit(struct extent_io_tree
*tree
,
557 struct extent_state
*state
,
558 unsigned *bits
, int wake
,
559 struct extent_changeset
*changeset
)
561 struct extent_state
*next
;
562 unsigned bits_to_clear
= *bits
& ~EXTENT_CTLBITS
;
565 if ((bits_to_clear
& EXTENT_DIRTY
) && (state
->state
& EXTENT_DIRTY
)) {
566 u64 range
= state
->end
- state
->start
+ 1;
567 WARN_ON(range
> tree
->dirty_bytes
);
568 tree
->dirty_bytes
-= range
;
571 if (tree
->private_data
&& is_data_inode(tree
->private_data
))
572 btrfs_clear_delalloc_extent(tree
->private_data
, state
, bits
);
574 ret
= add_extent_changeset(state
, bits_to_clear
, changeset
, 0);
576 state
->state
&= ~bits_to_clear
;
579 if (state
->state
== 0) {
580 next
= next_state(state
);
581 if (extent_state_in_tree(state
)) {
582 rb_erase(&state
->rb_node
, &tree
->state
);
583 RB_CLEAR_NODE(&state
->rb_node
);
584 free_extent_state(state
);
589 merge_state(tree
, state
);
590 next
= next_state(state
);
595 static struct extent_state
*
596 alloc_extent_state_atomic(struct extent_state
*prealloc
)
599 prealloc
= alloc_extent_state(GFP_ATOMIC
);
604 static void extent_io_tree_panic(struct extent_io_tree
*tree
, int err
)
606 struct inode
*inode
= tree
->private_data
;
608 btrfs_panic(btrfs_sb(inode
->i_sb
), err
,
609 "locking error: extent tree was modified by another thread while locked");
613 * clear some bits on a range in the tree. This may require splitting
614 * or inserting elements in the tree, so the gfp mask is used to
615 * indicate which allocations or sleeping are allowed.
617 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
618 * the given range from the tree regardless of state (ie for truncate).
620 * the range [start, end] is inclusive.
622 * This takes the tree lock, and returns 0 on success and < 0 on error.
624 int __clear_extent_bit(struct extent_io_tree
*tree
, u64 start
, u64 end
,
625 unsigned bits
, int wake
, int delete,
626 struct extent_state
**cached_state
,
627 gfp_t mask
, struct extent_changeset
*changeset
)
629 struct extent_state
*state
;
630 struct extent_state
*cached
;
631 struct extent_state
*prealloc
= NULL
;
632 struct rb_node
*node
;
637 btrfs_debug_check_extent_io_range(tree
, start
, end
);
638 trace_btrfs_clear_extent_bit(tree
, start
, end
- start
+ 1, bits
);
640 if (bits
& EXTENT_DELALLOC
)
641 bits
|= EXTENT_NORESERVE
;
644 bits
|= ~EXTENT_CTLBITS
;
646 if (bits
& (EXTENT_LOCKED
| EXTENT_BOUNDARY
))
649 if (!prealloc
&& gfpflags_allow_blocking(mask
)) {
651 * Don't care for allocation failure here because we might end
652 * up not needing the pre-allocated extent state at all, which
653 * is the case if we only have in the tree extent states that
654 * cover our input range and don't cover too any other range.
655 * If we end up needing a new extent state we allocate it later.
657 prealloc
= alloc_extent_state(mask
);
660 spin_lock(&tree
->lock
);
662 cached
= *cached_state
;
665 *cached_state
= NULL
;
669 if (cached
&& extent_state_in_tree(cached
) &&
670 cached
->start
<= start
&& cached
->end
> start
) {
672 refcount_dec(&cached
->refs
);
677 free_extent_state(cached
);
680 * this search will find the extents that end after
683 node
= tree_search(tree
, start
);
686 state
= rb_entry(node
, struct extent_state
, rb_node
);
688 if (state
->start
> end
)
690 WARN_ON(state
->end
< start
);
691 last_end
= state
->end
;
693 /* the state doesn't have the wanted bits, go ahead */
694 if (!(state
->state
& bits
)) {
695 state
= next_state(state
);
700 * | ---- desired range ---- |
702 * | ------------- state -------------- |
704 * We need to split the extent we found, and may flip
705 * bits on second half.
707 * If the extent we found extends past our range, we
708 * just split and search again. It'll get split again
709 * the next time though.
711 * If the extent we found is inside our range, we clear
712 * the desired bit on it.
715 if (state
->start
< start
) {
716 prealloc
= alloc_extent_state_atomic(prealloc
);
718 err
= split_state(tree
, state
, prealloc
, start
);
720 extent_io_tree_panic(tree
, err
);
725 if (state
->end
<= end
) {
726 state
= clear_state_bit(tree
, state
, &bits
, wake
,
733 * | ---- desired range ---- |
735 * We need to split the extent, and clear the bit
738 if (state
->start
<= end
&& state
->end
> end
) {
739 prealloc
= alloc_extent_state_atomic(prealloc
);
741 err
= split_state(tree
, state
, prealloc
, end
+ 1);
743 extent_io_tree_panic(tree
, err
);
748 clear_state_bit(tree
, prealloc
, &bits
, wake
, changeset
);
754 state
= clear_state_bit(tree
, state
, &bits
, wake
, changeset
);
756 if (last_end
== (u64
)-1)
758 start
= last_end
+ 1;
759 if (start
<= end
&& state
&& !need_resched())
765 spin_unlock(&tree
->lock
);
766 if (gfpflags_allow_blocking(mask
))
771 spin_unlock(&tree
->lock
);
773 free_extent_state(prealloc
);
779 static void wait_on_state(struct extent_io_tree
*tree
,
780 struct extent_state
*state
)
781 __releases(tree
->lock
)
782 __acquires(tree
->lock
)
785 prepare_to_wait(&state
->wq
, &wait
, TASK_UNINTERRUPTIBLE
);
786 spin_unlock(&tree
->lock
);
788 spin_lock(&tree
->lock
);
789 finish_wait(&state
->wq
, &wait
);
793 * waits for one or more bits to clear on a range in the state tree.
794 * The range [start, end] is inclusive.
795 * The tree lock is taken by this function
797 static void wait_extent_bit(struct extent_io_tree
*tree
, u64 start
, u64 end
,
800 struct extent_state
*state
;
801 struct rb_node
*node
;
803 btrfs_debug_check_extent_io_range(tree
, start
, end
);
805 spin_lock(&tree
->lock
);
809 * this search will find all the extents that end after
812 node
= tree_search(tree
, start
);
817 state
= rb_entry(node
, struct extent_state
, rb_node
);
819 if (state
->start
> end
)
822 if (state
->state
& bits
) {
823 start
= state
->start
;
824 refcount_inc(&state
->refs
);
825 wait_on_state(tree
, state
);
826 free_extent_state(state
);
829 start
= state
->end
+ 1;
834 if (!cond_resched_lock(&tree
->lock
)) {
835 node
= rb_next(node
);
840 spin_unlock(&tree
->lock
);
843 static void set_state_bits(struct extent_io_tree
*tree
,
844 struct extent_state
*state
,
845 unsigned *bits
, struct extent_changeset
*changeset
)
847 unsigned bits_to_set
= *bits
& ~EXTENT_CTLBITS
;
850 if (tree
->private_data
&& is_data_inode(tree
->private_data
))
851 btrfs_set_delalloc_extent(tree
->private_data
, state
, bits
);
853 if ((bits_to_set
& EXTENT_DIRTY
) && !(state
->state
& EXTENT_DIRTY
)) {
854 u64 range
= state
->end
- state
->start
+ 1;
855 tree
->dirty_bytes
+= range
;
857 ret
= add_extent_changeset(state
, bits_to_set
, changeset
, 1);
859 state
->state
|= bits_to_set
;
862 static void cache_state_if_flags(struct extent_state
*state
,
863 struct extent_state
**cached_ptr
,
866 if (cached_ptr
&& !(*cached_ptr
)) {
867 if (!flags
|| (state
->state
& flags
)) {
869 refcount_inc(&state
->refs
);
874 static void cache_state(struct extent_state
*state
,
875 struct extent_state
**cached_ptr
)
877 return cache_state_if_flags(state
, cached_ptr
,
878 EXTENT_LOCKED
| EXTENT_BOUNDARY
);
882 * set some bits on a range in the tree. This may require allocations or
883 * sleeping, so the gfp mask is used to indicate what is allowed.
885 * If any of the exclusive bits are set, this will fail with -EEXIST if some
886 * part of the range already has the desired bits set. The start of the
887 * existing range is returned in failed_start in this case.
889 * [start, end] is inclusive This takes the tree lock.
892 static int __must_check
893 __set_extent_bit(struct extent_io_tree
*tree
, u64 start
, u64 end
,
894 unsigned bits
, unsigned exclusive_bits
,
895 u64
*failed_start
, struct extent_state
**cached_state
,
896 gfp_t mask
, struct extent_changeset
*changeset
)
898 struct extent_state
*state
;
899 struct extent_state
*prealloc
= NULL
;
900 struct rb_node
*node
;
902 struct rb_node
*parent
;
907 btrfs_debug_check_extent_io_range(tree
, start
, end
);
908 trace_btrfs_set_extent_bit(tree
, start
, end
- start
+ 1, bits
);
911 if (!prealloc
&& gfpflags_allow_blocking(mask
)) {
913 * Don't care for allocation failure here because we might end
914 * up not needing the pre-allocated extent state at all, which
915 * is the case if we only have in the tree extent states that
916 * cover our input range and don't cover too any other range.
917 * If we end up needing a new extent state we allocate it later.
919 prealloc
= alloc_extent_state(mask
);
922 spin_lock(&tree
->lock
);
923 if (cached_state
&& *cached_state
) {
924 state
= *cached_state
;
925 if (state
->start
<= start
&& state
->end
> start
&&
926 extent_state_in_tree(state
)) {
927 node
= &state
->rb_node
;
932 * this search will find all the extents that end after
935 node
= tree_search_for_insert(tree
, start
, &p
, &parent
);
937 prealloc
= alloc_extent_state_atomic(prealloc
);
939 err
= insert_state(tree
, prealloc
, start
, end
,
940 &p
, &parent
, &bits
, changeset
);
942 extent_io_tree_panic(tree
, err
);
944 cache_state(prealloc
, cached_state
);
948 state
= rb_entry(node
, struct extent_state
, rb_node
);
950 last_start
= state
->start
;
951 last_end
= state
->end
;
954 * | ---- desired range ---- |
957 * Just lock what we found and keep going
959 if (state
->start
== start
&& state
->end
<= end
) {
960 if (state
->state
& exclusive_bits
) {
961 *failed_start
= state
->start
;
966 set_state_bits(tree
, state
, &bits
, changeset
);
967 cache_state(state
, cached_state
);
968 merge_state(tree
, state
);
969 if (last_end
== (u64
)-1)
971 start
= last_end
+ 1;
972 state
= next_state(state
);
973 if (start
< end
&& state
&& state
->start
== start
&&
980 * | ---- desired range ---- |
983 * | ------------- state -------------- |
985 * We need to split the extent we found, and may flip bits on
988 * If the extent we found extends past our
989 * range, we just split and search again. It'll get split
990 * again the next time though.
992 * If the extent we found is inside our range, we set the
995 if (state
->start
< start
) {
996 if (state
->state
& exclusive_bits
) {
997 *failed_start
= start
;
1002 prealloc
= alloc_extent_state_atomic(prealloc
);
1004 err
= split_state(tree
, state
, prealloc
, start
);
1006 extent_io_tree_panic(tree
, err
);
1011 if (state
->end
<= end
) {
1012 set_state_bits(tree
, state
, &bits
, changeset
);
1013 cache_state(state
, cached_state
);
1014 merge_state(tree
, state
);
1015 if (last_end
== (u64
)-1)
1017 start
= last_end
+ 1;
1018 state
= next_state(state
);
1019 if (start
< end
&& state
&& state
->start
== start
&&
1026 * | ---- desired range ---- |
1027 * | state | or | state |
1029 * There's a hole, we need to insert something in it and
1030 * ignore the extent we found.
1032 if (state
->start
> start
) {
1034 if (end
< last_start
)
1037 this_end
= last_start
- 1;
1039 prealloc
= alloc_extent_state_atomic(prealloc
);
1043 * Avoid to free 'prealloc' if it can be merged with
1046 err
= insert_state(tree
, prealloc
, start
, this_end
,
1047 NULL
, NULL
, &bits
, changeset
);
1049 extent_io_tree_panic(tree
, err
);
1051 cache_state(prealloc
, cached_state
);
1053 start
= this_end
+ 1;
1057 * | ---- desired range ---- |
1059 * We need to split the extent, and set the bit
1062 if (state
->start
<= end
&& state
->end
> end
) {
1063 if (state
->state
& exclusive_bits
) {
1064 *failed_start
= start
;
1069 prealloc
= alloc_extent_state_atomic(prealloc
);
1071 err
= split_state(tree
, state
, prealloc
, end
+ 1);
1073 extent_io_tree_panic(tree
, err
);
1075 set_state_bits(tree
, prealloc
, &bits
, changeset
);
1076 cache_state(prealloc
, cached_state
);
1077 merge_state(tree
, prealloc
);
1085 spin_unlock(&tree
->lock
);
1086 if (gfpflags_allow_blocking(mask
))
1091 spin_unlock(&tree
->lock
);
1093 free_extent_state(prealloc
);
1099 int set_extent_bit(struct extent_io_tree
*tree
, u64 start
, u64 end
,
1100 unsigned bits
, u64
* failed_start
,
1101 struct extent_state
**cached_state
, gfp_t mask
)
1103 return __set_extent_bit(tree
, start
, end
, bits
, 0, failed_start
,
1104 cached_state
, mask
, NULL
);
1109 * convert_extent_bit - convert all bits in a given range from one bit to
1111 * @tree: the io tree to search
1112 * @start: the start offset in bytes
1113 * @end: the end offset in bytes (inclusive)
1114 * @bits: the bits to set in this range
1115 * @clear_bits: the bits to clear in this range
1116 * @cached_state: state that we're going to cache
1118 * This will go through and set bits for the given range. If any states exist
1119 * already in this range they are set with the given bit and cleared of the
1120 * clear_bits. This is only meant to be used by things that are mergeable, ie
1121 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1122 * boundary bits like LOCK.
1124 * All allocations are done with GFP_NOFS.
1126 int convert_extent_bit(struct extent_io_tree
*tree
, u64 start
, u64 end
,
1127 unsigned bits
, unsigned clear_bits
,
1128 struct extent_state
**cached_state
)
1130 struct extent_state
*state
;
1131 struct extent_state
*prealloc
= NULL
;
1132 struct rb_node
*node
;
1134 struct rb_node
*parent
;
1138 bool first_iteration
= true;
1140 btrfs_debug_check_extent_io_range(tree
, start
, end
);
1141 trace_btrfs_convert_extent_bit(tree
, start
, end
- start
+ 1, bits
,
1147 * Best effort, don't worry if extent state allocation fails
1148 * here for the first iteration. We might have a cached state
1149 * that matches exactly the target range, in which case no
1150 * extent state allocations are needed. We'll only know this
1151 * after locking the tree.
1153 prealloc
= alloc_extent_state(GFP_NOFS
);
1154 if (!prealloc
&& !first_iteration
)
1158 spin_lock(&tree
->lock
);
1159 if (cached_state
&& *cached_state
) {
1160 state
= *cached_state
;
1161 if (state
->start
<= start
&& state
->end
> start
&&
1162 extent_state_in_tree(state
)) {
1163 node
= &state
->rb_node
;
1169 * this search will find all the extents that end after
1172 node
= tree_search_for_insert(tree
, start
, &p
, &parent
);
1174 prealloc
= alloc_extent_state_atomic(prealloc
);
1179 err
= insert_state(tree
, prealloc
, start
, end
,
1180 &p
, &parent
, &bits
, NULL
);
1182 extent_io_tree_panic(tree
, err
);
1183 cache_state(prealloc
, cached_state
);
1187 state
= rb_entry(node
, struct extent_state
, rb_node
);
1189 last_start
= state
->start
;
1190 last_end
= state
->end
;
1193 * | ---- desired range ---- |
1196 * Just lock what we found and keep going
1198 if (state
->start
== start
&& state
->end
<= end
) {
1199 set_state_bits(tree
, state
, &bits
, NULL
);
1200 cache_state(state
, cached_state
);
1201 state
= clear_state_bit(tree
, state
, &clear_bits
, 0, NULL
);
1202 if (last_end
== (u64
)-1)
1204 start
= last_end
+ 1;
1205 if (start
< end
&& state
&& state
->start
== start
&&
1212 * | ---- desired range ---- |
1215 * | ------------- state -------------- |
1217 * We need to split the extent we found, and may flip bits on
1220 * If the extent we found extends past our
1221 * range, we just split and search again. It'll get split
1222 * again the next time though.
1224 * If the extent we found is inside our range, we set the
1225 * desired bit on it.
1227 if (state
->start
< start
) {
1228 prealloc
= alloc_extent_state_atomic(prealloc
);
1233 err
= split_state(tree
, state
, prealloc
, start
);
1235 extent_io_tree_panic(tree
, err
);
1239 if (state
->end
<= end
) {
1240 set_state_bits(tree
, state
, &bits
, NULL
);
1241 cache_state(state
, cached_state
);
1242 state
= clear_state_bit(tree
, state
, &clear_bits
, 0,
1244 if (last_end
== (u64
)-1)
1246 start
= last_end
+ 1;
1247 if (start
< end
&& state
&& state
->start
== start
&&
1254 * | ---- desired range ---- |
1255 * | state | or | state |
1257 * There's a hole, we need to insert something in it and
1258 * ignore the extent we found.
1260 if (state
->start
> start
) {
1262 if (end
< last_start
)
1265 this_end
= last_start
- 1;
1267 prealloc
= alloc_extent_state_atomic(prealloc
);
1274 * Avoid to free 'prealloc' if it can be merged with
1277 err
= insert_state(tree
, prealloc
, start
, this_end
,
1278 NULL
, NULL
, &bits
, NULL
);
1280 extent_io_tree_panic(tree
, err
);
1281 cache_state(prealloc
, cached_state
);
1283 start
= this_end
+ 1;
1287 * | ---- desired range ---- |
1289 * We need to split the extent, and set the bit
1292 if (state
->start
<= end
&& state
->end
> end
) {
1293 prealloc
= alloc_extent_state_atomic(prealloc
);
1299 err
= split_state(tree
, state
, prealloc
, end
+ 1);
1301 extent_io_tree_panic(tree
, err
);
1303 set_state_bits(tree
, prealloc
, &bits
, NULL
);
1304 cache_state(prealloc
, cached_state
);
1305 clear_state_bit(tree
, prealloc
, &clear_bits
, 0, NULL
);
1313 spin_unlock(&tree
->lock
);
1315 first_iteration
= false;
1319 spin_unlock(&tree
->lock
);
1321 free_extent_state(prealloc
);
1326 /* wrappers around set/clear extent bit */
1327 int set_record_extent_bits(struct extent_io_tree
*tree
, u64 start
, u64 end
,
1328 unsigned bits
, struct extent_changeset
*changeset
)
1331 * We don't support EXTENT_LOCKED yet, as current changeset will
1332 * record any bits changed, so for EXTENT_LOCKED case, it will
1333 * either fail with -EEXIST or changeset will record the whole
1336 BUG_ON(bits
& EXTENT_LOCKED
);
1338 return __set_extent_bit(tree
, start
, end
, bits
, 0, NULL
, NULL
, GFP_NOFS
,
1342 int clear_extent_bit(struct extent_io_tree
*tree
, u64 start
, u64 end
,
1343 unsigned bits
, int wake
, int delete,
1344 struct extent_state
**cached
)
1346 return __clear_extent_bit(tree
, start
, end
, bits
, wake
, delete,
1347 cached
, GFP_NOFS
, NULL
);
1350 int clear_record_extent_bits(struct extent_io_tree
*tree
, u64 start
, u64 end
,
1351 unsigned bits
, struct extent_changeset
*changeset
)
1354 * Don't support EXTENT_LOCKED case, same reason as
1355 * set_record_extent_bits().
1357 BUG_ON(bits
& EXTENT_LOCKED
);
1359 return __clear_extent_bit(tree
, start
, end
, bits
, 0, 0, NULL
, GFP_NOFS
,
1364 * either insert or lock state struct between start and end use mask to tell
1365 * us if waiting is desired.
1367 int lock_extent_bits(struct extent_io_tree
*tree
, u64 start
, u64 end
,
1368 struct extent_state
**cached_state
)
1374 err
= __set_extent_bit(tree
, start
, end
, EXTENT_LOCKED
,
1375 EXTENT_LOCKED
, &failed_start
,
1376 cached_state
, GFP_NOFS
, NULL
);
1377 if (err
== -EEXIST
) {
1378 wait_extent_bit(tree
, failed_start
, end
, EXTENT_LOCKED
);
1379 start
= failed_start
;
1382 WARN_ON(start
> end
);
1387 int try_lock_extent(struct extent_io_tree
*tree
, u64 start
, u64 end
)
1392 err
= __set_extent_bit(tree
, start
, end
, EXTENT_LOCKED
, EXTENT_LOCKED
,
1393 &failed_start
, NULL
, GFP_NOFS
, NULL
);
1394 if (err
== -EEXIST
) {
1395 if (failed_start
> start
)
1396 clear_extent_bit(tree
, start
, failed_start
- 1,
1397 EXTENT_LOCKED
, 1, 0, NULL
);
1403 void extent_range_clear_dirty_for_io(struct inode
*inode
, u64 start
, u64 end
)
1405 unsigned long index
= start
>> PAGE_SHIFT
;
1406 unsigned long end_index
= end
>> PAGE_SHIFT
;
1409 while (index
<= end_index
) {
1410 page
= find_get_page(inode
->i_mapping
, index
);
1411 BUG_ON(!page
); /* Pages should be in the extent_io_tree */
1412 clear_page_dirty_for_io(page
);
1418 void extent_range_redirty_for_io(struct inode
*inode
, u64 start
, u64 end
)
1420 unsigned long index
= start
>> PAGE_SHIFT
;
1421 unsigned long end_index
= end
>> PAGE_SHIFT
;
1424 while (index
<= end_index
) {
1425 page
= find_get_page(inode
->i_mapping
, index
);
1426 BUG_ON(!page
); /* Pages should be in the extent_io_tree */
1427 __set_page_dirty_nobuffers(page
);
1428 account_page_redirty(page
);
1434 /* find the first state struct with 'bits' set after 'start', and
1435 * return it. tree->lock must be held. NULL will returned if
1436 * nothing was found after 'start'
1438 static struct extent_state
*
1439 find_first_extent_bit_state(struct extent_io_tree
*tree
,
1440 u64 start
, unsigned bits
)
1442 struct rb_node
*node
;
1443 struct extent_state
*state
;
1446 * this search will find all the extents that end after
1449 node
= tree_search(tree
, start
);
1454 state
= rb_entry(node
, struct extent_state
, rb_node
);
1455 if (state
->end
>= start
&& (state
->state
& bits
))
1458 node
= rb_next(node
);
1467 * find the first offset in the io tree with 'bits' set. zero is
1468 * returned if we find something, and *start_ret and *end_ret are
1469 * set to reflect the state struct that was found.
1471 * If nothing was found, 1 is returned. If found something, return 0.
1473 int find_first_extent_bit(struct extent_io_tree
*tree
, u64 start
,
1474 u64
*start_ret
, u64
*end_ret
, unsigned bits
,
1475 struct extent_state
**cached_state
)
1477 struct extent_state
*state
;
1480 spin_lock(&tree
->lock
);
1481 if (cached_state
&& *cached_state
) {
1482 state
= *cached_state
;
1483 if (state
->end
== start
- 1 && extent_state_in_tree(state
)) {
1484 while ((state
= next_state(state
)) != NULL
) {
1485 if (state
->state
& bits
)
1488 free_extent_state(*cached_state
);
1489 *cached_state
= NULL
;
1492 free_extent_state(*cached_state
);
1493 *cached_state
= NULL
;
1496 state
= find_first_extent_bit_state(tree
, start
, bits
);
1499 cache_state_if_flags(state
, cached_state
, 0);
1500 *start_ret
= state
->start
;
1501 *end_ret
= state
->end
;
1505 spin_unlock(&tree
->lock
);
1510 * find a contiguous range of bytes in the file marked as delalloc, not
1511 * more than 'max_bytes'. start and end are used to return the range,
1513 * true is returned if we find something, false if nothing was in the tree
1515 static noinline
bool find_delalloc_range(struct extent_io_tree
*tree
,
1516 u64
*start
, u64
*end
, u64 max_bytes
,
1517 struct extent_state
**cached_state
)
1519 struct rb_node
*node
;
1520 struct extent_state
*state
;
1521 u64 cur_start
= *start
;
1523 u64 total_bytes
= 0;
1525 spin_lock(&tree
->lock
);
1528 * this search will find all the extents that end after
1531 node
= tree_search(tree
, cur_start
);
1538 state
= rb_entry(node
, struct extent_state
, rb_node
);
1539 if (found
&& (state
->start
!= cur_start
||
1540 (state
->state
& EXTENT_BOUNDARY
))) {
1543 if (!(state
->state
& EXTENT_DELALLOC
)) {
1549 *start
= state
->start
;
1550 *cached_state
= state
;
1551 refcount_inc(&state
->refs
);
1555 cur_start
= state
->end
+ 1;
1556 node
= rb_next(node
);
1557 total_bytes
+= state
->end
- state
->start
+ 1;
1558 if (total_bytes
>= max_bytes
)
1564 spin_unlock(&tree
->lock
);
1568 static int __process_pages_contig(struct address_space
*mapping
,
1569 struct page
*locked_page
,
1570 pgoff_t start_index
, pgoff_t end_index
,
1571 unsigned long page_ops
, pgoff_t
*index_ret
);
1573 static noinline
void __unlock_for_delalloc(struct inode
*inode
,
1574 struct page
*locked_page
,
1577 unsigned long index
= start
>> PAGE_SHIFT
;
1578 unsigned long end_index
= end
>> PAGE_SHIFT
;
1580 ASSERT(locked_page
);
1581 if (index
== locked_page
->index
&& end_index
== index
)
1584 __process_pages_contig(inode
->i_mapping
, locked_page
, index
, end_index
,
1588 static noinline
int lock_delalloc_pages(struct inode
*inode
,
1589 struct page
*locked_page
,
1593 unsigned long index
= delalloc_start
>> PAGE_SHIFT
;
1594 unsigned long index_ret
= index
;
1595 unsigned long end_index
= delalloc_end
>> PAGE_SHIFT
;
1598 ASSERT(locked_page
);
1599 if (index
== locked_page
->index
&& index
== end_index
)
1602 ret
= __process_pages_contig(inode
->i_mapping
, locked_page
, index
,
1603 end_index
, PAGE_LOCK
, &index_ret
);
1605 __unlock_for_delalloc(inode
, locked_page
, delalloc_start
,
1606 (u64
)index_ret
<< PAGE_SHIFT
);
1611 * Find and lock a contiguous range of bytes in the file marked as delalloc, no
1612 * more than @max_bytes. @Start and @end are used to return the range,
1614 * Return: true if we find something
1615 * false if nothing was in the tree
1618 noinline_for_stack
bool find_lock_delalloc_range(struct inode
*inode
,
1619 struct extent_io_tree
*tree
,
1620 struct page
*locked_page
, u64
*start
,
1623 u64 max_bytes
= BTRFS_MAX_EXTENT_SIZE
;
1627 struct extent_state
*cached_state
= NULL
;
1632 /* step one, find a bunch of delalloc bytes starting at start */
1633 delalloc_start
= *start
;
1635 found
= find_delalloc_range(tree
, &delalloc_start
, &delalloc_end
,
1636 max_bytes
, &cached_state
);
1637 if (!found
|| delalloc_end
<= *start
) {
1638 *start
= delalloc_start
;
1639 *end
= delalloc_end
;
1640 free_extent_state(cached_state
);
1645 * start comes from the offset of locked_page. We have to lock
1646 * pages in order, so we can't process delalloc bytes before
1649 if (delalloc_start
< *start
)
1650 delalloc_start
= *start
;
1653 * make sure to limit the number of pages we try to lock down
1655 if (delalloc_end
+ 1 - delalloc_start
> max_bytes
)
1656 delalloc_end
= delalloc_start
+ max_bytes
- 1;
1658 /* step two, lock all the pages after the page that has start */
1659 ret
= lock_delalloc_pages(inode
, locked_page
,
1660 delalloc_start
, delalloc_end
);
1661 ASSERT(!ret
|| ret
== -EAGAIN
);
1662 if (ret
== -EAGAIN
) {
1663 /* some of the pages are gone, lets avoid looping by
1664 * shortening the size of the delalloc range we're searching
1666 free_extent_state(cached_state
);
1667 cached_state
= NULL
;
1669 max_bytes
= PAGE_SIZE
;
1678 /* step three, lock the state bits for the whole range */
1679 lock_extent_bits(tree
, delalloc_start
, delalloc_end
, &cached_state
);
1681 /* then test to make sure it is all still delalloc */
1682 ret
= test_range_bit(tree
, delalloc_start
, delalloc_end
,
1683 EXTENT_DELALLOC
, 1, cached_state
);
1685 unlock_extent_cached(tree
, delalloc_start
, delalloc_end
,
1687 __unlock_for_delalloc(inode
, locked_page
,
1688 delalloc_start
, delalloc_end
);
1692 free_extent_state(cached_state
);
1693 *start
= delalloc_start
;
1694 *end
= delalloc_end
;
1699 static int __process_pages_contig(struct address_space
*mapping
,
1700 struct page
*locked_page
,
1701 pgoff_t start_index
, pgoff_t end_index
,
1702 unsigned long page_ops
, pgoff_t
*index_ret
)
1704 unsigned long nr_pages
= end_index
- start_index
+ 1;
1705 unsigned long pages_locked
= 0;
1706 pgoff_t index
= start_index
;
1707 struct page
*pages
[16];
1712 if (page_ops
& PAGE_LOCK
) {
1713 ASSERT(page_ops
== PAGE_LOCK
);
1714 ASSERT(index_ret
&& *index_ret
== start_index
);
1717 if ((page_ops
& PAGE_SET_ERROR
) && nr_pages
> 0)
1718 mapping_set_error(mapping
, -EIO
);
1720 while (nr_pages
> 0) {
1721 ret
= find_get_pages_contig(mapping
, index
,
1722 min_t(unsigned long,
1723 nr_pages
, ARRAY_SIZE(pages
)), pages
);
1726 * Only if we're going to lock these pages,
1727 * can we find nothing at @index.
1729 ASSERT(page_ops
& PAGE_LOCK
);
1734 for (i
= 0; i
< ret
; i
++) {
1735 if (page_ops
& PAGE_SET_PRIVATE2
)
1736 SetPagePrivate2(pages
[i
]);
1738 if (pages
[i
] == locked_page
) {
1743 if (page_ops
& PAGE_CLEAR_DIRTY
)
1744 clear_page_dirty_for_io(pages
[i
]);
1745 if (page_ops
& PAGE_SET_WRITEBACK
)
1746 set_page_writeback(pages
[i
]);
1747 if (page_ops
& PAGE_SET_ERROR
)
1748 SetPageError(pages
[i
]);
1749 if (page_ops
& PAGE_END_WRITEBACK
)
1750 end_page_writeback(pages
[i
]);
1751 if (page_ops
& PAGE_UNLOCK
)
1752 unlock_page(pages
[i
]);
1753 if (page_ops
& PAGE_LOCK
) {
1754 lock_page(pages
[i
]);
1755 if (!PageDirty(pages
[i
]) ||
1756 pages
[i
]->mapping
!= mapping
) {
1757 unlock_page(pages
[i
]);
1771 if (err
&& index_ret
)
1772 *index_ret
= start_index
+ pages_locked
- 1;
1776 void extent_clear_unlock_delalloc(struct inode
*inode
, u64 start
, u64 end
,
1777 u64 delalloc_end
, struct page
*locked_page
,
1778 unsigned clear_bits
,
1779 unsigned long page_ops
)
1781 clear_extent_bit(&BTRFS_I(inode
)->io_tree
, start
, end
, clear_bits
, 1, 0,
1784 __process_pages_contig(inode
->i_mapping
, locked_page
,
1785 start
>> PAGE_SHIFT
, end
>> PAGE_SHIFT
,
1790 * count the number of bytes in the tree that have a given bit(s)
1791 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1792 * cached. The total number found is returned.
1794 u64
count_range_bits(struct extent_io_tree
*tree
,
1795 u64
*start
, u64 search_end
, u64 max_bytes
,
1796 unsigned bits
, int contig
)
1798 struct rb_node
*node
;
1799 struct extent_state
*state
;
1800 u64 cur_start
= *start
;
1801 u64 total_bytes
= 0;
1805 if (WARN_ON(search_end
<= cur_start
))
1808 spin_lock(&tree
->lock
);
1809 if (cur_start
== 0 && bits
== EXTENT_DIRTY
) {
1810 total_bytes
= tree
->dirty_bytes
;
1814 * this search will find all the extents that end after
1817 node
= tree_search(tree
, cur_start
);
1822 state
= rb_entry(node
, struct extent_state
, rb_node
);
1823 if (state
->start
> search_end
)
1825 if (contig
&& found
&& state
->start
> last
+ 1)
1827 if (state
->end
>= cur_start
&& (state
->state
& bits
) == bits
) {
1828 total_bytes
+= min(search_end
, state
->end
) + 1 -
1829 max(cur_start
, state
->start
);
1830 if (total_bytes
>= max_bytes
)
1833 *start
= max(cur_start
, state
->start
);
1837 } else if (contig
&& found
) {
1840 node
= rb_next(node
);
1845 spin_unlock(&tree
->lock
);
1850 * set the private field for a given byte offset in the tree. If there isn't
1851 * an extent_state there already, this does nothing.
1853 static noinline
int set_state_failrec(struct extent_io_tree
*tree
, u64 start
,
1854 struct io_failure_record
*failrec
)
1856 struct rb_node
*node
;
1857 struct extent_state
*state
;
1860 spin_lock(&tree
->lock
);
1862 * this search will find all the extents that end after
1865 node
= tree_search(tree
, start
);
1870 state
= rb_entry(node
, struct extent_state
, rb_node
);
1871 if (state
->start
!= start
) {
1875 state
->failrec
= failrec
;
1877 spin_unlock(&tree
->lock
);
1881 static noinline
int get_state_failrec(struct extent_io_tree
*tree
, u64 start
,
1882 struct io_failure_record
**failrec
)
1884 struct rb_node
*node
;
1885 struct extent_state
*state
;
1888 spin_lock(&tree
->lock
);
1890 * this search will find all the extents that end after
1893 node
= tree_search(tree
, start
);
1898 state
= rb_entry(node
, struct extent_state
, rb_node
);
1899 if (state
->start
!= start
) {
1903 *failrec
= state
->failrec
;
1905 spin_unlock(&tree
->lock
);
1910 * searches a range in the state tree for a given mask.
1911 * If 'filled' == 1, this returns 1 only if every extent in the tree
1912 * has the bits set. Otherwise, 1 is returned if any bit in the
1913 * range is found set.
1915 int test_range_bit(struct extent_io_tree
*tree
, u64 start
, u64 end
,
1916 unsigned bits
, int filled
, struct extent_state
*cached
)
1918 struct extent_state
*state
= NULL
;
1919 struct rb_node
*node
;
1922 spin_lock(&tree
->lock
);
1923 if (cached
&& extent_state_in_tree(cached
) && cached
->start
<= start
&&
1924 cached
->end
> start
)
1925 node
= &cached
->rb_node
;
1927 node
= tree_search(tree
, start
);
1928 while (node
&& start
<= end
) {
1929 state
= rb_entry(node
, struct extent_state
, rb_node
);
1931 if (filled
&& state
->start
> start
) {
1936 if (state
->start
> end
)
1939 if (state
->state
& bits
) {
1943 } else if (filled
) {
1948 if (state
->end
== (u64
)-1)
1951 start
= state
->end
+ 1;
1954 node
= rb_next(node
);
1961 spin_unlock(&tree
->lock
);
1966 * helper function to set a given page up to date if all the
1967 * extents in the tree for that page are up to date
1969 static void check_page_uptodate(struct extent_io_tree
*tree
, struct page
*page
)
1971 u64 start
= page_offset(page
);
1972 u64 end
= start
+ PAGE_SIZE
- 1;
1973 if (test_range_bit(tree
, start
, end
, EXTENT_UPTODATE
, 1, NULL
))
1974 SetPageUptodate(page
);
1977 int free_io_failure(struct extent_io_tree
*failure_tree
,
1978 struct extent_io_tree
*io_tree
,
1979 struct io_failure_record
*rec
)
1984 set_state_failrec(failure_tree
, rec
->start
, NULL
);
1985 ret
= clear_extent_bits(failure_tree
, rec
->start
,
1986 rec
->start
+ rec
->len
- 1,
1987 EXTENT_LOCKED
| EXTENT_DIRTY
);
1991 ret
= clear_extent_bits(io_tree
, rec
->start
,
1992 rec
->start
+ rec
->len
- 1,
2002 * this bypasses the standard btrfs submit functions deliberately, as
2003 * the standard behavior is to write all copies in a raid setup. here we only
2004 * want to write the one bad copy. so we do the mapping for ourselves and issue
2005 * submit_bio directly.
2006 * to avoid any synchronization issues, wait for the data after writing, which
2007 * actually prevents the read that triggered the error from finishing.
2008 * currently, there can be no more than two copies of every data bit. thus,
2009 * exactly one rewrite is required.
2011 int repair_io_failure(struct btrfs_fs_info
*fs_info
, u64 ino
, u64 start
,
2012 u64 length
, u64 logical
, struct page
*page
,
2013 unsigned int pg_offset
, int mirror_num
)
2016 struct btrfs_device
*dev
;
2019 struct btrfs_bio
*bbio
= NULL
;
2022 ASSERT(!(fs_info
->sb
->s_flags
& SB_RDONLY
));
2023 BUG_ON(!mirror_num
);
2025 bio
= btrfs_io_bio_alloc(1);
2026 bio
->bi_iter
.bi_size
= 0;
2027 map_length
= length
;
2030 * Avoid races with device replace and make sure our bbio has devices
2031 * associated to its stripes that don't go away while we are doing the
2032 * read repair operation.
2034 btrfs_bio_counter_inc_blocked(fs_info
);
2035 if (btrfs_is_parity_mirror(fs_info
, logical
, length
)) {
2037 * Note that we don't use BTRFS_MAP_WRITE because it's supposed
2038 * to update all raid stripes, but here we just want to correct
2039 * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
2040 * stripe's dev and sector.
2042 ret
= btrfs_map_block(fs_info
, BTRFS_MAP_READ
, logical
,
2043 &map_length
, &bbio
, 0);
2045 btrfs_bio_counter_dec(fs_info
);
2049 ASSERT(bbio
->mirror_num
== 1);
2051 ret
= btrfs_map_block(fs_info
, BTRFS_MAP_WRITE
, logical
,
2052 &map_length
, &bbio
, mirror_num
);
2054 btrfs_bio_counter_dec(fs_info
);
2058 BUG_ON(mirror_num
!= bbio
->mirror_num
);
2061 sector
= bbio
->stripes
[bbio
->mirror_num
- 1].physical
>> 9;
2062 bio
->bi_iter
.bi_sector
= sector
;
2063 dev
= bbio
->stripes
[bbio
->mirror_num
- 1].dev
;
2064 btrfs_put_bbio(bbio
);
2065 if (!dev
|| !dev
->bdev
||
2066 !test_bit(BTRFS_DEV_STATE_WRITEABLE
, &dev
->dev_state
)) {
2067 btrfs_bio_counter_dec(fs_info
);
2071 bio_set_dev(bio
, dev
->bdev
);
2072 bio
->bi_opf
= REQ_OP_WRITE
| REQ_SYNC
;
2073 bio_add_page(bio
, page
, length
, pg_offset
);
2075 if (btrfsic_submit_bio_wait(bio
)) {
2076 /* try to remap that extent elsewhere? */
2077 btrfs_bio_counter_dec(fs_info
);
2079 btrfs_dev_stat_inc_and_print(dev
, BTRFS_DEV_STAT_WRITE_ERRS
);
2083 btrfs_info_rl_in_rcu(fs_info
,
2084 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
2086 rcu_str_deref(dev
->name
), sector
);
2087 btrfs_bio_counter_dec(fs_info
);
2092 int repair_eb_io_failure(struct btrfs_fs_info
*fs_info
,
2093 struct extent_buffer
*eb
, int mirror_num
)
2095 u64 start
= eb
->start
;
2096 int i
, num_pages
= num_extent_pages(eb
);
2099 if (sb_rdonly(fs_info
->sb
))
2102 for (i
= 0; i
< num_pages
; i
++) {
2103 struct page
*p
= eb
->pages
[i
];
2105 ret
= repair_io_failure(fs_info
, 0, start
, PAGE_SIZE
, start
, p
,
2106 start
- page_offset(p
), mirror_num
);
2116 * each time an IO finishes, we do a fast check in the IO failure tree
2117 * to see if we need to process or clean up an io_failure_record
2119 int clean_io_failure(struct btrfs_fs_info
*fs_info
,
2120 struct extent_io_tree
*failure_tree
,
2121 struct extent_io_tree
*io_tree
, u64 start
,
2122 struct page
*page
, u64 ino
, unsigned int pg_offset
)
2125 struct io_failure_record
*failrec
;
2126 struct extent_state
*state
;
2131 ret
= count_range_bits(failure_tree
, &private, (u64
)-1, 1,
2136 ret
= get_state_failrec(failure_tree
, start
, &failrec
);
2140 BUG_ON(!failrec
->this_mirror
);
2142 if (failrec
->in_validation
) {
2143 /* there was no real error, just free the record */
2144 btrfs_debug(fs_info
,
2145 "clean_io_failure: freeing dummy error at %llu",
2149 if (sb_rdonly(fs_info
->sb
))
2152 spin_lock(&io_tree
->lock
);
2153 state
= find_first_extent_bit_state(io_tree
,
2156 spin_unlock(&io_tree
->lock
);
2158 if (state
&& state
->start
<= failrec
->start
&&
2159 state
->end
>= failrec
->start
+ failrec
->len
- 1) {
2160 num_copies
= btrfs_num_copies(fs_info
, failrec
->logical
,
2162 if (num_copies
> 1) {
2163 repair_io_failure(fs_info
, ino
, start
, failrec
->len
,
2164 failrec
->logical
, page
, pg_offset
,
2165 failrec
->failed_mirror
);
2170 free_io_failure(failure_tree
, io_tree
, failrec
);
2176 * Can be called when
2177 * - hold extent lock
2178 * - under ordered extent
2179 * - the inode is freeing
2181 void btrfs_free_io_failure_record(struct btrfs_inode
*inode
, u64 start
, u64 end
)
2183 struct extent_io_tree
*failure_tree
= &inode
->io_failure_tree
;
2184 struct io_failure_record
*failrec
;
2185 struct extent_state
*state
, *next
;
2187 if (RB_EMPTY_ROOT(&failure_tree
->state
))
2190 spin_lock(&failure_tree
->lock
);
2191 state
= find_first_extent_bit_state(failure_tree
, start
, EXTENT_DIRTY
);
2193 if (state
->start
> end
)
2196 ASSERT(state
->end
<= end
);
2198 next
= next_state(state
);
2200 failrec
= state
->failrec
;
2201 free_extent_state(state
);
2206 spin_unlock(&failure_tree
->lock
);
2209 int btrfs_get_io_failure_record(struct inode
*inode
, u64 start
, u64 end
,
2210 struct io_failure_record
**failrec_ret
)
2212 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2213 struct io_failure_record
*failrec
;
2214 struct extent_map
*em
;
2215 struct extent_io_tree
*failure_tree
= &BTRFS_I(inode
)->io_failure_tree
;
2216 struct extent_io_tree
*tree
= &BTRFS_I(inode
)->io_tree
;
2217 struct extent_map_tree
*em_tree
= &BTRFS_I(inode
)->extent_tree
;
2221 ret
= get_state_failrec(failure_tree
, start
, &failrec
);
2223 failrec
= kzalloc(sizeof(*failrec
), GFP_NOFS
);
2227 failrec
->start
= start
;
2228 failrec
->len
= end
- start
+ 1;
2229 failrec
->this_mirror
= 0;
2230 failrec
->bio_flags
= 0;
2231 failrec
->in_validation
= 0;
2233 read_lock(&em_tree
->lock
);
2234 em
= lookup_extent_mapping(em_tree
, start
, failrec
->len
);
2236 read_unlock(&em_tree
->lock
);
2241 if (em
->start
> start
|| em
->start
+ em
->len
<= start
) {
2242 free_extent_map(em
);
2245 read_unlock(&em_tree
->lock
);
2251 logical
= start
- em
->start
;
2252 logical
= em
->block_start
+ logical
;
2253 if (test_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
)) {
2254 logical
= em
->block_start
;
2255 failrec
->bio_flags
= EXTENT_BIO_COMPRESSED
;
2256 extent_set_compress_type(&failrec
->bio_flags
,
2260 btrfs_debug(fs_info
,
2261 "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2262 logical
, start
, failrec
->len
);
2264 failrec
->logical
= logical
;
2265 free_extent_map(em
);
2267 /* set the bits in the private failure tree */
2268 ret
= set_extent_bits(failure_tree
, start
, end
,
2269 EXTENT_LOCKED
| EXTENT_DIRTY
);
2271 ret
= set_state_failrec(failure_tree
, start
, failrec
);
2272 /* set the bits in the inode's tree */
2274 ret
= set_extent_bits(tree
, start
, end
, EXTENT_DAMAGED
);
2280 btrfs_debug(fs_info
,
2281 "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2282 failrec
->logical
, failrec
->start
, failrec
->len
,
2283 failrec
->in_validation
);
2285 * when data can be on disk more than twice, add to failrec here
2286 * (e.g. with a list for failed_mirror) to make
2287 * clean_io_failure() clean all those errors at once.
2291 *failrec_ret
= failrec
;
2296 bool btrfs_check_repairable(struct inode
*inode
, unsigned failed_bio_pages
,
2297 struct io_failure_record
*failrec
, int failed_mirror
)
2299 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2302 num_copies
= btrfs_num_copies(fs_info
, failrec
->logical
, failrec
->len
);
2303 if (num_copies
== 1) {
2305 * we only have a single copy of the data, so don't bother with
2306 * all the retry and error correction code that follows. no
2307 * matter what the error is, it is very likely to persist.
2309 btrfs_debug(fs_info
,
2310 "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2311 num_copies
, failrec
->this_mirror
, failed_mirror
);
2316 * there are two premises:
2317 * a) deliver good data to the caller
2318 * b) correct the bad sectors on disk
2320 if (failed_bio_pages
> 1) {
2322 * to fulfill b), we need to know the exact failing sectors, as
2323 * we don't want to rewrite any more than the failed ones. thus,
2324 * we need separate read requests for the failed bio
2326 * if the following BUG_ON triggers, our validation request got
2327 * merged. we need separate requests for our algorithm to work.
2329 BUG_ON(failrec
->in_validation
);
2330 failrec
->in_validation
= 1;
2331 failrec
->this_mirror
= failed_mirror
;
2334 * we're ready to fulfill a) and b) alongside. get a good copy
2335 * of the failed sector and if we succeed, we have setup
2336 * everything for repair_io_failure to do the rest for us.
2338 if (failrec
->in_validation
) {
2339 BUG_ON(failrec
->this_mirror
!= failed_mirror
);
2340 failrec
->in_validation
= 0;
2341 failrec
->this_mirror
= 0;
2343 failrec
->failed_mirror
= failed_mirror
;
2344 failrec
->this_mirror
++;
2345 if (failrec
->this_mirror
== failed_mirror
)
2346 failrec
->this_mirror
++;
2349 if (failrec
->this_mirror
> num_copies
) {
2350 btrfs_debug(fs_info
,
2351 "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2352 num_copies
, failrec
->this_mirror
, failed_mirror
);
2360 struct bio
*btrfs_create_repair_bio(struct inode
*inode
, struct bio
*failed_bio
,
2361 struct io_failure_record
*failrec
,
2362 struct page
*page
, int pg_offset
, int icsum
,
2363 bio_end_io_t
*endio_func
, void *data
)
2365 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2367 struct btrfs_io_bio
*btrfs_failed_bio
;
2368 struct btrfs_io_bio
*btrfs_bio
;
2370 bio
= btrfs_io_bio_alloc(1);
2371 bio
->bi_end_io
= endio_func
;
2372 bio
->bi_iter
.bi_sector
= failrec
->logical
>> 9;
2373 bio_set_dev(bio
, fs_info
->fs_devices
->latest_bdev
);
2374 bio
->bi_iter
.bi_size
= 0;
2375 bio
->bi_private
= data
;
2377 btrfs_failed_bio
= btrfs_io_bio(failed_bio
);
2378 if (btrfs_failed_bio
->csum
) {
2379 u16 csum_size
= btrfs_super_csum_size(fs_info
->super_copy
);
2381 btrfs_bio
= btrfs_io_bio(bio
);
2382 btrfs_bio
->csum
= btrfs_bio
->csum_inline
;
2384 memcpy(btrfs_bio
->csum
, btrfs_failed_bio
->csum
+ icsum
,
2388 bio_add_page(bio
, page
, failrec
->len
, pg_offset
);
2394 * This is a generic handler for readpage errors. If other copies exist, read
2395 * those and write back good data to the failed position. Does not investigate
2396 * in remapping the failed extent elsewhere, hoping the device will be smart
2397 * enough to do this as needed
2399 static int bio_readpage_error(struct bio
*failed_bio
, u64 phy_offset
,
2400 struct page
*page
, u64 start
, u64 end
,
2403 struct io_failure_record
*failrec
;
2404 struct inode
*inode
= page
->mapping
->host
;
2405 struct extent_io_tree
*tree
= &BTRFS_I(inode
)->io_tree
;
2406 struct extent_io_tree
*failure_tree
= &BTRFS_I(inode
)->io_failure_tree
;
2409 blk_status_t status
;
2411 unsigned failed_bio_pages
= failed_bio
->bi_iter
.bi_size
>> PAGE_SHIFT
;
2413 BUG_ON(bio_op(failed_bio
) == REQ_OP_WRITE
);
2415 ret
= btrfs_get_io_failure_record(inode
, start
, end
, &failrec
);
2419 if (!btrfs_check_repairable(inode
, failed_bio_pages
, failrec
,
2421 free_io_failure(failure_tree
, tree
, failrec
);
2425 if (failed_bio_pages
> 1)
2426 read_mode
|= REQ_FAILFAST_DEV
;
2428 phy_offset
>>= inode
->i_sb
->s_blocksize_bits
;
2429 bio
= btrfs_create_repair_bio(inode
, failed_bio
, failrec
, page
,
2430 start
- page_offset(page
),
2431 (int)phy_offset
, failed_bio
->bi_end_io
,
2433 bio
->bi_opf
= REQ_OP_READ
| read_mode
;
2435 btrfs_debug(btrfs_sb(inode
->i_sb
),
2436 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2437 read_mode
, failrec
->this_mirror
, failrec
->in_validation
);
2439 status
= tree
->ops
->submit_bio_hook(tree
->private_data
, bio
, failrec
->this_mirror
,
2440 failrec
->bio_flags
, 0);
2442 free_io_failure(failure_tree
, tree
, failrec
);
2444 ret
= blk_status_to_errno(status
);
2450 /* lots and lots of room for performance fixes in the end_bio funcs */
2452 void end_extent_writepage(struct page
*page
, int err
, u64 start
, u64 end
)
2454 int uptodate
= (err
== 0);
2457 btrfs_writepage_endio_finish_ordered(page
, start
, end
, uptodate
);
2460 ClearPageUptodate(page
);
2462 ret
= err
< 0 ? err
: -EIO
;
2463 mapping_set_error(page
->mapping
, ret
);
2468 * after a writepage IO is done, we need to:
2469 * clear the uptodate bits on error
2470 * clear the writeback bits in the extent tree for this IO
2471 * end_page_writeback if the page has no more pending IO
2473 * Scheduling is not allowed, so the extent state tree is expected
2474 * to have one and only one object corresponding to this IO.
2476 static void end_bio_extent_writepage(struct bio
*bio
)
2478 int error
= blk_status_to_errno(bio
->bi_status
);
2479 struct bio_vec
*bvec
;
2483 struct bvec_iter_all iter_all
;
2485 ASSERT(!bio_flagged(bio
, BIO_CLONED
));
2486 bio_for_each_segment_all(bvec
, bio
, i
, iter_all
) {
2487 struct page
*page
= bvec
->bv_page
;
2488 struct inode
*inode
= page
->mapping
->host
;
2489 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2491 /* We always issue full-page reads, but if some block
2492 * in a page fails to read, blk_update_request() will
2493 * advance bv_offset and adjust bv_len to compensate.
2494 * Print a warning for nonzero offsets, and an error
2495 * if they don't add up to a full page. */
2496 if (bvec
->bv_offset
|| bvec
->bv_len
!= PAGE_SIZE
) {
2497 if (bvec
->bv_offset
+ bvec
->bv_len
!= PAGE_SIZE
)
2499 "partial page write in btrfs with offset %u and length %u",
2500 bvec
->bv_offset
, bvec
->bv_len
);
2503 "incomplete page write in btrfs with offset %u and length %u",
2504 bvec
->bv_offset
, bvec
->bv_len
);
2507 start
= page_offset(page
);
2508 end
= start
+ bvec
->bv_offset
+ bvec
->bv_len
- 1;
2510 end_extent_writepage(page
, error
, start
, end
);
2511 end_page_writeback(page
);
2518 endio_readpage_release_extent(struct extent_io_tree
*tree
, u64 start
, u64 len
,
2521 struct extent_state
*cached
= NULL
;
2522 u64 end
= start
+ len
- 1;
2524 if (uptodate
&& tree
->track_uptodate
)
2525 set_extent_uptodate(tree
, start
, end
, &cached
, GFP_ATOMIC
);
2526 unlock_extent_cached_atomic(tree
, start
, end
, &cached
);
2530 * after a readpage IO is done, we need to:
2531 * clear the uptodate bits on error
2532 * set the uptodate bits if things worked
2533 * set the page up to date if all extents in the tree are uptodate
2534 * clear the lock bit in the extent tree
2535 * unlock the page if there are no other extents locked for it
2537 * Scheduling is not allowed, so the extent state tree is expected
2538 * to have one and only one object corresponding to this IO.
2540 static void end_bio_extent_readpage(struct bio
*bio
)
2542 struct bio_vec
*bvec
;
2543 int uptodate
= !bio
->bi_status
;
2544 struct btrfs_io_bio
*io_bio
= btrfs_io_bio(bio
);
2545 struct extent_io_tree
*tree
, *failure_tree
;
2550 u64 extent_start
= 0;
2555 struct bvec_iter_all iter_all
;
2557 ASSERT(!bio_flagged(bio
, BIO_CLONED
));
2558 bio_for_each_segment_all(bvec
, bio
, i
, iter_all
) {
2559 struct page
*page
= bvec
->bv_page
;
2560 struct inode
*inode
= page
->mapping
->host
;
2561 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2562 bool data_inode
= btrfs_ino(BTRFS_I(inode
))
2563 != BTRFS_BTREE_INODE_OBJECTID
;
2565 btrfs_debug(fs_info
,
2566 "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
2567 (u64
)bio
->bi_iter
.bi_sector
, bio
->bi_status
,
2568 io_bio
->mirror_num
);
2569 tree
= &BTRFS_I(inode
)->io_tree
;
2570 failure_tree
= &BTRFS_I(inode
)->io_failure_tree
;
2572 /* We always issue full-page reads, but if some block
2573 * in a page fails to read, blk_update_request() will
2574 * advance bv_offset and adjust bv_len to compensate.
2575 * Print a warning for nonzero offsets, and an error
2576 * if they don't add up to a full page. */
2577 if (bvec
->bv_offset
|| bvec
->bv_len
!= PAGE_SIZE
) {
2578 if (bvec
->bv_offset
+ bvec
->bv_len
!= PAGE_SIZE
)
2580 "partial page read in btrfs with offset %u and length %u",
2581 bvec
->bv_offset
, bvec
->bv_len
);
2584 "incomplete page read in btrfs with offset %u and length %u",
2585 bvec
->bv_offset
, bvec
->bv_len
);
2588 start
= page_offset(page
);
2589 end
= start
+ bvec
->bv_offset
+ bvec
->bv_len
- 1;
2592 mirror
= io_bio
->mirror_num
;
2593 if (likely(uptodate
)) {
2594 ret
= tree
->ops
->readpage_end_io_hook(io_bio
, offset
,
2600 clean_io_failure(BTRFS_I(inode
)->root
->fs_info
,
2601 failure_tree
, tree
, start
,
2603 btrfs_ino(BTRFS_I(inode
)), 0);
2606 if (likely(uptodate
))
2612 * The generic bio_readpage_error handles errors the
2613 * following way: If possible, new read requests are
2614 * created and submitted and will end up in
2615 * end_bio_extent_readpage as well (if we're lucky,
2616 * not in the !uptodate case). In that case it returns
2617 * 0 and we just go on with the next page in our bio.
2618 * If it can't handle the error it will return -EIO and
2619 * we remain responsible for that page.
2621 ret
= bio_readpage_error(bio
, offset
, page
, start
, end
,
2624 uptodate
= !bio
->bi_status
;
2629 struct extent_buffer
*eb
;
2631 eb
= (struct extent_buffer
*)page
->private;
2632 set_bit(EXTENT_BUFFER_READ_ERR
, &eb
->bflags
);
2633 eb
->read_mirror
= mirror
;
2634 atomic_dec(&eb
->io_pages
);
2635 if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD
,
2637 btree_readahead_hook(eb
, -EIO
);
2640 if (likely(uptodate
)) {
2641 loff_t i_size
= i_size_read(inode
);
2642 pgoff_t end_index
= i_size
>> PAGE_SHIFT
;
2645 /* Zero out the end if this page straddles i_size */
2646 off
= offset_in_page(i_size
);
2647 if (page
->index
== end_index
&& off
)
2648 zero_user_segment(page
, off
, PAGE_SIZE
);
2649 SetPageUptodate(page
);
2651 ClearPageUptodate(page
);
2657 if (unlikely(!uptodate
)) {
2659 endio_readpage_release_extent(tree
,
2665 endio_readpage_release_extent(tree
, start
,
2666 end
- start
+ 1, 0);
2667 } else if (!extent_len
) {
2668 extent_start
= start
;
2669 extent_len
= end
+ 1 - start
;
2670 } else if (extent_start
+ extent_len
== start
) {
2671 extent_len
+= end
+ 1 - start
;
2673 endio_readpage_release_extent(tree
, extent_start
,
2674 extent_len
, uptodate
);
2675 extent_start
= start
;
2676 extent_len
= end
+ 1 - start
;
2681 endio_readpage_release_extent(tree
, extent_start
, extent_len
,
2683 btrfs_io_bio_free_csum(io_bio
);
2688 * Initialize the members up to but not including 'bio'. Use after allocating a
2689 * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2690 * 'bio' because use of __GFP_ZERO is not supported.
2692 static inline void btrfs_io_bio_init(struct btrfs_io_bio
*btrfs_bio
)
2694 memset(btrfs_bio
, 0, offsetof(struct btrfs_io_bio
, bio
));
2698 * The following helpers allocate a bio. As it's backed by a bioset, it'll
2699 * never fail. We're returning a bio right now but you can call btrfs_io_bio
2700 * for the appropriate container_of magic
2702 struct bio
*btrfs_bio_alloc(struct block_device
*bdev
, u64 first_byte
)
2706 bio
= bio_alloc_bioset(GFP_NOFS
, BIO_MAX_PAGES
, &btrfs_bioset
);
2707 bio_set_dev(bio
, bdev
);
2708 bio
->bi_iter
.bi_sector
= first_byte
>> 9;
2709 btrfs_io_bio_init(btrfs_io_bio(bio
));
2713 struct bio
*btrfs_bio_clone(struct bio
*bio
)
2715 struct btrfs_io_bio
*btrfs_bio
;
2718 /* Bio allocation backed by a bioset does not fail */
2719 new = bio_clone_fast(bio
, GFP_NOFS
, &btrfs_bioset
);
2720 btrfs_bio
= btrfs_io_bio(new);
2721 btrfs_io_bio_init(btrfs_bio
);
2722 btrfs_bio
->iter
= bio
->bi_iter
;
2726 struct bio
*btrfs_io_bio_alloc(unsigned int nr_iovecs
)
2730 /* Bio allocation backed by a bioset does not fail */
2731 bio
= bio_alloc_bioset(GFP_NOFS
, nr_iovecs
, &btrfs_bioset
);
2732 btrfs_io_bio_init(btrfs_io_bio(bio
));
2736 struct bio
*btrfs_bio_clone_partial(struct bio
*orig
, int offset
, int size
)
2739 struct btrfs_io_bio
*btrfs_bio
;
2741 /* this will never fail when it's backed by a bioset */
2742 bio
= bio_clone_fast(orig
, GFP_NOFS
, &btrfs_bioset
);
2745 btrfs_bio
= btrfs_io_bio(bio
);
2746 btrfs_io_bio_init(btrfs_bio
);
2748 bio_trim(bio
, offset
>> 9, size
>> 9);
2749 btrfs_bio
->iter
= bio
->bi_iter
;
2754 * @opf: bio REQ_OP_* and REQ_* flags as one value
2755 * @tree: tree so we can call our merge_bio hook
2756 * @wbc: optional writeback control for io accounting
2757 * @page: page to add to the bio
2758 * @pg_offset: offset of the new bio or to check whether we are adding
2759 * a contiguous page to the previous one
2760 * @size: portion of page that we want to write
2761 * @offset: starting offset in the page
2762 * @bdev: attach newly created bios to this bdev
2763 * @bio_ret: must be valid pointer, newly allocated bio will be stored there
2764 * @end_io_func: end_io callback for new bio
2765 * @mirror_num: desired mirror to read/write
2766 * @prev_bio_flags: flags of previous bio to see if we can merge the current one
2767 * @bio_flags: flags of the current bio to see if we can merge them
2769 static int submit_extent_page(unsigned int opf
, struct extent_io_tree
*tree
,
2770 struct writeback_control
*wbc
,
2771 struct page
*page
, u64 offset
,
2772 size_t size
, unsigned long pg_offset
,
2773 struct block_device
*bdev
,
2774 struct bio
**bio_ret
,
2775 bio_end_io_t end_io_func
,
2777 unsigned long prev_bio_flags
,
2778 unsigned long bio_flags
,
2779 bool force_bio_submit
)
2783 size_t page_size
= min_t(size_t, size
, PAGE_SIZE
);
2784 sector_t sector
= offset
>> 9;
2790 bool can_merge
= true;
2793 if (prev_bio_flags
& EXTENT_BIO_COMPRESSED
)
2794 contig
= bio
->bi_iter
.bi_sector
== sector
;
2796 contig
= bio_end_sector(bio
) == sector
;
2799 if (btrfs_bio_fits_in_stripe(page
, page_size
, bio
, bio_flags
))
2802 if (prev_bio_flags
!= bio_flags
|| !contig
|| !can_merge
||
2804 bio_add_page(bio
, page
, page_size
, pg_offset
) < page_size
) {
2805 ret
= submit_one_bio(bio
, mirror_num
, prev_bio_flags
);
2813 wbc_account_io(wbc
, page
, page_size
);
2818 bio
= btrfs_bio_alloc(bdev
, offset
);
2819 bio_add_page(bio
, page
, page_size
, pg_offset
);
2820 bio
->bi_end_io
= end_io_func
;
2821 bio
->bi_private
= tree
;
2822 bio
->bi_write_hint
= page
->mapping
->host
->i_write_hint
;
2825 wbc_init_bio(wbc
, bio
);
2826 wbc_account_io(wbc
, page
, page_size
);
2834 static void attach_extent_buffer_page(struct extent_buffer
*eb
,
2837 if (!PagePrivate(page
)) {
2838 SetPagePrivate(page
);
2840 set_page_private(page
, (unsigned long)eb
);
2842 WARN_ON(page
->private != (unsigned long)eb
);
2846 void set_page_extent_mapped(struct page
*page
)
2848 if (!PagePrivate(page
)) {
2849 SetPagePrivate(page
);
2851 set_page_private(page
, EXTENT_PAGE_PRIVATE
);
2855 static struct extent_map
*
2856 __get_extent_map(struct inode
*inode
, struct page
*page
, size_t pg_offset
,
2857 u64 start
, u64 len
, get_extent_t
*get_extent
,
2858 struct extent_map
**em_cached
)
2860 struct extent_map
*em
;
2862 if (em_cached
&& *em_cached
) {
2864 if (extent_map_in_tree(em
) && start
>= em
->start
&&
2865 start
< extent_map_end(em
)) {
2866 refcount_inc(&em
->refs
);
2870 free_extent_map(em
);
2874 em
= get_extent(BTRFS_I(inode
), page
, pg_offset
, start
, len
, 0);
2875 if (em_cached
&& !IS_ERR_OR_NULL(em
)) {
2877 refcount_inc(&em
->refs
);
2883 * basic readpage implementation. Locked extent state structs are inserted
2884 * into the tree that are removed when the IO is done (by the end_io
2886 * XXX JDM: This needs looking at to ensure proper page locking
2887 * return 0 on success, otherwise return error
2889 static int __do_readpage(struct extent_io_tree
*tree
,
2891 get_extent_t
*get_extent
,
2892 struct extent_map
**em_cached
,
2893 struct bio
**bio
, int mirror_num
,
2894 unsigned long *bio_flags
, unsigned int read_flags
,
2897 struct inode
*inode
= page
->mapping
->host
;
2898 u64 start
= page_offset(page
);
2899 const u64 end
= start
+ PAGE_SIZE
- 1;
2902 u64 last_byte
= i_size_read(inode
);
2905 struct extent_map
*em
;
2906 struct block_device
*bdev
;
2909 size_t pg_offset
= 0;
2911 size_t disk_io_size
;
2912 size_t blocksize
= inode
->i_sb
->s_blocksize
;
2913 unsigned long this_bio_flag
= 0;
2915 set_page_extent_mapped(page
);
2917 if (!PageUptodate(page
)) {
2918 if (cleancache_get_page(page
) == 0) {
2919 BUG_ON(blocksize
!= PAGE_SIZE
);
2920 unlock_extent(tree
, start
, end
);
2925 if (page
->index
== last_byte
>> PAGE_SHIFT
) {
2927 size_t zero_offset
= offset_in_page(last_byte
);
2930 iosize
= PAGE_SIZE
- zero_offset
;
2931 userpage
= kmap_atomic(page
);
2932 memset(userpage
+ zero_offset
, 0, iosize
);
2933 flush_dcache_page(page
);
2934 kunmap_atomic(userpage
);
2937 while (cur
<= end
) {
2938 bool force_bio_submit
= false;
2941 if (cur
>= last_byte
) {
2943 struct extent_state
*cached
= NULL
;
2945 iosize
= PAGE_SIZE
- pg_offset
;
2946 userpage
= kmap_atomic(page
);
2947 memset(userpage
+ pg_offset
, 0, iosize
);
2948 flush_dcache_page(page
);
2949 kunmap_atomic(userpage
);
2950 set_extent_uptodate(tree
, cur
, cur
+ iosize
- 1,
2952 unlock_extent_cached(tree
, cur
,
2953 cur
+ iosize
- 1, &cached
);
2956 em
= __get_extent_map(inode
, page
, pg_offset
, cur
,
2957 end
- cur
+ 1, get_extent
, em_cached
);
2958 if (IS_ERR_OR_NULL(em
)) {
2960 unlock_extent(tree
, cur
, end
);
2963 extent_offset
= cur
- em
->start
;
2964 BUG_ON(extent_map_end(em
) <= cur
);
2967 if (test_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
)) {
2968 this_bio_flag
|= EXTENT_BIO_COMPRESSED
;
2969 extent_set_compress_type(&this_bio_flag
,
2973 iosize
= min(extent_map_end(em
) - cur
, end
- cur
+ 1);
2974 cur_end
= min(extent_map_end(em
) - 1, end
);
2975 iosize
= ALIGN(iosize
, blocksize
);
2976 if (this_bio_flag
& EXTENT_BIO_COMPRESSED
) {
2977 disk_io_size
= em
->block_len
;
2978 offset
= em
->block_start
;
2980 offset
= em
->block_start
+ extent_offset
;
2981 disk_io_size
= iosize
;
2984 block_start
= em
->block_start
;
2985 if (test_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
))
2986 block_start
= EXTENT_MAP_HOLE
;
2989 * If we have a file range that points to a compressed extent
2990 * and it's followed by a consecutive file range that points to
2991 * to the same compressed extent (possibly with a different
2992 * offset and/or length, so it either points to the whole extent
2993 * or only part of it), we must make sure we do not submit a
2994 * single bio to populate the pages for the 2 ranges because
2995 * this makes the compressed extent read zero out the pages
2996 * belonging to the 2nd range. Imagine the following scenario:
2999 * [0 - 8K] [8K - 24K]
3002 * points to extent X, points to extent X,
3003 * offset 4K, length of 8K offset 0, length 16K
3005 * [extent X, compressed length = 4K uncompressed length = 16K]
3007 * If the bio to read the compressed extent covers both ranges,
3008 * it will decompress extent X into the pages belonging to the
3009 * first range and then it will stop, zeroing out the remaining
3010 * pages that belong to the other range that points to extent X.
3011 * So here we make sure we submit 2 bios, one for the first
3012 * range and another one for the third range. Both will target
3013 * the same physical extent from disk, but we can't currently
3014 * make the compressed bio endio callback populate the pages
3015 * for both ranges because each compressed bio is tightly
3016 * coupled with a single extent map, and each range can have
3017 * an extent map with a different offset value relative to the
3018 * uncompressed data of our extent and different lengths. This
3019 * is a corner case so we prioritize correctness over
3020 * non-optimal behavior (submitting 2 bios for the same extent).
3022 if (test_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
) &&
3023 prev_em_start
&& *prev_em_start
!= (u64
)-1 &&
3024 *prev_em_start
!= em
->start
)
3025 force_bio_submit
= true;
3028 *prev_em_start
= em
->start
;
3030 free_extent_map(em
);
3033 /* we've found a hole, just zero and go on */
3034 if (block_start
== EXTENT_MAP_HOLE
) {
3036 struct extent_state
*cached
= NULL
;
3038 userpage
= kmap_atomic(page
);
3039 memset(userpage
+ pg_offset
, 0, iosize
);
3040 flush_dcache_page(page
);
3041 kunmap_atomic(userpage
);
3043 set_extent_uptodate(tree
, cur
, cur
+ iosize
- 1,
3045 unlock_extent_cached(tree
, cur
,
3046 cur
+ iosize
- 1, &cached
);
3048 pg_offset
+= iosize
;
3051 /* the get_extent function already copied into the page */
3052 if (test_range_bit(tree
, cur
, cur_end
,
3053 EXTENT_UPTODATE
, 1, NULL
)) {
3054 check_page_uptodate(tree
, page
);
3055 unlock_extent(tree
, cur
, cur
+ iosize
- 1);
3057 pg_offset
+= iosize
;
3060 /* we have an inline extent but it didn't get marked up
3061 * to date. Error out
3063 if (block_start
== EXTENT_MAP_INLINE
) {
3065 unlock_extent(tree
, cur
, cur
+ iosize
- 1);
3067 pg_offset
+= iosize
;
3071 ret
= submit_extent_page(REQ_OP_READ
| read_flags
, tree
, NULL
,
3072 page
, offset
, disk_io_size
,
3073 pg_offset
, bdev
, bio
,
3074 end_bio_extent_readpage
, mirror_num
,
3080 *bio_flags
= this_bio_flag
;
3083 unlock_extent(tree
, cur
, cur
+ iosize
- 1);
3087 pg_offset
+= iosize
;
3091 if (!PageError(page
))
3092 SetPageUptodate(page
);
3098 static inline void contiguous_readpages(struct extent_io_tree
*tree
,
3099 struct page
*pages
[], int nr_pages
,
3101 struct extent_map
**em_cached
,
3103 unsigned long *bio_flags
,
3106 struct inode
*inode
;
3107 struct btrfs_ordered_extent
*ordered
;
3110 inode
= pages
[0]->mapping
->host
;
3112 lock_extent(tree
, start
, end
);
3113 ordered
= btrfs_lookup_ordered_range(BTRFS_I(inode
), start
,
3117 unlock_extent(tree
, start
, end
);
3118 btrfs_start_ordered_extent(inode
, ordered
, 1);
3119 btrfs_put_ordered_extent(ordered
);
3122 for (index
= 0; index
< nr_pages
; index
++) {
3123 __do_readpage(tree
, pages
[index
], btrfs_get_extent
, em_cached
,
3124 bio
, 0, bio_flags
, REQ_RAHEAD
, prev_em_start
);
3125 put_page(pages
[index
]);
3129 static int __extent_read_full_page(struct extent_io_tree
*tree
,
3131 get_extent_t
*get_extent
,
3132 struct bio
**bio
, int mirror_num
,
3133 unsigned long *bio_flags
,
3134 unsigned int read_flags
)
3136 struct inode
*inode
= page
->mapping
->host
;
3137 struct btrfs_ordered_extent
*ordered
;
3138 u64 start
= page_offset(page
);
3139 u64 end
= start
+ PAGE_SIZE
- 1;
3143 lock_extent(tree
, start
, end
);
3144 ordered
= btrfs_lookup_ordered_range(BTRFS_I(inode
), start
,
3148 unlock_extent(tree
, start
, end
);
3149 btrfs_start_ordered_extent(inode
, ordered
, 1);
3150 btrfs_put_ordered_extent(ordered
);
3153 ret
= __do_readpage(tree
, page
, get_extent
, NULL
, bio
, mirror_num
,
3154 bio_flags
, read_flags
, NULL
);
3158 int extent_read_full_page(struct extent_io_tree
*tree
, struct page
*page
,
3159 get_extent_t
*get_extent
, int mirror_num
)
3161 struct bio
*bio
= NULL
;
3162 unsigned long bio_flags
= 0;
3165 ret
= __extent_read_full_page(tree
, page
, get_extent
, &bio
, mirror_num
,
3168 ret
= submit_one_bio(bio
, mirror_num
, bio_flags
);
3172 static void update_nr_written(struct writeback_control
*wbc
,
3173 unsigned long nr_written
)
3175 wbc
->nr_to_write
-= nr_written
;
3179 * helper for __extent_writepage, doing all of the delayed allocation setup.
3181 * This returns 1 if btrfs_run_delalloc_range function did all the work required
3182 * to write the page (copy into inline extent). In this case the IO has
3183 * been started and the page is already unlocked.
3185 * This returns 0 if all went well (page still locked)
3186 * This returns < 0 if there were errors (page still locked)
3188 static noinline_for_stack
int writepage_delalloc(struct inode
*inode
,
3189 struct page
*page
, struct writeback_control
*wbc
,
3190 u64 delalloc_start
, unsigned long *nr_written
)
3192 struct extent_io_tree
*tree
= &BTRFS_I(inode
)->io_tree
;
3193 u64 page_end
= delalloc_start
+ PAGE_SIZE
- 1;
3195 u64 delalloc_to_write
= 0;
3196 u64 delalloc_end
= 0;
3198 int page_started
= 0;
3201 while (delalloc_end
< page_end
) {
3202 found
= find_lock_delalloc_range(inode
, tree
,
3207 delalloc_start
= delalloc_end
+ 1;
3210 ret
= btrfs_run_delalloc_range(inode
, page
, delalloc_start
,
3211 delalloc_end
, &page_started
, nr_written
, wbc
);
3212 /* File system has been set read-only */
3216 * btrfs_run_delalloc_range should return < 0 for error
3217 * but just in case, we use > 0 here meaning the IO is
3218 * started, so we don't want to return > 0 unless
3219 * things are going well.
3221 ret
= ret
< 0 ? ret
: -EIO
;
3225 * delalloc_end is already one less than the total length, so
3226 * we don't subtract one from PAGE_SIZE
3228 delalloc_to_write
+= (delalloc_end
- delalloc_start
+
3229 PAGE_SIZE
) >> PAGE_SHIFT
;
3230 delalloc_start
= delalloc_end
+ 1;
3232 if (wbc
->nr_to_write
< delalloc_to_write
) {
3235 if (delalloc_to_write
< thresh
* 2)
3236 thresh
= delalloc_to_write
;
3237 wbc
->nr_to_write
= min_t(u64
, delalloc_to_write
,
3241 /* did the fill delalloc function already unlock and start
3246 * we've unlocked the page, so we can't update
3247 * the mapping's writeback index, just update
3250 wbc
->nr_to_write
-= *nr_written
;
3261 * helper for __extent_writepage. This calls the writepage start hooks,
3262 * and does the loop to map the page into extents and bios.
3264 * We return 1 if the IO is started and the page is unlocked,
3265 * 0 if all went well (page still locked)
3266 * < 0 if there were errors (page still locked)
3268 static noinline_for_stack
int __extent_writepage_io(struct inode
*inode
,
3270 struct writeback_control
*wbc
,
3271 struct extent_page_data
*epd
,
3273 unsigned long nr_written
,
3274 unsigned int write_flags
, int *nr_ret
)
3276 struct extent_io_tree
*tree
= epd
->tree
;
3277 u64 start
= page_offset(page
);
3278 u64 page_end
= start
+ PAGE_SIZE
- 1;
3284 struct extent_map
*em
;
3285 struct block_device
*bdev
;
3286 size_t pg_offset
= 0;
3292 ret
= btrfs_writepage_cow_fixup(page
, start
, page_end
);
3294 /* Fixup worker will requeue */
3296 wbc
->pages_skipped
++;
3298 redirty_page_for_writepage(wbc
, page
);
3300 update_nr_written(wbc
, nr_written
);
3306 * we don't want to touch the inode after unlocking the page,
3307 * so we update the mapping writeback index now
3309 update_nr_written(wbc
, nr_written
+ 1);
3312 if (i_size
<= start
) {
3313 btrfs_writepage_endio_finish_ordered(page
, start
, page_end
, 1);
3317 blocksize
= inode
->i_sb
->s_blocksize
;
3319 while (cur
<= end
) {
3323 if (cur
>= i_size
) {
3324 btrfs_writepage_endio_finish_ordered(page
, cur
,
3328 em
= btrfs_get_extent(BTRFS_I(inode
), page
, pg_offset
, cur
,
3330 if (IS_ERR_OR_NULL(em
)) {
3332 ret
= PTR_ERR_OR_ZERO(em
);
3336 extent_offset
= cur
- em
->start
;
3337 em_end
= extent_map_end(em
);
3338 BUG_ON(em_end
<= cur
);
3340 iosize
= min(em_end
- cur
, end
- cur
+ 1);
3341 iosize
= ALIGN(iosize
, blocksize
);
3342 offset
= em
->block_start
+ extent_offset
;
3344 block_start
= em
->block_start
;
3345 compressed
= test_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
);
3346 free_extent_map(em
);
3350 * compressed and inline extents are written through other
3353 if (compressed
|| block_start
== EXTENT_MAP_HOLE
||
3354 block_start
== EXTENT_MAP_INLINE
) {
3356 * end_io notification does not happen here for
3357 * compressed extents
3360 btrfs_writepage_endio_finish_ordered(page
, cur
,
3363 else if (compressed
) {
3364 /* we don't want to end_page_writeback on
3365 * a compressed extent. this happens
3372 pg_offset
+= iosize
;
3376 btrfs_set_range_writeback(tree
, cur
, cur
+ iosize
- 1);
3377 if (!PageWriteback(page
)) {
3378 btrfs_err(BTRFS_I(inode
)->root
->fs_info
,
3379 "page %lu not writeback, cur %llu end %llu",
3380 page
->index
, cur
, end
);
3383 ret
= submit_extent_page(REQ_OP_WRITE
| write_flags
, tree
, wbc
,
3384 page
, offset
, iosize
, pg_offset
,
3386 end_bio_extent_writepage
,
3390 if (PageWriteback(page
))
3391 end_page_writeback(page
);
3395 pg_offset
+= iosize
;
3404 * the writepage semantics are similar to regular writepage. extent
3405 * records are inserted to lock ranges in the tree, and as dirty areas
3406 * are found, they are marked writeback. Then the lock bits are removed
3407 * and the end_io handler clears the writeback ranges
3409 * Return 0 if everything goes well.
3410 * Return <0 for error.
3412 static int __extent_writepage(struct page
*page
, struct writeback_control
*wbc
,
3413 struct extent_page_data
*epd
)
3415 struct inode
*inode
= page
->mapping
->host
;
3416 u64 start
= page_offset(page
);
3417 u64 page_end
= start
+ PAGE_SIZE
- 1;
3420 size_t pg_offset
= 0;
3421 loff_t i_size
= i_size_read(inode
);
3422 unsigned long end_index
= i_size
>> PAGE_SHIFT
;
3423 unsigned int write_flags
= 0;
3424 unsigned long nr_written
= 0;
3426 write_flags
= wbc_to_write_flags(wbc
);
3428 trace___extent_writepage(page
, inode
, wbc
);
3430 WARN_ON(!PageLocked(page
));
3432 ClearPageError(page
);
3434 pg_offset
= offset_in_page(i_size
);
3435 if (page
->index
> end_index
||
3436 (page
->index
== end_index
&& !pg_offset
)) {
3437 page
->mapping
->a_ops
->invalidatepage(page
, 0, PAGE_SIZE
);
3442 if (page
->index
== end_index
) {
3445 userpage
= kmap_atomic(page
);
3446 memset(userpage
+ pg_offset
, 0,
3447 PAGE_SIZE
- pg_offset
);
3448 kunmap_atomic(userpage
);
3449 flush_dcache_page(page
);
3454 set_page_extent_mapped(page
);
3456 if (!epd
->extent_locked
) {
3457 ret
= writepage_delalloc(inode
, page
, wbc
, start
, &nr_written
);
3464 ret
= __extent_writepage_io(inode
, page
, wbc
, epd
,
3465 i_size
, nr_written
, write_flags
, &nr
);
3471 /* make sure the mapping tag for page dirty gets cleared */
3472 set_page_writeback(page
);
3473 end_page_writeback(page
);
3475 if (PageError(page
)) {
3476 ret
= ret
< 0 ? ret
: -EIO
;
3477 end_extent_writepage(page
, ret
, start
, page_end
);
3487 void wait_on_extent_buffer_writeback(struct extent_buffer
*eb
)
3489 wait_on_bit_io(&eb
->bflags
, EXTENT_BUFFER_WRITEBACK
,
3490 TASK_UNINTERRUPTIBLE
);
3494 * Lock eb pages and flush the bio if we can't the locks
3496 * Return 0 if nothing went wrong
3497 * Return >0 is same as 0, except bio is not submitted
3498 * Return <0 if something went wrong, no page is locked
3500 static noinline_for_stack
int lock_extent_buffer_for_io(struct extent_buffer
*eb
,
3501 struct extent_page_data
*epd
)
3503 struct btrfs_fs_info
*fs_info
= eb
->fs_info
;
3504 int i
, num_pages
, failed_page_nr
;
3508 if (!btrfs_try_tree_write_lock(eb
)) {
3509 ret
= flush_write_bio(epd
);
3513 btrfs_tree_lock(eb
);
3516 if (test_bit(EXTENT_BUFFER_WRITEBACK
, &eb
->bflags
)) {
3517 btrfs_tree_unlock(eb
);
3521 ret
= flush_write_bio(epd
);
3527 wait_on_extent_buffer_writeback(eb
);
3528 btrfs_tree_lock(eb
);
3529 if (!test_bit(EXTENT_BUFFER_WRITEBACK
, &eb
->bflags
))
3531 btrfs_tree_unlock(eb
);
3536 * We need to do this to prevent races in people who check if the eb is
3537 * under IO since we can end up having no IO bits set for a short period
3540 spin_lock(&eb
->refs_lock
);
3541 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY
, &eb
->bflags
)) {
3542 set_bit(EXTENT_BUFFER_WRITEBACK
, &eb
->bflags
);
3543 spin_unlock(&eb
->refs_lock
);
3544 btrfs_set_header_flag(eb
, BTRFS_HEADER_FLAG_WRITTEN
);
3545 percpu_counter_add_batch(&fs_info
->dirty_metadata_bytes
,
3547 fs_info
->dirty_metadata_batch
);
3550 spin_unlock(&eb
->refs_lock
);
3553 btrfs_tree_unlock(eb
);
3558 num_pages
= num_extent_pages(eb
);
3559 for (i
= 0; i
< num_pages
; i
++) {
3560 struct page
*p
= eb
->pages
[i
];
3562 if (!trylock_page(p
)) {
3564 ret
= flush_write_bio(epd
);
3577 /* Unlock already locked pages */
3578 for (i
= 0; i
< failed_page_nr
; i
++)
3579 unlock_page(eb
->pages
[i
]);
3583 static void end_extent_buffer_writeback(struct extent_buffer
*eb
)
3585 clear_bit(EXTENT_BUFFER_WRITEBACK
, &eb
->bflags
);
3586 smp_mb__after_atomic();
3587 wake_up_bit(&eb
->bflags
, EXTENT_BUFFER_WRITEBACK
);
3590 static void set_btree_ioerr(struct page
*page
)
3592 struct extent_buffer
*eb
= (struct extent_buffer
*)page
->private;
3595 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR
, &eb
->bflags
))
3599 * If writeback for a btree extent that doesn't belong to a log tree
3600 * failed, increment the counter transaction->eb_write_errors.
3601 * We do this because while the transaction is running and before it's
3602 * committing (when we call filemap_fdata[write|wait]_range against
3603 * the btree inode), we might have
3604 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3605 * returns an error or an error happens during writeback, when we're
3606 * committing the transaction we wouldn't know about it, since the pages
3607 * can be no longer dirty nor marked anymore for writeback (if a
3608 * subsequent modification to the extent buffer didn't happen before the
3609 * transaction commit), which makes filemap_fdata[write|wait]_range not
3610 * able to find the pages tagged with SetPageError at transaction
3611 * commit time. So if this happens we must abort the transaction,
3612 * otherwise we commit a super block with btree roots that point to
3613 * btree nodes/leafs whose content on disk is invalid - either garbage
3614 * or the content of some node/leaf from a past generation that got
3615 * cowed or deleted and is no longer valid.
3617 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3618 * not be enough - we need to distinguish between log tree extents vs
3619 * non-log tree extents, and the next filemap_fdatawait_range() call
3620 * will catch and clear such errors in the mapping - and that call might
3621 * be from a log sync and not from a transaction commit. Also, checking
3622 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3623 * not done and would not be reliable - the eb might have been released
3624 * from memory and reading it back again means that flag would not be
3625 * set (since it's a runtime flag, not persisted on disk).
3627 * Using the flags below in the btree inode also makes us achieve the
3628 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3629 * writeback for all dirty pages and before filemap_fdatawait_range()
3630 * is called, the writeback for all dirty pages had already finished
3631 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3632 * filemap_fdatawait_range() would return success, as it could not know
3633 * that writeback errors happened (the pages were no longer tagged for
3636 switch (eb
->log_index
) {
3638 set_bit(BTRFS_FS_BTREE_ERR
, &eb
->fs_info
->flags
);
3641 set_bit(BTRFS_FS_LOG1_ERR
, &eb
->fs_info
->flags
);
3644 set_bit(BTRFS_FS_LOG2_ERR
, &eb
->fs_info
->flags
);
3647 BUG(); /* unexpected, logic error */
3651 static void end_bio_extent_buffer_writepage(struct bio
*bio
)
3653 struct bio_vec
*bvec
;
3654 struct extent_buffer
*eb
;
3656 struct bvec_iter_all iter_all
;
3658 ASSERT(!bio_flagged(bio
, BIO_CLONED
));
3659 bio_for_each_segment_all(bvec
, bio
, i
, iter_all
) {
3660 struct page
*page
= bvec
->bv_page
;
3662 eb
= (struct extent_buffer
*)page
->private;
3664 done
= atomic_dec_and_test(&eb
->io_pages
);
3666 if (bio
->bi_status
||
3667 test_bit(EXTENT_BUFFER_WRITE_ERR
, &eb
->bflags
)) {
3668 ClearPageUptodate(page
);
3669 set_btree_ioerr(page
);
3672 end_page_writeback(page
);
3677 end_extent_buffer_writeback(eb
);
3683 static noinline_for_stack
int write_one_eb(struct extent_buffer
*eb
,
3684 struct btrfs_fs_info
*fs_info
,
3685 struct writeback_control
*wbc
,
3686 struct extent_page_data
*epd
)
3688 struct block_device
*bdev
= fs_info
->fs_devices
->latest_bdev
;
3689 struct extent_io_tree
*tree
= &BTRFS_I(fs_info
->btree_inode
)->io_tree
;
3690 u64 offset
= eb
->start
;
3693 unsigned long start
, end
;
3694 unsigned int write_flags
= wbc_to_write_flags(wbc
) | REQ_META
;
3697 clear_bit(EXTENT_BUFFER_WRITE_ERR
, &eb
->bflags
);
3698 num_pages
= num_extent_pages(eb
);
3699 atomic_set(&eb
->io_pages
, num_pages
);
3701 /* set btree blocks beyond nritems with 0 to avoid stale content. */
3702 nritems
= btrfs_header_nritems(eb
);
3703 if (btrfs_header_level(eb
) > 0) {
3704 end
= btrfs_node_key_ptr_offset(nritems
);
3706 memzero_extent_buffer(eb
, end
, eb
->len
- end
);
3710 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
3712 start
= btrfs_item_nr_offset(nritems
);
3713 end
= BTRFS_LEAF_DATA_OFFSET
+ leaf_data_end(fs_info
, eb
);
3714 memzero_extent_buffer(eb
, start
, end
- start
);
3717 for (i
= 0; i
< num_pages
; i
++) {
3718 struct page
*p
= eb
->pages
[i
];
3720 clear_page_dirty_for_io(p
);
3721 set_page_writeback(p
);
3722 ret
= submit_extent_page(REQ_OP_WRITE
| write_flags
, tree
, wbc
,
3723 p
, offset
, PAGE_SIZE
, 0, bdev
,
3725 end_bio_extent_buffer_writepage
,
3729 if (PageWriteback(p
))
3730 end_page_writeback(p
);
3731 if (atomic_sub_and_test(num_pages
- i
, &eb
->io_pages
))
3732 end_extent_buffer_writeback(eb
);
3736 offset
+= PAGE_SIZE
;
3737 update_nr_written(wbc
, 1);
3741 if (unlikely(ret
)) {
3742 for (; i
< num_pages
; i
++) {
3743 struct page
*p
= eb
->pages
[i
];
3744 clear_page_dirty_for_io(p
);
3752 int btree_write_cache_pages(struct address_space
*mapping
,
3753 struct writeback_control
*wbc
)
3755 struct extent_io_tree
*tree
= &BTRFS_I(mapping
->host
)->io_tree
;
3756 struct btrfs_fs_info
*fs_info
= BTRFS_I(mapping
->host
)->root
->fs_info
;
3757 struct extent_buffer
*eb
, *prev_eb
= NULL
;
3758 struct extent_page_data epd
= {
3762 .sync_io
= wbc
->sync_mode
== WB_SYNC_ALL
,
3766 int nr_to_write_done
= 0;
3767 struct pagevec pvec
;
3770 pgoff_t end
; /* Inclusive */
3774 pagevec_init(&pvec
);
3775 if (wbc
->range_cyclic
) {
3776 index
= mapping
->writeback_index
; /* Start from prev offset */
3779 index
= wbc
->range_start
>> PAGE_SHIFT
;
3780 end
= wbc
->range_end
>> PAGE_SHIFT
;
3783 if (wbc
->sync_mode
== WB_SYNC_ALL
)
3784 tag
= PAGECACHE_TAG_TOWRITE
;
3786 tag
= PAGECACHE_TAG_DIRTY
;
3788 if (wbc
->sync_mode
== WB_SYNC_ALL
)
3789 tag_pages_for_writeback(mapping
, index
, end
);
3790 while (!done
&& !nr_to_write_done
&& (index
<= end
) &&
3791 (nr_pages
= pagevec_lookup_range_tag(&pvec
, mapping
, &index
, end
,
3796 for (i
= 0; i
< nr_pages
; i
++) {
3797 struct page
*page
= pvec
.pages
[i
];
3799 if (!PagePrivate(page
))
3802 spin_lock(&mapping
->private_lock
);
3803 if (!PagePrivate(page
)) {
3804 spin_unlock(&mapping
->private_lock
);
3808 eb
= (struct extent_buffer
*)page
->private;
3811 * Shouldn't happen and normally this would be a BUG_ON
3812 * but no sense in crashing the users box for something
3813 * we can survive anyway.
3816 spin_unlock(&mapping
->private_lock
);
3820 if (eb
== prev_eb
) {
3821 spin_unlock(&mapping
->private_lock
);
3825 ret
= atomic_inc_not_zero(&eb
->refs
);
3826 spin_unlock(&mapping
->private_lock
);
3831 ret
= lock_extent_buffer_for_io(eb
, &epd
);
3833 free_extent_buffer(eb
);
3837 ret
= write_one_eb(eb
, fs_info
, wbc
, &epd
);
3840 free_extent_buffer(eb
);
3843 free_extent_buffer(eb
);
3846 * the filesystem may choose to bump up nr_to_write.
3847 * We have to make sure to honor the new nr_to_write
3850 nr_to_write_done
= wbc
->nr_to_write
<= 0;
3852 pagevec_release(&pvec
);
3855 if (!scanned
&& !done
) {
3857 * We hit the last page and there is more work to be done: wrap
3858 * back to the start of the file
3866 end_write_bio(&epd
, ret
);
3869 ret
= flush_write_bio(&epd
);
3874 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3875 * @mapping: address space structure to write
3876 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3877 * @data: data passed to __extent_writepage function
3879 * If a page is already under I/O, write_cache_pages() skips it, even
3880 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3881 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3882 * and msync() need to guarantee that all the data which was dirty at the time
3883 * the call was made get new I/O started against them. If wbc->sync_mode is
3884 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3885 * existing IO to complete.
3887 static int extent_write_cache_pages(struct address_space
*mapping
,
3888 struct writeback_control
*wbc
,
3889 struct extent_page_data
*epd
)
3891 struct inode
*inode
= mapping
->host
;
3894 int nr_to_write_done
= 0;
3895 struct pagevec pvec
;
3898 pgoff_t end
; /* Inclusive */
3900 int range_whole
= 0;
3905 * We have to hold onto the inode so that ordered extents can do their
3906 * work when the IO finishes. The alternative to this is failing to add
3907 * an ordered extent if the igrab() fails there and that is a huge pain
3908 * to deal with, so instead just hold onto the inode throughout the
3909 * writepages operation. If it fails here we are freeing up the inode
3910 * anyway and we'd rather not waste our time writing out stuff that is
3911 * going to be truncated anyway.
3916 pagevec_init(&pvec
);
3917 if (wbc
->range_cyclic
) {
3918 index
= mapping
->writeback_index
; /* Start from prev offset */
3921 index
= wbc
->range_start
>> PAGE_SHIFT
;
3922 end
= wbc
->range_end
>> PAGE_SHIFT
;
3923 if (wbc
->range_start
== 0 && wbc
->range_end
== LLONG_MAX
)
3929 * We do the tagged writepage as long as the snapshot flush bit is set
3930 * and we are the first one who do the filemap_flush() on this inode.
3932 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
3933 * not race in and drop the bit.
3935 if (range_whole
&& wbc
->nr_to_write
== LONG_MAX
&&
3936 test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH
,
3937 &BTRFS_I(inode
)->runtime_flags
))
3938 wbc
->tagged_writepages
= 1;
3940 if (wbc
->sync_mode
== WB_SYNC_ALL
|| wbc
->tagged_writepages
)
3941 tag
= PAGECACHE_TAG_TOWRITE
;
3943 tag
= PAGECACHE_TAG_DIRTY
;
3945 if (wbc
->sync_mode
== WB_SYNC_ALL
|| wbc
->tagged_writepages
)
3946 tag_pages_for_writeback(mapping
, index
, end
);
3948 while (!done
&& !nr_to_write_done
&& (index
<= end
) &&
3949 (nr_pages
= pagevec_lookup_range_tag(&pvec
, mapping
,
3950 &index
, end
, tag
))) {
3954 for (i
= 0; i
< nr_pages
; i
++) {
3955 struct page
*page
= pvec
.pages
[i
];
3957 done_index
= page
->index
;
3959 * At this point we hold neither the i_pages lock nor
3960 * the page lock: the page may be truncated or
3961 * invalidated (changing page->mapping to NULL),
3962 * or even swizzled back from swapper_space to
3963 * tmpfs file mapping
3965 if (!trylock_page(page
)) {
3966 ret
= flush_write_bio(epd
);
3971 if (unlikely(page
->mapping
!= mapping
)) {
3976 if (wbc
->sync_mode
!= WB_SYNC_NONE
) {
3977 if (PageWriteback(page
)) {
3978 ret
= flush_write_bio(epd
);
3981 wait_on_page_writeback(page
);
3984 if (PageWriteback(page
) ||
3985 !clear_page_dirty_for_io(page
)) {
3990 ret
= __extent_writepage(page
, wbc
, epd
);
3993 * done_index is set past this page,
3994 * so media errors will not choke
3995 * background writeout for the entire
3996 * file. This has consequences for
3997 * range_cyclic semantics (ie. it may
3998 * not be suitable for data integrity
4001 done_index
= page
->index
+ 1;
4007 * the filesystem may choose to bump up nr_to_write.
4008 * We have to make sure to honor the new nr_to_write
4011 nr_to_write_done
= wbc
->nr_to_write
<= 0;
4013 pagevec_release(&pvec
);
4016 if (!scanned
&& !done
) {
4018 * We hit the last page and there is more work to be done: wrap
4019 * back to the start of the file
4026 if (wbc
->range_cyclic
|| (wbc
->nr_to_write
> 0 && range_whole
))
4027 mapping
->writeback_index
= done_index
;
4029 btrfs_add_delayed_iput(inode
);
4033 int extent_write_full_page(struct page
*page
, struct writeback_control
*wbc
)
4036 struct extent_page_data epd
= {
4038 .tree
= &BTRFS_I(page
->mapping
->host
)->io_tree
,
4040 .sync_io
= wbc
->sync_mode
== WB_SYNC_ALL
,
4043 ret
= __extent_writepage(page
, wbc
, &epd
);
4046 end_write_bio(&epd
, ret
);
4050 ret
= flush_write_bio(&epd
);
4055 int extent_write_locked_range(struct inode
*inode
, u64 start
, u64 end
,
4059 struct address_space
*mapping
= inode
->i_mapping
;
4060 struct extent_io_tree
*tree
= &BTRFS_I(inode
)->io_tree
;
4062 unsigned long nr_pages
= (end
- start
+ PAGE_SIZE
) >>
4065 struct extent_page_data epd
= {
4069 .sync_io
= mode
== WB_SYNC_ALL
,
4071 struct writeback_control wbc_writepages
= {
4073 .nr_to_write
= nr_pages
* 2,
4074 .range_start
= start
,
4075 .range_end
= end
+ 1,
4078 while (start
<= end
) {
4079 page
= find_get_page(mapping
, start
>> PAGE_SHIFT
);
4080 if (clear_page_dirty_for_io(page
))
4081 ret
= __extent_writepage(page
, &wbc_writepages
, &epd
);
4083 btrfs_writepage_endio_finish_ordered(page
, start
,
4084 start
+ PAGE_SIZE
- 1, 1);
4093 end_write_bio(&epd
, ret
);
4096 ret
= flush_write_bio(&epd
);
4100 int extent_writepages(struct address_space
*mapping
,
4101 struct writeback_control
*wbc
)
4104 struct extent_page_data epd
= {
4106 .tree
= &BTRFS_I(mapping
->host
)->io_tree
,
4108 .sync_io
= wbc
->sync_mode
== WB_SYNC_ALL
,
4111 ret
= extent_write_cache_pages(mapping
, wbc
, &epd
);
4114 end_write_bio(&epd
, ret
);
4117 ret
= flush_write_bio(&epd
);
4121 int extent_readpages(struct address_space
*mapping
, struct list_head
*pages
,
4124 struct bio
*bio
= NULL
;
4125 unsigned long bio_flags
= 0;
4126 struct page
*pagepool
[16];
4127 struct extent_map
*em_cached
= NULL
;
4128 struct extent_io_tree
*tree
= &BTRFS_I(mapping
->host
)->io_tree
;
4130 u64 prev_em_start
= (u64
)-1;
4132 while (!list_empty(pages
)) {
4135 for (nr
= 0; nr
< ARRAY_SIZE(pagepool
) && !list_empty(pages
);) {
4136 struct page
*page
= lru_to_page(pages
);
4138 prefetchw(&page
->flags
);
4139 list_del(&page
->lru
);
4140 if (add_to_page_cache_lru(page
, mapping
, page
->index
,
4141 readahead_gfp_mask(mapping
))) {
4146 pagepool
[nr
++] = page
;
4147 contig_end
= page_offset(page
) + PAGE_SIZE
- 1;
4151 u64 contig_start
= page_offset(pagepool
[0]);
4153 ASSERT(contig_start
+ nr
* PAGE_SIZE
- 1 == contig_end
);
4155 contiguous_readpages(tree
, pagepool
, nr
, contig_start
,
4156 contig_end
, &em_cached
, &bio
, &bio_flags
,
4162 free_extent_map(em_cached
);
4165 return submit_one_bio(bio
, 0, bio_flags
);
4170 * basic invalidatepage code, this waits on any locked or writeback
4171 * ranges corresponding to the page, and then deletes any extent state
4172 * records from the tree
4174 int extent_invalidatepage(struct extent_io_tree
*tree
,
4175 struct page
*page
, unsigned long offset
)
4177 struct extent_state
*cached_state
= NULL
;
4178 u64 start
= page_offset(page
);
4179 u64 end
= start
+ PAGE_SIZE
- 1;
4180 size_t blocksize
= page
->mapping
->host
->i_sb
->s_blocksize
;
4182 start
+= ALIGN(offset
, blocksize
);
4186 lock_extent_bits(tree
, start
, end
, &cached_state
);
4187 wait_on_page_writeback(page
);
4188 clear_extent_bit(tree
, start
, end
,
4189 EXTENT_LOCKED
| EXTENT_DIRTY
| EXTENT_DELALLOC
|
4190 EXTENT_DO_ACCOUNTING
,
4191 1, 1, &cached_state
);
4196 * a helper for releasepage, this tests for areas of the page that
4197 * are locked or under IO and drops the related state bits if it is safe
4200 static int try_release_extent_state(struct extent_io_tree
*tree
,
4201 struct page
*page
, gfp_t mask
)
4203 u64 start
= page_offset(page
);
4204 u64 end
= start
+ PAGE_SIZE
- 1;
4207 if (test_range_bit(tree
, start
, end
, EXTENT_LOCKED
, 0, NULL
)) {
4211 * at this point we can safely clear everything except the
4212 * locked bit and the nodatasum bit
4214 ret
= __clear_extent_bit(tree
, start
, end
,
4215 ~(EXTENT_LOCKED
| EXTENT_NODATASUM
),
4216 0, 0, NULL
, mask
, NULL
);
4218 /* if clear_extent_bit failed for enomem reasons,
4219 * we can't allow the release to continue.
4230 * a helper for releasepage. As long as there are no locked extents
4231 * in the range corresponding to the page, both state records and extent
4232 * map records are removed
4234 int try_release_extent_mapping(struct page
*page
, gfp_t mask
)
4236 struct extent_map
*em
;
4237 u64 start
= page_offset(page
);
4238 u64 end
= start
+ PAGE_SIZE
- 1;
4239 struct btrfs_inode
*btrfs_inode
= BTRFS_I(page
->mapping
->host
);
4240 struct extent_io_tree
*tree
= &btrfs_inode
->io_tree
;
4241 struct extent_map_tree
*map
= &btrfs_inode
->extent_tree
;
4243 if (gfpflags_allow_blocking(mask
) &&
4244 page
->mapping
->host
->i_size
> SZ_16M
) {
4246 while (start
<= end
) {
4247 len
= end
- start
+ 1;
4248 write_lock(&map
->lock
);
4249 em
= lookup_extent_mapping(map
, start
, len
);
4251 write_unlock(&map
->lock
);
4254 if (test_bit(EXTENT_FLAG_PINNED
, &em
->flags
) ||
4255 em
->start
!= start
) {
4256 write_unlock(&map
->lock
);
4257 free_extent_map(em
);
4260 if (!test_range_bit(tree
, em
->start
,
4261 extent_map_end(em
) - 1,
4262 EXTENT_LOCKED
, 0, NULL
)) {
4263 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC
,
4264 &btrfs_inode
->runtime_flags
);
4265 remove_extent_mapping(map
, em
);
4266 /* once for the rb tree */
4267 free_extent_map(em
);
4269 start
= extent_map_end(em
);
4270 write_unlock(&map
->lock
);
4273 free_extent_map(em
);
4276 return try_release_extent_state(tree
, page
, mask
);
4280 * helper function for fiemap, which doesn't want to see any holes.
4281 * This maps until we find something past 'last'
4283 static struct extent_map
*get_extent_skip_holes(struct inode
*inode
,
4284 u64 offset
, u64 last
)
4286 u64 sectorsize
= btrfs_inode_sectorsize(inode
);
4287 struct extent_map
*em
;
4294 len
= last
- offset
;
4297 len
= ALIGN(len
, sectorsize
);
4298 em
= btrfs_get_extent_fiemap(BTRFS_I(inode
), offset
, len
);
4299 if (IS_ERR_OR_NULL(em
))
4302 /* if this isn't a hole return it */
4303 if (em
->block_start
!= EXTENT_MAP_HOLE
)
4306 /* this is a hole, advance to the next extent */
4307 offset
= extent_map_end(em
);
4308 free_extent_map(em
);
4316 * To cache previous fiemap extent
4318 * Will be used for merging fiemap extent
4320 struct fiemap_cache
{
4329 * Helper to submit fiemap extent.
4331 * Will try to merge current fiemap extent specified by @offset, @phys,
4332 * @len and @flags with cached one.
4333 * And only when we fails to merge, cached one will be submitted as
4336 * Return value is the same as fiemap_fill_next_extent().
4338 static int emit_fiemap_extent(struct fiemap_extent_info
*fieinfo
,
4339 struct fiemap_cache
*cache
,
4340 u64 offset
, u64 phys
, u64 len
, u32 flags
)
4348 * Sanity check, extent_fiemap() should have ensured that new
4349 * fiemap extent won't overlap with cached one.
4352 * NOTE: Physical address can overlap, due to compression
4354 if (cache
->offset
+ cache
->len
> offset
) {
4360 * Only merges fiemap extents if
4361 * 1) Their logical addresses are continuous
4363 * 2) Their physical addresses are continuous
4364 * So truly compressed (physical size smaller than logical size)
4365 * extents won't get merged with each other
4367 * 3) Share same flags except FIEMAP_EXTENT_LAST
4368 * So regular extent won't get merged with prealloc extent
4370 if (cache
->offset
+ cache
->len
== offset
&&
4371 cache
->phys
+ cache
->len
== phys
&&
4372 (cache
->flags
& ~FIEMAP_EXTENT_LAST
) ==
4373 (flags
& ~FIEMAP_EXTENT_LAST
)) {
4375 cache
->flags
|= flags
;
4376 goto try_submit_last
;
4379 /* Not mergeable, need to submit cached one */
4380 ret
= fiemap_fill_next_extent(fieinfo
, cache
->offset
, cache
->phys
,
4381 cache
->len
, cache
->flags
);
4382 cache
->cached
= false;
4386 cache
->cached
= true;
4387 cache
->offset
= offset
;
4390 cache
->flags
= flags
;
4392 if (cache
->flags
& FIEMAP_EXTENT_LAST
) {
4393 ret
= fiemap_fill_next_extent(fieinfo
, cache
->offset
,
4394 cache
->phys
, cache
->len
, cache
->flags
);
4395 cache
->cached
= false;
4401 * Emit last fiemap cache
4403 * The last fiemap cache may still be cached in the following case:
4405 * |<- Fiemap range ->|
4406 * |<------------ First extent ----------->|
4408 * In this case, the first extent range will be cached but not emitted.
4409 * So we must emit it before ending extent_fiemap().
4411 static int emit_last_fiemap_cache(struct btrfs_fs_info
*fs_info
,
4412 struct fiemap_extent_info
*fieinfo
,
4413 struct fiemap_cache
*cache
)
4420 ret
= fiemap_fill_next_extent(fieinfo
, cache
->offset
, cache
->phys
,
4421 cache
->len
, cache
->flags
);
4422 cache
->cached
= false;
4428 int extent_fiemap(struct inode
*inode
, struct fiemap_extent_info
*fieinfo
,
4429 __u64 start
, __u64 len
)
4433 u64 max
= start
+ len
;
4437 u64 last_for_get_extent
= 0;
4439 u64 isize
= i_size_read(inode
);
4440 struct btrfs_key found_key
;
4441 struct extent_map
*em
= NULL
;
4442 struct extent_state
*cached_state
= NULL
;
4443 struct btrfs_path
*path
;
4444 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
4445 struct fiemap_cache cache
= { 0 };
4454 path
= btrfs_alloc_path();
4457 path
->leave_spinning
= 1;
4459 start
= round_down(start
, btrfs_inode_sectorsize(inode
));
4460 len
= round_up(max
, btrfs_inode_sectorsize(inode
)) - start
;
4463 * lookup the last file extent. We're not using i_size here
4464 * because there might be preallocation past i_size
4466 ret
= btrfs_lookup_file_extent(NULL
, root
, path
,
4467 btrfs_ino(BTRFS_I(inode
)), -1, 0);
4469 btrfs_free_path(path
);
4478 btrfs_item_key_to_cpu(path
->nodes
[0], &found_key
, path
->slots
[0]);
4479 found_type
= found_key
.type
;
4481 /* No extents, but there might be delalloc bits */
4482 if (found_key
.objectid
!= btrfs_ino(BTRFS_I(inode
)) ||
4483 found_type
!= BTRFS_EXTENT_DATA_KEY
) {
4484 /* have to trust i_size as the end */
4486 last_for_get_extent
= isize
;
4489 * remember the start of the last extent. There are a
4490 * bunch of different factors that go into the length of the
4491 * extent, so its much less complex to remember where it started
4493 last
= found_key
.offset
;
4494 last_for_get_extent
= last
+ 1;
4496 btrfs_release_path(path
);
4499 * we might have some extents allocated but more delalloc past those
4500 * extents. so, we trust isize unless the start of the last extent is
4505 last_for_get_extent
= isize
;
4508 lock_extent_bits(&BTRFS_I(inode
)->io_tree
, start
, start
+ len
- 1,
4511 em
= get_extent_skip_holes(inode
, start
, last_for_get_extent
);
4520 u64 offset_in_extent
= 0;
4522 /* break if the extent we found is outside the range */
4523 if (em
->start
>= max
|| extent_map_end(em
) < off
)
4527 * get_extent may return an extent that starts before our
4528 * requested range. We have to make sure the ranges
4529 * we return to fiemap always move forward and don't
4530 * overlap, so adjust the offsets here
4532 em_start
= max(em
->start
, off
);
4535 * record the offset from the start of the extent
4536 * for adjusting the disk offset below. Only do this if the
4537 * extent isn't compressed since our in ram offset may be past
4538 * what we have actually allocated on disk.
4540 if (!test_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
))
4541 offset_in_extent
= em_start
- em
->start
;
4542 em_end
= extent_map_end(em
);
4543 em_len
= em_end
- em_start
;
4545 if (em
->block_start
< EXTENT_MAP_LAST_BYTE
)
4546 disko
= em
->block_start
+ offset_in_extent
;
4551 * bump off for our next call to get_extent
4553 off
= extent_map_end(em
);
4557 if (em
->block_start
== EXTENT_MAP_LAST_BYTE
) {
4559 flags
|= FIEMAP_EXTENT_LAST
;
4560 } else if (em
->block_start
== EXTENT_MAP_INLINE
) {
4561 flags
|= (FIEMAP_EXTENT_DATA_INLINE
|
4562 FIEMAP_EXTENT_NOT_ALIGNED
);
4563 } else if (em
->block_start
== EXTENT_MAP_DELALLOC
) {
4564 flags
|= (FIEMAP_EXTENT_DELALLOC
|
4565 FIEMAP_EXTENT_UNKNOWN
);
4566 } else if (fieinfo
->fi_extents_max
) {
4567 u64 bytenr
= em
->block_start
-
4568 (em
->start
- em
->orig_start
);
4571 * As btrfs supports shared space, this information
4572 * can be exported to userspace tools via
4573 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0
4574 * then we're just getting a count and we can skip the
4577 ret
= btrfs_check_shared(root
,
4578 btrfs_ino(BTRFS_I(inode
)),
4583 flags
|= FIEMAP_EXTENT_SHARED
;
4586 if (test_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
))
4587 flags
|= FIEMAP_EXTENT_ENCODED
;
4588 if (test_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
))
4589 flags
|= FIEMAP_EXTENT_UNWRITTEN
;
4591 free_extent_map(em
);
4593 if ((em_start
>= last
) || em_len
== (u64
)-1 ||
4594 (last
== (u64
)-1 && isize
<= em_end
)) {
4595 flags
|= FIEMAP_EXTENT_LAST
;
4599 /* now scan forward to see if this is really the last extent. */
4600 em
= get_extent_skip_holes(inode
, off
, last_for_get_extent
);
4606 flags
|= FIEMAP_EXTENT_LAST
;
4609 ret
= emit_fiemap_extent(fieinfo
, &cache
, em_start
, disko
,
4619 ret
= emit_last_fiemap_cache(root
->fs_info
, fieinfo
, &cache
);
4620 free_extent_map(em
);
4622 btrfs_free_path(path
);
4623 unlock_extent_cached(&BTRFS_I(inode
)->io_tree
, start
, start
+ len
- 1,
4628 static void __free_extent_buffer(struct extent_buffer
*eb
)
4630 btrfs_leak_debug_del(&eb
->leak_list
);
4631 kmem_cache_free(extent_buffer_cache
, eb
);
4634 int extent_buffer_under_io(struct extent_buffer
*eb
)
4636 return (atomic_read(&eb
->io_pages
) ||
4637 test_bit(EXTENT_BUFFER_WRITEBACK
, &eb
->bflags
) ||
4638 test_bit(EXTENT_BUFFER_DIRTY
, &eb
->bflags
));
4642 * Release all pages attached to the extent buffer.
4644 static void btrfs_release_extent_buffer_pages(struct extent_buffer
*eb
)
4648 int mapped
= !test_bit(EXTENT_BUFFER_UNMAPPED
, &eb
->bflags
);
4650 BUG_ON(extent_buffer_under_io(eb
));
4652 num_pages
= num_extent_pages(eb
);
4653 for (i
= 0; i
< num_pages
; i
++) {
4654 struct page
*page
= eb
->pages
[i
];
4659 spin_lock(&page
->mapping
->private_lock
);
4661 * We do this since we'll remove the pages after we've
4662 * removed the eb from the radix tree, so we could race
4663 * and have this page now attached to the new eb. So
4664 * only clear page_private if it's still connected to
4667 if (PagePrivate(page
) &&
4668 page
->private == (unsigned long)eb
) {
4669 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY
, &eb
->bflags
));
4670 BUG_ON(PageDirty(page
));
4671 BUG_ON(PageWriteback(page
));
4673 * We need to make sure we haven't be attached
4676 ClearPagePrivate(page
);
4677 set_page_private(page
, 0);
4678 /* One for the page private */
4683 spin_unlock(&page
->mapping
->private_lock
);
4685 /* One for when we allocated the page */
4691 * Helper for releasing the extent buffer.
4693 static inline void btrfs_release_extent_buffer(struct extent_buffer
*eb
)
4695 btrfs_release_extent_buffer_pages(eb
);
4696 __free_extent_buffer(eb
);
4699 static struct extent_buffer
*
4700 __alloc_extent_buffer(struct btrfs_fs_info
*fs_info
, u64 start
,
4703 struct extent_buffer
*eb
= NULL
;
4705 eb
= kmem_cache_zalloc(extent_buffer_cache
, GFP_NOFS
|__GFP_NOFAIL
);
4708 eb
->fs_info
= fs_info
;
4710 rwlock_init(&eb
->lock
);
4711 atomic_set(&eb
->blocking_readers
, 0);
4712 atomic_set(&eb
->blocking_writers
, 0);
4713 eb
->lock_nested
= false;
4714 init_waitqueue_head(&eb
->write_lock_wq
);
4715 init_waitqueue_head(&eb
->read_lock_wq
);
4717 btrfs_leak_debug_add(&eb
->leak_list
, &buffers
);
4719 spin_lock_init(&eb
->refs_lock
);
4720 atomic_set(&eb
->refs
, 1);
4721 atomic_set(&eb
->io_pages
, 0);
4724 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4726 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4727 > MAX_INLINE_EXTENT_BUFFER_SIZE
);
4728 BUG_ON(len
> MAX_INLINE_EXTENT_BUFFER_SIZE
);
4730 #ifdef CONFIG_BTRFS_DEBUG
4731 atomic_set(&eb
->spinning_writers
, 0);
4732 atomic_set(&eb
->spinning_readers
, 0);
4733 atomic_set(&eb
->read_locks
, 0);
4734 atomic_set(&eb
->write_locks
, 0);
4740 struct extent_buffer
*btrfs_clone_extent_buffer(struct extent_buffer
*src
)
4744 struct extent_buffer
*new;
4745 int num_pages
= num_extent_pages(src
);
4747 new = __alloc_extent_buffer(src
->fs_info
, src
->start
, src
->len
);
4751 for (i
= 0; i
< num_pages
; i
++) {
4752 p
= alloc_page(GFP_NOFS
);
4754 btrfs_release_extent_buffer(new);
4757 attach_extent_buffer_page(new, p
);
4758 WARN_ON(PageDirty(p
));
4761 copy_page(page_address(p
), page_address(src
->pages
[i
]));
4764 set_bit(EXTENT_BUFFER_UPTODATE
, &new->bflags
);
4765 set_bit(EXTENT_BUFFER_UNMAPPED
, &new->bflags
);
4770 struct extent_buffer
*__alloc_dummy_extent_buffer(struct btrfs_fs_info
*fs_info
,
4771 u64 start
, unsigned long len
)
4773 struct extent_buffer
*eb
;
4777 eb
= __alloc_extent_buffer(fs_info
, start
, len
);
4781 num_pages
= num_extent_pages(eb
);
4782 for (i
= 0; i
< num_pages
; i
++) {
4783 eb
->pages
[i
] = alloc_page(GFP_NOFS
);
4787 set_extent_buffer_uptodate(eb
);
4788 btrfs_set_header_nritems(eb
, 0);
4789 set_bit(EXTENT_BUFFER_UNMAPPED
, &eb
->bflags
);
4794 __free_page(eb
->pages
[i
- 1]);
4795 __free_extent_buffer(eb
);
4799 struct extent_buffer
*alloc_dummy_extent_buffer(struct btrfs_fs_info
*fs_info
,
4802 return __alloc_dummy_extent_buffer(fs_info
, start
, fs_info
->nodesize
);
4805 static void check_buffer_tree_ref(struct extent_buffer
*eb
)
4808 /* the ref bit is tricky. We have to make sure it is set
4809 * if we have the buffer dirty. Otherwise the
4810 * code to free a buffer can end up dropping a dirty
4813 * Once the ref bit is set, it won't go away while the
4814 * buffer is dirty or in writeback, and it also won't
4815 * go away while we have the reference count on the
4818 * We can't just set the ref bit without bumping the
4819 * ref on the eb because free_extent_buffer might
4820 * see the ref bit and try to clear it. If this happens
4821 * free_extent_buffer might end up dropping our original
4822 * ref by mistake and freeing the page before we are able
4823 * to add one more ref.
4825 * So bump the ref count first, then set the bit. If someone
4826 * beat us to it, drop the ref we added.
4828 refs
= atomic_read(&eb
->refs
);
4829 if (refs
>= 2 && test_bit(EXTENT_BUFFER_TREE_REF
, &eb
->bflags
))
4832 spin_lock(&eb
->refs_lock
);
4833 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF
, &eb
->bflags
))
4834 atomic_inc(&eb
->refs
);
4835 spin_unlock(&eb
->refs_lock
);
4838 static void mark_extent_buffer_accessed(struct extent_buffer
*eb
,
4839 struct page
*accessed
)
4843 check_buffer_tree_ref(eb
);
4845 num_pages
= num_extent_pages(eb
);
4846 for (i
= 0; i
< num_pages
; i
++) {
4847 struct page
*p
= eb
->pages
[i
];
4850 mark_page_accessed(p
);
4854 struct extent_buffer
*find_extent_buffer(struct btrfs_fs_info
*fs_info
,
4857 struct extent_buffer
*eb
;
4860 eb
= radix_tree_lookup(&fs_info
->buffer_radix
,
4861 start
>> PAGE_SHIFT
);
4862 if (eb
&& atomic_inc_not_zero(&eb
->refs
)) {
4865 * Lock our eb's refs_lock to avoid races with
4866 * free_extent_buffer. When we get our eb it might be flagged
4867 * with EXTENT_BUFFER_STALE and another task running
4868 * free_extent_buffer might have seen that flag set,
4869 * eb->refs == 2, that the buffer isn't under IO (dirty and
4870 * writeback flags not set) and it's still in the tree (flag
4871 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
4872 * of decrementing the extent buffer's reference count twice.
4873 * So here we could race and increment the eb's reference count,
4874 * clear its stale flag, mark it as dirty and drop our reference
4875 * before the other task finishes executing free_extent_buffer,
4876 * which would later result in an attempt to free an extent
4877 * buffer that is dirty.
4879 if (test_bit(EXTENT_BUFFER_STALE
, &eb
->bflags
)) {
4880 spin_lock(&eb
->refs_lock
);
4881 spin_unlock(&eb
->refs_lock
);
4883 mark_extent_buffer_accessed(eb
, NULL
);
4891 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4892 struct extent_buffer
*alloc_test_extent_buffer(struct btrfs_fs_info
*fs_info
,
4895 struct extent_buffer
*eb
, *exists
= NULL
;
4898 eb
= find_extent_buffer(fs_info
, start
);
4901 eb
= alloc_dummy_extent_buffer(fs_info
, start
);
4904 eb
->fs_info
= fs_info
;
4906 ret
= radix_tree_preload(GFP_NOFS
);
4909 spin_lock(&fs_info
->buffer_lock
);
4910 ret
= radix_tree_insert(&fs_info
->buffer_radix
,
4911 start
>> PAGE_SHIFT
, eb
);
4912 spin_unlock(&fs_info
->buffer_lock
);
4913 radix_tree_preload_end();
4914 if (ret
== -EEXIST
) {
4915 exists
= find_extent_buffer(fs_info
, start
);
4921 check_buffer_tree_ref(eb
);
4922 set_bit(EXTENT_BUFFER_IN_TREE
, &eb
->bflags
);
4926 btrfs_release_extent_buffer(eb
);
4931 struct extent_buffer
*alloc_extent_buffer(struct btrfs_fs_info
*fs_info
,
4934 unsigned long len
= fs_info
->nodesize
;
4937 unsigned long index
= start
>> PAGE_SHIFT
;
4938 struct extent_buffer
*eb
;
4939 struct extent_buffer
*exists
= NULL
;
4941 struct address_space
*mapping
= fs_info
->btree_inode
->i_mapping
;
4945 if (!IS_ALIGNED(start
, fs_info
->sectorsize
)) {
4946 btrfs_err(fs_info
, "bad tree block start %llu", start
);
4947 return ERR_PTR(-EINVAL
);
4950 eb
= find_extent_buffer(fs_info
, start
);
4954 eb
= __alloc_extent_buffer(fs_info
, start
, len
);
4956 return ERR_PTR(-ENOMEM
);
4958 num_pages
= num_extent_pages(eb
);
4959 for (i
= 0; i
< num_pages
; i
++, index
++) {
4960 p
= find_or_create_page(mapping
, index
, GFP_NOFS
|__GFP_NOFAIL
);
4962 exists
= ERR_PTR(-ENOMEM
);
4966 spin_lock(&mapping
->private_lock
);
4967 if (PagePrivate(p
)) {
4969 * We could have already allocated an eb for this page
4970 * and attached one so lets see if we can get a ref on
4971 * the existing eb, and if we can we know it's good and
4972 * we can just return that one, else we know we can just
4973 * overwrite page->private.
4975 exists
= (struct extent_buffer
*)p
->private;
4976 if (atomic_inc_not_zero(&exists
->refs
)) {
4977 spin_unlock(&mapping
->private_lock
);
4980 mark_extent_buffer_accessed(exists
, p
);
4986 * Do this so attach doesn't complain and we need to
4987 * drop the ref the old guy had.
4989 ClearPagePrivate(p
);
4990 WARN_ON(PageDirty(p
));
4993 attach_extent_buffer_page(eb
, p
);
4994 spin_unlock(&mapping
->private_lock
);
4995 WARN_ON(PageDirty(p
));
4997 if (!PageUptodate(p
))
5001 * We can't unlock the pages just yet since the extent buffer
5002 * hasn't been properly inserted in the radix tree, this
5003 * opens a race with btree_releasepage which can free a page
5004 * while we are still filling in all pages for the buffer and
5009 set_bit(EXTENT_BUFFER_UPTODATE
, &eb
->bflags
);
5011 ret
= radix_tree_preload(GFP_NOFS
);
5013 exists
= ERR_PTR(ret
);
5017 spin_lock(&fs_info
->buffer_lock
);
5018 ret
= radix_tree_insert(&fs_info
->buffer_radix
,
5019 start
>> PAGE_SHIFT
, eb
);
5020 spin_unlock(&fs_info
->buffer_lock
);
5021 radix_tree_preload_end();
5022 if (ret
== -EEXIST
) {
5023 exists
= find_extent_buffer(fs_info
, start
);
5029 /* add one reference for the tree */
5030 check_buffer_tree_ref(eb
);
5031 set_bit(EXTENT_BUFFER_IN_TREE
, &eb
->bflags
);
5034 * Now it's safe to unlock the pages because any calls to
5035 * btree_releasepage will correctly detect that a page belongs to a
5036 * live buffer and won't free them prematurely.
5038 for (i
= 0; i
< num_pages
; i
++)
5039 unlock_page(eb
->pages
[i
]);
5043 WARN_ON(!atomic_dec_and_test(&eb
->refs
));
5044 for (i
= 0; i
< num_pages
; i
++) {
5046 unlock_page(eb
->pages
[i
]);
5049 btrfs_release_extent_buffer(eb
);
5053 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head
*head
)
5055 struct extent_buffer
*eb
=
5056 container_of(head
, struct extent_buffer
, rcu_head
);
5058 __free_extent_buffer(eb
);
5061 static int release_extent_buffer(struct extent_buffer
*eb
)
5063 lockdep_assert_held(&eb
->refs_lock
);
5065 WARN_ON(atomic_read(&eb
->refs
) == 0);
5066 if (atomic_dec_and_test(&eb
->refs
)) {
5067 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE
, &eb
->bflags
)) {
5068 struct btrfs_fs_info
*fs_info
= eb
->fs_info
;
5070 spin_unlock(&eb
->refs_lock
);
5072 spin_lock(&fs_info
->buffer_lock
);
5073 radix_tree_delete(&fs_info
->buffer_radix
,
5074 eb
->start
>> PAGE_SHIFT
);
5075 spin_unlock(&fs_info
->buffer_lock
);
5077 spin_unlock(&eb
->refs_lock
);
5080 /* Should be safe to release our pages at this point */
5081 btrfs_release_extent_buffer_pages(eb
);
5082 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
5083 if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED
, &eb
->bflags
))) {
5084 __free_extent_buffer(eb
);
5088 call_rcu(&eb
->rcu_head
, btrfs_release_extent_buffer_rcu
);
5091 spin_unlock(&eb
->refs_lock
);
5096 void free_extent_buffer(struct extent_buffer
*eb
)
5104 refs
= atomic_read(&eb
->refs
);
5105 if ((!test_bit(EXTENT_BUFFER_UNMAPPED
, &eb
->bflags
) && refs
<= 3)
5106 || (test_bit(EXTENT_BUFFER_UNMAPPED
, &eb
->bflags
) &&
5109 old
= atomic_cmpxchg(&eb
->refs
, refs
, refs
- 1);
5114 spin_lock(&eb
->refs_lock
);
5115 if (atomic_read(&eb
->refs
) == 2 &&
5116 test_bit(EXTENT_BUFFER_STALE
, &eb
->bflags
) &&
5117 !extent_buffer_under_io(eb
) &&
5118 test_and_clear_bit(EXTENT_BUFFER_TREE_REF
, &eb
->bflags
))
5119 atomic_dec(&eb
->refs
);
5122 * I know this is terrible, but it's temporary until we stop tracking
5123 * the uptodate bits and such for the extent buffers.
5125 release_extent_buffer(eb
);
5128 void free_extent_buffer_stale(struct extent_buffer
*eb
)
5133 spin_lock(&eb
->refs_lock
);
5134 set_bit(EXTENT_BUFFER_STALE
, &eb
->bflags
);
5136 if (atomic_read(&eb
->refs
) == 2 && !extent_buffer_under_io(eb
) &&
5137 test_and_clear_bit(EXTENT_BUFFER_TREE_REF
, &eb
->bflags
))
5138 atomic_dec(&eb
->refs
);
5139 release_extent_buffer(eb
);
5142 void clear_extent_buffer_dirty(struct extent_buffer
*eb
)
5148 num_pages
= num_extent_pages(eb
);
5150 for (i
= 0; i
< num_pages
; i
++) {
5151 page
= eb
->pages
[i
];
5152 if (!PageDirty(page
))
5156 WARN_ON(!PagePrivate(page
));
5158 clear_page_dirty_for_io(page
);
5159 xa_lock_irq(&page
->mapping
->i_pages
);
5160 if (!PageDirty(page
))
5161 __xa_clear_mark(&page
->mapping
->i_pages
,
5162 page_index(page
), PAGECACHE_TAG_DIRTY
);
5163 xa_unlock_irq(&page
->mapping
->i_pages
);
5164 ClearPageError(page
);
5167 WARN_ON(atomic_read(&eb
->refs
) == 0);
5170 bool set_extent_buffer_dirty(struct extent_buffer
*eb
)
5176 check_buffer_tree_ref(eb
);
5178 was_dirty
= test_and_set_bit(EXTENT_BUFFER_DIRTY
, &eb
->bflags
);
5180 num_pages
= num_extent_pages(eb
);
5181 WARN_ON(atomic_read(&eb
->refs
) == 0);
5182 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF
, &eb
->bflags
));
5185 for (i
= 0; i
< num_pages
; i
++)
5186 set_page_dirty(eb
->pages
[i
]);
5188 #ifdef CONFIG_BTRFS_DEBUG
5189 for (i
= 0; i
< num_pages
; i
++)
5190 ASSERT(PageDirty(eb
->pages
[i
]));
5196 void clear_extent_buffer_uptodate(struct extent_buffer
*eb
)
5202 clear_bit(EXTENT_BUFFER_UPTODATE
, &eb
->bflags
);
5203 num_pages
= num_extent_pages(eb
);
5204 for (i
= 0; i
< num_pages
; i
++) {
5205 page
= eb
->pages
[i
];
5207 ClearPageUptodate(page
);
5211 void set_extent_buffer_uptodate(struct extent_buffer
*eb
)
5217 set_bit(EXTENT_BUFFER_UPTODATE
, &eb
->bflags
);
5218 num_pages
= num_extent_pages(eb
);
5219 for (i
= 0; i
< num_pages
; i
++) {
5220 page
= eb
->pages
[i
];
5221 SetPageUptodate(page
);
5225 int read_extent_buffer_pages(struct extent_io_tree
*tree
,
5226 struct extent_buffer
*eb
, int wait
, int mirror_num
)
5232 int locked_pages
= 0;
5233 int all_uptodate
= 1;
5235 unsigned long num_reads
= 0;
5236 struct bio
*bio
= NULL
;
5237 unsigned long bio_flags
= 0;
5239 if (test_bit(EXTENT_BUFFER_UPTODATE
, &eb
->bflags
))
5242 num_pages
= num_extent_pages(eb
);
5243 for (i
= 0; i
< num_pages
; i
++) {
5244 page
= eb
->pages
[i
];
5245 if (wait
== WAIT_NONE
) {
5246 if (!trylock_page(page
))
5254 * We need to firstly lock all pages to make sure that
5255 * the uptodate bit of our pages won't be affected by
5256 * clear_extent_buffer_uptodate().
5258 for (i
= 0; i
< num_pages
; i
++) {
5259 page
= eb
->pages
[i
];
5260 if (!PageUptodate(page
)) {
5267 set_bit(EXTENT_BUFFER_UPTODATE
, &eb
->bflags
);
5271 clear_bit(EXTENT_BUFFER_READ_ERR
, &eb
->bflags
);
5272 eb
->read_mirror
= 0;
5273 atomic_set(&eb
->io_pages
, num_reads
);
5274 for (i
= 0; i
< num_pages
; i
++) {
5275 page
= eb
->pages
[i
];
5277 if (!PageUptodate(page
)) {
5279 atomic_dec(&eb
->io_pages
);
5284 ClearPageError(page
);
5285 err
= __extent_read_full_page(tree
, page
,
5286 btree_get_extent
, &bio
,
5287 mirror_num
, &bio_flags
,
5292 * We use &bio in above __extent_read_full_page,
5293 * so we ensure that if it returns error, the
5294 * current page fails to add itself to bio and
5295 * it's been unlocked.
5297 * We must dec io_pages by ourselves.
5299 atomic_dec(&eb
->io_pages
);
5307 err
= submit_one_bio(bio
, mirror_num
, bio_flags
);
5312 if (ret
|| wait
!= WAIT_COMPLETE
)
5315 for (i
= 0; i
< num_pages
; i
++) {
5316 page
= eb
->pages
[i
];
5317 wait_on_page_locked(page
);
5318 if (!PageUptodate(page
))
5325 while (locked_pages
> 0) {
5327 page
= eb
->pages
[locked_pages
];
5333 void read_extent_buffer(const struct extent_buffer
*eb
, void *dstv
,
5334 unsigned long start
, unsigned long len
)
5340 char *dst
= (char *)dstv
;
5341 size_t start_offset
= offset_in_page(eb
->start
);
5342 unsigned long i
= (start_offset
+ start
) >> PAGE_SHIFT
;
5344 if (start
+ len
> eb
->len
) {
5345 WARN(1, KERN_ERR
"btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5346 eb
->start
, eb
->len
, start
, len
);
5347 memset(dst
, 0, len
);
5351 offset
= offset_in_page(start_offset
+ start
);
5354 page
= eb
->pages
[i
];
5356 cur
= min(len
, (PAGE_SIZE
- offset
));
5357 kaddr
= page_address(page
);
5358 memcpy(dst
, kaddr
+ offset
, cur
);
5367 int read_extent_buffer_to_user(const struct extent_buffer
*eb
,
5369 unsigned long start
, unsigned long len
)
5375 char __user
*dst
= (char __user
*)dstv
;
5376 size_t start_offset
= offset_in_page(eb
->start
);
5377 unsigned long i
= (start_offset
+ start
) >> PAGE_SHIFT
;
5380 WARN_ON(start
> eb
->len
);
5381 WARN_ON(start
+ len
> eb
->start
+ eb
->len
);
5383 offset
= offset_in_page(start_offset
+ start
);
5386 page
= eb
->pages
[i
];
5388 cur
= min(len
, (PAGE_SIZE
- offset
));
5389 kaddr
= page_address(page
);
5390 if (copy_to_user(dst
, kaddr
+ offset
, cur
)) {
5405 * return 0 if the item is found within a page.
5406 * return 1 if the item spans two pages.
5407 * return -EINVAL otherwise.
5409 int map_private_extent_buffer(const struct extent_buffer
*eb
,
5410 unsigned long start
, unsigned long min_len
,
5411 char **map
, unsigned long *map_start
,
5412 unsigned long *map_len
)
5417 size_t start_offset
= offset_in_page(eb
->start
);
5418 unsigned long i
= (start_offset
+ start
) >> PAGE_SHIFT
;
5419 unsigned long end_i
= (start_offset
+ start
+ min_len
- 1) >>
5422 if (start
+ min_len
> eb
->len
) {
5423 WARN(1, KERN_ERR
"btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
5424 eb
->start
, eb
->len
, start
, min_len
);
5432 offset
= start_offset
;
5436 *map_start
= ((u64
)i
<< PAGE_SHIFT
) - start_offset
;
5440 kaddr
= page_address(p
);
5441 *map
= kaddr
+ offset
;
5442 *map_len
= PAGE_SIZE
- offset
;
5446 int memcmp_extent_buffer(const struct extent_buffer
*eb
, const void *ptrv
,
5447 unsigned long start
, unsigned long len
)
5453 char *ptr
= (char *)ptrv
;
5454 size_t start_offset
= offset_in_page(eb
->start
);
5455 unsigned long i
= (start_offset
+ start
) >> PAGE_SHIFT
;
5458 WARN_ON(start
> eb
->len
);
5459 WARN_ON(start
+ len
> eb
->start
+ eb
->len
);
5461 offset
= offset_in_page(start_offset
+ start
);
5464 page
= eb
->pages
[i
];
5466 cur
= min(len
, (PAGE_SIZE
- offset
));
5468 kaddr
= page_address(page
);
5469 ret
= memcmp(ptr
, kaddr
+ offset
, cur
);
5481 void write_extent_buffer_chunk_tree_uuid(struct extent_buffer
*eb
,
5486 WARN_ON(!PageUptodate(eb
->pages
[0]));
5487 kaddr
= page_address(eb
->pages
[0]);
5488 memcpy(kaddr
+ offsetof(struct btrfs_header
, chunk_tree_uuid
), srcv
,
5492 void write_extent_buffer_fsid(struct extent_buffer
*eb
, const void *srcv
)
5496 WARN_ON(!PageUptodate(eb
->pages
[0]));
5497 kaddr
= page_address(eb
->pages
[0]);
5498 memcpy(kaddr
+ offsetof(struct btrfs_header
, fsid
), srcv
,
5502 void write_extent_buffer(struct extent_buffer
*eb
, const void *srcv
,
5503 unsigned long start
, unsigned long len
)
5509 char *src
= (char *)srcv
;
5510 size_t start_offset
= offset_in_page(eb
->start
);
5511 unsigned long i
= (start_offset
+ start
) >> PAGE_SHIFT
;
5513 WARN_ON(start
> eb
->len
);
5514 WARN_ON(start
+ len
> eb
->start
+ eb
->len
);
5516 offset
= offset_in_page(start_offset
+ start
);
5519 page
= eb
->pages
[i
];
5520 WARN_ON(!PageUptodate(page
));
5522 cur
= min(len
, PAGE_SIZE
- offset
);
5523 kaddr
= page_address(page
);
5524 memcpy(kaddr
+ offset
, src
, cur
);
5533 void memzero_extent_buffer(struct extent_buffer
*eb
, unsigned long start
,
5540 size_t start_offset
= offset_in_page(eb
->start
);
5541 unsigned long i
= (start_offset
+ start
) >> PAGE_SHIFT
;
5543 WARN_ON(start
> eb
->len
);
5544 WARN_ON(start
+ len
> eb
->start
+ eb
->len
);
5546 offset
= offset_in_page(start_offset
+ start
);
5549 page
= eb
->pages
[i
];
5550 WARN_ON(!PageUptodate(page
));
5552 cur
= min(len
, PAGE_SIZE
- offset
);
5553 kaddr
= page_address(page
);
5554 memset(kaddr
+ offset
, 0, cur
);
5562 void copy_extent_buffer_full(struct extent_buffer
*dst
,
5563 struct extent_buffer
*src
)
5568 ASSERT(dst
->len
== src
->len
);
5570 num_pages
= num_extent_pages(dst
);
5571 for (i
= 0; i
< num_pages
; i
++)
5572 copy_page(page_address(dst
->pages
[i
]),
5573 page_address(src
->pages
[i
]));
5576 void copy_extent_buffer(struct extent_buffer
*dst
, struct extent_buffer
*src
,
5577 unsigned long dst_offset
, unsigned long src_offset
,
5580 u64 dst_len
= dst
->len
;
5585 size_t start_offset
= offset_in_page(dst
->start
);
5586 unsigned long i
= (start_offset
+ dst_offset
) >> PAGE_SHIFT
;
5588 WARN_ON(src
->len
!= dst_len
);
5590 offset
= offset_in_page(start_offset
+ dst_offset
);
5593 page
= dst
->pages
[i
];
5594 WARN_ON(!PageUptodate(page
));
5596 cur
= min(len
, (unsigned long)(PAGE_SIZE
- offset
));
5598 kaddr
= page_address(page
);
5599 read_extent_buffer(src
, kaddr
+ offset
, src_offset
, cur
);
5609 * eb_bitmap_offset() - calculate the page and offset of the byte containing the
5611 * @eb: the extent buffer
5612 * @start: offset of the bitmap item in the extent buffer
5614 * @page_index: return index of the page in the extent buffer that contains the
5616 * @page_offset: return offset into the page given by page_index
5618 * This helper hides the ugliness of finding the byte in an extent buffer which
5619 * contains a given bit.
5621 static inline void eb_bitmap_offset(struct extent_buffer
*eb
,
5622 unsigned long start
, unsigned long nr
,
5623 unsigned long *page_index
,
5624 size_t *page_offset
)
5626 size_t start_offset
= offset_in_page(eb
->start
);
5627 size_t byte_offset
= BIT_BYTE(nr
);
5631 * The byte we want is the offset of the extent buffer + the offset of
5632 * the bitmap item in the extent buffer + the offset of the byte in the
5635 offset
= start_offset
+ start
+ byte_offset
;
5637 *page_index
= offset
>> PAGE_SHIFT
;
5638 *page_offset
= offset_in_page(offset
);
5642 * extent_buffer_test_bit - determine whether a bit in a bitmap item is set
5643 * @eb: the extent buffer
5644 * @start: offset of the bitmap item in the extent buffer
5645 * @nr: bit number to test
5647 int extent_buffer_test_bit(struct extent_buffer
*eb
, unsigned long start
,
5655 eb_bitmap_offset(eb
, start
, nr
, &i
, &offset
);
5656 page
= eb
->pages
[i
];
5657 WARN_ON(!PageUptodate(page
));
5658 kaddr
= page_address(page
);
5659 return 1U & (kaddr
[offset
] >> (nr
& (BITS_PER_BYTE
- 1)));
5663 * extent_buffer_bitmap_set - set an area of a bitmap
5664 * @eb: the extent buffer
5665 * @start: offset of the bitmap item in the extent buffer
5666 * @pos: bit number of the first bit
5667 * @len: number of bits to set
5669 void extent_buffer_bitmap_set(struct extent_buffer
*eb
, unsigned long start
,
5670 unsigned long pos
, unsigned long len
)
5676 const unsigned int size
= pos
+ len
;
5677 int bits_to_set
= BITS_PER_BYTE
- (pos
% BITS_PER_BYTE
);
5678 u8 mask_to_set
= BITMAP_FIRST_BYTE_MASK(pos
);
5680 eb_bitmap_offset(eb
, start
, pos
, &i
, &offset
);
5681 page
= eb
->pages
[i
];
5682 WARN_ON(!PageUptodate(page
));
5683 kaddr
= page_address(page
);
5685 while (len
>= bits_to_set
) {
5686 kaddr
[offset
] |= mask_to_set
;
5688 bits_to_set
= BITS_PER_BYTE
;
5690 if (++offset
>= PAGE_SIZE
&& len
> 0) {
5692 page
= eb
->pages
[++i
];
5693 WARN_ON(!PageUptodate(page
));
5694 kaddr
= page_address(page
);
5698 mask_to_set
&= BITMAP_LAST_BYTE_MASK(size
);
5699 kaddr
[offset
] |= mask_to_set
;
5705 * extent_buffer_bitmap_clear - clear an area of a bitmap
5706 * @eb: the extent buffer
5707 * @start: offset of the bitmap item in the extent buffer
5708 * @pos: bit number of the first bit
5709 * @len: number of bits to clear
5711 void extent_buffer_bitmap_clear(struct extent_buffer
*eb
, unsigned long start
,
5712 unsigned long pos
, unsigned long len
)
5718 const unsigned int size
= pos
+ len
;
5719 int bits_to_clear
= BITS_PER_BYTE
- (pos
% BITS_PER_BYTE
);
5720 u8 mask_to_clear
= BITMAP_FIRST_BYTE_MASK(pos
);
5722 eb_bitmap_offset(eb
, start
, pos
, &i
, &offset
);
5723 page
= eb
->pages
[i
];
5724 WARN_ON(!PageUptodate(page
));
5725 kaddr
= page_address(page
);
5727 while (len
>= bits_to_clear
) {
5728 kaddr
[offset
] &= ~mask_to_clear
;
5729 len
-= bits_to_clear
;
5730 bits_to_clear
= BITS_PER_BYTE
;
5732 if (++offset
>= PAGE_SIZE
&& len
> 0) {
5734 page
= eb
->pages
[++i
];
5735 WARN_ON(!PageUptodate(page
));
5736 kaddr
= page_address(page
);
5740 mask_to_clear
&= BITMAP_LAST_BYTE_MASK(size
);
5741 kaddr
[offset
] &= ~mask_to_clear
;
5745 static inline bool areas_overlap(unsigned long src
, unsigned long dst
, unsigned long len
)
5747 unsigned long distance
= (src
> dst
) ? src
- dst
: dst
- src
;
5748 return distance
< len
;
5751 static void copy_pages(struct page
*dst_page
, struct page
*src_page
,
5752 unsigned long dst_off
, unsigned long src_off
,
5755 char *dst_kaddr
= page_address(dst_page
);
5757 int must_memmove
= 0;
5759 if (dst_page
!= src_page
) {
5760 src_kaddr
= page_address(src_page
);
5762 src_kaddr
= dst_kaddr
;
5763 if (areas_overlap(src_off
, dst_off
, len
))
5768 memmove(dst_kaddr
+ dst_off
, src_kaddr
+ src_off
, len
);
5770 memcpy(dst_kaddr
+ dst_off
, src_kaddr
+ src_off
, len
);
5773 void memcpy_extent_buffer(struct extent_buffer
*dst
, unsigned long dst_offset
,
5774 unsigned long src_offset
, unsigned long len
)
5776 struct btrfs_fs_info
*fs_info
= dst
->fs_info
;
5778 size_t dst_off_in_page
;
5779 size_t src_off_in_page
;
5780 size_t start_offset
= offset_in_page(dst
->start
);
5781 unsigned long dst_i
;
5782 unsigned long src_i
;
5784 if (src_offset
+ len
> dst
->len
) {
5786 "memmove bogus src_offset %lu move len %lu dst len %lu",
5787 src_offset
, len
, dst
->len
);
5790 if (dst_offset
+ len
> dst
->len
) {
5792 "memmove bogus dst_offset %lu move len %lu dst len %lu",
5793 dst_offset
, len
, dst
->len
);
5798 dst_off_in_page
= offset_in_page(start_offset
+ dst_offset
);
5799 src_off_in_page
= offset_in_page(start_offset
+ src_offset
);
5801 dst_i
= (start_offset
+ dst_offset
) >> PAGE_SHIFT
;
5802 src_i
= (start_offset
+ src_offset
) >> PAGE_SHIFT
;
5804 cur
= min(len
, (unsigned long)(PAGE_SIZE
-
5806 cur
= min_t(unsigned long, cur
,
5807 (unsigned long)(PAGE_SIZE
- dst_off_in_page
));
5809 copy_pages(dst
->pages
[dst_i
], dst
->pages
[src_i
],
5810 dst_off_in_page
, src_off_in_page
, cur
);
5818 void memmove_extent_buffer(struct extent_buffer
*dst
, unsigned long dst_offset
,
5819 unsigned long src_offset
, unsigned long len
)
5821 struct btrfs_fs_info
*fs_info
= dst
->fs_info
;
5823 size_t dst_off_in_page
;
5824 size_t src_off_in_page
;
5825 unsigned long dst_end
= dst_offset
+ len
- 1;
5826 unsigned long src_end
= src_offset
+ len
- 1;
5827 size_t start_offset
= offset_in_page(dst
->start
);
5828 unsigned long dst_i
;
5829 unsigned long src_i
;
5831 if (src_offset
+ len
> dst
->len
) {
5833 "memmove bogus src_offset %lu move len %lu len %lu",
5834 src_offset
, len
, dst
->len
);
5837 if (dst_offset
+ len
> dst
->len
) {
5839 "memmove bogus dst_offset %lu move len %lu len %lu",
5840 dst_offset
, len
, dst
->len
);
5843 if (dst_offset
< src_offset
) {
5844 memcpy_extent_buffer(dst
, dst_offset
, src_offset
, len
);
5848 dst_i
= (start_offset
+ dst_end
) >> PAGE_SHIFT
;
5849 src_i
= (start_offset
+ src_end
) >> PAGE_SHIFT
;
5851 dst_off_in_page
= offset_in_page(start_offset
+ dst_end
);
5852 src_off_in_page
= offset_in_page(start_offset
+ src_end
);
5854 cur
= min_t(unsigned long, len
, src_off_in_page
+ 1);
5855 cur
= min(cur
, dst_off_in_page
+ 1);
5856 copy_pages(dst
->pages
[dst_i
], dst
->pages
[src_i
],
5857 dst_off_in_page
- cur
+ 1,
5858 src_off_in_page
- cur
+ 1, cur
);
5866 int try_release_extent_buffer(struct page
*page
)
5868 struct extent_buffer
*eb
;
5871 * We need to make sure nobody is attaching this page to an eb right
5874 spin_lock(&page
->mapping
->private_lock
);
5875 if (!PagePrivate(page
)) {
5876 spin_unlock(&page
->mapping
->private_lock
);
5880 eb
= (struct extent_buffer
*)page
->private;
5884 * This is a little awful but should be ok, we need to make sure that
5885 * the eb doesn't disappear out from under us while we're looking at
5888 spin_lock(&eb
->refs_lock
);
5889 if (atomic_read(&eb
->refs
) != 1 || extent_buffer_under_io(eb
)) {
5890 spin_unlock(&eb
->refs_lock
);
5891 spin_unlock(&page
->mapping
->private_lock
);
5894 spin_unlock(&page
->mapping
->private_lock
);
5897 * If tree ref isn't set then we know the ref on this eb is a real ref,
5898 * so just return, this page will likely be freed soon anyway.
5900 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF
, &eb
->bflags
)) {
5901 spin_unlock(&eb
->refs_lock
);
5905 return release_extent_buffer(eb
);