1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2011, 2012 STRATO. All rights reserved.
6 #include <linux/blkdev.h>
7 #include <linux/ratelimit.h>
8 #include <linux/sched/mm.h>
9 #include <crypto/hash.h>
13 #include "ordered-data.h"
14 #include "transaction.h"
16 #include "extent_io.h"
17 #include "dev-replace.h"
18 #include "check-integrity.h"
19 #include "rcu-string.h"
21 #include "block-group.h"
24 * This is only the first step towards a full-features scrub. It reads all
25 * extent and super block and verifies the checksums. In case a bad checksum
26 * is found or the extent cannot be read, good data will be written back if
29 * Future enhancements:
30 * - In case an unrepairable extent is encountered, track which files are
31 * affected and report them
32 * - track and record media errors, throw out bad devices
33 * - add a mode to also read unallocated space
40 * the following three values only influence the performance.
41 * The last one configures the number of parallel and outstanding I/O
42 * operations. The first two values configure an upper limit for the number
43 * of (dynamically allocated) pages that are added to a bio.
45 #define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */
46 #define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */
47 #define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */
50 * the following value times PAGE_SIZE needs to be large enough to match the
51 * largest node/leaf/sector size that shall be supported.
52 * Values larger than BTRFS_STRIPE_LEN are not supported.
54 #define SCRUB_MAX_PAGES_PER_BLOCK 16 /* 64k per node/leaf/sector */
56 struct scrub_recover
{
58 struct btrfs_bio
*bbio
;
63 struct scrub_block
*sblock
;
65 struct btrfs_device
*dev
;
66 struct list_head list
;
67 u64 flags
; /* extent flags */
71 u64 physical_for_dev_replace
;
74 unsigned int mirror_num
:8;
75 unsigned int have_csum
:1;
76 unsigned int io_error
:1;
78 u8 csum
[BTRFS_CSUM_SIZE
];
80 struct scrub_recover
*recover
;
85 struct scrub_ctx
*sctx
;
86 struct btrfs_device
*dev
;
91 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
92 struct scrub_page
*pagev
[SCRUB_PAGES_PER_WR_BIO
];
94 struct scrub_page
*pagev
[SCRUB_PAGES_PER_RD_BIO
];
98 struct btrfs_work work
;
102 struct scrub_page
*pagev
[SCRUB_MAX_PAGES_PER_BLOCK
];
104 atomic_t outstanding_pages
;
105 refcount_t refs
; /* free mem on transition to zero */
106 struct scrub_ctx
*sctx
;
107 struct scrub_parity
*sparity
;
109 unsigned int header_error
:1;
110 unsigned int checksum_error
:1;
111 unsigned int no_io_error_seen
:1;
112 unsigned int generation_error
:1; /* also sets header_error */
114 /* The following is for the data used to check parity */
115 /* It is for the data with checksum */
116 unsigned int data_corrected
:1;
118 struct btrfs_work work
;
121 /* Used for the chunks with parity stripe such RAID5/6 */
122 struct scrub_parity
{
123 struct scrub_ctx
*sctx
;
125 struct btrfs_device
*scrub_dev
;
137 struct list_head spages
;
139 /* Work of parity check and repair */
140 struct btrfs_work work
;
142 /* Mark the parity blocks which have data */
143 unsigned long *dbitmap
;
146 * Mark the parity blocks which have data, but errors happen when
147 * read data or check data
149 unsigned long *ebitmap
;
151 unsigned long bitmap
[0];
155 struct scrub_bio
*bios
[SCRUB_BIOS_PER_SCTX
];
156 struct btrfs_fs_info
*fs_info
;
159 atomic_t bios_in_flight
;
160 atomic_t workers_pending
;
161 spinlock_t list_lock
;
162 wait_queue_head_t list_wait
;
164 struct list_head csum_list
;
167 int pages_per_rd_bio
;
171 struct scrub_bio
*wr_curr_bio
;
172 struct mutex wr_lock
;
173 int pages_per_wr_bio
; /* <= SCRUB_PAGES_PER_WR_BIO */
174 struct btrfs_device
*wr_tgtdev
;
175 bool flush_all_writes
;
180 struct btrfs_scrub_progress stat
;
181 spinlock_t stat_lock
;
184 * Use a ref counter to avoid use-after-free issues. Scrub workers
185 * decrement bios_in_flight and workers_pending and then do a wakeup
186 * on the list_wait wait queue. We must ensure the main scrub task
187 * doesn't free the scrub context before or while the workers are
188 * doing the wakeup() call.
193 struct scrub_warning
{
194 struct btrfs_path
*path
;
195 u64 extent_item_size
;
199 struct btrfs_device
*dev
;
202 struct full_stripe_lock
{
209 static void scrub_pending_bio_inc(struct scrub_ctx
*sctx
);
210 static void scrub_pending_bio_dec(struct scrub_ctx
*sctx
);
211 static int scrub_handle_errored_block(struct scrub_block
*sblock_to_check
);
212 static int scrub_setup_recheck_block(struct scrub_block
*original_sblock
,
213 struct scrub_block
*sblocks_for_recheck
);
214 static void scrub_recheck_block(struct btrfs_fs_info
*fs_info
,
215 struct scrub_block
*sblock
,
216 int retry_failed_mirror
);
217 static void scrub_recheck_block_checksum(struct scrub_block
*sblock
);
218 static int scrub_repair_block_from_good_copy(struct scrub_block
*sblock_bad
,
219 struct scrub_block
*sblock_good
);
220 static int scrub_repair_page_from_good_copy(struct scrub_block
*sblock_bad
,
221 struct scrub_block
*sblock_good
,
222 int page_num
, int force_write
);
223 static void scrub_write_block_to_dev_replace(struct scrub_block
*sblock
);
224 static int scrub_write_page_to_dev_replace(struct scrub_block
*sblock
,
226 static int scrub_checksum_data(struct scrub_block
*sblock
);
227 static int scrub_checksum_tree_block(struct scrub_block
*sblock
);
228 static int scrub_checksum_super(struct scrub_block
*sblock
);
229 static void scrub_block_get(struct scrub_block
*sblock
);
230 static void scrub_block_put(struct scrub_block
*sblock
);
231 static void scrub_page_get(struct scrub_page
*spage
);
232 static void scrub_page_put(struct scrub_page
*spage
);
233 static void scrub_parity_get(struct scrub_parity
*sparity
);
234 static void scrub_parity_put(struct scrub_parity
*sparity
);
235 static int scrub_add_page_to_rd_bio(struct scrub_ctx
*sctx
,
236 struct scrub_page
*spage
);
237 static int scrub_pages(struct scrub_ctx
*sctx
, u64 logical
, u64 len
,
238 u64 physical
, struct btrfs_device
*dev
, u64 flags
,
239 u64 gen
, int mirror_num
, u8
*csum
, int force
,
240 u64 physical_for_dev_replace
);
241 static void scrub_bio_end_io(struct bio
*bio
);
242 static void scrub_bio_end_io_worker(struct btrfs_work
*work
);
243 static void scrub_block_complete(struct scrub_block
*sblock
);
244 static void scrub_remap_extent(struct btrfs_fs_info
*fs_info
,
245 u64 extent_logical
, u64 extent_len
,
246 u64
*extent_physical
,
247 struct btrfs_device
**extent_dev
,
248 int *extent_mirror_num
);
249 static int scrub_add_page_to_wr_bio(struct scrub_ctx
*sctx
,
250 struct scrub_page
*spage
);
251 static void scrub_wr_submit(struct scrub_ctx
*sctx
);
252 static void scrub_wr_bio_end_io(struct bio
*bio
);
253 static void scrub_wr_bio_end_io_worker(struct btrfs_work
*work
);
254 static void __scrub_blocked_if_needed(struct btrfs_fs_info
*fs_info
);
255 static void scrub_blocked_if_needed(struct btrfs_fs_info
*fs_info
);
256 static void scrub_put_ctx(struct scrub_ctx
*sctx
);
258 static inline int scrub_is_page_on_raid56(struct scrub_page
*page
)
260 return page
->recover
&&
261 (page
->recover
->bbio
->map_type
& BTRFS_BLOCK_GROUP_RAID56_MASK
);
264 static void scrub_pending_bio_inc(struct scrub_ctx
*sctx
)
266 refcount_inc(&sctx
->refs
);
267 atomic_inc(&sctx
->bios_in_flight
);
270 static void scrub_pending_bio_dec(struct scrub_ctx
*sctx
)
272 atomic_dec(&sctx
->bios_in_flight
);
273 wake_up(&sctx
->list_wait
);
277 static void __scrub_blocked_if_needed(struct btrfs_fs_info
*fs_info
)
279 while (atomic_read(&fs_info
->scrub_pause_req
)) {
280 mutex_unlock(&fs_info
->scrub_lock
);
281 wait_event(fs_info
->scrub_pause_wait
,
282 atomic_read(&fs_info
->scrub_pause_req
) == 0);
283 mutex_lock(&fs_info
->scrub_lock
);
287 static void scrub_pause_on(struct btrfs_fs_info
*fs_info
)
289 atomic_inc(&fs_info
->scrubs_paused
);
290 wake_up(&fs_info
->scrub_pause_wait
);
293 static void scrub_pause_off(struct btrfs_fs_info
*fs_info
)
295 mutex_lock(&fs_info
->scrub_lock
);
296 __scrub_blocked_if_needed(fs_info
);
297 atomic_dec(&fs_info
->scrubs_paused
);
298 mutex_unlock(&fs_info
->scrub_lock
);
300 wake_up(&fs_info
->scrub_pause_wait
);
303 static void scrub_blocked_if_needed(struct btrfs_fs_info
*fs_info
)
305 scrub_pause_on(fs_info
);
306 scrub_pause_off(fs_info
);
310 * Insert new full stripe lock into full stripe locks tree
312 * Return pointer to existing or newly inserted full_stripe_lock structure if
313 * everything works well.
314 * Return ERR_PTR(-ENOMEM) if we failed to allocate memory
316 * NOTE: caller must hold full_stripe_locks_root->lock before calling this
319 static struct full_stripe_lock
*insert_full_stripe_lock(
320 struct btrfs_full_stripe_locks_tree
*locks_root
,
324 struct rb_node
*parent
= NULL
;
325 struct full_stripe_lock
*entry
;
326 struct full_stripe_lock
*ret
;
328 lockdep_assert_held(&locks_root
->lock
);
330 p
= &locks_root
->root
.rb_node
;
333 entry
= rb_entry(parent
, struct full_stripe_lock
, node
);
334 if (fstripe_logical
< entry
->logical
) {
336 } else if (fstripe_logical
> entry
->logical
) {
347 ret
= kmalloc(sizeof(*ret
), GFP_KERNEL
);
349 return ERR_PTR(-ENOMEM
);
350 ret
->logical
= fstripe_logical
;
352 mutex_init(&ret
->mutex
);
354 rb_link_node(&ret
->node
, parent
, p
);
355 rb_insert_color(&ret
->node
, &locks_root
->root
);
360 * Search for a full stripe lock of a block group
362 * Return pointer to existing full stripe lock if found
363 * Return NULL if not found
365 static struct full_stripe_lock
*search_full_stripe_lock(
366 struct btrfs_full_stripe_locks_tree
*locks_root
,
369 struct rb_node
*node
;
370 struct full_stripe_lock
*entry
;
372 lockdep_assert_held(&locks_root
->lock
);
374 node
= locks_root
->root
.rb_node
;
376 entry
= rb_entry(node
, struct full_stripe_lock
, node
);
377 if (fstripe_logical
< entry
->logical
)
378 node
= node
->rb_left
;
379 else if (fstripe_logical
> entry
->logical
)
380 node
= node
->rb_right
;
388 * Helper to get full stripe logical from a normal bytenr.
390 * Caller must ensure @cache is a RAID56 block group.
392 static u64
get_full_stripe_logical(struct btrfs_block_group
*cache
, u64 bytenr
)
397 * Due to chunk item size limit, full stripe length should not be
398 * larger than U32_MAX. Just a sanity check here.
400 WARN_ON_ONCE(cache
->full_stripe_len
>= U32_MAX
);
403 * round_down() can only handle power of 2, while RAID56 full
404 * stripe length can be 64KiB * n, so we need to manually round down.
406 ret
= div64_u64(bytenr
- cache
->start
, cache
->full_stripe_len
) *
407 cache
->full_stripe_len
+ cache
->start
;
412 * Lock a full stripe to avoid concurrency of recovery and read
414 * It's only used for profiles with parities (RAID5/6), for other profiles it
417 * Return 0 if we locked full stripe covering @bytenr, with a mutex held.
418 * So caller must call unlock_full_stripe() at the same context.
420 * Return <0 if encounters error.
422 static int lock_full_stripe(struct btrfs_fs_info
*fs_info
, u64 bytenr
,
425 struct btrfs_block_group
*bg_cache
;
426 struct btrfs_full_stripe_locks_tree
*locks_root
;
427 struct full_stripe_lock
*existing
;
432 bg_cache
= btrfs_lookup_block_group(fs_info
, bytenr
);
438 /* Profiles not based on parity don't need full stripe lock */
439 if (!(bg_cache
->flags
& BTRFS_BLOCK_GROUP_RAID56_MASK
))
441 locks_root
= &bg_cache
->full_stripe_locks_root
;
443 fstripe_start
= get_full_stripe_logical(bg_cache
, bytenr
);
445 /* Now insert the full stripe lock */
446 mutex_lock(&locks_root
->lock
);
447 existing
= insert_full_stripe_lock(locks_root
, fstripe_start
);
448 mutex_unlock(&locks_root
->lock
);
449 if (IS_ERR(existing
)) {
450 ret
= PTR_ERR(existing
);
453 mutex_lock(&existing
->mutex
);
456 btrfs_put_block_group(bg_cache
);
461 * Unlock a full stripe.
463 * NOTE: Caller must ensure it's the same context calling corresponding
464 * lock_full_stripe().
466 * Return 0 if we unlock full stripe without problem.
467 * Return <0 for error
469 static int unlock_full_stripe(struct btrfs_fs_info
*fs_info
, u64 bytenr
,
472 struct btrfs_block_group
*bg_cache
;
473 struct btrfs_full_stripe_locks_tree
*locks_root
;
474 struct full_stripe_lock
*fstripe_lock
;
479 /* If we didn't acquire full stripe lock, no need to continue */
483 bg_cache
= btrfs_lookup_block_group(fs_info
, bytenr
);
488 if (!(bg_cache
->flags
& BTRFS_BLOCK_GROUP_RAID56_MASK
))
491 locks_root
= &bg_cache
->full_stripe_locks_root
;
492 fstripe_start
= get_full_stripe_logical(bg_cache
, bytenr
);
494 mutex_lock(&locks_root
->lock
);
495 fstripe_lock
= search_full_stripe_lock(locks_root
, fstripe_start
);
496 /* Unpaired unlock_full_stripe() detected */
500 mutex_unlock(&locks_root
->lock
);
504 if (fstripe_lock
->refs
== 0) {
506 btrfs_warn(fs_info
, "full stripe lock at %llu refcount underflow",
507 fstripe_lock
->logical
);
509 fstripe_lock
->refs
--;
512 if (fstripe_lock
->refs
== 0) {
513 rb_erase(&fstripe_lock
->node
, &locks_root
->root
);
516 mutex_unlock(&locks_root
->lock
);
518 mutex_unlock(&fstripe_lock
->mutex
);
522 btrfs_put_block_group(bg_cache
);
526 static void scrub_free_csums(struct scrub_ctx
*sctx
)
528 while (!list_empty(&sctx
->csum_list
)) {
529 struct btrfs_ordered_sum
*sum
;
530 sum
= list_first_entry(&sctx
->csum_list
,
531 struct btrfs_ordered_sum
, list
);
532 list_del(&sum
->list
);
537 static noinline_for_stack
void scrub_free_ctx(struct scrub_ctx
*sctx
)
544 /* this can happen when scrub is cancelled */
545 if (sctx
->curr
!= -1) {
546 struct scrub_bio
*sbio
= sctx
->bios
[sctx
->curr
];
548 for (i
= 0; i
< sbio
->page_count
; i
++) {
549 WARN_ON(!sbio
->pagev
[i
]->page
);
550 scrub_block_put(sbio
->pagev
[i
]->sblock
);
555 for (i
= 0; i
< SCRUB_BIOS_PER_SCTX
; ++i
) {
556 struct scrub_bio
*sbio
= sctx
->bios
[i
];
563 kfree(sctx
->wr_curr_bio
);
564 scrub_free_csums(sctx
);
568 static void scrub_put_ctx(struct scrub_ctx
*sctx
)
570 if (refcount_dec_and_test(&sctx
->refs
))
571 scrub_free_ctx(sctx
);
574 static noinline_for_stack
struct scrub_ctx
*scrub_setup_ctx(
575 struct btrfs_fs_info
*fs_info
, int is_dev_replace
)
577 struct scrub_ctx
*sctx
;
580 sctx
= kzalloc(sizeof(*sctx
), GFP_KERNEL
);
583 refcount_set(&sctx
->refs
, 1);
584 sctx
->is_dev_replace
= is_dev_replace
;
585 sctx
->pages_per_rd_bio
= SCRUB_PAGES_PER_RD_BIO
;
587 sctx
->fs_info
= fs_info
;
588 INIT_LIST_HEAD(&sctx
->csum_list
);
589 for (i
= 0; i
< SCRUB_BIOS_PER_SCTX
; ++i
) {
590 struct scrub_bio
*sbio
;
592 sbio
= kzalloc(sizeof(*sbio
), GFP_KERNEL
);
595 sctx
->bios
[i
] = sbio
;
599 sbio
->page_count
= 0;
600 btrfs_init_work(&sbio
->work
, scrub_bio_end_io_worker
, NULL
,
603 if (i
!= SCRUB_BIOS_PER_SCTX
- 1)
604 sctx
->bios
[i
]->next_free
= i
+ 1;
606 sctx
->bios
[i
]->next_free
= -1;
608 sctx
->first_free
= 0;
609 atomic_set(&sctx
->bios_in_flight
, 0);
610 atomic_set(&sctx
->workers_pending
, 0);
611 atomic_set(&sctx
->cancel_req
, 0);
612 sctx
->csum_size
= btrfs_super_csum_size(fs_info
->super_copy
);
614 spin_lock_init(&sctx
->list_lock
);
615 spin_lock_init(&sctx
->stat_lock
);
616 init_waitqueue_head(&sctx
->list_wait
);
618 WARN_ON(sctx
->wr_curr_bio
!= NULL
);
619 mutex_init(&sctx
->wr_lock
);
620 sctx
->wr_curr_bio
= NULL
;
621 if (is_dev_replace
) {
622 WARN_ON(!fs_info
->dev_replace
.tgtdev
);
623 sctx
->pages_per_wr_bio
= SCRUB_PAGES_PER_WR_BIO
;
624 sctx
->wr_tgtdev
= fs_info
->dev_replace
.tgtdev
;
625 sctx
->flush_all_writes
= false;
631 scrub_free_ctx(sctx
);
632 return ERR_PTR(-ENOMEM
);
635 static int scrub_print_warning_inode(u64 inum
, u64 offset
, u64 root
,
643 struct extent_buffer
*eb
;
644 struct btrfs_inode_item
*inode_item
;
645 struct scrub_warning
*swarn
= warn_ctx
;
646 struct btrfs_fs_info
*fs_info
= swarn
->dev
->fs_info
;
647 struct inode_fs_paths
*ipath
= NULL
;
648 struct btrfs_root
*local_root
;
649 struct btrfs_key root_key
;
650 struct btrfs_key key
;
652 root_key
.objectid
= root
;
653 root_key
.type
= BTRFS_ROOT_ITEM_KEY
;
654 root_key
.offset
= (u64
)-1;
655 local_root
= btrfs_read_fs_root_no_name(fs_info
, &root_key
);
656 if (IS_ERR(local_root
)) {
657 ret
= PTR_ERR(local_root
);
662 * this makes the path point to (inum INODE_ITEM ioff)
665 key
.type
= BTRFS_INODE_ITEM_KEY
;
668 ret
= btrfs_search_slot(NULL
, local_root
, &key
, swarn
->path
, 0, 0);
670 btrfs_release_path(swarn
->path
);
674 eb
= swarn
->path
->nodes
[0];
675 inode_item
= btrfs_item_ptr(eb
, swarn
->path
->slots
[0],
676 struct btrfs_inode_item
);
677 isize
= btrfs_inode_size(eb
, inode_item
);
678 nlink
= btrfs_inode_nlink(eb
, inode_item
);
679 btrfs_release_path(swarn
->path
);
682 * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub
683 * uses GFP_NOFS in this context, so we keep it consistent but it does
684 * not seem to be strictly necessary.
686 nofs_flag
= memalloc_nofs_save();
687 ipath
= init_ipath(4096, local_root
, swarn
->path
);
688 memalloc_nofs_restore(nofs_flag
);
690 ret
= PTR_ERR(ipath
);
694 ret
= paths_from_inode(inum
, ipath
);
700 * we deliberately ignore the bit ipath might have been too small to
701 * hold all of the paths here
703 for (i
= 0; i
< ipath
->fspath
->elem_cnt
; ++i
)
704 btrfs_warn_in_rcu(fs_info
,
705 "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)",
706 swarn
->errstr
, swarn
->logical
,
707 rcu_str_deref(swarn
->dev
->name
),
710 min(isize
- offset
, (u64
)PAGE_SIZE
), nlink
,
711 (char *)(unsigned long)ipath
->fspath
->val
[i
]);
717 btrfs_warn_in_rcu(fs_info
,
718 "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d",
719 swarn
->errstr
, swarn
->logical
,
720 rcu_str_deref(swarn
->dev
->name
),
722 root
, inum
, offset
, ret
);
728 static void scrub_print_warning(const char *errstr
, struct scrub_block
*sblock
)
730 struct btrfs_device
*dev
;
731 struct btrfs_fs_info
*fs_info
;
732 struct btrfs_path
*path
;
733 struct btrfs_key found_key
;
734 struct extent_buffer
*eb
;
735 struct btrfs_extent_item
*ei
;
736 struct scrub_warning swarn
;
737 unsigned long ptr
= 0;
745 WARN_ON(sblock
->page_count
< 1);
746 dev
= sblock
->pagev
[0]->dev
;
747 fs_info
= sblock
->sctx
->fs_info
;
749 path
= btrfs_alloc_path();
753 swarn
.physical
= sblock
->pagev
[0]->physical
;
754 swarn
.logical
= sblock
->pagev
[0]->logical
;
755 swarn
.errstr
= errstr
;
758 ret
= extent_from_logical(fs_info
, swarn
.logical
, path
, &found_key
,
763 extent_item_pos
= swarn
.logical
- found_key
.objectid
;
764 swarn
.extent_item_size
= found_key
.offset
;
767 ei
= btrfs_item_ptr(eb
, path
->slots
[0], struct btrfs_extent_item
);
768 item_size
= btrfs_item_size_nr(eb
, path
->slots
[0]);
770 if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
772 ret
= tree_backref_for_extent(&ptr
, eb
, &found_key
, ei
,
773 item_size
, &ref_root
,
775 btrfs_warn_in_rcu(fs_info
,
776 "%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",
777 errstr
, swarn
.logical
,
778 rcu_str_deref(dev
->name
),
780 ref_level
? "node" : "leaf",
781 ret
< 0 ? -1 : ref_level
,
782 ret
< 0 ? -1 : ref_root
);
784 btrfs_release_path(path
);
786 btrfs_release_path(path
);
789 iterate_extent_inodes(fs_info
, found_key
.objectid
,
791 scrub_print_warning_inode
, &swarn
, false);
795 btrfs_free_path(path
);
798 static inline void scrub_get_recover(struct scrub_recover
*recover
)
800 refcount_inc(&recover
->refs
);
803 static inline void scrub_put_recover(struct btrfs_fs_info
*fs_info
,
804 struct scrub_recover
*recover
)
806 if (refcount_dec_and_test(&recover
->refs
)) {
807 btrfs_bio_counter_dec(fs_info
);
808 btrfs_put_bbio(recover
->bbio
);
814 * scrub_handle_errored_block gets called when either verification of the
815 * pages failed or the bio failed to read, e.g. with EIO. In the latter
816 * case, this function handles all pages in the bio, even though only one
818 * The goal of this function is to repair the errored block by using the
819 * contents of one of the mirrors.
821 static int scrub_handle_errored_block(struct scrub_block
*sblock_to_check
)
823 struct scrub_ctx
*sctx
= sblock_to_check
->sctx
;
824 struct btrfs_device
*dev
;
825 struct btrfs_fs_info
*fs_info
;
827 unsigned int failed_mirror_index
;
828 unsigned int is_metadata
;
829 unsigned int have_csum
;
830 struct scrub_block
*sblocks_for_recheck
; /* holds one for each mirror */
831 struct scrub_block
*sblock_bad
;
836 bool full_stripe_locked
;
837 unsigned int nofs_flag
;
838 static DEFINE_RATELIMIT_STATE(_rs
, DEFAULT_RATELIMIT_INTERVAL
,
839 DEFAULT_RATELIMIT_BURST
);
841 BUG_ON(sblock_to_check
->page_count
< 1);
842 fs_info
= sctx
->fs_info
;
843 if (sblock_to_check
->pagev
[0]->flags
& BTRFS_EXTENT_FLAG_SUPER
) {
845 * if we find an error in a super block, we just report it.
846 * They will get written with the next transaction commit
849 spin_lock(&sctx
->stat_lock
);
850 ++sctx
->stat
.super_errors
;
851 spin_unlock(&sctx
->stat_lock
);
854 logical
= sblock_to_check
->pagev
[0]->logical
;
855 BUG_ON(sblock_to_check
->pagev
[0]->mirror_num
< 1);
856 failed_mirror_index
= sblock_to_check
->pagev
[0]->mirror_num
- 1;
857 is_metadata
= !(sblock_to_check
->pagev
[0]->flags
&
858 BTRFS_EXTENT_FLAG_DATA
);
859 have_csum
= sblock_to_check
->pagev
[0]->have_csum
;
860 dev
= sblock_to_check
->pagev
[0]->dev
;
863 * We must use GFP_NOFS because the scrub task might be waiting for a
864 * worker task executing this function and in turn a transaction commit
865 * might be waiting the scrub task to pause (which needs to wait for all
866 * the worker tasks to complete before pausing).
867 * We do allocations in the workers through insert_full_stripe_lock()
868 * and scrub_add_page_to_wr_bio(), which happens down the call chain of
871 nofs_flag
= memalloc_nofs_save();
873 * For RAID5/6, race can happen for a different device scrub thread.
874 * For data corruption, Parity and Data threads will both try
875 * to recovery the data.
876 * Race can lead to doubly added csum error, or even unrecoverable
879 ret
= lock_full_stripe(fs_info
, logical
, &full_stripe_locked
);
881 memalloc_nofs_restore(nofs_flag
);
882 spin_lock(&sctx
->stat_lock
);
884 sctx
->stat
.malloc_errors
++;
885 sctx
->stat
.read_errors
++;
886 sctx
->stat
.uncorrectable_errors
++;
887 spin_unlock(&sctx
->stat_lock
);
892 * read all mirrors one after the other. This includes to
893 * re-read the extent or metadata block that failed (that was
894 * the cause that this fixup code is called) another time,
895 * page by page this time in order to know which pages
896 * caused I/O errors and which ones are good (for all mirrors).
897 * It is the goal to handle the situation when more than one
898 * mirror contains I/O errors, but the errors do not
899 * overlap, i.e. the data can be repaired by selecting the
900 * pages from those mirrors without I/O error on the
901 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
902 * would be that mirror #1 has an I/O error on the first page,
903 * the second page is good, and mirror #2 has an I/O error on
904 * the second page, but the first page is good.
905 * Then the first page of the first mirror can be repaired by
906 * taking the first page of the second mirror, and the
907 * second page of the second mirror can be repaired by
908 * copying the contents of the 2nd page of the 1st mirror.
909 * One more note: if the pages of one mirror contain I/O
910 * errors, the checksum cannot be verified. In order to get
911 * the best data for repairing, the first attempt is to find
912 * a mirror without I/O errors and with a validated checksum.
913 * Only if this is not possible, the pages are picked from
914 * mirrors with I/O errors without considering the checksum.
915 * If the latter is the case, at the end, the checksum of the
916 * repaired area is verified in order to correctly maintain
920 sblocks_for_recheck
= kcalloc(BTRFS_MAX_MIRRORS
,
921 sizeof(*sblocks_for_recheck
), GFP_KERNEL
);
922 if (!sblocks_for_recheck
) {
923 spin_lock(&sctx
->stat_lock
);
924 sctx
->stat
.malloc_errors
++;
925 sctx
->stat
.read_errors
++;
926 sctx
->stat
.uncorrectable_errors
++;
927 spin_unlock(&sctx
->stat_lock
);
928 btrfs_dev_stat_inc_and_print(dev
, BTRFS_DEV_STAT_READ_ERRS
);
932 /* setup the context, map the logical blocks and alloc the pages */
933 ret
= scrub_setup_recheck_block(sblock_to_check
, sblocks_for_recheck
);
935 spin_lock(&sctx
->stat_lock
);
936 sctx
->stat
.read_errors
++;
937 sctx
->stat
.uncorrectable_errors
++;
938 spin_unlock(&sctx
->stat_lock
);
939 btrfs_dev_stat_inc_and_print(dev
, BTRFS_DEV_STAT_READ_ERRS
);
942 BUG_ON(failed_mirror_index
>= BTRFS_MAX_MIRRORS
);
943 sblock_bad
= sblocks_for_recheck
+ failed_mirror_index
;
945 /* build and submit the bios for the failed mirror, check checksums */
946 scrub_recheck_block(fs_info
, sblock_bad
, 1);
948 if (!sblock_bad
->header_error
&& !sblock_bad
->checksum_error
&&
949 sblock_bad
->no_io_error_seen
) {
951 * the error disappeared after reading page by page, or
952 * the area was part of a huge bio and other parts of the
953 * bio caused I/O errors, or the block layer merged several
954 * read requests into one and the error is caused by a
955 * different bio (usually one of the two latter cases is
958 spin_lock(&sctx
->stat_lock
);
959 sctx
->stat
.unverified_errors
++;
960 sblock_to_check
->data_corrected
= 1;
961 spin_unlock(&sctx
->stat_lock
);
963 if (sctx
->is_dev_replace
)
964 scrub_write_block_to_dev_replace(sblock_bad
);
968 if (!sblock_bad
->no_io_error_seen
) {
969 spin_lock(&sctx
->stat_lock
);
970 sctx
->stat
.read_errors
++;
971 spin_unlock(&sctx
->stat_lock
);
972 if (__ratelimit(&_rs
))
973 scrub_print_warning("i/o error", sblock_to_check
);
974 btrfs_dev_stat_inc_and_print(dev
, BTRFS_DEV_STAT_READ_ERRS
);
975 } else if (sblock_bad
->checksum_error
) {
976 spin_lock(&sctx
->stat_lock
);
977 sctx
->stat
.csum_errors
++;
978 spin_unlock(&sctx
->stat_lock
);
979 if (__ratelimit(&_rs
))
980 scrub_print_warning("checksum error", sblock_to_check
);
981 btrfs_dev_stat_inc_and_print(dev
,
982 BTRFS_DEV_STAT_CORRUPTION_ERRS
);
983 } else if (sblock_bad
->header_error
) {
984 spin_lock(&sctx
->stat_lock
);
985 sctx
->stat
.verify_errors
++;
986 spin_unlock(&sctx
->stat_lock
);
987 if (__ratelimit(&_rs
))
988 scrub_print_warning("checksum/header error",
990 if (sblock_bad
->generation_error
)
991 btrfs_dev_stat_inc_and_print(dev
,
992 BTRFS_DEV_STAT_GENERATION_ERRS
);
994 btrfs_dev_stat_inc_and_print(dev
,
995 BTRFS_DEV_STAT_CORRUPTION_ERRS
);
998 if (sctx
->readonly
) {
999 ASSERT(!sctx
->is_dev_replace
);
1004 * now build and submit the bios for the other mirrors, check
1006 * First try to pick the mirror which is completely without I/O
1007 * errors and also does not have a checksum error.
1008 * If one is found, and if a checksum is present, the full block
1009 * that is known to contain an error is rewritten. Afterwards
1010 * the block is known to be corrected.
1011 * If a mirror is found which is completely correct, and no
1012 * checksum is present, only those pages are rewritten that had
1013 * an I/O error in the block to be repaired, since it cannot be
1014 * determined, which copy of the other pages is better (and it
1015 * could happen otherwise that a correct page would be
1016 * overwritten by a bad one).
1018 for (mirror_index
= 0; ;mirror_index
++) {
1019 struct scrub_block
*sblock_other
;
1021 if (mirror_index
== failed_mirror_index
)
1024 /* raid56's mirror can be more than BTRFS_MAX_MIRRORS */
1025 if (!scrub_is_page_on_raid56(sblock_bad
->pagev
[0])) {
1026 if (mirror_index
>= BTRFS_MAX_MIRRORS
)
1028 if (!sblocks_for_recheck
[mirror_index
].page_count
)
1031 sblock_other
= sblocks_for_recheck
+ mirror_index
;
1033 struct scrub_recover
*r
= sblock_bad
->pagev
[0]->recover
;
1034 int max_allowed
= r
->bbio
->num_stripes
-
1035 r
->bbio
->num_tgtdevs
;
1037 if (mirror_index
>= max_allowed
)
1039 if (!sblocks_for_recheck
[1].page_count
)
1042 ASSERT(failed_mirror_index
== 0);
1043 sblock_other
= sblocks_for_recheck
+ 1;
1044 sblock_other
->pagev
[0]->mirror_num
= 1 + mirror_index
;
1047 /* build and submit the bios, check checksums */
1048 scrub_recheck_block(fs_info
, sblock_other
, 0);
1050 if (!sblock_other
->header_error
&&
1051 !sblock_other
->checksum_error
&&
1052 sblock_other
->no_io_error_seen
) {
1053 if (sctx
->is_dev_replace
) {
1054 scrub_write_block_to_dev_replace(sblock_other
);
1055 goto corrected_error
;
1057 ret
= scrub_repair_block_from_good_copy(
1058 sblock_bad
, sblock_other
);
1060 goto corrected_error
;
1065 if (sblock_bad
->no_io_error_seen
&& !sctx
->is_dev_replace
)
1066 goto did_not_correct_error
;
1069 * In case of I/O errors in the area that is supposed to be
1070 * repaired, continue by picking good copies of those pages.
1071 * Select the good pages from mirrors to rewrite bad pages from
1072 * the area to fix. Afterwards verify the checksum of the block
1073 * that is supposed to be repaired. This verification step is
1074 * only done for the purpose of statistic counting and for the
1075 * final scrub report, whether errors remain.
1076 * A perfect algorithm could make use of the checksum and try
1077 * all possible combinations of pages from the different mirrors
1078 * until the checksum verification succeeds. For example, when
1079 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1080 * of mirror #2 is readable but the final checksum test fails,
1081 * then the 2nd page of mirror #3 could be tried, whether now
1082 * the final checksum succeeds. But this would be a rare
1083 * exception and is therefore not implemented. At least it is
1084 * avoided that the good copy is overwritten.
1085 * A more useful improvement would be to pick the sectors
1086 * without I/O error based on sector sizes (512 bytes on legacy
1087 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1088 * mirror could be repaired by taking 512 byte of a different
1089 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1090 * area are unreadable.
1093 for (page_num
= 0; page_num
< sblock_bad
->page_count
;
1095 struct scrub_page
*page_bad
= sblock_bad
->pagev
[page_num
];
1096 struct scrub_block
*sblock_other
= NULL
;
1098 /* skip no-io-error page in scrub */
1099 if (!page_bad
->io_error
&& !sctx
->is_dev_replace
)
1102 if (scrub_is_page_on_raid56(sblock_bad
->pagev
[0])) {
1104 * In case of dev replace, if raid56 rebuild process
1105 * didn't work out correct data, then copy the content
1106 * in sblock_bad to make sure target device is identical
1107 * to source device, instead of writing garbage data in
1108 * sblock_for_recheck array to target device.
1110 sblock_other
= NULL
;
1111 } else if (page_bad
->io_error
) {
1112 /* try to find no-io-error page in mirrors */
1113 for (mirror_index
= 0;
1114 mirror_index
< BTRFS_MAX_MIRRORS
&&
1115 sblocks_for_recheck
[mirror_index
].page_count
> 0;
1117 if (!sblocks_for_recheck
[mirror_index
].
1118 pagev
[page_num
]->io_error
) {
1119 sblock_other
= sblocks_for_recheck
+
1128 if (sctx
->is_dev_replace
) {
1130 * did not find a mirror to fetch the page
1131 * from. scrub_write_page_to_dev_replace()
1132 * handles this case (page->io_error), by
1133 * filling the block with zeros before
1134 * submitting the write request
1137 sblock_other
= sblock_bad
;
1139 if (scrub_write_page_to_dev_replace(sblock_other
,
1142 &fs_info
->dev_replace
.num_write_errors
);
1145 } else if (sblock_other
) {
1146 ret
= scrub_repair_page_from_good_copy(sblock_bad
,
1150 page_bad
->io_error
= 0;
1156 if (success
&& !sctx
->is_dev_replace
) {
1157 if (is_metadata
|| have_csum
) {
1159 * need to verify the checksum now that all
1160 * sectors on disk are repaired (the write
1161 * request for data to be repaired is on its way).
1162 * Just be lazy and use scrub_recheck_block()
1163 * which re-reads the data before the checksum
1164 * is verified, but most likely the data comes out
1165 * of the page cache.
1167 scrub_recheck_block(fs_info
, sblock_bad
, 1);
1168 if (!sblock_bad
->header_error
&&
1169 !sblock_bad
->checksum_error
&&
1170 sblock_bad
->no_io_error_seen
)
1171 goto corrected_error
;
1173 goto did_not_correct_error
;
1176 spin_lock(&sctx
->stat_lock
);
1177 sctx
->stat
.corrected_errors
++;
1178 sblock_to_check
->data_corrected
= 1;
1179 spin_unlock(&sctx
->stat_lock
);
1180 btrfs_err_rl_in_rcu(fs_info
,
1181 "fixed up error at logical %llu on dev %s",
1182 logical
, rcu_str_deref(dev
->name
));
1185 did_not_correct_error
:
1186 spin_lock(&sctx
->stat_lock
);
1187 sctx
->stat
.uncorrectable_errors
++;
1188 spin_unlock(&sctx
->stat_lock
);
1189 btrfs_err_rl_in_rcu(fs_info
,
1190 "unable to fixup (regular) error at logical %llu on dev %s",
1191 logical
, rcu_str_deref(dev
->name
));
1195 if (sblocks_for_recheck
) {
1196 for (mirror_index
= 0; mirror_index
< BTRFS_MAX_MIRRORS
;
1198 struct scrub_block
*sblock
= sblocks_for_recheck
+
1200 struct scrub_recover
*recover
;
1203 for (page_index
= 0; page_index
< sblock
->page_count
;
1205 sblock
->pagev
[page_index
]->sblock
= NULL
;
1206 recover
= sblock
->pagev
[page_index
]->recover
;
1208 scrub_put_recover(fs_info
, recover
);
1209 sblock
->pagev
[page_index
]->recover
=
1212 scrub_page_put(sblock
->pagev
[page_index
]);
1215 kfree(sblocks_for_recheck
);
1218 ret
= unlock_full_stripe(fs_info
, logical
, full_stripe_locked
);
1219 memalloc_nofs_restore(nofs_flag
);
1225 static inline int scrub_nr_raid_mirrors(struct btrfs_bio
*bbio
)
1227 if (bbio
->map_type
& BTRFS_BLOCK_GROUP_RAID5
)
1229 else if (bbio
->map_type
& BTRFS_BLOCK_GROUP_RAID6
)
1232 return (int)bbio
->num_stripes
;
1235 static inline void scrub_stripe_index_and_offset(u64 logical
, u64 map_type
,
1238 int nstripes
, int mirror
,
1244 if (map_type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
1246 for (i
= 0; i
< nstripes
; i
++) {
1247 if (raid_map
[i
] == RAID6_Q_STRIPE
||
1248 raid_map
[i
] == RAID5_P_STRIPE
)
1251 if (logical
>= raid_map
[i
] &&
1252 logical
< raid_map
[i
] + mapped_length
)
1257 *stripe_offset
= logical
- raid_map
[i
];
1259 /* The other RAID type */
1260 *stripe_index
= mirror
;
1265 static int scrub_setup_recheck_block(struct scrub_block
*original_sblock
,
1266 struct scrub_block
*sblocks_for_recheck
)
1268 struct scrub_ctx
*sctx
= original_sblock
->sctx
;
1269 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
1270 u64 length
= original_sblock
->page_count
* PAGE_SIZE
;
1271 u64 logical
= original_sblock
->pagev
[0]->logical
;
1272 u64 generation
= original_sblock
->pagev
[0]->generation
;
1273 u64 flags
= original_sblock
->pagev
[0]->flags
;
1274 u64 have_csum
= original_sblock
->pagev
[0]->have_csum
;
1275 struct scrub_recover
*recover
;
1276 struct btrfs_bio
*bbio
;
1287 * note: the two members refs and outstanding_pages
1288 * are not used (and not set) in the blocks that are used for
1289 * the recheck procedure
1292 while (length
> 0) {
1293 sublen
= min_t(u64
, length
, PAGE_SIZE
);
1294 mapped_length
= sublen
;
1298 * with a length of PAGE_SIZE, each returned stripe
1299 * represents one mirror
1301 btrfs_bio_counter_inc_blocked(fs_info
);
1302 ret
= btrfs_map_sblock(fs_info
, BTRFS_MAP_GET_READ_MIRRORS
,
1303 logical
, &mapped_length
, &bbio
);
1304 if (ret
|| !bbio
|| mapped_length
< sublen
) {
1305 btrfs_put_bbio(bbio
);
1306 btrfs_bio_counter_dec(fs_info
);
1310 recover
= kzalloc(sizeof(struct scrub_recover
), GFP_NOFS
);
1312 btrfs_put_bbio(bbio
);
1313 btrfs_bio_counter_dec(fs_info
);
1317 refcount_set(&recover
->refs
, 1);
1318 recover
->bbio
= bbio
;
1319 recover
->map_length
= mapped_length
;
1321 BUG_ON(page_index
>= SCRUB_MAX_PAGES_PER_BLOCK
);
1323 nmirrors
= min(scrub_nr_raid_mirrors(bbio
), BTRFS_MAX_MIRRORS
);
1325 for (mirror_index
= 0; mirror_index
< nmirrors
;
1327 struct scrub_block
*sblock
;
1328 struct scrub_page
*page
;
1330 sblock
= sblocks_for_recheck
+ mirror_index
;
1331 sblock
->sctx
= sctx
;
1333 page
= kzalloc(sizeof(*page
), GFP_NOFS
);
1336 spin_lock(&sctx
->stat_lock
);
1337 sctx
->stat
.malloc_errors
++;
1338 spin_unlock(&sctx
->stat_lock
);
1339 scrub_put_recover(fs_info
, recover
);
1342 scrub_page_get(page
);
1343 sblock
->pagev
[page_index
] = page
;
1344 page
->sblock
= sblock
;
1345 page
->flags
= flags
;
1346 page
->generation
= generation
;
1347 page
->logical
= logical
;
1348 page
->have_csum
= have_csum
;
1351 original_sblock
->pagev
[0]->csum
,
1354 scrub_stripe_index_and_offset(logical
,
1363 page
->physical
= bbio
->stripes
[stripe_index
].physical
+
1365 page
->dev
= bbio
->stripes
[stripe_index
].dev
;
1367 BUG_ON(page_index
>= original_sblock
->page_count
);
1368 page
->physical_for_dev_replace
=
1369 original_sblock
->pagev
[page_index
]->
1370 physical_for_dev_replace
;
1371 /* for missing devices, dev->bdev is NULL */
1372 page
->mirror_num
= mirror_index
+ 1;
1373 sblock
->page_count
++;
1374 page
->page
= alloc_page(GFP_NOFS
);
1378 scrub_get_recover(recover
);
1379 page
->recover
= recover
;
1381 scrub_put_recover(fs_info
, recover
);
1390 static void scrub_bio_wait_endio(struct bio
*bio
)
1392 complete(bio
->bi_private
);
1395 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info
*fs_info
,
1397 struct scrub_page
*page
)
1399 DECLARE_COMPLETION_ONSTACK(done
);
1403 bio
->bi_iter
.bi_sector
= page
->logical
>> 9;
1404 bio
->bi_private
= &done
;
1405 bio
->bi_end_io
= scrub_bio_wait_endio
;
1407 mirror_num
= page
->sblock
->pagev
[0]->mirror_num
;
1408 ret
= raid56_parity_recover(fs_info
, bio
, page
->recover
->bbio
,
1409 page
->recover
->map_length
,
1414 wait_for_completion_io(&done
);
1415 return blk_status_to_errno(bio
->bi_status
);
1418 static void scrub_recheck_block_on_raid56(struct btrfs_fs_info
*fs_info
,
1419 struct scrub_block
*sblock
)
1421 struct scrub_page
*first_page
= sblock
->pagev
[0];
1425 /* All pages in sblock belong to the same stripe on the same device. */
1426 ASSERT(first_page
->dev
);
1427 if (!first_page
->dev
->bdev
)
1430 bio
= btrfs_io_bio_alloc(BIO_MAX_PAGES
);
1431 bio_set_dev(bio
, first_page
->dev
->bdev
);
1433 for (page_num
= 0; page_num
< sblock
->page_count
; page_num
++) {
1434 struct scrub_page
*page
= sblock
->pagev
[page_num
];
1436 WARN_ON(!page
->page
);
1437 bio_add_page(bio
, page
->page
, PAGE_SIZE
, 0);
1440 if (scrub_submit_raid56_bio_wait(fs_info
, bio
, first_page
)) {
1447 scrub_recheck_block_checksum(sblock
);
1451 for (page_num
= 0; page_num
< sblock
->page_count
; page_num
++)
1452 sblock
->pagev
[page_num
]->io_error
= 1;
1454 sblock
->no_io_error_seen
= 0;
1458 * this function will check the on disk data for checksum errors, header
1459 * errors and read I/O errors. If any I/O errors happen, the exact pages
1460 * which are errored are marked as being bad. The goal is to enable scrub
1461 * to take those pages that are not errored from all the mirrors so that
1462 * the pages that are errored in the just handled mirror can be repaired.
1464 static void scrub_recheck_block(struct btrfs_fs_info
*fs_info
,
1465 struct scrub_block
*sblock
,
1466 int retry_failed_mirror
)
1470 sblock
->no_io_error_seen
= 1;
1472 /* short cut for raid56 */
1473 if (!retry_failed_mirror
&& scrub_is_page_on_raid56(sblock
->pagev
[0]))
1474 return scrub_recheck_block_on_raid56(fs_info
, sblock
);
1476 for (page_num
= 0; page_num
< sblock
->page_count
; page_num
++) {
1478 struct scrub_page
*page
= sblock
->pagev
[page_num
];
1480 if (page
->dev
->bdev
== NULL
) {
1482 sblock
->no_io_error_seen
= 0;
1486 WARN_ON(!page
->page
);
1487 bio
= btrfs_io_bio_alloc(1);
1488 bio_set_dev(bio
, page
->dev
->bdev
);
1490 bio_add_page(bio
, page
->page
, PAGE_SIZE
, 0);
1491 bio
->bi_iter
.bi_sector
= page
->physical
>> 9;
1492 bio
->bi_opf
= REQ_OP_READ
;
1494 if (btrfsic_submit_bio_wait(bio
)) {
1496 sblock
->no_io_error_seen
= 0;
1502 if (sblock
->no_io_error_seen
)
1503 scrub_recheck_block_checksum(sblock
);
1506 static inline int scrub_check_fsid(u8 fsid
[],
1507 struct scrub_page
*spage
)
1509 struct btrfs_fs_devices
*fs_devices
= spage
->dev
->fs_devices
;
1512 ret
= memcmp(fsid
, fs_devices
->fsid
, BTRFS_FSID_SIZE
);
1516 static void scrub_recheck_block_checksum(struct scrub_block
*sblock
)
1518 sblock
->header_error
= 0;
1519 sblock
->checksum_error
= 0;
1520 sblock
->generation_error
= 0;
1522 if (sblock
->pagev
[0]->flags
& BTRFS_EXTENT_FLAG_DATA
)
1523 scrub_checksum_data(sblock
);
1525 scrub_checksum_tree_block(sblock
);
1528 static int scrub_repair_block_from_good_copy(struct scrub_block
*sblock_bad
,
1529 struct scrub_block
*sblock_good
)
1534 for (page_num
= 0; page_num
< sblock_bad
->page_count
; page_num
++) {
1537 ret_sub
= scrub_repair_page_from_good_copy(sblock_bad
,
1547 static int scrub_repair_page_from_good_copy(struct scrub_block
*sblock_bad
,
1548 struct scrub_block
*sblock_good
,
1549 int page_num
, int force_write
)
1551 struct scrub_page
*page_bad
= sblock_bad
->pagev
[page_num
];
1552 struct scrub_page
*page_good
= sblock_good
->pagev
[page_num
];
1553 struct btrfs_fs_info
*fs_info
= sblock_bad
->sctx
->fs_info
;
1555 BUG_ON(page_bad
->page
== NULL
);
1556 BUG_ON(page_good
->page
== NULL
);
1557 if (force_write
|| sblock_bad
->header_error
||
1558 sblock_bad
->checksum_error
|| page_bad
->io_error
) {
1562 if (!page_bad
->dev
->bdev
) {
1563 btrfs_warn_rl(fs_info
,
1564 "scrub_repair_page_from_good_copy(bdev == NULL) is unexpected");
1568 bio
= btrfs_io_bio_alloc(1);
1569 bio_set_dev(bio
, page_bad
->dev
->bdev
);
1570 bio
->bi_iter
.bi_sector
= page_bad
->physical
>> 9;
1571 bio
->bi_opf
= REQ_OP_WRITE
;
1573 ret
= bio_add_page(bio
, page_good
->page
, PAGE_SIZE
, 0);
1574 if (PAGE_SIZE
!= ret
) {
1579 if (btrfsic_submit_bio_wait(bio
)) {
1580 btrfs_dev_stat_inc_and_print(page_bad
->dev
,
1581 BTRFS_DEV_STAT_WRITE_ERRS
);
1582 atomic64_inc(&fs_info
->dev_replace
.num_write_errors
);
1592 static void scrub_write_block_to_dev_replace(struct scrub_block
*sblock
)
1594 struct btrfs_fs_info
*fs_info
= sblock
->sctx
->fs_info
;
1598 * This block is used for the check of the parity on the source device,
1599 * so the data needn't be written into the destination device.
1601 if (sblock
->sparity
)
1604 for (page_num
= 0; page_num
< sblock
->page_count
; page_num
++) {
1607 ret
= scrub_write_page_to_dev_replace(sblock
, page_num
);
1609 atomic64_inc(&fs_info
->dev_replace
.num_write_errors
);
1613 static int scrub_write_page_to_dev_replace(struct scrub_block
*sblock
,
1616 struct scrub_page
*spage
= sblock
->pagev
[page_num
];
1618 BUG_ON(spage
->page
== NULL
);
1619 if (spage
->io_error
) {
1620 void *mapped_buffer
= kmap_atomic(spage
->page
);
1622 clear_page(mapped_buffer
);
1623 flush_dcache_page(spage
->page
);
1624 kunmap_atomic(mapped_buffer
);
1626 return scrub_add_page_to_wr_bio(sblock
->sctx
, spage
);
1629 static int scrub_add_page_to_wr_bio(struct scrub_ctx
*sctx
,
1630 struct scrub_page
*spage
)
1632 struct scrub_bio
*sbio
;
1635 mutex_lock(&sctx
->wr_lock
);
1637 if (!sctx
->wr_curr_bio
) {
1638 sctx
->wr_curr_bio
= kzalloc(sizeof(*sctx
->wr_curr_bio
),
1640 if (!sctx
->wr_curr_bio
) {
1641 mutex_unlock(&sctx
->wr_lock
);
1644 sctx
->wr_curr_bio
->sctx
= sctx
;
1645 sctx
->wr_curr_bio
->page_count
= 0;
1647 sbio
= sctx
->wr_curr_bio
;
1648 if (sbio
->page_count
== 0) {
1651 sbio
->physical
= spage
->physical_for_dev_replace
;
1652 sbio
->logical
= spage
->logical
;
1653 sbio
->dev
= sctx
->wr_tgtdev
;
1656 bio
= btrfs_io_bio_alloc(sctx
->pages_per_wr_bio
);
1660 bio
->bi_private
= sbio
;
1661 bio
->bi_end_io
= scrub_wr_bio_end_io
;
1662 bio_set_dev(bio
, sbio
->dev
->bdev
);
1663 bio
->bi_iter
.bi_sector
= sbio
->physical
>> 9;
1664 bio
->bi_opf
= REQ_OP_WRITE
;
1666 } else if (sbio
->physical
+ sbio
->page_count
* PAGE_SIZE
!=
1667 spage
->physical_for_dev_replace
||
1668 sbio
->logical
+ sbio
->page_count
* PAGE_SIZE
!=
1670 scrub_wr_submit(sctx
);
1674 ret
= bio_add_page(sbio
->bio
, spage
->page
, PAGE_SIZE
, 0);
1675 if (ret
!= PAGE_SIZE
) {
1676 if (sbio
->page_count
< 1) {
1679 mutex_unlock(&sctx
->wr_lock
);
1682 scrub_wr_submit(sctx
);
1686 sbio
->pagev
[sbio
->page_count
] = spage
;
1687 scrub_page_get(spage
);
1689 if (sbio
->page_count
== sctx
->pages_per_wr_bio
)
1690 scrub_wr_submit(sctx
);
1691 mutex_unlock(&sctx
->wr_lock
);
1696 static void scrub_wr_submit(struct scrub_ctx
*sctx
)
1698 struct scrub_bio
*sbio
;
1700 if (!sctx
->wr_curr_bio
)
1703 sbio
= sctx
->wr_curr_bio
;
1704 sctx
->wr_curr_bio
= NULL
;
1705 WARN_ON(!sbio
->bio
->bi_disk
);
1706 scrub_pending_bio_inc(sctx
);
1707 /* process all writes in a single worker thread. Then the block layer
1708 * orders the requests before sending them to the driver which
1709 * doubled the write performance on spinning disks when measured
1711 btrfsic_submit_bio(sbio
->bio
);
1714 static void scrub_wr_bio_end_io(struct bio
*bio
)
1716 struct scrub_bio
*sbio
= bio
->bi_private
;
1717 struct btrfs_fs_info
*fs_info
= sbio
->dev
->fs_info
;
1719 sbio
->status
= bio
->bi_status
;
1722 btrfs_init_work(&sbio
->work
, scrub_wr_bio_end_io_worker
, NULL
, NULL
);
1723 btrfs_queue_work(fs_info
->scrub_wr_completion_workers
, &sbio
->work
);
1726 static void scrub_wr_bio_end_io_worker(struct btrfs_work
*work
)
1728 struct scrub_bio
*sbio
= container_of(work
, struct scrub_bio
, work
);
1729 struct scrub_ctx
*sctx
= sbio
->sctx
;
1732 WARN_ON(sbio
->page_count
> SCRUB_PAGES_PER_WR_BIO
);
1734 struct btrfs_dev_replace
*dev_replace
=
1735 &sbio
->sctx
->fs_info
->dev_replace
;
1737 for (i
= 0; i
< sbio
->page_count
; i
++) {
1738 struct scrub_page
*spage
= sbio
->pagev
[i
];
1740 spage
->io_error
= 1;
1741 atomic64_inc(&dev_replace
->num_write_errors
);
1745 for (i
= 0; i
< sbio
->page_count
; i
++)
1746 scrub_page_put(sbio
->pagev
[i
]);
1750 scrub_pending_bio_dec(sctx
);
1753 static int scrub_checksum(struct scrub_block
*sblock
)
1759 * No need to initialize these stats currently,
1760 * because this function only use return value
1761 * instead of these stats value.
1766 sblock
->header_error
= 0;
1767 sblock
->generation_error
= 0;
1768 sblock
->checksum_error
= 0;
1770 WARN_ON(sblock
->page_count
< 1);
1771 flags
= sblock
->pagev
[0]->flags
;
1773 if (flags
& BTRFS_EXTENT_FLAG_DATA
)
1774 ret
= scrub_checksum_data(sblock
);
1775 else if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
)
1776 ret
= scrub_checksum_tree_block(sblock
);
1777 else if (flags
& BTRFS_EXTENT_FLAG_SUPER
)
1778 (void)scrub_checksum_super(sblock
);
1782 scrub_handle_errored_block(sblock
);
1787 static int scrub_checksum_data(struct scrub_block
*sblock
)
1789 struct scrub_ctx
*sctx
= sblock
->sctx
;
1790 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
1791 SHASH_DESC_ON_STACK(shash
, fs_info
->csum_shash
);
1792 u8 csum
[BTRFS_CSUM_SIZE
];
1799 BUG_ON(sblock
->page_count
< 1);
1800 if (!sblock
->pagev
[0]->have_csum
)
1803 shash
->tfm
= fs_info
->csum_shash
;
1804 crypto_shash_init(shash
);
1806 on_disk_csum
= sblock
->pagev
[0]->csum
;
1807 page
= sblock
->pagev
[0]->page
;
1808 buffer
= kmap_atomic(page
);
1810 len
= sctx
->fs_info
->sectorsize
;
1813 u64 l
= min_t(u64
, len
, PAGE_SIZE
);
1815 crypto_shash_update(shash
, buffer
, l
);
1816 kunmap_atomic(buffer
);
1821 BUG_ON(index
>= sblock
->page_count
);
1822 BUG_ON(!sblock
->pagev
[index
]->page
);
1823 page
= sblock
->pagev
[index
]->page
;
1824 buffer
= kmap_atomic(page
);
1827 crypto_shash_final(shash
, csum
);
1828 if (memcmp(csum
, on_disk_csum
, sctx
->csum_size
))
1829 sblock
->checksum_error
= 1;
1831 return sblock
->checksum_error
;
1834 static int scrub_checksum_tree_block(struct scrub_block
*sblock
)
1836 struct scrub_ctx
*sctx
= sblock
->sctx
;
1837 struct btrfs_header
*h
;
1838 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
1839 SHASH_DESC_ON_STACK(shash
, fs_info
->csum_shash
);
1840 u8 calculated_csum
[BTRFS_CSUM_SIZE
];
1841 u8 on_disk_csum
[BTRFS_CSUM_SIZE
];
1843 void *mapped_buffer
;
1849 shash
->tfm
= fs_info
->csum_shash
;
1850 crypto_shash_init(shash
);
1852 BUG_ON(sblock
->page_count
< 1);
1853 page
= sblock
->pagev
[0]->page
;
1854 mapped_buffer
= kmap_atomic(page
);
1855 h
= (struct btrfs_header
*)mapped_buffer
;
1856 memcpy(on_disk_csum
, h
->csum
, sctx
->csum_size
);
1859 * we don't use the getter functions here, as we
1860 * a) don't have an extent buffer and
1861 * b) the page is already kmapped
1863 if (sblock
->pagev
[0]->logical
!= btrfs_stack_header_bytenr(h
))
1864 sblock
->header_error
= 1;
1866 if (sblock
->pagev
[0]->generation
!= btrfs_stack_header_generation(h
)) {
1867 sblock
->header_error
= 1;
1868 sblock
->generation_error
= 1;
1871 if (!scrub_check_fsid(h
->fsid
, sblock
->pagev
[0]))
1872 sblock
->header_error
= 1;
1874 if (memcmp(h
->chunk_tree_uuid
, fs_info
->chunk_tree_uuid
,
1876 sblock
->header_error
= 1;
1878 len
= sctx
->fs_info
->nodesize
- BTRFS_CSUM_SIZE
;
1879 mapped_size
= PAGE_SIZE
- BTRFS_CSUM_SIZE
;
1880 p
= ((u8
*)mapped_buffer
) + BTRFS_CSUM_SIZE
;
1883 u64 l
= min_t(u64
, len
, mapped_size
);
1885 crypto_shash_update(shash
, p
, l
);
1886 kunmap_atomic(mapped_buffer
);
1891 BUG_ON(index
>= sblock
->page_count
);
1892 BUG_ON(!sblock
->pagev
[index
]->page
);
1893 page
= sblock
->pagev
[index
]->page
;
1894 mapped_buffer
= kmap_atomic(page
);
1895 mapped_size
= PAGE_SIZE
;
1899 crypto_shash_final(shash
, calculated_csum
);
1900 if (memcmp(calculated_csum
, on_disk_csum
, sctx
->csum_size
))
1901 sblock
->checksum_error
= 1;
1903 return sblock
->header_error
|| sblock
->checksum_error
;
1906 static int scrub_checksum_super(struct scrub_block
*sblock
)
1908 struct btrfs_super_block
*s
;
1909 struct scrub_ctx
*sctx
= sblock
->sctx
;
1910 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
1911 SHASH_DESC_ON_STACK(shash
, fs_info
->csum_shash
);
1912 u8 calculated_csum
[BTRFS_CSUM_SIZE
];
1913 u8 on_disk_csum
[BTRFS_CSUM_SIZE
];
1915 void *mapped_buffer
;
1923 shash
->tfm
= fs_info
->csum_shash
;
1924 crypto_shash_init(shash
);
1926 BUG_ON(sblock
->page_count
< 1);
1927 page
= sblock
->pagev
[0]->page
;
1928 mapped_buffer
= kmap_atomic(page
);
1929 s
= (struct btrfs_super_block
*)mapped_buffer
;
1930 memcpy(on_disk_csum
, s
->csum
, sctx
->csum_size
);
1932 if (sblock
->pagev
[0]->logical
!= btrfs_super_bytenr(s
))
1935 if (sblock
->pagev
[0]->generation
!= btrfs_super_generation(s
))
1938 if (!scrub_check_fsid(s
->fsid
, sblock
->pagev
[0]))
1941 len
= BTRFS_SUPER_INFO_SIZE
- BTRFS_CSUM_SIZE
;
1942 mapped_size
= PAGE_SIZE
- BTRFS_CSUM_SIZE
;
1943 p
= ((u8
*)mapped_buffer
) + BTRFS_CSUM_SIZE
;
1946 u64 l
= min_t(u64
, len
, mapped_size
);
1948 crypto_shash_update(shash
, p
, l
);
1949 kunmap_atomic(mapped_buffer
);
1954 BUG_ON(index
>= sblock
->page_count
);
1955 BUG_ON(!sblock
->pagev
[index
]->page
);
1956 page
= sblock
->pagev
[index
]->page
;
1957 mapped_buffer
= kmap_atomic(page
);
1958 mapped_size
= PAGE_SIZE
;
1962 crypto_shash_final(shash
, calculated_csum
);
1963 if (memcmp(calculated_csum
, on_disk_csum
, sctx
->csum_size
))
1966 if (fail_cor
+ fail_gen
) {
1968 * if we find an error in a super block, we just report it.
1969 * They will get written with the next transaction commit
1972 spin_lock(&sctx
->stat_lock
);
1973 ++sctx
->stat
.super_errors
;
1974 spin_unlock(&sctx
->stat_lock
);
1976 btrfs_dev_stat_inc_and_print(sblock
->pagev
[0]->dev
,
1977 BTRFS_DEV_STAT_CORRUPTION_ERRS
);
1979 btrfs_dev_stat_inc_and_print(sblock
->pagev
[0]->dev
,
1980 BTRFS_DEV_STAT_GENERATION_ERRS
);
1983 return fail_cor
+ fail_gen
;
1986 static void scrub_block_get(struct scrub_block
*sblock
)
1988 refcount_inc(&sblock
->refs
);
1991 static void scrub_block_put(struct scrub_block
*sblock
)
1993 if (refcount_dec_and_test(&sblock
->refs
)) {
1996 if (sblock
->sparity
)
1997 scrub_parity_put(sblock
->sparity
);
1999 for (i
= 0; i
< sblock
->page_count
; i
++)
2000 scrub_page_put(sblock
->pagev
[i
]);
2005 static void scrub_page_get(struct scrub_page
*spage
)
2007 atomic_inc(&spage
->refs
);
2010 static void scrub_page_put(struct scrub_page
*spage
)
2012 if (atomic_dec_and_test(&spage
->refs
)) {
2014 __free_page(spage
->page
);
2019 static void scrub_submit(struct scrub_ctx
*sctx
)
2021 struct scrub_bio
*sbio
;
2023 if (sctx
->curr
== -1)
2026 sbio
= sctx
->bios
[sctx
->curr
];
2028 scrub_pending_bio_inc(sctx
);
2029 btrfsic_submit_bio(sbio
->bio
);
2032 static int scrub_add_page_to_rd_bio(struct scrub_ctx
*sctx
,
2033 struct scrub_page
*spage
)
2035 struct scrub_block
*sblock
= spage
->sblock
;
2036 struct scrub_bio
*sbio
;
2041 * grab a fresh bio or wait for one to become available
2043 while (sctx
->curr
== -1) {
2044 spin_lock(&sctx
->list_lock
);
2045 sctx
->curr
= sctx
->first_free
;
2046 if (sctx
->curr
!= -1) {
2047 sctx
->first_free
= sctx
->bios
[sctx
->curr
]->next_free
;
2048 sctx
->bios
[sctx
->curr
]->next_free
= -1;
2049 sctx
->bios
[sctx
->curr
]->page_count
= 0;
2050 spin_unlock(&sctx
->list_lock
);
2052 spin_unlock(&sctx
->list_lock
);
2053 wait_event(sctx
->list_wait
, sctx
->first_free
!= -1);
2056 sbio
= sctx
->bios
[sctx
->curr
];
2057 if (sbio
->page_count
== 0) {
2060 sbio
->physical
= spage
->physical
;
2061 sbio
->logical
= spage
->logical
;
2062 sbio
->dev
= spage
->dev
;
2065 bio
= btrfs_io_bio_alloc(sctx
->pages_per_rd_bio
);
2069 bio
->bi_private
= sbio
;
2070 bio
->bi_end_io
= scrub_bio_end_io
;
2071 bio_set_dev(bio
, sbio
->dev
->bdev
);
2072 bio
->bi_iter
.bi_sector
= sbio
->physical
>> 9;
2073 bio
->bi_opf
= REQ_OP_READ
;
2075 } else if (sbio
->physical
+ sbio
->page_count
* PAGE_SIZE
!=
2077 sbio
->logical
+ sbio
->page_count
* PAGE_SIZE
!=
2079 sbio
->dev
!= spage
->dev
) {
2084 sbio
->pagev
[sbio
->page_count
] = spage
;
2085 ret
= bio_add_page(sbio
->bio
, spage
->page
, PAGE_SIZE
, 0);
2086 if (ret
!= PAGE_SIZE
) {
2087 if (sbio
->page_count
< 1) {
2096 scrub_block_get(sblock
); /* one for the page added to the bio */
2097 atomic_inc(&sblock
->outstanding_pages
);
2099 if (sbio
->page_count
== sctx
->pages_per_rd_bio
)
2105 static void scrub_missing_raid56_end_io(struct bio
*bio
)
2107 struct scrub_block
*sblock
= bio
->bi_private
;
2108 struct btrfs_fs_info
*fs_info
= sblock
->sctx
->fs_info
;
2111 sblock
->no_io_error_seen
= 0;
2115 btrfs_queue_work(fs_info
->scrub_workers
, &sblock
->work
);
2118 static void scrub_missing_raid56_worker(struct btrfs_work
*work
)
2120 struct scrub_block
*sblock
= container_of(work
, struct scrub_block
, work
);
2121 struct scrub_ctx
*sctx
= sblock
->sctx
;
2122 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
2124 struct btrfs_device
*dev
;
2126 logical
= sblock
->pagev
[0]->logical
;
2127 dev
= sblock
->pagev
[0]->dev
;
2129 if (sblock
->no_io_error_seen
)
2130 scrub_recheck_block_checksum(sblock
);
2132 if (!sblock
->no_io_error_seen
) {
2133 spin_lock(&sctx
->stat_lock
);
2134 sctx
->stat
.read_errors
++;
2135 spin_unlock(&sctx
->stat_lock
);
2136 btrfs_err_rl_in_rcu(fs_info
,
2137 "IO error rebuilding logical %llu for dev %s",
2138 logical
, rcu_str_deref(dev
->name
));
2139 } else if (sblock
->header_error
|| sblock
->checksum_error
) {
2140 spin_lock(&sctx
->stat_lock
);
2141 sctx
->stat
.uncorrectable_errors
++;
2142 spin_unlock(&sctx
->stat_lock
);
2143 btrfs_err_rl_in_rcu(fs_info
,
2144 "failed to rebuild valid logical %llu for dev %s",
2145 logical
, rcu_str_deref(dev
->name
));
2147 scrub_write_block_to_dev_replace(sblock
);
2150 if (sctx
->is_dev_replace
&& sctx
->flush_all_writes
) {
2151 mutex_lock(&sctx
->wr_lock
);
2152 scrub_wr_submit(sctx
);
2153 mutex_unlock(&sctx
->wr_lock
);
2156 scrub_block_put(sblock
);
2157 scrub_pending_bio_dec(sctx
);
2160 static void scrub_missing_raid56_pages(struct scrub_block
*sblock
)
2162 struct scrub_ctx
*sctx
= sblock
->sctx
;
2163 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
2164 u64 length
= sblock
->page_count
* PAGE_SIZE
;
2165 u64 logical
= sblock
->pagev
[0]->logical
;
2166 struct btrfs_bio
*bbio
= NULL
;
2168 struct btrfs_raid_bio
*rbio
;
2172 btrfs_bio_counter_inc_blocked(fs_info
);
2173 ret
= btrfs_map_sblock(fs_info
, BTRFS_MAP_GET_READ_MIRRORS
, logical
,
2175 if (ret
|| !bbio
|| !bbio
->raid_map
)
2178 if (WARN_ON(!sctx
->is_dev_replace
||
2179 !(bbio
->map_type
& BTRFS_BLOCK_GROUP_RAID56_MASK
))) {
2181 * We shouldn't be scrubbing a missing device. Even for dev
2182 * replace, we should only get here for RAID 5/6. We either
2183 * managed to mount something with no mirrors remaining or
2184 * there's a bug in scrub_remap_extent()/btrfs_map_block().
2189 bio
= btrfs_io_bio_alloc(0);
2190 bio
->bi_iter
.bi_sector
= logical
>> 9;
2191 bio
->bi_private
= sblock
;
2192 bio
->bi_end_io
= scrub_missing_raid56_end_io
;
2194 rbio
= raid56_alloc_missing_rbio(fs_info
, bio
, bbio
, length
);
2198 for (i
= 0; i
< sblock
->page_count
; i
++) {
2199 struct scrub_page
*spage
= sblock
->pagev
[i
];
2201 raid56_add_scrub_pages(rbio
, spage
->page
, spage
->logical
);
2204 btrfs_init_work(&sblock
->work
, scrub_missing_raid56_worker
, NULL
, NULL
);
2205 scrub_block_get(sblock
);
2206 scrub_pending_bio_inc(sctx
);
2207 raid56_submit_missing_rbio(rbio
);
2213 btrfs_bio_counter_dec(fs_info
);
2214 btrfs_put_bbio(bbio
);
2215 spin_lock(&sctx
->stat_lock
);
2216 sctx
->stat
.malloc_errors
++;
2217 spin_unlock(&sctx
->stat_lock
);
2220 static int scrub_pages(struct scrub_ctx
*sctx
, u64 logical
, u64 len
,
2221 u64 physical
, struct btrfs_device
*dev
, u64 flags
,
2222 u64 gen
, int mirror_num
, u8
*csum
, int force
,
2223 u64 physical_for_dev_replace
)
2225 struct scrub_block
*sblock
;
2228 sblock
= kzalloc(sizeof(*sblock
), GFP_KERNEL
);
2230 spin_lock(&sctx
->stat_lock
);
2231 sctx
->stat
.malloc_errors
++;
2232 spin_unlock(&sctx
->stat_lock
);
2236 /* one ref inside this function, plus one for each page added to
2238 refcount_set(&sblock
->refs
, 1);
2239 sblock
->sctx
= sctx
;
2240 sblock
->no_io_error_seen
= 1;
2242 for (index
= 0; len
> 0; index
++) {
2243 struct scrub_page
*spage
;
2244 u64 l
= min_t(u64
, len
, PAGE_SIZE
);
2246 spage
= kzalloc(sizeof(*spage
), GFP_KERNEL
);
2249 spin_lock(&sctx
->stat_lock
);
2250 sctx
->stat
.malloc_errors
++;
2251 spin_unlock(&sctx
->stat_lock
);
2252 scrub_block_put(sblock
);
2255 BUG_ON(index
>= SCRUB_MAX_PAGES_PER_BLOCK
);
2256 scrub_page_get(spage
);
2257 sblock
->pagev
[index
] = spage
;
2258 spage
->sblock
= sblock
;
2260 spage
->flags
= flags
;
2261 spage
->generation
= gen
;
2262 spage
->logical
= logical
;
2263 spage
->physical
= physical
;
2264 spage
->physical_for_dev_replace
= physical_for_dev_replace
;
2265 spage
->mirror_num
= mirror_num
;
2267 spage
->have_csum
= 1;
2268 memcpy(spage
->csum
, csum
, sctx
->csum_size
);
2270 spage
->have_csum
= 0;
2272 sblock
->page_count
++;
2273 spage
->page
= alloc_page(GFP_KERNEL
);
2279 physical_for_dev_replace
+= l
;
2282 WARN_ON(sblock
->page_count
== 0);
2283 if (test_bit(BTRFS_DEV_STATE_MISSING
, &dev
->dev_state
)) {
2285 * This case should only be hit for RAID 5/6 device replace. See
2286 * the comment in scrub_missing_raid56_pages() for details.
2288 scrub_missing_raid56_pages(sblock
);
2290 for (index
= 0; index
< sblock
->page_count
; index
++) {
2291 struct scrub_page
*spage
= sblock
->pagev
[index
];
2294 ret
= scrub_add_page_to_rd_bio(sctx
, spage
);
2296 scrub_block_put(sblock
);
2305 /* last one frees, either here or in bio completion for last page */
2306 scrub_block_put(sblock
);
2310 static void scrub_bio_end_io(struct bio
*bio
)
2312 struct scrub_bio
*sbio
= bio
->bi_private
;
2313 struct btrfs_fs_info
*fs_info
= sbio
->dev
->fs_info
;
2315 sbio
->status
= bio
->bi_status
;
2318 btrfs_queue_work(fs_info
->scrub_workers
, &sbio
->work
);
2321 static void scrub_bio_end_io_worker(struct btrfs_work
*work
)
2323 struct scrub_bio
*sbio
= container_of(work
, struct scrub_bio
, work
);
2324 struct scrub_ctx
*sctx
= sbio
->sctx
;
2327 BUG_ON(sbio
->page_count
> SCRUB_PAGES_PER_RD_BIO
);
2329 for (i
= 0; i
< sbio
->page_count
; i
++) {
2330 struct scrub_page
*spage
= sbio
->pagev
[i
];
2332 spage
->io_error
= 1;
2333 spage
->sblock
->no_io_error_seen
= 0;
2337 /* now complete the scrub_block items that have all pages completed */
2338 for (i
= 0; i
< sbio
->page_count
; i
++) {
2339 struct scrub_page
*spage
= sbio
->pagev
[i
];
2340 struct scrub_block
*sblock
= spage
->sblock
;
2342 if (atomic_dec_and_test(&sblock
->outstanding_pages
))
2343 scrub_block_complete(sblock
);
2344 scrub_block_put(sblock
);
2349 spin_lock(&sctx
->list_lock
);
2350 sbio
->next_free
= sctx
->first_free
;
2351 sctx
->first_free
= sbio
->index
;
2352 spin_unlock(&sctx
->list_lock
);
2354 if (sctx
->is_dev_replace
&& sctx
->flush_all_writes
) {
2355 mutex_lock(&sctx
->wr_lock
);
2356 scrub_wr_submit(sctx
);
2357 mutex_unlock(&sctx
->wr_lock
);
2360 scrub_pending_bio_dec(sctx
);
2363 static inline void __scrub_mark_bitmap(struct scrub_parity
*sparity
,
2364 unsigned long *bitmap
,
2370 int sectorsize
= sparity
->sctx
->fs_info
->sectorsize
;
2372 if (len
>= sparity
->stripe_len
) {
2373 bitmap_set(bitmap
, 0, sparity
->nsectors
);
2377 start
-= sparity
->logic_start
;
2378 start
= div64_u64_rem(start
, sparity
->stripe_len
, &offset
);
2379 offset
= div_u64(offset
, sectorsize
);
2380 nsectors64
= div_u64(len
, sectorsize
);
2382 ASSERT(nsectors64
< UINT_MAX
);
2383 nsectors
= (u32
)nsectors64
;
2385 if (offset
+ nsectors
<= sparity
->nsectors
) {
2386 bitmap_set(bitmap
, offset
, nsectors
);
2390 bitmap_set(bitmap
, offset
, sparity
->nsectors
- offset
);
2391 bitmap_set(bitmap
, 0, nsectors
- (sparity
->nsectors
- offset
));
2394 static inline void scrub_parity_mark_sectors_error(struct scrub_parity
*sparity
,
2397 __scrub_mark_bitmap(sparity
, sparity
->ebitmap
, start
, len
);
2400 static inline void scrub_parity_mark_sectors_data(struct scrub_parity
*sparity
,
2403 __scrub_mark_bitmap(sparity
, sparity
->dbitmap
, start
, len
);
2406 static void scrub_block_complete(struct scrub_block
*sblock
)
2410 if (!sblock
->no_io_error_seen
) {
2412 scrub_handle_errored_block(sblock
);
2415 * if has checksum error, write via repair mechanism in
2416 * dev replace case, otherwise write here in dev replace
2419 corrupted
= scrub_checksum(sblock
);
2420 if (!corrupted
&& sblock
->sctx
->is_dev_replace
)
2421 scrub_write_block_to_dev_replace(sblock
);
2424 if (sblock
->sparity
&& corrupted
&& !sblock
->data_corrected
) {
2425 u64 start
= sblock
->pagev
[0]->logical
;
2426 u64 end
= sblock
->pagev
[sblock
->page_count
- 1]->logical
+
2429 scrub_parity_mark_sectors_error(sblock
->sparity
,
2430 start
, end
- start
);
2434 static int scrub_find_csum(struct scrub_ctx
*sctx
, u64 logical
, u8
*csum
)
2436 struct btrfs_ordered_sum
*sum
= NULL
;
2437 unsigned long index
;
2438 unsigned long num_sectors
;
2440 while (!list_empty(&sctx
->csum_list
)) {
2441 sum
= list_first_entry(&sctx
->csum_list
,
2442 struct btrfs_ordered_sum
, list
);
2443 if (sum
->bytenr
> logical
)
2445 if (sum
->bytenr
+ sum
->len
> logical
)
2448 ++sctx
->stat
.csum_discards
;
2449 list_del(&sum
->list
);
2456 index
= div_u64(logical
- sum
->bytenr
, sctx
->fs_info
->sectorsize
);
2457 ASSERT(index
< UINT_MAX
);
2459 num_sectors
= sum
->len
/ sctx
->fs_info
->sectorsize
;
2460 memcpy(csum
, sum
->sums
+ index
* sctx
->csum_size
, sctx
->csum_size
);
2461 if (index
== num_sectors
- 1) {
2462 list_del(&sum
->list
);
2468 /* scrub extent tries to collect up to 64 kB for each bio */
2469 static int scrub_extent(struct scrub_ctx
*sctx
, struct map_lookup
*map
,
2470 u64 logical
, u64 len
,
2471 u64 physical
, struct btrfs_device
*dev
, u64 flags
,
2472 u64 gen
, int mirror_num
, u64 physical_for_dev_replace
)
2475 u8 csum
[BTRFS_CSUM_SIZE
];
2478 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
2479 if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
)
2480 blocksize
= map
->stripe_len
;
2482 blocksize
= sctx
->fs_info
->sectorsize
;
2483 spin_lock(&sctx
->stat_lock
);
2484 sctx
->stat
.data_extents_scrubbed
++;
2485 sctx
->stat
.data_bytes_scrubbed
+= len
;
2486 spin_unlock(&sctx
->stat_lock
);
2487 } else if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
2488 if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
)
2489 blocksize
= map
->stripe_len
;
2491 blocksize
= sctx
->fs_info
->nodesize
;
2492 spin_lock(&sctx
->stat_lock
);
2493 sctx
->stat
.tree_extents_scrubbed
++;
2494 sctx
->stat
.tree_bytes_scrubbed
+= len
;
2495 spin_unlock(&sctx
->stat_lock
);
2497 blocksize
= sctx
->fs_info
->sectorsize
;
2502 u64 l
= min_t(u64
, len
, blocksize
);
2505 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
2506 /* push csums to sbio */
2507 have_csum
= scrub_find_csum(sctx
, logical
, csum
);
2509 ++sctx
->stat
.no_csum
;
2511 ret
= scrub_pages(sctx
, logical
, l
, physical
, dev
, flags
, gen
,
2512 mirror_num
, have_csum
? csum
: NULL
, 0,
2513 physical_for_dev_replace
);
2519 physical_for_dev_replace
+= l
;
2524 static int scrub_pages_for_parity(struct scrub_parity
*sparity
,
2525 u64 logical
, u64 len
,
2526 u64 physical
, struct btrfs_device
*dev
,
2527 u64 flags
, u64 gen
, int mirror_num
, u8
*csum
)
2529 struct scrub_ctx
*sctx
= sparity
->sctx
;
2530 struct scrub_block
*sblock
;
2533 sblock
= kzalloc(sizeof(*sblock
), GFP_KERNEL
);
2535 spin_lock(&sctx
->stat_lock
);
2536 sctx
->stat
.malloc_errors
++;
2537 spin_unlock(&sctx
->stat_lock
);
2541 /* one ref inside this function, plus one for each page added to
2543 refcount_set(&sblock
->refs
, 1);
2544 sblock
->sctx
= sctx
;
2545 sblock
->no_io_error_seen
= 1;
2546 sblock
->sparity
= sparity
;
2547 scrub_parity_get(sparity
);
2549 for (index
= 0; len
> 0; index
++) {
2550 struct scrub_page
*spage
;
2551 u64 l
= min_t(u64
, len
, PAGE_SIZE
);
2553 spage
= kzalloc(sizeof(*spage
), GFP_KERNEL
);
2556 spin_lock(&sctx
->stat_lock
);
2557 sctx
->stat
.malloc_errors
++;
2558 spin_unlock(&sctx
->stat_lock
);
2559 scrub_block_put(sblock
);
2562 BUG_ON(index
>= SCRUB_MAX_PAGES_PER_BLOCK
);
2563 /* For scrub block */
2564 scrub_page_get(spage
);
2565 sblock
->pagev
[index
] = spage
;
2566 /* For scrub parity */
2567 scrub_page_get(spage
);
2568 list_add_tail(&spage
->list
, &sparity
->spages
);
2569 spage
->sblock
= sblock
;
2571 spage
->flags
= flags
;
2572 spage
->generation
= gen
;
2573 spage
->logical
= logical
;
2574 spage
->physical
= physical
;
2575 spage
->mirror_num
= mirror_num
;
2577 spage
->have_csum
= 1;
2578 memcpy(spage
->csum
, csum
, sctx
->csum_size
);
2580 spage
->have_csum
= 0;
2582 sblock
->page_count
++;
2583 spage
->page
= alloc_page(GFP_KERNEL
);
2591 WARN_ON(sblock
->page_count
== 0);
2592 for (index
= 0; index
< sblock
->page_count
; index
++) {
2593 struct scrub_page
*spage
= sblock
->pagev
[index
];
2596 ret
= scrub_add_page_to_rd_bio(sctx
, spage
);
2598 scrub_block_put(sblock
);
2603 /* last one frees, either here or in bio completion for last page */
2604 scrub_block_put(sblock
);
2608 static int scrub_extent_for_parity(struct scrub_parity
*sparity
,
2609 u64 logical
, u64 len
,
2610 u64 physical
, struct btrfs_device
*dev
,
2611 u64 flags
, u64 gen
, int mirror_num
)
2613 struct scrub_ctx
*sctx
= sparity
->sctx
;
2615 u8 csum
[BTRFS_CSUM_SIZE
];
2618 if (test_bit(BTRFS_DEV_STATE_MISSING
, &dev
->dev_state
)) {
2619 scrub_parity_mark_sectors_error(sparity
, logical
, len
);
2623 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
2624 blocksize
= sparity
->stripe_len
;
2625 } else if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
2626 blocksize
= sparity
->stripe_len
;
2628 blocksize
= sctx
->fs_info
->sectorsize
;
2633 u64 l
= min_t(u64
, len
, blocksize
);
2636 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
2637 /* push csums to sbio */
2638 have_csum
= scrub_find_csum(sctx
, logical
, csum
);
2642 ret
= scrub_pages_for_parity(sparity
, logical
, l
, physical
, dev
,
2643 flags
, gen
, mirror_num
,
2644 have_csum
? csum
: NULL
);
2656 * Given a physical address, this will calculate it's
2657 * logical offset. if this is a parity stripe, it will return
2658 * the most left data stripe's logical offset.
2660 * return 0 if it is a data stripe, 1 means parity stripe.
2662 static int get_raid56_logic_offset(u64 physical
, int num
,
2663 struct map_lookup
*map
, u64
*offset
,
2672 const int data_stripes
= nr_data_stripes(map
);
2674 last_offset
= (physical
- map
->stripes
[num
].physical
) * data_stripes
;
2676 *stripe_start
= last_offset
;
2678 *offset
= last_offset
;
2679 for (i
= 0; i
< data_stripes
; i
++) {
2680 *offset
= last_offset
+ i
* map
->stripe_len
;
2682 stripe_nr
= div64_u64(*offset
, map
->stripe_len
);
2683 stripe_nr
= div_u64(stripe_nr
, data_stripes
);
2685 /* Work out the disk rotation on this stripe-set */
2686 stripe_nr
= div_u64_rem(stripe_nr
, map
->num_stripes
, &rot
);
2687 /* calculate which stripe this data locates */
2689 stripe_index
= rot
% map
->num_stripes
;
2690 if (stripe_index
== num
)
2692 if (stripe_index
< num
)
2695 *offset
= last_offset
+ j
* map
->stripe_len
;
2699 static void scrub_free_parity(struct scrub_parity
*sparity
)
2701 struct scrub_ctx
*sctx
= sparity
->sctx
;
2702 struct scrub_page
*curr
, *next
;
2705 nbits
= bitmap_weight(sparity
->ebitmap
, sparity
->nsectors
);
2707 spin_lock(&sctx
->stat_lock
);
2708 sctx
->stat
.read_errors
+= nbits
;
2709 sctx
->stat
.uncorrectable_errors
+= nbits
;
2710 spin_unlock(&sctx
->stat_lock
);
2713 list_for_each_entry_safe(curr
, next
, &sparity
->spages
, list
) {
2714 list_del_init(&curr
->list
);
2715 scrub_page_put(curr
);
2721 static void scrub_parity_bio_endio_worker(struct btrfs_work
*work
)
2723 struct scrub_parity
*sparity
= container_of(work
, struct scrub_parity
,
2725 struct scrub_ctx
*sctx
= sparity
->sctx
;
2727 scrub_free_parity(sparity
);
2728 scrub_pending_bio_dec(sctx
);
2731 static void scrub_parity_bio_endio(struct bio
*bio
)
2733 struct scrub_parity
*sparity
= (struct scrub_parity
*)bio
->bi_private
;
2734 struct btrfs_fs_info
*fs_info
= sparity
->sctx
->fs_info
;
2737 bitmap_or(sparity
->ebitmap
, sparity
->ebitmap
, sparity
->dbitmap
,
2742 btrfs_init_work(&sparity
->work
, scrub_parity_bio_endio_worker
, NULL
,
2744 btrfs_queue_work(fs_info
->scrub_parity_workers
, &sparity
->work
);
2747 static void scrub_parity_check_and_repair(struct scrub_parity
*sparity
)
2749 struct scrub_ctx
*sctx
= sparity
->sctx
;
2750 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
2752 struct btrfs_raid_bio
*rbio
;
2753 struct btrfs_bio
*bbio
= NULL
;
2757 if (!bitmap_andnot(sparity
->dbitmap
, sparity
->dbitmap
, sparity
->ebitmap
,
2761 length
= sparity
->logic_end
- sparity
->logic_start
;
2763 btrfs_bio_counter_inc_blocked(fs_info
);
2764 ret
= btrfs_map_sblock(fs_info
, BTRFS_MAP_WRITE
, sparity
->logic_start
,
2766 if (ret
|| !bbio
|| !bbio
->raid_map
)
2769 bio
= btrfs_io_bio_alloc(0);
2770 bio
->bi_iter
.bi_sector
= sparity
->logic_start
>> 9;
2771 bio
->bi_private
= sparity
;
2772 bio
->bi_end_io
= scrub_parity_bio_endio
;
2774 rbio
= raid56_parity_alloc_scrub_rbio(fs_info
, bio
, bbio
,
2775 length
, sparity
->scrub_dev
,
2781 scrub_pending_bio_inc(sctx
);
2782 raid56_parity_submit_scrub_rbio(rbio
);
2788 btrfs_bio_counter_dec(fs_info
);
2789 btrfs_put_bbio(bbio
);
2790 bitmap_or(sparity
->ebitmap
, sparity
->ebitmap
, sparity
->dbitmap
,
2792 spin_lock(&sctx
->stat_lock
);
2793 sctx
->stat
.malloc_errors
++;
2794 spin_unlock(&sctx
->stat_lock
);
2796 scrub_free_parity(sparity
);
2799 static inline int scrub_calc_parity_bitmap_len(int nsectors
)
2801 return DIV_ROUND_UP(nsectors
, BITS_PER_LONG
) * sizeof(long);
2804 static void scrub_parity_get(struct scrub_parity
*sparity
)
2806 refcount_inc(&sparity
->refs
);
2809 static void scrub_parity_put(struct scrub_parity
*sparity
)
2811 if (!refcount_dec_and_test(&sparity
->refs
))
2814 scrub_parity_check_and_repair(sparity
);
2817 static noinline_for_stack
int scrub_raid56_parity(struct scrub_ctx
*sctx
,
2818 struct map_lookup
*map
,
2819 struct btrfs_device
*sdev
,
2820 struct btrfs_path
*path
,
2824 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
2825 struct btrfs_root
*root
= fs_info
->extent_root
;
2826 struct btrfs_root
*csum_root
= fs_info
->csum_root
;
2827 struct btrfs_extent_item
*extent
;
2828 struct btrfs_bio
*bbio
= NULL
;
2832 struct extent_buffer
*l
;
2833 struct btrfs_key key
;
2836 u64 extent_physical
;
2839 struct btrfs_device
*extent_dev
;
2840 struct scrub_parity
*sparity
;
2843 int extent_mirror_num
;
2846 nsectors
= div_u64(map
->stripe_len
, fs_info
->sectorsize
);
2847 bitmap_len
= scrub_calc_parity_bitmap_len(nsectors
);
2848 sparity
= kzalloc(sizeof(struct scrub_parity
) + 2 * bitmap_len
,
2851 spin_lock(&sctx
->stat_lock
);
2852 sctx
->stat
.malloc_errors
++;
2853 spin_unlock(&sctx
->stat_lock
);
2857 sparity
->stripe_len
= map
->stripe_len
;
2858 sparity
->nsectors
= nsectors
;
2859 sparity
->sctx
= sctx
;
2860 sparity
->scrub_dev
= sdev
;
2861 sparity
->logic_start
= logic_start
;
2862 sparity
->logic_end
= logic_end
;
2863 refcount_set(&sparity
->refs
, 1);
2864 INIT_LIST_HEAD(&sparity
->spages
);
2865 sparity
->dbitmap
= sparity
->bitmap
;
2866 sparity
->ebitmap
= (void *)sparity
->bitmap
+ bitmap_len
;
2869 while (logic_start
< logic_end
) {
2870 if (btrfs_fs_incompat(fs_info
, SKINNY_METADATA
))
2871 key
.type
= BTRFS_METADATA_ITEM_KEY
;
2873 key
.type
= BTRFS_EXTENT_ITEM_KEY
;
2874 key
.objectid
= logic_start
;
2875 key
.offset
= (u64
)-1;
2877 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
2882 ret
= btrfs_previous_extent_item(root
, path
, 0);
2886 btrfs_release_path(path
);
2887 ret
= btrfs_search_slot(NULL
, root
, &key
,
2899 slot
= path
->slots
[0];
2900 if (slot
>= btrfs_header_nritems(l
)) {
2901 ret
= btrfs_next_leaf(root
, path
);
2910 btrfs_item_key_to_cpu(l
, &key
, slot
);
2912 if (key
.type
!= BTRFS_EXTENT_ITEM_KEY
&&
2913 key
.type
!= BTRFS_METADATA_ITEM_KEY
)
2916 if (key
.type
== BTRFS_METADATA_ITEM_KEY
)
2917 bytes
= fs_info
->nodesize
;
2921 if (key
.objectid
+ bytes
<= logic_start
)
2924 if (key
.objectid
>= logic_end
) {
2929 while (key
.objectid
>= logic_start
+ map
->stripe_len
)
2930 logic_start
+= map
->stripe_len
;
2932 extent
= btrfs_item_ptr(l
, slot
,
2933 struct btrfs_extent_item
);
2934 flags
= btrfs_extent_flags(l
, extent
);
2935 generation
= btrfs_extent_generation(l
, extent
);
2937 if ((flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) &&
2938 (key
.objectid
< logic_start
||
2939 key
.objectid
+ bytes
>
2940 logic_start
+ map
->stripe_len
)) {
2942 "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
2943 key
.objectid
, logic_start
);
2944 spin_lock(&sctx
->stat_lock
);
2945 sctx
->stat
.uncorrectable_errors
++;
2946 spin_unlock(&sctx
->stat_lock
);
2950 extent_logical
= key
.objectid
;
2953 if (extent_logical
< logic_start
) {
2954 extent_len
-= logic_start
- extent_logical
;
2955 extent_logical
= logic_start
;
2958 if (extent_logical
+ extent_len
>
2959 logic_start
+ map
->stripe_len
)
2960 extent_len
= logic_start
+ map
->stripe_len
-
2963 scrub_parity_mark_sectors_data(sparity
, extent_logical
,
2966 mapped_length
= extent_len
;
2968 ret
= btrfs_map_block(fs_info
, BTRFS_MAP_READ
,
2969 extent_logical
, &mapped_length
, &bbio
,
2972 if (!bbio
|| mapped_length
< extent_len
)
2976 btrfs_put_bbio(bbio
);
2979 extent_physical
= bbio
->stripes
[0].physical
;
2980 extent_mirror_num
= bbio
->mirror_num
;
2981 extent_dev
= bbio
->stripes
[0].dev
;
2982 btrfs_put_bbio(bbio
);
2984 ret
= btrfs_lookup_csums_range(csum_root
,
2986 extent_logical
+ extent_len
- 1,
2987 &sctx
->csum_list
, 1);
2991 ret
= scrub_extent_for_parity(sparity
, extent_logical
,
2998 scrub_free_csums(sctx
);
3003 if (extent_logical
+ extent_len
<
3004 key
.objectid
+ bytes
) {
3005 logic_start
+= map
->stripe_len
;
3007 if (logic_start
>= logic_end
) {
3012 if (logic_start
< key
.objectid
+ bytes
) {
3021 btrfs_release_path(path
);
3026 logic_start
+= map
->stripe_len
;
3030 scrub_parity_mark_sectors_error(sparity
, logic_start
,
3031 logic_end
- logic_start
);
3032 scrub_parity_put(sparity
);
3034 mutex_lock(&sctx
->wr_lock
);
3035 scrub_wr_submit(sctx
);
3036 mutex_unlock(&sctx
->wr_lock
);
3038 btrfs_release_path(path
);
3039 return ret
< 0 ? ret
: 0;
3042 static noinline_for_stack
int scrub_stripe(struct scrub_ctx
*sctx
,
3043 struct map_lookup
*map
,
3044 struct btrfs_device
*scrub_dev
,
3045 int num
, u64 base
, u64 length
)
3047 struct btrfs_path
*path
, *ppath
;
3048 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
3049 struct btrfs_root
*root
= fs_info
->extent_root
;
3050 struct btrfs_root
*csum_root
= fs_info
->csum_root
;
3051 struct btrfs_extent_item
*extent
;
3052 struct blk_plug plug
;
3057 struct extent_buffer
*l
;
3064 struct reada_control
*reada1
;
3065 struct reada_control
*reada2
;
3066 struct btrfs_key key
;
3067 struct btrfs_key key_end
;
3068 u64 increment
= map
->stripe_len
;
3071 u64 extent_physical
;
3075 struct btrfs_device
*extent_dev
;
3076 int extent_mirror_num
;
3079 physical
= map
->stripes
[num
].physical
;
3081 nstripes
= div64_u64(length
, map
->stripe_len
);
3082 if (map
->type
& BTRFS_BLOCK_GROUP_RAID0
) {
3083 offset
= map
->stripe_len
* num
;
3084 increment
= map
->stripe_len
* map
->num_stripes
;
3086 } else if (map
->type
& BTRFS_BLOCK_GROUP_RAID10
) {
3087 int factor
= map
->num_stripes
/ map
->sub_stripes
;
3088 offset
= map
->stripe_len
* (num
/ map
->sub_stripes
);
3089 increment
= map
->stripe_len
* factor
;
3090 mirror_num
= num
% map
->sub_stripes
+ 1;
3091 } else if (map
->type
& BTRFS_BLOCK_GROUP_RAID1_MASK
) {
3092 increment
= map
->stripe_len
;
3093 mirror_num
= num
% map
->num_stripes
+ 1;
3094 } else if (map
->type
& BTRFS_BLOCK_GROUP_DUP
) {
3095 increment
= map
->stripe_len
;
3096 mirror_num
= num
% map
->num_stripes
+ 1;
3097 } else if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
3098 get_raid56_logic_offset(physical
, num
, map
, &offset
, NULL
);
3099 increment
= map
->stripe_len
* nr_data_stripes(map
);
3102 increment
= map
->stripe_len
;
3106 path
= btrfs_alloc_path();
3110 ppath
= btrfs_alloc_path();
3112 btrfs_free_path(path
);
3117 * work on commit root. The related disk blocks are static as
3118 * long as COW is applied. This means, it is save to rewrite
3119 * them to repair disk errors without any race conditions
3121 path
->search_commit_root
= 1;
3122 path
->skip_locking
= 1;
3124 ppath
->search_commit_root
= 1;
3125 ppath
->skip_locking
= 1;
3127 * trigger the readahead for extent tree csum tree and wait for
3128 * completion. During readahead, the scrub is officially paused
3129 * to not hold off transaction commits
3131 logical
= base
+ offset
;
3132 physical_end
= physical
+ nstripes
* map
->stripe_len
;
3133 if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
3134 get_raid56_logic_offset(physical_end
, num
,
3135 map
, &logic_end
, NULL
);
3138 logic_end
= logical
+ increment
* nstripes
;
3140 wait_event(sctx
->list_wait
,
3141 atomic_read(&sctx
->bios_in_flight
) == 0);
3142 scrub_blocked_if_needed(fs_info
);
3144 /* FIXME it might be better to start readahead at commit root */
3145 key
.objectid
= logical
;
3146 key
.type
= BTRFS_EXTENT_ITEM_KEY
;
3147 key
.offset
= (u64
)0;
3148 key_end
.objectid
= logic_end
;
3149 key_end
.type
= BTRFS_METADATA_ITEM_KEY
;
3150 key_end
.offset
= (u64
)-1;
3151 reada1
= btrfs_reada_add(root
, &key
, &key_end
);
3153 key
.objectid
= BTRFS_EXTENT_CSUM_OBJECTID
;
3154 key
.type
= BTRFS_EXTENT_CSUM_KEY
;
3155 key
.offset
= logical
;
3156 key_end
.objectid
= BTRFS_EXTENT_CSUM_OBJECTID
;
3157 key_end
.type
= BTRFS_EXTENT_CSUM_KEY
;
3158 key_end
.offset
= logic_end
;
3159 reada2
= btrfs_reada_add(csum_root
, &key
, &key_end
);
3161 if (!IS_ERR(reada1
))
3162 btrfs_reada_wait(reada1
);
3163 if (!IS_ERR(reada2
))
3164 btrfs_reada_wait(reada2
);
3168 * collect all data csums for the stripe to avoid seeking during
3169 * the scrub. This might currently (crc32) end up to be about 1MB
3171 blk_start_plug(&plug
);
3174 * now find all extents for each stripe and scrub them
3177 while (physical
< physical_end
) {
3181 if (atomic_read(&fs_info
->scrub_cancel_req
) ||
3182 atomic_read(&sctx
->cancel_req
)) {
3187 * check to see if we have to pause
3189 if (atomic_read(&fs_info
->scrub_pause_req
)) {
3190 /* push queued extents */
3191 sctx
->flush_all_writes
= true;
3193 mutex_lock(&sctx
->wr_lock
);
3194 scrub_wr_submit(sctx
);
3195 mutex_unlock(&sctx
->wr_lock
);
3196 wait_event(sctx
->list_wait
,
3197 atomic_read(&sctx
->bios_in_flight
) == 0);
3198 sctx
->flush_all_writes
= false;
3199 scrub_blocked_if_needed(fs_info
);
3202 if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
3203 ret
= get_raid56_logic_offset(physical
, num
, map
,
3208 /* it is parity strip */
3209 stripe_logical
+= base
;
3210 stripe_end
= stripe_logical
+ increment
;
3211 ret
= scrub_raid56_parity(sctx
, map
, scrub_dev
,
3212 ppath
, stripe_logical
,
3220 if (btrfs_fs_incompat(fs_info
, SKINNY_METADATA
))
3221 key
.type
= BTRFS_METADATA_ITEM_KEY
;
3223 key
.type
= BTRFS_EXTENT_ITEM_KEY
;
3224 key
.objectid
= logical
;
3225 key
.offset
= (u64
)-1;
3227 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
3232 ret
= btrfs_previous_extent_item(root
, path
, 0);
3236 /* there's no smaller item, so stick with the
3238 btrfs_release_path(path
);
3239 ret
= btrfs_search_slot(NULL
, root
, &key
,
3251 slot
= path
->slots
[0];
3252 if (slot
>= btrfs_header_nritems(l
)) {
3253 ret
= btrfs_next_leaf(root
, path
);
3262 btrfs_item_key_to_cpu(l
, &key
, slot
);
3264 if (key
.type
!= BTRFS_EXTENT_ITEM_KEY
&&
3265 key
.type
!= BTRFS_METADATA_ITEM_KEY
)
3268 if (key
.type
== BTRFS_METADATA_ITEM_KEY
)
3269 bytes
= fs_info
->nodesize
;
3273 if (key
.objectid
+ bytes
<= logical
)
3276 if (key
.objectid
>= logical
+ map
->stripe_len
) {
3277 /* out of this device extent */
3278 if (key
.objectid
>= logic_end
)
3283 extent
= btrfs_item_ptr(l
, slot
,
3284 struct btrfs_extent_item
);
3285 flags
= btrfs_extent_flags(l
, extent
);
3286 generation
= btrfs_extent_generation(l
, extent
);
3288 if ((flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) &&
3289 (key
.objectid
< logical
||
3290 key
.objectid
+ bytes
>
3291 logical
+ map
->stripe_len
)) {
3293 "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3294 key
.objectid
, logical
);
3295 spin_lock(&sctx
->stat_lock
);
3296 sctx
->stat
.uncorrectable_errors
++;
3297 spin_unlock(&sctx
->stat_lock
);
3302 extent_logical
= key
.objectid
;
3306 * trim extent to this stripe
3308 if (extent_logical
< logical
) {
3309 extent_len
-= logical
- extent_logical
;
3310 extent_logical
= logical
;
3312 if (extent_logical
+ extent_len
>
3313 logical
+ map
->stripe_len
) {
3314 extent_len
= logical
+ map
->stripe_len
-
3318 extent_physical
= extent_logical
- logical
+ physical
;
3319 extent_dev
= scrub_dev
;
3320 extent_mirror_num
= mirror_num
;
3321 if (sctx
->is_dev_replace
)
3322 scrub_remap_extent(fs_info
, extent_logical
,
3323 extent_len
, &extent_physical
,
3325 &extent_mirror_num
);
3327 ret
= btrfs_lookup_csums_range(csum_root
,
3331 &sctx
->csum_list
, 1);
3335 ret
= scrub_extent(sctx
, map
, extent_logical
, extent_len
,
3336 extent_physical
, extent_dev
, flags
,
3337 generation
, extent_mirror_num
,
3338 extent_logical
- logical
+ physical
);
3340 scrub_free_csums(sctx
);
3345 if (extent_logical
+ extent_len
<
3346 key
.objectid
+ bytes
) {
3347 if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
3349 * loop until we find next data stripe
3350 * or we have finished all stripes.
3353 physical
+= map
->stripe_len
;
3354 ret
= get_raid56_logic_offset(physical
,
3359 if (ret
&& physical
< physical_end
) {
3360 stripe_logical
+= base
;
3361 stripe_end
= stripe_logical
+
3363 ret
= scrub_raid56_parity(sctx
,
3364 map
, scrub_dev
, ppath
,
3372 physical
+= map
->stripe_len
;
3373 logical
+= increment
;
3375 if (logical
< key
.objectid
+ bytes
) {
3380 if (physical
>= physical_end
) {
3388 btrfs_release_path(path
);
3390 logical
+= increment
;
3391 physical
+= map
->stripe_len
;
3392 spin_lock(&sctx
->stat_lock
);
3394 sctx
->stat
.last_physical
= map
->stripes
[num
].physical
+
3397 sctx
->stat
.last_physical
= physical
;
3398 spin_unlock(&sctx
->stat_lock
);
3403 /* push queued extents */
3405 mutex_lock(&sctx
->wr_lock
);
3406 scrub_wr_submit(sctx
);
3407 mutex_unlock(&sctx
->wr_lock
);
3409 blk_finish_plug(&plug
);
3410 btrfs_free_path(path
);
3411 btrfs_free_path(ppath
);
3412 return ret
< 0 ? ret
: 0;
3415 static noinline_for_stack
int scrub_chunk(struct scrub_ctx
*sctx
,
3416 struct btrfs_device
*scrub_dev
,
3417 u64 chunk_offset
, u64 length
,
3419 struct btrfs_block_group
*cache
)
3421 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
3422 struct extent_map_tree
*map_tree
= &fs_info
->mapping_tree
;
3423 struct map_lookup
*map
;
3424 struct extent_map
*em
;
3428 read_lock(&map_tree
->lock
);
3429 em
= lookup_extent_mapping(map_tree
, chunk_offset
, 1);
3430 read_unlock(&map_tree
->lock
);
3434 * Might have been an unused block group deleted by the cleaner
3435 * kthread or relocation.
3437 spin_lock(&cache
->lock
);
3438 if (!cache
->removed
)
3440 spin_unlock(&cache
->lock
);
3445 map
= em
->map_lookup
;
3446 if (em
->start
!= chunk_offset
)
3449 if (em
->len
< length
)
3452 for (i
= 0; i
< map
->num_stripes
; ++i
) {
3453 if (map
->stripes
[i
].dev
->bdev
== scrub_dev
->bdev
&&
3454 map
->stripes
[i
].physical
== dev_offset
) {
3455 ret
= scrub_stripe(sctx
, map
, scrub_dev
, i
,
3456 chunk_offset
, length
);
3462 free_extent_map(em
);
3467 static noinline_for_stack
3468 int scrub_enumerate_chunks(struct scrub_ctx
*sctx
,
3469 struct btrfs_device
*scrub_dev
, u64 start
, u64 end
)
3471 struct btrfs_dev_extent
*dev_extent
= NULL
;
3472 struct btrfs_path
*path
;
3473 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
3474 struct btrfs_root
*root
= fs_info
->dev_root
;
3480 struct extent_buffer
*l
;
3481 struct btrfs_key key
;
3482 struct btrfs_key found_key
;
3483 struct btrfs_block_group
*cache
;
3484 struct btrfs_dev_replace
*dev_replace
= &fs_info
->dev_replace
;
3486 path
= btrfs_alloc_path();
3490 path
->reada
= READA_FORWARD
;
3491 path
->search_commit_root
= 1;
3492 path
->skip_locking
= 1;
3494 key
.objectid
= scrub_dev
->devid
;
3496 key
.type
= BTRFS_DEV_EXTENT_KEY
;
3499 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
3503 if (path
->slots
[0] >=
3504 btrfs_header_nritems(path
->nodes
[0])) {
3505 ret
= btrfs_next_leaf(root
, path
);
3518 slot
= path
->slots
[0];
3520 btrfs_item_key_to_cpu(l
, &found_key
, slot
);
3522 if (found_key
.objectid
!= scrub_dev
->devid
)
3525 if (found_key
.type
!= BTRFS_DEV_EXTENT_KEY
)
3528 if (found_key
.offset
>= end
)
3531 if (found_key
.offset
< key
.offset
)
3534 dev_extent
= btrfs_item_ptr(l
, slot
, struct btrfs_dev_extent
);
3535 length
= btrfs_dev_extent_length(l
, dev_extent
);
3537 if (found_key
.offset
+ length
<= start
)
3540 chunk_offset
= btrfs_dev_extent_chunk_offset(l
, dev_extent
);
3543 * get a reference on the corresponding block group to prevent
3544 * the chunk from going away while we scrub it
3546 cache
= btrfs_lookup_block_group(fs_info
, chunk_offset
);
3548 /* some chunks are removed but not committed to disk yet,
3549 * continue scrubbing */
3554 * we need call btrfs_inc_block_group_ro() with scrubs_paused,
3555 * to avoid deadlock caused by:
3556 * btrfs_inc_block_group_ro()
3557 * -> btrfs_wait_for_commit()
3558 * -> btrfs_commit_transaction()
3559 * -> btrfs_scrub_pause()
3561 scrub_pause_on(fs_info
);
3564 * Don't do chunk preallocation for scrub.
3566 * This is especially important for SYSTEM bgs, or we can hit
3567 * -EFBIG from btrfs_finish_chunk_alloc() like:
3568 * 1. The only SYSTEM bg is marked RO.
3569 * Since SYSTEM bg is small, that's pretty common.
3570 * 2. New SYSTEM bg will be allocated
3571 * Due to regular version will allocate new chunk.
3572 * 3. New SYSTEM bg is empty and will get cleaned up
3573 * Before cleanup really happens, it's marked RO again.
3574 * 4. Empty SYSTEM bg get scrubbed
3577 * This can easily boost the amount of SYSTEM chunks if cleaner
3578 * thread can't be triggered fast enough, and use up all space
3579 * of btrfs_super_block::sys_chunk_array
3581 ret
= btrfs_inc_block_group_ro(cache
, false);
3582 if (!ret
&& sctx
->is_dev_replace
) {
3584 * If we are doing a device replace wait for any tasks
3585 * that started delalloc right before we set the block
3586 * group to RO mode, as they might have just allocated
3587 * an extent from it or decided they could do a nocow
3588 * write. And if any such tasks did that, wait for their
3589 * ordered extents to complete and then commit the
3590 * current transaction, so that we can later see the new
3591 * extent items in the extent tree - the ordered extents
3592 * create delayed data references (for cow writes) when
3593 * they complete, which will be run and insert the
3594 * corresponding extent items into the extent tree when
3595 * we commit the transaction they used when running
3596 * inode.c:btrfs_finish_ordered_io(). We later use
3597 * the commit root of the extent tree to find extents
3598 * to copy from the srcdev into the tgtdev, and we don't
3599 * want to miss any new extents.
3601 btrfs_wait_block_group_reservations(cache
);
3602 btrfs_wait_nocow_writers(cache
);
3603 ret
= btrfs_wait_ordered_roots(fs_info
, U64_MAX
,
3607 struct btrfs_trans_handle
*trans
;
3609 trans
= btrfs_join_transaction(root
);
3611 ret
= PTR_ERR(trans
);
3613 ret
= btrfs_commit_transaction(trans
);
3615 scrub_pause_off(fs_info
);
3616 btrfs_put_block_group(cache
);
3621 scrub_pause_off(fs_info
);
3625 } else if (ret
== -ENOSPC
) {
3627 * btrfs_inc_block_group_ro return -ENOSPC when it
3628 * failed in creating new chunk for metadata.
3629 * It is not a problem for scrub/replace, because
3630 * metadata are always cowed, and our scrub paused
3631 * commit_transactions.
3636 "failed setting block group ro: %d", ret
);
3637 btrfs_put_block_group(cache
);
3641 down_write(&dev_replace
->rwsem
);
3642 dev_replace
->cursor_right
= found_key
.offset
+ length
;
3643 dev_replace
->cursor_left
= found_key
.offset
;
3644 dev_replace
->item_needs_writeback
= 1;
3645 up_write(&dev_replace
->rwsem
);
3647 ret
= scrub_chunk(sctx
, scrub_dev
, chunk_offset
, length
,
3648 found_key
.offset
, cache
);
3651 * flush, submit all pending read and write bios, afterwards
3653 * Note that in the dev replace case, a read request causes
3654 * write requests that are submitted in the read completion
3655 * worker. Therefore in the current situation, it is required
3656 * that all write requests are flushed, so that all read and
3657 * write requests are really completed when bios_in_flight
3660 sctx
->flush_all_writes
= true;
3662 mutex_lock(&sctx
->wr_lock
);
3663 scrub_wr_submit(sctx
);
3664 mutex_unlock(&sctx
->wr_lock
);
3666 wait_event(sctx
->list_wait
,
3667 atomic_read(&sctx
->bios_in_flight
) == 0);
3669 scrub_pause_on(fs_info
);
3672 * must be called before we decrease @scrub_paused.
3673 * make sure we don't block transaction commit while
3674 * we are waiting pending workers finished.
3676 wait_event(sctx
->list_wait
,
3677 atomic_read(&sctx
->workers_pending
) == 0);
3678 sctx
->flush_all_writes
= false;
3680 scrub_pause_off(fs_info
);
3682 down_write(&dev_replace
->rwsem
);
3683 dev_replace
->cursor_left
= dev_replace
->cursor_right
;
3684 dev_replace
->item_needs_writeback
= 1;
3685 up_write(&dev_replace
->rwsem
);
3688 btrfs_dec_block_group_ro(cache
);
3691 * We might have prevented the cleaner kthread from deleting
3692 * this block group if it was already unused because we raced
3693 * and set it to RO mode first. So add it back to the unused
3694 * list, otherwise it might not ever be deleted unless a manual
3695 * balance is triggered or it becomes used and unused again.
3697 spin_lock(&cache
->lock
);
3698 if (!cache
->removed
&& !cache
->ro
&& cache
->reserved
== 0 &&
3700 spin_unlock(&cache
->lock
);
3701 btrfs_mark_bg_unused(cache
);
3703 spin_unlock(&cache
->lock
);
3706 btrfs_put_block_group(cache
);
3709 if (sctx
->is_dev_replace
&&
3710 atomic64_read(&dev_replace
->num_write_errors
) > 0) {
3714 if (sctx
->stat
.malloc_errors
> 0) {
3719 key
.offset
= found_key
.offset
+ length
;
3720 btrfs_release_path(path
);
3723 btrfs_free_path(path
);
3728 static noinline_for_stack
int scrub_supers(struct scrub_ctx
*sctx
,
3729 struct btrfs_device
*scrub_dev
)
3735 struct btrfs_fs_info
*fs_info
= sctx
->fs_info
;
3737 if (test_bit(BTRFS_FS_STATE_ERROR
, &fs_info
->fs_state
))
3740 /* Seed devices of a new filesystem has their own generation. */
3741 if (scrub_dev
->fs_devices
!= fs_info
->fs_devices
)
3742 gen
= scrub_dev
->generation
;
3744 gen
= fs_info
->last_trans_committed
;
3746 for (i
= 0; i
< BTRFS_SUPER_MIRROR_MAX
; i
++) {
3747 bytenr
= btrfs_sb_offset(i
);
3748 if (bytenr
+ BTRFS_SUPER_INFO_SIZE
>
3749 scrub_dev
->commit_total_bytes
)
3752 ret
= scrub_pages(sctx
, bytenr
, BTRFS_SUPER_INFO_SIZE
, bytenr
,
3753 scrub_dev
, BTRFS_EXTENT_FLAG_SUPER
, gen
, i
,
3758 wait_event(sctx
->list_wait
, atomic_read(&sctx
->bios_in_flight
) == 0);
3764 * get a reference count on fs_info->scrub_workers. start worker if necessary
3766 static noinline_for_stack
int scrub_workers_get(struct btrfs_fs_info
*fs_info
,
3769 unsigned int flags
= WQ_FREEZABLE
| WQ_UNBOUND
;
3770 int max_active
= fs_info
->thread_pool_size
;
3772 lockdep_assert_held(&fs_info
->scrub_lock
);
3774 if (refcount_read(&fs_info
->scrub_workers_refcnt
) == 0) {
3775 ASSERT(fs_info
->scrub_workers
== NULL
);
3776 fs_info
->scrub_workers
= btrfs_alloc_workqueue(fs_info
, "scrub",
3777 flags
, is_dev_replace
? 1 : max_active
, 4);
3778 if (!fs_info
->scrub_workers
)
3779 goto fail_scrub_workers
;
3781 ASSERT(fs_info
->scrub_wr_completion_workers
== NULL
);
3782 fs_info
->scrub_wr_completion_workers
=
3783 btrfs_alloc_workqueue(fs_info
, "scrubwrc", flags
,
3785 if (!fs_info
->scrub_wr_completion_workers
)
3786 goto fail_scrub_wr_completion_workers
;
3788 ASSERT(fs_info
->scrub_parity_workers
== NULL
);
3789 fs_info
->scrub_parity_workers
=
3790 btrfs_alloc_workqueue(fs_info
, "scrubparity", flags
,
3792 if (!fs_info
->scrub_parity_workers
)
3793 goto fail_scrub_parity_workers
;
3795 refcount_set(&fs_info
->scrub_workers_refcnt
, 1);
3797 refcount_inc(&fs_info
->scrub_workers_refcnt
);
3801 fail_scrub_parity_workers
:
3802 btrfs_destroy_workqueue(fs_info
->scrub_wr_completion_workers
);
3803 fail_scrub_wr_completion_workers
:
3804 btrfs_destroy_workqueue(fs_info
->scrub_workers
);
3809 int btrfs_scrub_dev(struct btrfs_fs_info
*fs_info
, u64 devid
, u64 start
,
3810 u64 end
, struct btrfs_scrub_progress
*progress
,
3811 int readonly
, int is_dev_replace
)
3813 struct scrub_ctx
*sctx
;
3815 struct btrfs_device
*dev
;
3816 unsigned int nofs_flag
;
3817 struct btrfs_workqueue
*scrub_workers
= NULL
;
3818 struct btrfs_workqueue
*scrub_wr_comp
= NULL
;
3819 struct btrfs_workqueue
*scrub_parity
= NULL
;
3821 if (btrfs_fs_closing(fs_info
))
3824 if (fs_info
->nodesize
> BTRFS_STRIPE_LEN
) {
3826 * in this case scrub is unable to calculate the checksum
3827 * the way scrub is implemented. Do not handle this
3828 * situation at all because it won't ever happen.
3831 "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
3837 if (fs_info
->sectorsize
!= PAGE_SIZE
) {
3838 /* not supported for data w/o checksums */
3839 btrfs_err_rl(fs_info
,
3840 "scrub: size assumption sectorsize != PAGE_SIZE (%d != %lu) fails",
3841 fs_info
->sectorsize
, PAGE_SIZE
);
3845 if (fs_info
->nodesize
>
3846 PAGE_SIZE
* SCRUB_MAX_PAGES_PER_BLOCK
||
3847 fs_info
->sectorsize
> PAGE_SIZE
* SCRUB_MAX_PAGES_PER_BLOCK
) {
3849 * would exhaust the array bounds of pagev member in
3850 * struct scrub_block
3853 "scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
3855 SCRUB_MAX_PAGES_PER_BLOCK
,
3856 fs_info
->sectorsize
,
3857 SCRUB_MAX_PAGES_PER_BLOCK
);
3861 /* Allocate outside of device_list_mutex */
3862 sctx
= scrub_setup_ctx(fs_info
, is_dev_replace
);
3864 return PTR_ERR(sctx
);
3866 mutex_lock(&fs_info
->fs_devices
->device_list_mutex
);
3867 dev
= btrfs_find_device(fs_info
->fs_devices
, devid
, NULL
, NULL
, true);
3868 if (!dev
|| (test_bit(BTRFS_DEV_STATE_MISSING
, &dev
->dev_state
) &&
3870 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3875 if (!is_dev_replace
&& !readonly
&&
3876 !test_bit(BTRFS_DEV_STATE_WRITEABLE
, &dev
->dev_state
)) {
3877 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3878 btrfs_err_in_rcu(fs_info
, "scrub: device %s is not writable",
3879 rcu_str_deref(dev
->name
));
3884 mutex_lock(&fs_info
->scrub_lock
);
3885 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA
, &dev
->dev_state
) ||
3886 test_bit(BTRFS_DEV_STATE_REPLACE_TGT
, &dev
->dev_state
)) {
3887 mutex_unlock(&fs_info
->scrub_lock
);
3888 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3893 down_read(&fs_info
->dev_replace
.rwsem
);
3894 if (dev
->scrub_ctx
||
3896 btrfs_dev_replace_is_ongoing(&fs_info
->dev_replace
))) {
3897 up_read(&fs_info
->dev_replace
.rwsem
);
3898 mutex_unlock(&fs_info
->scrub_lock
);
3899 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3903 up_read(&fs_info
->dev_replace
.rwsem
);
3905 ret
= scrub_workers_get(fs_info
, is_dev_replace
);
3907 mutex_unlock(&fs_info
->scrub_lock
);
3908 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3912 sctx
->readonly
= readonly
;
3913 dev
->scrub_ctx
= sctx
;
3914 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3917 * checking @scrub_pause_req here, we can avoid
3918 * race between committing transaction and scrubbing.
3920 __scrub_blocked_if_needed(fs_info
);
3921 atomic_inc(&fs_info
->scrubs_running
);
3922 mutex_unlock(&fs_info
->scrub_lock
);
3925 * In order to avoid deadlock with reclaim when there is a transaction
3926 * trying to pause scrub, make sure we use GFP_NOFS for all the
3927 * allocations done at btrfs_scrub_pages() and scrub_pages_for_parity()
3928 * invoked by our callees. The pausing request is done when the
3929 * transaction commit starts, and it blocks the transaction until scrub
3930 * is paused (done at specific points at scrub_stripe() or right above
3931 * before incrementing fs_info->scrubs_running).
3933 nofs_flag
= memalloc_nofs_save();
3934 if (!is_dev_replace
) {
3935 btrfs_info(fs_info
, "scrub: started on devid %llu", devid
);
3937 * by holding device list mutex, we can
3938 * kick off writing super in log tree sync.
3940 mutex_lock(&fs_info
->fs_devices
->device_list_mutex
);
3941 ret
= scrub_supers(sctx
, dev
);
3942 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3946 ret
= scrub_enumerate_chunks(sctx
, dev
, start
, end
);
3947 memalloc_nofs_restore(nofs_flag
);
3949 wait_event(sctx
->list_wait
, atomic_read(&sctx
->bios_in_flight
) == 0);
3950 atomic_dec(&fs_info
->scrubs_running
);
3951 wake_up(&fs_info
->scrub_pause_wait
);
3953 wait_event(sctx
->list_wait
, atomic_read(&sctx
->workers_pending
) == 0);
3956 memcpy(progress
, &sctx
->stat
, sizeof(*progress
));
3958 if (!is_dev_replace
)
3959 btrfs_info(fs_info
, "scrub: %s on devid %llu with status: %d",
3960 ret
? "not finished" : "finished", devid
, ret
);
3962 mutex_lock(&fs_info
->scrub_lock
);
3963 dev
->scrub_ctx
= NULL
;
3964 if (refcount_dec_and_test(&fs_info
->scrub_workers_refcnt
)) {
3965 scrub_workers
= fs_info
->scrub_workers
;
3966 scrub_wr_comp
= fs_info
->scrub_wr_completion_workers
;
3967 scrub_parity
= fs_info
->scrub_parity_workers
;
3969 fs_info
->scrub_workers
= NULL
;
3970 fs_info
->scrub_wr_completion_workers
= NULL
;
3971 fs_info
->scrub_parity_workers
= NULL
;
3973 mutex_unlock(&fs_info
->scrub_lock
);
3975 btrfs_destroy_workqueue(scrub_workers
);
3976 btrfs_destroy_workqueue(scrub_wr_comp
);
3977 btrfs_destroy_workqueue(scrub_parity
);
3978 scrub_put_ctx(sctx
);
3983 scrub_free_ctx(sctx
);
3988 void btrfs_scrub_pause(struct btrfs_fs_info
*fs_info
)
3990 mutex_lock(&fs_info
->scrub_lock
);
3991 atomic_inc(&fs_info
->scrub_pause_req
);
3992 while (atomic_read(&fs_info
->scrubs_paused
) !=
3993 atomic_read(&fs_info
->scrubs_running
)) {
3994 mutex_unlock(&fs_info
->scrub_lock
);
3995 wait_event(fs_info
->scrub_pause_wait
,
3996 atomic_read(&fs_info
->scrubs_paused
) ==
3997 atomic_read(&fs_info
->scrubs_running
));
3998 mutex_lock(&fs_info
->scrub_lock
);
4000 mutex_unlock(&fs_info
->scrub_lock
);
4003 void btrfs_scrub_continue(struct btrfs_fs_info
*fs_info
)
4005 atomic_dec(&fs_info
->scrub_pause_req
);
4006 wake_up(&fs_info
->scrub_pause_wait
);
4009 int btrfs_scrub_cancel(struct btrfs_fs_info
*fs_info
)
4011 mutex_lock(&fs_info
->scrub_lock
);
4012 if (!atomic_read(&fs_info
->scrubs_running
)) {
4013 mutex_unlock(&fs_info
->scrub_lock
);
4017 atomic_inc(&fs_info
->scrub_cancel_req
);
4018 while (atomic_read(&fs_info
->scrubs_running
)) {
4019 mutex_unlock(&fs_info
->scrub_lock
);
4020 wait_event(fs_info
->scrub_pause_wait
,
4021 atomic_read(&fs_info
->scrubs_running
) == 0);
4022 mutex_lock(&fs_info
->scrub_lock
);
4024 atomic_dec(&fs_info
->scrub_cancel_req
);
4025 mutex_unlock(&fs_info
->scrub_lock
);
4030 int btrfs_scrub_cancel_dev(struct btrfs_device
*dev
)
4032 struct btrfs_fs_info
*fs_info
= dev
->fs_info
;
4033 struct scrub_ctx
*sctx
;
4035 mutex_lock(&fs_info
->scrub_lock
);
4036 sctx
= dev
->scrub_ctx
;
4038 mutex_unlock(&fs_info
->scrub_lock
);
4041 atomic_inc(&sctx
->cancel_req
);
4042 while (dev
->scrub_ctx
) {
4043 mutex_unlock(&fs_info
->scrub_lock
);
4044 wait_event(fs_info
->scrub_pause_wait
,
4045 dev
->scrub_ctx
== NULL
);
4046 mutex_lock(&fs_info
->scrub_lock
);
4048 mutex_unlock(&fs_info
->scrub_lock
);
4053 int btrfs_scrub_progress(struct btrfs_fs_info
*fs_info
, u64 devid
,
4054 struct btrfs_scrub_progress
*progress
)
4056 struct btrfs_device
*dev
;
4057 struct scrub_ctx
*sctx
= NULL
;
4059 mutex_lock(&fs_info
->fs_devices
->device_list_mutex
);
4060 dev
= btrfs_find_device(fs_info
->fs_devices
, devid
, NULL
, NULL
, true);
4062 sctx
= dev
->scrub_ctx
;
4064 memcpy(progress
, &sctx
->stat
, sizeof(*progress
));
4065 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
4067 return dev
? (sctx
? 0 : -ENOTCONN
) : -ENODEV
;
4070 static void scrub_remap_extent(struct btrfs_fs_info
*fs_info
,
4071 u64 extent_logical
, u64 extent_len
,
4072 u64
*extent_physical
,
4073 struct btrfs_device
**extent_dev
,
4074 int *extent_mirror_num
)
4077 struct btrfs_bio
*bbio
= NULL
;
4080 mapped_length
= extent_len
;
4081 ret
= btrfs_map_block(fs_info
, BTRFS_MAP_READ
, extent_logical
,
4082 &mapped_length
, &bbio
, 0);
4083 if (ret
|| !bbio
|| mapped_length
< extent_len
||
4084 !bbio
->stripes
[0].dev
->bdev
) {
4085 btrfs_put_bbio(bbio
);
4089 *extent_physical
= bbio
->stripes
[0].physical
;
4090 *extent_mirror_num
= bbio
->mirror_num
;
4091 *extent_dev
= bbio
->stripes
[0].dev
;
4092 btrfs_put_bbio(bbio
);