2 * Copyright (C) 2011, 2012 STRATO. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
19 #include <linux/blkdev.h>
20 #include <linux/ratelimit.h>
24 #include "ordered-data.h"
25 #include "transaction.h"
27 #include "extent_io.h"
28 #include "dev-replace.h"
29 #include "check-integrity.h"
30 #include "rcu-string.h"
34 * This is only the first step towards a full-features scrub. It reads all
35 * extent and super block and verifies the checksums. In case a bad checksum
36 * is found or the extent cannot be read, good data will be written back if
39 * Future enhancements:
40 * - In case an unrepairable extent is encountered, track which files are
41 * affected and report them
42 * - track and record media errors, throw out bad devices
43 * - add a mode to also read unallocated space
50 * the following three values only influence the performance.
51 * The last one configures the number of parallel and outstanding I/O
52 * operations. The first two values configure an upper limit for the number
53 * of (dynamically allocated) pages that are added to a bio.
55 #define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */
56 #define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */
57 #define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */
60 * the following value times PAGE_SIZE needs to be large enough to match the
61 * largest node/leaf/sector size that shall be supported.
62 * Values larger than BTRFS_STRIPE_LEN are not supported.
64 #define SCRUB_MAX_PAGES_PER_BLOCK 16 /* 64k per node/leaf/sector */
66 struct scrub_recover
{
68 struct btrfs_bio
*bbio
;
73 struct scrub_block
*sblock
;
75 struct btrfs_device
*dev
;
76 struct list_head list
;
77 u64 flags
; /* extent flags */
81 u64 physical_for_dev_replace
;
84 unsigned int mirror_num
:8;
85 unsigned int have_csum
:1;
86 unsigned int io_error
:1;
88 u8 csum
[BTRFS_CSUM_SIZE
];
90 struct scrub_recover
*recover
;
95 struct scrub_ctx
*sctx
;
96 struct btrfs_device
*dev
;
101 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
102 struct scrub_page
*pagev
[SCRUB_PAGES_PER_WR_BIO
];
104 struct scrub_page
*pagev
[SCRUB_PAGES_PER_RD_BIO
];
108 struct btrfs_work work
;
112 struct scrub_page
*pagev
[SCRUB_MAX_PAGES_PER_BLOCK
];
114 atomic_t outstanding_pages
;
115 atomic_t refs
; /* free mem on transition to zero */
116 struct scrub_ctx
*sctx
;
117 struct scrub_parity
*sparity
;
119 unsigned int header_error
:1;
120 unsigned int checksum_error
:1;
121 unsigned int no_io_error_seen
:1;
122 unsigned int generation_error
:1; /* also sets header_error */
124 /* The following is for the data used to check parity */
125 /* It is for the data with checksum */
126 unsigned int data_corrected
:1;
128 struct btrfs_work work
;
131 /* Used for the chunks with parity stripe such RAID5/6 */
132 struct scrub_parity
{
133 struct scrub_ctx
*sctx
;
135 struct btrfs_device
*scrub_dev
;
147 struct list_head spages
;
149 /* Work of parity check and repair */
150 struct btrfs_work work
;
152 /* Mark the parity blocks which have data */
153 unsigned long *dbitmap
;
156 * Mark the parity blocks which have data, but errors happen when
157 * read data or check data
159 unsigned long *ebitmap
;
161 unsigned long bitmap
[0];
164 struct scrub_wr_ctx
{
165 struct scrub_bio
*wr_curr_bio
;
166 struct btrfs_device
*tgtdev
;
167 int pages_per_wr_bio
; /* <= SCRUB_PAGES_PER_WR_BIO */
168 atomic_t flush_all_writes
;
169 struct mutex wr_lock
;
173 struct scrub_bio
*bios
[SCRUB_BIOS_PER_SCTX
];
174 struct btrfs_root
*dev_root
;
177 atomic_t bios_in_flight
;
178 atomic_t workers_pending
;
179 spinlock_t list_lock
;
180 wait_queue_head_t list_wait
;
182 struct list_head csum_list
;
185 int pages_per_rd_bio
;
190 struct scrub_wr_ctx wr_ctx
;
195 struct btrfs_scrub_progress stat
;
196 spinlock_t stat_lock
;
199 * Use a ref counter to avoid use-after-free issues. Scrub workers
200 * decrement bios_in_flight and workers_pending and then do a wakeup
201 * on the list_wait wait queue. We must ensure the main scrub task
202 * doesn't free the scrub context before or while the workers are
203 * doing the wakeup() call.
208 struct scrub_fixup_nodatasum
{
209 struct scrub_ctx
*sctx
;
210 struct btrfs_device
*dev
;
212 struct btrfs_root
*root
;
213 struct btrfs_work work
;
217 struct scrub_nocow_inode
{
221 struct list_head list
;
224 struct scrub_copy_nocow_ctx
{
225 struct scrub_ctx
*sctx
;
229 u64 physical_for_dev_replace
;
230 struct list_head inodes
;
231 struct btrfs_work work
;
234 struct scrub_warning
{
235 struct btrfs_path
*path
;
236 u64 extent_item_size
;
240 struct btrfs_device
*dev
;
243 static void scrub_pending_bio_inc(struct scrub_ctx
*sctx
);
244 static void scrub_pending_bio_dec(struct scrub_ctx
*sctx
);
245 static void scrub_pending_trans_workers_inc(struct scrub_ctx
*sctx
);
246 static void scrub_pending_trans_workers_dec(struct scrub_ctx
*sctx
);
247 static int scrub_handle_errored_block(struct scrub_block
*sblock_to_check
);
248 static int scrub_setup_recheck_block(struct scrub_block
*original_sblock
,
249 struct scrub_block
*sblocks_for_recheck
);
250 static void scrub_recheck_block(struct btrfs_fs_info
*fs_info
,
251 struct scrub_block
*sblock
,
252 int retry_failed_mirror
);
253 static void scrub_recheck_block_checksum(struct scrub_block
*sblock
);
254 static int scrub_repair_block_from_good_copy(struct scrub_block
*sblock_bad
,
255 struct scrub_block
*sblock_good
);
256 static int scrub_repair_page_from_good_copy(struct scrub_block
*sblock_bad
,
257 struct scrub_block
*sblock_good
,
258 int page_num
, int force_write
);
259 static void scrub_write_block_to_dev_replace(struct scrub_block
*sblock
);
260 static int scrub_write_page_to_dev_replace(struct scrub_block
*sblock
,
262 static int scrub_checksum_data(struct scrub_block
*sblock
);
263 static int scrub_checksum_tree_block(struct scrub_block
*sblock
);
264 static int scrub_checksum_super(struct scrub_block
*sblock
);
265 static void scrub_block_get(struct scrub_block
*sblock
);
266 static void scrub_block_put(struct scrub_block
*sblock
);
267 static void scrub_page_get(struct scrub_page
*spage
);
268 static void scrub_page_put(struct scrub_page
*spage
);
269 static void scrub_parity_get(struct scrub_parity
*sparity
);
270 static void scrub_parity_put(struct scrub_parity
*sparity
);
271 static int scrub_add_page_to_rd_bio(struct scrub_ctx
*sctx
,
272 struct scrub_page
*spage
);
273 static int scrub_pages(struct scrub_ctx
*sctx
, u64 logical
, u64 len
,
274 u64 physical
, struct btrfs_device
*dev
, u64 flags
,
275 u64 gen
, int mirror_num
, u8
*csum
, int force
,
276 u64 physical_for_dev_replace
);
277 static void scrub_bio_end_io(struct bio
*bio
);
278 static void scrub_bio_end_io_worker(struct btrfs_work
*work
);
279 static void scrub_block_complete(struct scrub_block
*sblock
);
280 static void scrub_remap_extent(struct btrfs_fs_info
*fs_info
,
281 u64 extent_logical
, u64 extent_len
,
282 u64
*extent_physical
,
283 struct btrfs_device
**extent_dev
,
284 int *extent_mirror_num
);
285 static int scrub_setup_wr_ctx(struct scrub_ctx
*sctx
,
286 struct scrub_wr_ctx
*wr_ctx
,
287 struct btrfs_fs_info
*fs_info
,
288 struct btrfs_device
*dev
,
290 static void scrub_free_wr_ctx(struct scrub_wr_ctx
*wr_ctx
);
291 static int scrub_add_page_to_wr_bio(struct scrub_ctx
*sctx
,
292 struct scrub_page
*spage
);
293 static void scrub_wr_submit(struct scrub_ctx
*sctx
);
294 static void scrub_wr_bio_end_io(struct bio
*bio
);
295 static void scrub_wr_bio_end_io_worker(struct btrfs_work
*work
);
296 static int write_page_nocow(struct scrub_ctx
*sctx
,
297 u64 physical_for_dev_replace
, struct page
*page
);
298 static int copy_nocow_pages_for_inode(u64 inum
, u64 offset
, u64 root
,
299 struct scrub_copy_nocow_ctx
*ctx
);
300 static int copy_nocow_pages(struct scrub_ctx
*sctx
, u64 logical
, u64 len
,
301 int mirror_num
, u64 physical_for_dev_replace
);
302 static void copy_nocow_pages_worker(struct btrfs_work
*work
);
303 static void __scrub_blocked_if_needed(struct btrfs_fs_info
*fs_info
);
304 static void scrub_blocked_if_needed(struct btrfs_fs_info
*fs_info
);
305 static void scrub_put_ctx(struct scrub_ctx
*sctx
);
308 static void scrub_pending_bio_inc(struct scrub_ctx
*sctx
)
310 atomic_inc(&sctx
->refs
);
311 atomic_inc(&sctx
->bios_in_flight
);
314 static void scrub_pending_bio_dec(struct scrub_ctx
*sctx
)
316 atomic_dec(&sctx
->bios_in_flight
);
317 wake_up(&sctx
->list_wait
);
321 static void __scrub_blocked_if_needed(struct btrfs_fs_info
*fs_info
)
323 while (atomic_read(&fs_info
->scrub_pause_req
)) {
324 mutex_unlock(&fs_info
->scrub_lock
);
325 wait_event(fs_info
->scrub_pause_wait
,
326 atomic_read(&fs_info
->scrub_pause_req
) == 0);
327 mutex_lock(&fs_info
->scrub_lock
);
331 static void scrub_pause_on(struct btrfs_fs_info
*fs_info
)
333 atomic_inc(&fs_info
->scrubs_paused
);
334 wake_up(&fs_info
->scrub_pause_wait
);
337 static void scrub_pause_off(struct btrfs_fs_info
*fs_info
)
339 mutex_lock(&fs_info
->scrub_lock
);
340 __scrub_blocked_if_needed(fs_info
);
341 atomic_dec(&fs_info
->scrubs_paused
);
342 mutex_unlock(&fs_info
->scrub_lock
);
344 wake_up(&fs_info
->scrub_pause_wait
);
347 static void scrub_blocked_if_needed(struct btrfs_fs_info
*fs_info
)
349 scrub_pause_on(fs_info
);
350 scrub_pause_off(fs_info
);
354 * used for workers that require transaction commits (i.e., for the
357 static void scrub_pending_trans_workers_inc(struct scrub_ctx
*sctx
)
359 struct btrfs_fs_info
*fs_info
= sctx
->dev_root
->fs_info
;
361 atomic_inc(&sctx
->refs
);
363 * increment scrubs_running to prevent cancel requests from
364 * completing as long as a worker is running. we must also
365 * increment scrubs_paused to prevent deadlocking on pause
366 * requests used for transactions commits (as the worker uses a
367 * transaction context). it is safe to regard the worker
368 * as paused for all matters practical. effectively, we only
369 * avoid cancellation requests from completing.
371 mutex_lock(&fs_info
->scrub_lock
);
372 atomic_inc(&fs_info
->scrubs_running
);
373 atomic_inc(&fs_info
->scrubs_paused
);
374 mutex_unlock(&fs_info
->scrub_lock
);
377 * check if @scrubs_running=@scrubs_paused condition
378 * inside wait_event() is not an atomic operation.
379 * which means we may inc/dec @scrub_running/paused
380 * at any time. Let's wake up @scrub_pause_wait as
381 * much as we can to let commit transaction blocked less.
383 wake_up(&fs_info
->scrub_pause_wait
);
385 atomic_inc(&sctx
->workers_pending
);
388 /* used for workers that require transaction commits */
389 static void scrub_pending_trans_workers_dec(struct scrub_ctx
*sctx
)
391 struct btrfs_fs_info
*fs_info
= sctx
->dev_root
->fs_info
;
394 * see scrub_pending_trans_workers_inc() why we're pretending
395 * to be paused in the scrub counters
397 mutex_lock(&fs_info
->scrub_lock
);
398 atomic_dec(&fs_info
->scrubs_running
);
399 atomic_dec(&fs_info
->scrubs_paused
);
400 mutex_unlock(&fs_info
->scrub_lock
);
401 atomic_dec(&sctx
->workers_pending
);
402 wake_up(&fs_info
->scrub_pause_wait
);
403 wake_up(&sctx
->list_wait
);
407 static void scrub_free_csums(struct scrub_ctx
*sctx
)
409 while (!list_empty(&sctx
->csum_list
)) {
410 struct btrfs_ordered_sum
*sum
;
411 sum
= list_first_entry(&sctx
->csum_list
,
412 struct btrfs_ordered_sum
, list
);
413 list_del(&sum
->list
);
418 static noinline_for_stack
void scrub_free_ctx(struct scrub_ctx
*sctx
)
425 scrub_free_wr_ctx(&sctx
->wr_ctx
);
427 /* this can happen when scrub is cancelled */
428 if (sctx
->curr
!= -1) {
429 struct scrub_bio
*sbio
= sctx
->bios
[sctx
->curr
];
431 for (i
= 0; i
< sbio
->page_count
; i
++) {
432 WARN_ON(!sbio
->pagev
[i
]->page
);
433 scrub_block_put(sbio
->pagev
[i
]->sblock
);
438 for (i
= 0; i
< SCRUB_BIOS_PER_SCTX
; ++i
) {
439 struct scrub_bio
*sbio
= sctx
->bios
[i
];
446 scrub_free_csums(sctx
);
450 static void scrub_put_ctx(struct scrub_ctx
*sctx
)
452 if (atomic_dec_and_test(&sctx
->refs
))
453 scrub_free_ctx(sctx
);
456 static noinline_for_stack
457 struct scrub_ctx
*scrub_setup_ctx(struct btrfs_device
*dev
, int is_dev_replace
)
459 struct scrub_ctx
*sctx
;
461 struct btrfs_fs_info
*fs_info
= dev
->dev_root
->fs_info
;
464 sctx
= kzalloc(sizeof(*sctx
), GFP_NOFS
);
467 atomic_set(&sctx
->refs
, 1);
468 sctx
->is_dev_replace
= is_dev_replace
;
469 sctx
->pages_per_rd_bio
= SCRUB_PAGES_PER_RD_BIO
;
471 sctx
->dev_root
= dev
->dev_root
;
472 for (i
= 0; i
< SCRUB_BIOS_PER_SCTX
; ++i
) {
473 struct scrub_bio
*sbio
;
475 sbio
= kzalloc(sizeof(*sbio
), GFP_NOFS
);
478 sctx
->bios
[i
] = sbio
;
482 sbio
->page_count
= 0;
483 btrfs_init_work(&sbio
->work
, btrfs_scrub_helper
,
484 scrub_bio_end_io_worker
, NULL
, NULL
);
486 if (i
!= SCRUB_BIOS_PER_SCTX
- 1)
487 sctx
->bios
[i
]->next_free
= i
+ 1;
489 sctx
->bios
[i
]->next_free
= -1;
491 sctx
->first_free
= 0;
492 sctx
->nodesize
= dev
->dev_root
->nodesize
;
493 sctx
->sectorsize
= dev
->dev_root
->sectorsize
;
494 atomic_set(&sctx
->bios_in_flight
, 0);
495 atomic_set(&sctx
->workers_pending
, 0);
496 atomic_set(&sctx
->cancel_req
, 0);
497 sctx
->csum_size
= btrfs_super_csum_size(fs_info
->super_copy
);
498 INIT_LIST_HEAD(&sctx
->csum_list
);
500 spin_lock_init(&sctx
->list_lock
);
501 spin_lock_init(&sctx
->stat_lock
);
502 init_waitqueue_head(&sctx
->list_wait
);
504 ret
= scrub_setup_wr_ctx(sctx
, &sctx
->wr_ctx
, fs_info
,
505 fs_info
->dev_replace
.tgtdev
, is_dev_replace
);
507 scrub_free_ctx(sctx
);
513 scrub_free_ctx(sctx
);
514 return ERR_PTR(-ENOMEM
);
517 static int scrub_print_warning_inode(u64 inum
, u64 offset
, u64 root
,
524 struct extent_buffer
*eb
;
525 struct btrfs_inode_item
*inode_item
;
526 struct scrub_warning
*swarn
= warn_ctx
;
527 struct btrfs_fs_info
*fs_info
= swarn
->dev
->dev_root
->fs_info
;
528 struct inode_fs_paths
*ipath
= NULL
;
529 struct btrfs_root
*local_root
;
530 struct btrfs_key root_key
;
531 struct btrfs_key key
;
533 root_key
.objectid
= root
;
534 root_key
.type
= BTRFS_ROOT_ITEM_KEY
;
535 root_key
.offset
= (u64
)-1;
536 local_root
= btrfs_read_fs_root_no_name(fs_info
, &root_key
);
537 if (IS_ERR(local_root
)) {
538 ret
= PTR_ERR(local_root
);
543 * this makes the path point to (inum INODE_ITEM ioff)
546 key
.type
= BTRFS_INODE_ITEM_KEY
;
549 ret
= btrfs_search_slot(NULL
, local_root
, &key
, swarn
->path
, 0, 0);
551 btrfs_release_path(swarn
->path
);
555 eb
= swarn
->path
->nodes
[0];
556 inode_item
= btrfs_item_ptr(eb
, swarn
->path
->slots
[0],
557 struct btrfs_inode_item
);
558 isize
= btrfs_inode_size(eb
, inode_item
);
559 nlink
= btrfs_inode_nlink(eb
, inode_item
);
560 btrfs_release_path(swarn
->path
);
562 ipath
= init_ipath(4096, local_root
, swarn
->path
);
564 ret
= PTR_ERR(ipath
);
568 ret
= paths_from_inode(inum
, ipath
);
574 * we deliberately ignore the bit ipath might have been too small to
575 * hold all of the paths here
577 for (i
= 0; i
< ipath
->fspath
->elem_cnt
; ++i
)
578 btrfs_warn_in_rcu(fs_info
, "%s at logical %llu on dev "
579 "%s, sector %llu, root %llu, inode %llu, offset %llu, "
580 "length %llu, links %u (path: %s)", swarn
->errstr
,
581 swarn
->logical
, rcu_str_deref(swarn
->dev
->name
),
582 (unsigned long long)swarn
->sector
, root
, inum
, offset
,
583 min(isize
- offset
, (u64
)PAGE_SIZE
), nlink
,
584 (char *)(unsigned long)ipath
->fspath
->val
[i
]);
590 btrfs_warn_in_rcu(fs_info
, "%s at logical %llu on dev "
591 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
592 "resolving failed with ret=%d", swarn
->errstr
,
593 swarn
->logical
, rcu_str_deref(swarn
->dev
->name
),
594 (unsigned long long)swarn
->sector
, root
, inum
, offset
, ret
);
600 static void scrub_print_warning(const char *errstr
, struct scrub_block
*sblock
)
602 struct btrfs_device
*dev
;
603 struct btrfs_fs_info
*fs_info
;
604 struct btrfs_path
*path
;
605 struct btrfs_key found_key
;
606 struct extent_buffer
*eb
;
607 struct btrfs_extent_item
*ei
;
608 struct scrub_warning swarn
;
609 unsigned long ptr
= 0;
617 WARN_ON(sblock
->page_count
< 1);
618 dev
= sblock
->pagev
[0]->dev
;
619 fs_info
= sblock
->sctx
->dev_root
->fs_info
;
621 path
= btrfs_alloc_path();
625 swarn
.sector
= (sblock
->pagev
[0]->physical
) >> 9;
626 swarn
.logical
= sblock
->pagev
[0]->logical
;
627 swarn
.errstr
= errstr
;
630 ret
= extent_from_logical(fs_info
, swarn
.logical
, path
, &found_key
,
635 extent_item_pos
= swarn
.logical
- found_key
.objectid
;
636 swarn
.extent_item_size
= found_key
.offset
;
639 ei
= btrfs_item_ptr(eb
, path
->slots
[0], struct btrfs_extent_item
);
640 item_size
= btrfs_item_size_nr(eb
, path
->slots
[0]);
642 if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
644 ret
= tree_backref_for_extent(&ptr
, eb
, &found_key
, ei
,
645 item_size
, &ref_root
,
647 btrfs_warn_in_rcu(fs_info
,
648 "%s at logical %llu on dev %s, "
649 "sector %llu: metadata %s (level %d) in tree "
650 "%llu", errstr
, swarn
.logical
,
651 rcu_str_deref(dev
->name
),
652 (unsigned long long)swarn
.sector
,
653 ref_level
? "node" : "leaf",
654 ret
< 0 ? -1 : ref_level
,
655 ret
< 0 ? -1 : ref_root
);
657 btrfs_release_path(path
);
659 btrfs_release_path(path
);
662 iterate_extent_inodes(fs_info
, found_key
.objectid
,
664 scrub_print_warning_inode
, &swarn
);
668 btrfs_free_path(path
);
671 static int scrub_fixup_readpage(u64 inum
, u64 offset
, u64 root
, void *fixup_ctx
)
673 struct page
*page
= NULL
;
675 struct scrub_fixup_nodatasum
*fixup
= fixup_ctx
;
678 struct btrfs_key key
;
679 struct inode
*inode
= NULL
;
680 struct btrfs_fs_info
*fs_info
;
681 u64 end
= offset
+ PAGE_SIZE
- 1;
682 struct btrfs_root
*local_root
;
686 key
.type
= BTRFS_ROOT_ITEM_KEY
;
687 key
.offset
= (u64
)-1;
689 fs_info
= fixup
->root
->fs_info
;
690 srcu_index
= srcu_read_lock(&fs_info
->subvol_srcu
);
692 local_root
= btrfs_read_fs_root_no_name(fs_info
, &key
);
693 if (IS_ERR(local_root
)) {
694 srcu_read_unlock(&fs_info
->subvol_srcu
, srcu_index
);
695 return PTR_ERR(local_root
);
698 key
.type
= BTRFS_INODE_ITEM_KEY
;
701 inode
= btrfs_iget(fs_info
->sb
, &key
, local_root
, NULL
);
702 srcu_read_unlock(&fs_info
->subvol_srcu
, srcu_index
);
704 return PTR_ERR(inode
);
706 index
= offset
>> PAGE_CACHE_SHIFT
;
708 page
= find_or_create_page(inode
->i_mapping
, index
, GFP_NOFS
);
714 if (PageUptodate(page
)) {
715 if (PageDirty(page
)) {
717 * we need to write the data to the defect sector. the
718 * data that was in that sector is not in memory,
719 * because the page was modified. we must not write the
720 * modified page to that sector.
722 * TODO: what could be done here: wait for the delalloc
723 * runner to write out that page (might involve
724 * COW) and see whether the sector is still
725 * referenced afterwards.
727 * For the meantime, we'll treat this error
728 * incorrectable, although there is a chance that a
729 * later scrub will find the bad sector again and that
730 * there's no dirty page in memory, then.
735 ret
= repair_io_failure(inode
, offset
, PAGE_SIZE
,
736 fixup
->logical
, page
,
737 offset
- page_offset(page
),
743 * we need to get good data first. the general readpage path
744 * will call repair_io_failure for us, we just have to make
745 * sure we read the bad mirror.
747 ret
= set_extent_bits(&BTRFS_I(inode
)->io_tree
, offset
, end
,
748 EXTENT_DAMAGED
, GFP_NOFS
);
750 /* set_extent_bits should give proper error */
757 ret
= extent_read_full_page(&BTRFS_I(inode
)->io_tree
, page
,
760 wait_on_page_locked(page
);
762 corrected
= !test_range_bit(&BTRFS_I(inode
)->io_tree
, offset
,
763 end
, EXTENT_DAMAGED
, 0, NULL
);
765 clear_extent_bits(&BTRFS_I(inode
)->io_tree
, offset
, end
,
766 EXTENT_DAMAGED
, GFP_NOFS
);
778 if (ret
== 0 && corrected
) {
780 * we only need to call readpage for one of the inodes belonging
781 * to this extent. so make iterate_extent_inodes stop
789 static void scrub_fixup_nodatasum(struct btrfs_work
*work
)
792 struct scrub_fixup_nodatasum
*fixup
;
793 struct scrub_ctx
*sctx
;
794 struct btrfs_trans_handle
*trans
= NULL
;
795 struct btrfs_path
*path
;
796 int uncorrectable
= 0;
798 fixup
= container_of(work
, struct scrub_fixup_nodatasum
, work
);
801 path
= btrfs_alloc_path();
803 spin_lock(&sctx
->stat_lock
);
804 ++sctx
->stat
.malloc_errors
;
805 spin_unlock(&sctx
->stat_lock
);
810 trans
= btrfs_join_transaction(fixup
->root
);
817 * the idea is to trigger a regular read through the standard path. we
818 * read a page from the (failed) logical address by specifying the
819 * corresponding copynum of the failed sector. thus, that readpage is
821 * that is the point where on-the-fly error correction will kick in
822 * (once it's finished) and rewrite the failed sector if a good copy
825 ret
= iterate_inodes_from_logical(fixup
->logical
, fixup
->root
->fs_info
,
826 path
, scrub_fixup_readpage
,
834 spin_lock(&sctx
->stat_lock
);
835 ++sctx
->stat
.corrected_errors
;
836 spin_unlock(&sctx
->stat_lock
);
839 if (trans
&& !IS_ERR(trans
))
840 btrfs_end_transaction(trans
, fixup
->root
);
842 spin_lock(&sctx
->stat_lock
);
843 ++sctx
->stat
.uncorrectable_errors
;
844 spin_unlock(&sctx
->stat_lock
);
845 btrfs_dev_replace_stats_inc(
846 &sctx
->dev_root
->fs_info
->dev_replace
.
847 num_uncorrectable_read_errors
);
848 btrfs_err_rl_in_rcu(sctx
->dev_root
->fs_info
,
849 "unable to fixup (nodatasum) error at logical %llu on dev %s",
850 fixup
->logical
, rcu_str_deref(fixup
->dev
->name
));
853 btrfs_free_path(path
);
856 scrub_pending_trans_workers_dec(sctx
);
859 static inline void scrub_get_recover(struct scrub_recover
*recover
)
861 atomic_inc(&recover
->refs
);
864 static inline void scrub_put_recover(struct scrub_recover
*recover
)
866 if (atomic_dec_and_test(&recover
->refs
)) {
867 btrfs_put_bbio(recover
->bbio
);
873 * scrub_handle_errored_block gets called when either verification of the
874 * pages failed or the bio failed to read, e.g. with EIO. In the latter
875 * case, this function handles all pages in the bio, even though only one
877 * The goal of this function is to repair the errored block by using the
878 * contents of one of the mirrors.
880 static int scrub_handle_errored_block(struct scrub_block
*sblock_to_check
)
882 struct scrub_ctx
*sctx
= sblock_to_check
->sctx
;
883 struct btrfs_device
*dev
;
884 struct btrfs_fs_info
*fs_info
;
887 unsigned int failed_mirror_index
;
888 unsigned int is_metadata
;
889 unsigned int have_csum
;
890 struct scrub_block
*sblocks_for_recheck
; /* holds one for each mirror */
891 struct scrub_block
*sblock_bad
;
896 static DEFINE_RATELIMIT_STATE(_rs
, DEFAULT_RATELIMIT_INTERVAL
,
897 DEFAULT_RATELIMIT_BURST
);
899 BUG_ON(sblock_to_check
->page_count
< 1);
900 fs_info
= sctx
->dev_root
->fs_info
;
901 if (sblock_to_check
->pagev
[0]->flags
& BTRFS_EXTENT_FLAG_SUPER
) {
903 * if we find an error in a super block, we just report it.
904 * They will get written with the next transaction commit
907 spin_lock(&sctx
->stat_lock
);
908 ++sctx
->stat
.super_errors
;
909 spin_unlock(&sctx
->stat_lock
);
912 length
= sblock_to_check
->page_count
* PAGE_SIZE
;
913 logical
= sblock_to_check
->pagev
[0]->logical
;
914 BUG_ON(sblock_to_check
->pagev
[0]->mirror_num
< 1);
915 failed_mirror_index
= sblock_to_check
->pagev
[0]->mirror_num
- 1;
916 is_metadata
= !(sblock_to_check
->pagev
[0]->flags
&
917 BTRFS_EXTENT_FLAG_DATA
);
918 have_csum
= sblock_to_check
->pagev
[0]->have_csum
;
919 dev
= sblock_to_check
->pagev
[0]->dev
;
921 if (sctx
->is_dev_replace
&& !is_metadata
&& !have_csum
) {
922 sblocks_for_recheck
= NULL
;
927 * read all mirrors one after the other. This includes to
928 * re-read the extent or metadata block that failed (that was
929 * the cause that this fixup code is called) another time,
930 * page by page this time in order to know which pages
931 * caused I/O errors and which ones are good (for all mirrors).
932 * It is the goal to handle the situation when more than one
933 * mirror contains I/O errors, but the errors do not
934 * overlap, i.e. the data can be repaired by selecting the
935 * pages from those mirrors without I/O error on the
936 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
937 * would be that mirror #1 has an I/O error on the first page,
938 * the second page is good, and mirror #2 has an I/O error on
939 * the second page, but the first page is good.
940 * Then the first page of the first mirror can be repaired by
941 * taking the first page of the second mirror, and the
942 * second page of the second mirror can be repaired by
943 * copying the contents of the 2nd page of the 1st mirror.
944 * One more note: if the pages of one mirror contain I/O
945 * errors, the checksum cannot be verified. In order to get
946 * the best data for repairing, the first attempt is to find
947 * a mirror without I/O errors and with a validated checksum.
948 * Only if this is not possible, the pages are picked from
949 * mirrors with I/O errors without considering the checksum.
950 * If the latter is the case, at the end, the checksum of the
951 * repaired area is verified in order to correctly maintain
955 sblocks_for_recheck
= kcalloc(BTRFS_MAX_MIRRORS
,
956 sizeof(*sblocks_for_recheck
), GFP_NOFS
);
957 if (!sblocks_for_recheck
) {
958 spin_lock(&sctx
->stat_lock
);
959 sctx
->stat
.malloc_errors
++;
960 sctx
->stat
.read_errors
++;
961 sctx
->stat
.uncorrectable_errors
++;
962 spin_unlock(&sctx
->stat_lock
);
963 btrfs_dev_stat_inc_and_print(dev
, BTRFS_DEV_STAT_READ_ERRS
);
967 /* setup the context, map the logical blocks and alloc the pages */
968 ret
= scrub_setup_recheck_block(sblock_to_check
, sblocks_for_recheck
);
970 spin_lock(&sctx
->stat_lock
);
971 sctx
->stat
.read_errors
++;
972 sctx
->stat
.uncorrectable_errors
++;
973 spin_unlock(&sctx
->stat_lock
);
974 btrfs_dev_stat_inc_and_print(dev
, BTRFS_DEV_STAT_READ_ERRS
);
977 BUG_ON(failed_mirror_index
>= BTRFS_MAX_MIRRORS
);
978 sblock_bad
= sblocks_for_recheck
+ failed_mirror_index
;
980 /* build and submit the bios for the failed mirror, check checksums */
981 scrub_recheck_block(fs_info
, sblock_bad
, 1);
983 if (!sblock_bad
->header_error
&& !sblock_bad
->checksum_error
&&
984 sblock_bad
->no_io_error_seen
) {
986 * the error disappeared after reading page by page, or
987 * the area was part of a huge bio and other parts of the
988 * bio caused I/O errors, or the block layer merged several
989 * read requests into one and the error is caused by a
990 * different bio (usually one of the two latter cases is
993 spin_lock(&sctx
->stat_lock
);
994 sctx
->stat
.unverified_errors
++;
995 sblock_to_check
->data_corrected
= 1;
996 spin_unlock(&sctx
->stat_lock
);
998 if (sctx
->is_dev_replace
)
999 scrub_write_block_to_dev_replace(sblock_bad
);
1003 if (!sblock_bad
->no_io_error_seen
) {
1004 spin_lock(&sctx
->stat_lock
);
1005 sctx
->stat
.read_errors
++;
1006 spin_unlock(&sctx
->stat_lock
);
1007 if (__ratelimit(&_rs
))
1008 scrub_print_warning("i/o error", sblock_to_check
);
1009 btrfs_dev_stat_inc_and_print(dev
, BTRFS_DEV_STAT_READ_ERRS
);
1010 } else if (sblock_bad
->checksum_error
) {
1011 spin_lock(&sctx
->stat_lock
);
1012 sctx
->stat
.csum_errors
++;
1013 spin_unlock(&sctx
->stat_lock
);
1014 if (__ratelimit(&_rs
))
1015 scrub_print_warning("checksum error", sblock_to_check
);
1016 btrfs_dev_stat_inc_and_print(dev
,
1017 BTRFS_DEV_STAT_CORRUPTION_ERRS
);
1018 } else if (sblock_bad
->header_error
) {
1019 spin_lock(&sctx
->stat_lock
);
1020 sctx
->stat
.verify_errors
++;
1021 spin_unlock(&sctx
->stat_lock
);
1022 if (__ratelimit(&_rs
))
1023 scrub_print_warning("checksum/header error",
1025 if (sblock_bad
->generation_error
)
1026 btrfs_dev_stat_inc_and_print(dev
,
1027 BTRFS_DEV_STAT_GENERATION_ERRS
);
1029 btrfs_dev_stat_inc_and_print(dev
,
1030 BTRFS_DEV_STAT_CORRUPTION_ERRS
);
1033 if (sctx
->readonly
) {
1034 ASSERT(!sctx
->is_dev_replace
);
1038 if (!is_metadata
&& !have_csum
) {
1039 struct scrub_fixup_nodatasum
*fixup_nodatasum
;
1041 WARN_ON(sctx
->is_dev_replace
);
1046 * !is_metadata and !have_csum, this means that the data
1047 * might not be COW'ed, that it might be modified
1048 * concurrently. The general strategy to work on the
1049 * commit root does not help in the case when COW is not
1052 fixup_nodatasum
= kzalloc(sizeof(*fixup_nodatasum
), GFP_NOFS
);
1053 if (!fixup_nodatasum
)
1054 goto did_not_correct_error
;
1055 fixup_nodatasum
->sctx
= sctx
;
1056 fixup_nodatasum
->dev
= dev
;
1057 fixup_nodatasum
->logical
= logical
;
1058 fixup_nodatasum
->root
= fs_info
->extent_root
;
1059 fixup_nodatasum
->mirror_num
= failed_mirror_index
+ 1;
1060 scrub_pending_trans_workers_inc(sctx
);
1061 btrfs_init_work(&fixup_nodatasum
->work
, btrfs_scrub_helper
,
1062 scrub_fixup_nodatasum
, NULL
, NULL
);
1063 btrfs_queue_work(fs_info
->scrub_workers
,
1064 &fixup_nodatasum
->work
);
1069 * now build and submit the bios for the other mirrors, check
1071 * First try to pick the mirror which is completely without I/O
1072 * errors and also does not have a checksum error.
1073 * If one is found, and if a checksum is present, the full block
1074 * that is known to contain an error is rewritten. Afterwards
1075 * the block is known to be corrected.
1076 * If a mirror is found which is completely correct, and no
1077 * checksum is present, only those pages are rewritten that had
1078 * an I/O error in the block to be repaired, since it cannot be
1079 * determined, which copy of the other pages is better (and it
1080 * could happen otherwise that a correct page would be
1081 * overwritten by a bad one).
1083 for (mirror_index
= 0;
1084 mirror_index
< BTRFS_MAX_MIRRORS
&&
1085 sblocks_for_recheck
[mirror_index
].page_count
> 0;
1087 struct scrub_block
*sblock_other
;
1089 if (mirror_index
== failed_mirror_index
)
1091 sblock_other
= sblocks_for_recheck
+ mirror_index
;
1093 /* build and submit the bios, check checksums */
1094 scrub_recheck_block(fs_info
, sblock_other
, 0);
1096 if (!sblock_other
->header_error
&&
1097 !sblock_other
->checksum_error
&&
1098 sblock_other
->no_io_error_seen
) {
1099 if (sctx
->is_dev_replace
) {
1100 scrub_write_block_to_dev_replace(sblock_other
);
1101 goto corrected_error
;
1103 ret
= scrub_repair_block_from_good_copy(
1104 sblock_bad
, sblock_other
);
1106 goto corrected_error
;
1111 if (sblock_bad
->no_io_error_seen
&& !sctx
->is_dev_replace
)
1112 goto did_not_correct_error
;
1115 * In case of I/O errors in the area that is supposed to be
1116 * repaired, continue by picking good copies of those pages.
1117 * Select the good pages from mirrors to rewrite bad pages from
1118 * the area to fix. Afterwards verify the checksum of the block
1119 * that is supposed to be repaired. This verification step is
1120 * only done for the purpose of statistic counting and for the
1121 * final scrub report, whether errors remain.
1122 * A perfect algorithm could make use of the checksum and try
1123 * all possible combinations of pages from the different mirrors
1124 * until the checksum verification succeeds. For example, when
1125 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1126 * of mirror #2 is readable but the final checksum test fails,
1127 * then the 2nd page of mirror #3 could be tried, whether now
1128 * the final checksum succeedes. But this would be a rare
1129 * exception and is therefore not implemented. At least it is
1130 * avoided that the good copy is overwritten.
1131 * A more useful improvement would be to pick the sectors
1132 * without I/O error based on sector sizes (512 bytes on legacy
1133 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1134 * mirror could be repaired by taking 512 byte of a different
1135 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1136 * area are unreadable.
1139 for (page_num
= 0; page_num
< sblock_bad
->page_count
;
1141 struct scrub_page
*page_bad
= sblock_bad
->pagev
[page_num
];
1142 struct scrub_block
*sblock_other
= NULL
;
1144 /* skip no-io-error page in scrub */
1145 if (!page_bad
->io_error
&& !sctx
->is_dev_replace
)
1148 /* try to find no-io-error page in mirrors */
1149 if (page_bad
->io_error
) {
1150 for (mirror_index
= 0;
1151 mirror_index
< BTRFS_MAX_MIRRORS
&&
1152 sblocks_for_recheck
[mirror_index
].page_count
> 0;
1154 if (!sblocks_for_recheck
[mirror_index
].
1155 pagev
[page_num
]->io_error
) {
1156 sblock_other
= sblocks_for_recheck
+
1165 if (sctx
->is_dev_replace
) {
1167 * did not find a mirror to fetch the page
1168 * from. scrub_write_page_to_dev_replace()
1169 * handles this case (page->io_error), by
1170 * filling the block with zeros before
1171 * submitting the write request
1174 sblock_other
= sblock_bad
;
1176 if (scrub_write_page_to_dev_replace(sblock_other
,
1178 btrfs_dev_replace_stats_inc(
1180 fs_info
->dev_replace
.
1184 } else if (sblock_other
) {
1185 ret
= scrub_repair_page_from_good_copy(sblock_bad
,
1189 page_bad
->io_error
= 0;
1195 if (success
&& !sctx
->is_dev_replace
) {
1196 if (is_metadata
|| have_csum
) {
1198 * need to verify the checksum now that all
1199 * sectors on disk are repaired (the write
1200 * request for data to be repaired is on its way).
1201 * Just be lazy and use scrub_recheck_block()
1202 * which re-reads the data before the checksum
1203 * is verified, but most likely the data comes out
1204 * of the page cache.
1206 scrub_recheck_block(fs_info
, sblock_bad
, 1);
1207 if (!sblock_bad
->header_error
&&
1208 !sblock_bad
->checksum_error
&&
1209 sblock_bad
->no_io_error_seen
)
1210 goto corrected_error
;
1212 goto did_not_correct_error
;
1215 spin_lock(&sctx
->stat_lock
);
1216 sctx
->stat
.corrected_errors
++;
1217 sblock_to_check
->data_corrected
= 1;
1218 spin_unlock(&sctx
->stat_lock
);
1219 btrfs_err_rl_in_rcu(fs_info
,
1220 "fixed up error at logical %llu on dev %s",
1221 logical
, rcu_str_deref(dev
->name
));
1224 did_not_correct_error
:
1225 spin_lock(&sctx
->stat_lock
);
1226 sctx
->stat
.uncorrectable_errors
++;
1227 spin_unlock(&sctx
->stat_lock
);
1228 btrfs_err_rl_in_rcu(fs_info
,
1229 "unable to fixup (regular) error at logical %llu on dev %s",
1230 logical
, rcu_str_deref(dev
->name
));
1234 if (sblocks_for_recheck
) {
1235 for (mirror_index
= 0; mirror_index
< BTRFS_MAX_MIRRORS
;
1237 struct scrub_block
*sblock
= sblocks_for_recheck
+
1239 struct scrub_recover
*recover
;
1242 for (page_index
= 0; page_index
< sblock
->page_count
;
1244 sblock
->pagev
[page_index
]->sblock
= NULL
;
1245 recover
= sblock
->pagev
[page_index
]->recover
;
1247 scrub_put_recover(recover
);
1248 sblock
->pagev
[page_index
]->recover
=
1251 scrub_page_put(sblock
->pagev
[page_index
]);
1254 kfree(sblocks_for_recheck
);
1260 static inline int scrub_nr_raid_mirrors(struct btrfs_bio
*bbio
)
1262 if (bbio
->map_type
& BTRFS_BLOCK_GROUP_RAID5
)
1264 else if (bbio
->map_type
& BTRFS_BLOCK_GROUP_RAID6
)
1267 return (int)bbio
->num_stripes
;
1270 static inline void scrub_stripe_index_and_offset(u64 logical
, u64 map_type
,
1273 int nstripes
, int mirror
,
1279 if (map_type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
1281 for (i
= 0; i
< nstripes
; i
++) {
1282 if (raid_map
[i
] == RAID6_Q_STRIPE
||
1283 raid_map
[i
] == RAID5_P_STRIPE
)
1286 if (logical
>= raid_map
[i
] &&
1287 logical
< raid_map
[i
] + mapped_length
)
1292 *stripe_offset
= logical
- raid_map
[i
];
1294 /* The other RAID type */
1295 *stripe_index
= mirror
;
1300 static int scrub_setup_recheck_block(struct scrub_block
*original_sblock
,
1301 struct scrub_block
*sblocks_for_recheck
)
1303 struct scrub_ctx
*sctx
= original_sblock
->sctx
;
1304 struct btrfs_fs_info
*fs_info
= sctx
->dev_root
->fs_info
;
1305 u64 length
= original_sblock
->page_count
* PAGE_SIZE
;
1306 u64 logical
= original_sblock
->pagev
[0]->logical
;
1307 u64 generation
= original_sblock
->pagev
[0]->generation
;
1308 u64 flags
= original_sblock
->pagev
[0]->flags
;
1309 u64 have_csum
= original_sblock
->pagev
[0]->have_csum
;
1310 struct scrub_recover
*recover
;
1311 struct btrfs_bio
*bbio
;
1322 * note: the two members refs and outstanding_pages
1323 * are not used (and not set) in the blocks that are used for
1324 * the recheck procedure
1327 while (length
> 0) {
1328 sublen
= min_t(u64
, length
, PAGE_SIZE
);
1329 mapped_length
= sublen
;
1333 * with a length of PAGE_SIZE, each returned stripe
1334 * represents one mirror
1336 ret
= btrfs_map_sblock(fs_info
, REQ_GET_READ_MIRRORS
, logical
,
1337 &mapped_length
, &bbio
, 0, 1);
1338 if (ret
|| !bbio
|| mapped_length
< sublen
) {
1339 btrfs_put_bbio(bbio
);
1343 recover
= kzalloc(sizeof(struct scrub_recover
), GFP_NOFS
);
1345 btrfs_put_bbio(bbio
);
1349 atomic_set(&recover
->refs
, 1);
1350 recover
->bbio
= bbio
;
1351 recover
->map_length
= mapped_length
;
1353 BUG_ON(page_index
>= SCRUB_PAGES_PER_RD_BIO
);
1355 nmirrors
= min(scrub_nr_raid_mirrors(bbio
), BTRFS_MAX_MIRRORS
);
1357 for (mirror_index
= 0; mirror_index
< nmirrors
;
1359 struct scrub_block
*sblock
;
1360 struct scrub_page
*page
;
1362 sblock
= sblocks_for_recheck
+ mirror_index
;
1363 sblock
->sctx
= sctx
;
1365 page
= kzalloc(sizeof(*page
), GFP_NOFS
);
1368 spin_lock(&sctx
->stat_lock
);
1369 sctx
->stat
.malloc_errors
++;
1370 spin_unlock(&sctx
->stat_lock
);
1371 scrub_put_recover(recover
);
1374 scrub_page_get(page
);
1375 sblock
->pagev
[page_index
] = page
;
1376 page
->sblock
= sblock
;
1377 page
->flags
= flags
;
1378 page
->generation
= generation
;
1379 page
->logical
= logical
;
1380 page
->have_csum
= have_csum
;
1383 original_sblock
->pagev
[0]->csum
,
1386 scrub_stripe_index_and_offset(logical
,
1395 page
->physical
= bbio
->stripes
[stripe_index
].physical
+
1397 page
->dev
= bbio
->stripes
[stripe_index
].dev
;
1399 BUG_ON(page_index
>= original_sblock
->page_count
);
1400 page
->physical_for_dev_replace
=
1401 original_sblock
->pagev
[page_index
]->
1402 physical_for_dev_replace
;
1403 /* for missing devices, dev->bdev is NULL */
1404 page
->mirror_num
= mirror_index
+ 1;
1405 sblock
->page_count
++;
1406 page
->page
= alloc_page(GFP_NOFS
);
1410 scrub_get_recover(recover
);
1411 page
->recover
= recover
;
1413 scrub_put_recover(recover
);
1422 struct scrub_bio_ret
{
1423 struct completion event
;
1427 static void scrub_bio_wait_endio(struct bio
*bio
)
1429 struct scrub_bio_ret
*ret
= bio
->bi_private
;
1431 ret
->error
= bio
->bi_error
;
1432 complete(&ret
->event
);
1435 static inline int scrub_is_page_on_raid56(struct scrub_page
*page
)
1437 return page
->recover
&&
1438 (page
->recover
->bbio
->map_type
& BTRFS_BLOCK_GROUP_RAID56_MASK
);
1441 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info
*fs_info
,
1443 struct scrub_page
*page
)
1445 struct scrub_bio_ret done
;
1448 init_completion(&done
.event
);
1450 bio
->bi_iter
.bi_sector
= page
->logical
>> 9;
1451 bio
->bi_private
= &done
;
1452 bio
->bi_end_io
= scrub_bio_wait_endio
;
1454 ret
= raid56_parity_recover(fs_info
->fs_root
, bio
, page
->recover
->bbio
,
1455 page
->recover
->map_length
,
1456 page
->mirror_num
, 0);
1460 wait_for_completion(&done
.event
);
1468 * this function will check the on disk data for checksum errors, header
1469 * errors and read I/O errors. If any I/O errors happen, the exact pages
1470 * which are errored are marked as being bad. The goal is to enable scrub
1471 * to take those pages that are not errored from all the mirrors so that
1472 * the pages that are errored in the just handled mirror can be repaired.
1474 static void scrub_recheck_block(struct btrfs_fs_info
*fs_info
,
1475 struct scrub_block
*sblock
,
1476 int retry_failed_mirror
)
1480 sblock
->no_io_error_seen
= 1;
1482 for (page_num
= 0; page_num
< sblock
->page_count
; page_num
++) {
1484 struct scrub_page
*page
= sblock
->pagev
[page_num
];
1486 if (page
->dev
->bdev
== NULL
) {
1488 sblock
->no_io_error_seen
= 0;
1492 WARN_ON(!page
->page
);
1493 bio
= btrfs_io_bio_alloc(GFP_NOFS
, 1);
1496 sblock
->no_io_error_seen
= 0;
1499 bio
->bi_bdev
= page
->dev
->bdev
;
1501 bio_add_page(bio
, page
->page
, PAGE_SIZE
, 0);
1502 if (!retry_failed_mirror
&& scrub_is_page_on_raid56(page
)) {
1503 if (scrub_submit_raid56_bio_wait(fs_info
, bio
, page
))
1504 sblock
->no_io_error_seen
= 0;
1506 bio
->bi_iter
.bi_sector
= page
->physical
>> 9;
1508 if (btrfsic_submit_bio_wait(READ
, bio
))
1509 sblock
->no_io_error_seen
= 0;
1515 if (sblock
->no_io_error_seen
)
1516 scrub_recheck_block_checksum(sblock
);
1519 static inline int scrub_check_fsid(u8 fsid
[],
1520 struct scrub_page
*spage
)
1522 struct btrfs_fs_devices
*fs_devices
= spage
->dev
->fs_devices
;
1525 ret
= memcmp(fsid
, fs_devices
->fsid
, BTRFS_UUID_SIZE
);
1529 static void scrub_recheck_block_checksum(struct scrub_block
*sblock
)
1531 sblock
->header_error
= 0;
1532 sblock
->checksum_error
= 0;
1533 sblock
->generation_error
= 0;
1535 if (sblock
->pagev
[0]->flags
& BTRFS_EXTENT_FLAG_DATA
)
1536 scrub_checksum_data(sblock
);
1538 scrub_checksum_tree_block(sblock
);
1541 static int scrub_repair_block_from_good_copy(struct scrub_block
*sblock_bad
,
1542 struct scrub_block
*sblock_good
)
1547 for (page_num
= 0; page_num
< sblock_bad
->page_count
; page_num
++) {
1550 ret_sub
= scrub_repair_page_from_good_copy(sblock_bad
,
1560 static int scrub_repair_page_from_good_copy(struct scrub_block
*sblock_bad
,
1561 struct scrub_block
*sblock_good
,
1562 int page_num
, int force_write
)
1564 struct scrub_page
*page_bad
= sblock_bad
->pagev
[page_num
];
1565 struct scrub_page
*page_good
= sblock_good
->pagev
[page_num
];
1567 BUG_ON(page_bad
->page
== NULL
);
1568 BUG_ON(page_good
->page
== NULL
);
1569 if (force_write
|| sblock_bad
->header_error
||
1570 sblock_bad
->checksum_error
|| page_bad
->io_error
) {
1574 if (!page_bad
->dev
->bdev
) {
1575 btrfs_warn_rl(sblock_bad
->sctx
->dev_root
->fs_info
,
1576 "scrub_repair_page_from_good_copy(bdev == NULL) "
1581 bio
= btrfs_io_bio_alloc(GFP_NOFS
, 1);
1584 bio
->bi_bdev
= page_bad
->dev
->bdev
;
1585 bio
->bi_iter
.bi_sector
= page_bad
->physical
>> 9;
1587 ret
= bio_add_page(bio
, page_good
->page
, PAGE_SIZE
, 0);
1588 if (PAGE_SIZE
!= ret
) {
1593 if (btrfsic_submit_bio_wait(WRITE
, bio
)) {
1594 btrfs_dev_stat_inc_and_print(page_bad
->dev
,
1595 BTRFS_DEV_STAT_WRITE_ERRS
);
1596 btrfs_dev_replace_stats_inc(
1597 &sblock_bad
->sctx
->dev_root
->fs_info
->
1598 dev_replace
.num_write_errors
);
1608 static void scrub_write_block_to_dev_replace(struct scrub_block
*sblock
)
1613 * This block is used for the check of the parity on the source device,
1614 * so the data needn't be written into the destination device.
1616 if (sblock
->sparity
)
1619 for (page_num
= 0; page_num
< sblock
->page_count
; page_num
++) {
1622 ret
= scrub_write_page_to_dev_replace(sblock
, page_num
);
1624 btrfs_dev_replace_stats_inc(
1625 &sblock
->sctx
->dev_root
->fs_info
->dev_replace
.
1630 static int scrub_write_page_to_dev_replace(struct scrub_block
*sblock
,
1633 struct scrub_page
*spage
= sblock
->pagev
[page_num
];
1635 BUG_ON(spage
->page
== NULL
);
1636 if (spage
->io_error
) {
1637 void *mapped_buffer
= kmap_atomic(spage
->page
);
1639 memset(mapped_buffer
, 0, PAGE_CACHE_SIZE
);
1640 flush_dcache_page(spage
->page
);
1641 kunmap_atomic(mapped_buffer
);
1643 return scrub_add_page_to_wr_bio(sblock
->sctx
, spage
);
1646 static int scrub_add_page_to_wr_bio(struct scrub_ctx
*sctx
,
1647 struct scrub_page
*spage
)
1649 struct scrub_wr_ctx
*wr_ctx
= &sctx
->wr_ctx
;
1650 struct scrub_bio
*sbio
;
1653 mutex_lock(&wr_ctx
->wr_lock
);
1655 if (!wr_ctx
->wr_curr_bio
) {
1656 wr_ctx
->wr_curr_bio
= kzalloc(sizeof(*wr_ctx
->wr_curr_bio
),
1658 if (!wr_ctx
->wr_curr_bio
) {
1659 mutex_unlock(&wr_ctx
->wr_lock
);
1662 wr_ctx
->wr_curr_bio
->sctx
= sctx
;
1663 wr_ctx
->wr_curr_bio
->page_count
= 0;
1665 sbio
= wr_ctx
->wr_curr_bio
;
1666 if (sbio
->page_count
== 0) {
1669 sbio
->physical
= spage
->physical_for_dev_replace
;
1670 sbio
->logical
= spage
->logical
;
1671 sbio
->dev
= wr_ctx
->tgtdev
;
1674 bio
= btrfs_io_bio_alloc(GFP_NOFS
, wr_ctx
->pages_per_wr_bio
);
1676 mutex_unlock(&wr_ctx
->wr_lock
);
1682 bio
->bi_private
= sbio
;
1683 bio
->bi_end_io
= scrub_wr_bio_end_io
;
1684 bio
->bi_bdev
= sbio
->dev
->bdev
;
1685 bio
->bi_iter
.bi_sector
= sbio
->physical
>> 9;
1687 } else if (sbio
->physical
+ sbio
->page_count
* PAGE_SIZE
!=
1688 spage
->physical_for_dev_replace
||
1689 sbio
->logical
+ sbio
->page_count
* PAGE_SIZE
!=
1691 scrub_wr_submit(sctx
);
1695 ret
= bio_add_page(sbio
->bio
, spage
->page
, PAGE_SIZE
, 0);
1696 if (ret
!= PAGE_SIZE
) {
1697 if (sbio
->page_count
< 1) {
1700 mutex_unlock(&wr_ctx
->wr_lock
);
1703 scrub_wr_submit(sctx
);
1707 sbio
->pagev
[sbio
->page_count
] = spage
;
1708 scrub_page_get(spage
);
1710 if (sbio
->page_count
== wr_ctx
->pages_per_wr_bio
)
1711 scrub_wr_submit(sctx
);
1712 mutex_unlock(&wr_ctx
->wr_lock
);
1717 static void scrub_wr_submit(struct scrub_ctx
*sctx
)
1719 struct scrub_wr_ctx
*wr_ctx
= &sctx
->wr_ctx
;
1720 struct scrub_bio
*sbio
;
1722 if (!wr_ctx
->wr_curr_bio
)
1725 sbio
= wr_ctx
->wr_curr_bio
;
1726 wr_ctx
->wr_curr_bio
= NULL
;
1727 WARN_ON(!sbio
->bio
->bi_bdev
);
1728 scrub_pending_bio_inc(sctx
);
1729 /* process all writes in a single worker thread. Then the block layer
1730 * orders the requests before sending them to the driver which
1731 * doubled the write performance on spinning disks when measured
1733 btrfsic_submit_bio(WRITE
, sbio
->bio
);
1736 static void scrub_wr_bio_end_io(struct bio
*bio
)
1738 struct scrub_bio
*sbio
= bio
->bi_private
;
1739 struct btrfs_fs_info
*fs_info
= sbio
->dev
->dev_root
->fs_info
;
1741 sbio
->err
= bio
->bi_error
;
1744 btrfs_init_work(&sbio
->work
, btrfs_scrubwrc_helper
,
1745 scrub_wr_bio_end_io_worker
, NULL
, NULL
);
1746 btrfs_queue_work(fs_info
->scrub_wr_completion_workers
, &sbio
->work
);
1749 static void scrub_wr_bio_end_io_worker(struct btrfs_work
*work
)
1751 struct scrub_bio
*sbio
= container_of(work
, struct scrub_bio
, work
);
1752 struct scrub_ctx
*sctx
= sbio
->sctx
;
1755 WARN_ON(sbio
->page_count
> SCRUB_PAGES_PER_WR_BIO
);
1757 struct btrfs_dev_replace
*dev_replace
=
1758 &sbio
->sctx
->dev_root
->fs_info
->dev_replace
;
1760 for (i
= 0; i
< sbio
->page_count
; i
++) {
1761 struct scrub_page
*spage
= sbio
->pagev
[i
];
1763 spage
->io_error
= 1;
1764 btrfs_dev_replace_stats_inc(&dev_replace
->
1769 for (i
= 0; i
< sbio
->page_count
; i
++)
1770 scrub_page_put(sbio
->pagev
[i
]);
1774 scrub_pending_bio_dec(sctx
);
1777 static int scrub_checksum(struct scrub_block
*sblock
)
1783 * No need to initialize these stats currently,
1784 * because this function only use return value
1785 * instead of these stats value.
1790 sblock
->header_error
= 0;
1791 sblock
->generation_error
= 0;
1792 sblock
->checksum_error
= 0;
1794 WARN_ON(sblock
->page_count
< 1);
1795 flags
= sblock
->pagev
[0]->flags
;
1797 if (flags
& BTRFS_EXTENT_FLAG_DATA
)
1798 ret
= scrub_checksum_data(sblock
);
1799 else if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
)
1800 ret
= scrub_checksum_tree_block(sblock
);
1801 else if (flags
& BTRFS_EXTENT_FLAG_SUPER
)
1802 (void)scrub_checksum_super(sblock
);
1806 scrub_handle_errored_block(sblock
);
1811 static int scrub_checksum_data(struct scrub_block
*sblock
)
1813 struct scrub_ctx
*sctx
= sblock
->sctx
;
1814 u8 csum
[BTRFS_CSUM_SIZE
];
1822 BUG_ON(sblock
->page_count
< 1);
1823 if (!sblock
->pagev
[0]->have_csum
)
1826 on_disk_csum
= sblock
->pagev
[0]->csum
;
1827 page
= sblock
->pagev
[0]->page
;
1828 buffer
= kmap_atomic(page
);
1830 len
= sctx
->sectorsize
;
1833 u64 l
= min_t(u64
, len
, PAGE_SIZE
);
1835 crc
= btrfs_csum_data(buffer
, crc
, l
);
1836 kunmap_atomic(buffer
);
1841 BUG_ON(index
>= sblock
->page_count
);
1842 BUG_ON(!sblock
->pagev
[index
]->page
);
1843 page
= sblock
->pagev
[index
]->page
;
1844 buffer
= kmap_atomic(page
);
1847 btrfs_csum_final(crc
, csum
);
1848 if (memcmp(csum
, on_disk_csum
, sctx
->csum_size
))
1849 sblock
->checksum_error
= 1;
1851 return sblock
->checksum_error
;
1854 static int scrub_checksum_tree_block(struct scrub_block
*sblock
)
1856 struct scrub_ctx
*sctx
= sblock
->sctx
;
1857 struct btrfs_header
*h
;
1858 struct btrfs_root
*root
= sctx
->dev_root
;
1859 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
1860 u8 calculated_csum
[BTRFS_CSUM_SIZE
];
1861 u8 on_disk_csum
[BTRFS_CSUM_SIZE
];
1863 void *mapped_buffer
;
1870 BUG_ON(sblock
->page_count
< 1);
1871 page
= sblock
->pagev
[0]->page
;
1872 mapped_buffer
= kmap_atomic(page
);
1873 h
= (struct btrfs_header
*)mapped_buffer
;
1874 memcpy(on_disk_csum
, h
->csum
, sctx
->csum_size
);
1877 * we don't use the getter functions here, as we
1878 * a) don't have an extent buffer and
1879 * b) the page is already kmapped
1881 if (sblock
->pagev
[0]->logical
!= btrfs_stack_header_bytenr(h
))
1882 sblock
->header_error
= 1;
1884 if (sblock
->pagev
[0]->generation
!= btrfs_stack_header_generation(h
)) {
1885 sblock
->header_error
= 1;
1886 sblock
->generation_error
= 1;
1889 if (!scrub_check_fsid(h
->fsid
, sblock
->pagev
[0]))
1890 sblock
->header_error
= 1;
1892 if (memcmp(h
->chunk_tree_uuid
, fs_info
->chunk_tree_uuid
,
1894 sblock
->header_error
= 1;
1896 len
= sctx
->nodesize
- BTRFS_CSUM_SIZE
;
1897 mapped_size
= PAGE_SIZE
- BTRFS_CSUM_SIZE
;
1898 p
= ((u8
*)mapped_buffer
) + BTRFS_CSUM_SIZE
;
1901 u64 l
= min_t(u64
, len
, mapped_size
);
1903 crc
= btrfs_csum_data(p
, crc
, l
);
1904 kunmap_atomic(mapped_buffer
);
1909 BUG_ON(index
>= sblock
->page_count
);
1910 BUG_ON(!sblock
->pagev
[index
]->page
);
1911 page
= sblock
->pagev
[index
]->page
;
1912 mapped_buffer
= kmap_atomic(page
);
1913 mapped_size
= PAGE_SIZE
;
1917 btrfs_csum_final(crc
, calculated_csum
);
1918 if (memcmp(calculated_csum
, on_disk_csum
, sctx
->csum_size
))
1919 sblock
->checksum_error
= 1;
1921 return sblock
->header_error
|| sblock
->checksum_error
;
1924 static int scrub_checksum_super(struct scrub_block
*sblock
)
1926 struct btrfs_super_block
*s
;
1927 struct scrub_ctx
*sctx
= sblock
->sctx
;
1928 u8 calculated_csum
[BTRFS_CSUM_SIZE
];
1929 u8 on_disk_csum
[BTRFS_CSUM_SIZE
];
1931 void *mapped_buffer
;
1940 BUG_ON(sblock
->page_count
< 1);
1941 page
= sblock
->pagev
[0]->page
;
1942 mapped_buffer
= kmap_atomic(page
);
1943 s
= (struct btrfs_super_block
*)mapped_buffer
;
1944 memcpy(on_disk_csum
, s
->csum
, sctx
->csum_size
);
1946 if (sblock
->pagev
[0]->logical
!= btrfs_super_bytenr(s
))
1949 if (sblock
->pagev
[0]->generation
!= btrfs_super_generation(s
))
1952 if (!scrub_check_fsid(s
->fsid
, sblock
->pagev
[0]))
1955 len
= BTRFS_SUPER_INFO_SIZE
- BTRFS_CSUM_SIZE
;
1956 mapped_size
= PAGE_SIZE
- BTRFS_CSUM_SIZE
;
1957 p
= ((u8
*)mapped_buffer
) + BTRFS_CSUM_SIZE
;
1960 u64 l
= min_t(u64
, len
, mapped_size
);
1962 crc
= btrfs_csum_data(p
, crc
, l
);
1963 kunmap_atomic(mapped_buffer
);
1968 BUG_ON(index
>= sblock
->page_count
);
1969 BUG_ON(!sblock
->pagev
[index
]->page
);
1970 page
= sblock
->pagev
[index
]->page
;
1971 mapped_buffer
= kmap_atomic(page
);
1972 mapped_size
= PAGE_SIZE
;
1976 btrfs_csum_final(crc
, calculated_csum
);
1977 if (memcmp(calculated_csum
, on_disk_csum
, sctx
->csum_size
))
1980 if (fail_cor
+ fail_gen
) {
1982 * if we find an error in a super block, we just report it.
1983 * They will get written with the next transaction commit
1986 spin_lock(&sctx
->stat_lock
);
1987 ++sctx
->stat
.super_errors
;
1988 spin_unlock(&sctx
->stat_lock
);
1990 btrfs_dev_stat_inc_and_print(sblock
->pagev
[0]->dev
,
1991 BTRFS_DEV_STAT_CORRUPTION_ERRS
);
1993 btrfs_dev_stat_inc_and_print(sblock
->pagev
[0]->dev
,
1994 BTRFS_DEV_STAT_GENERATION_ERRS
);
1997 return fail_cor
+ fail_gen
;
2000 static void scrub_block_get(struct scrub_block
*sblock
)
2002 atomic_inc(&sblock
->refs
);
2005 static void scrub_block_put(struct scrub_block
*sblock
)
2007 if (atomic_dec_and_test(&sblock
->refs
)) {
2010 if (sblock
->sparity
)
2011 scrub_parity_put(sblock
->sparity
);
2013 for (i
= 0; i
< sblock
->page_count
; i
++)
2014 scrub_page_put(sblock
->pagev
[i
]);
2019 static void scrub_page_get(struct scrub_page
*spage
)
2021 atomic_inc(&spage
->refs
);
2024 static void scrub_page_put(struct scrub_page
*spage
)
2026 if (atomic_dec_and_test(&spage
->refs
)) {
2028 __free_page(spage
->page
);
2033 static void scrub_submit(struct scrub_ctx
*sctx
)
2035 struct scrub_bio
*sbio
;
2037 if (sctx
->curr
== -1)
2040 sbio
= sctx
->bios
[sctx
->curr
];
2042 scrub_pending_bio_inc(sctx
);
2043 btrfsic_submit_bio(READ
, sbio
->bio
);
2046 static int scrub_add_page_to_rd_bio(struct scrub_ctx
*sctx
,
2047 struct scrub_page
*spage
)
2049 struct scrub_block
*sblock
= spage
->sblock
;
2050 struct scrub_bio
*sbio
;
2055 * grab a fresh bio or wait for one to become available
2057 while (sctx
->curr
== -1) {
2058 spin_lock(&sctx
->list_lock
);
2059 sctx
->curr
= sctx
->first_free
;
2060 if (sctx
->curr
!= -1) {
2061 sctx
->first_free
= sctx
->bios
[sctx
->curr
]->next_free
;
2062 sctx
->bios
[sctx
->curr
]->next_free
= -1;
2063 sctx
->bios
[sctx
->curr
]->page_count
= 0;
2064 spin_unlock(&sctx
->list_lock
);
2066 spin_unlock(&sctx
->list_lock
);
2067 wait_event(sctx
->list_wait
, sctx
->first_free
!= -1);
2070 sbio
= sctx
->bios
[sctx
->curr
];
2071 if (sbio
->page_count
== 0) {
2074 sbio
->physical
= spage
->physical
;
2075 sbio
->logical
= spage
->logical
;
2076 sbio
->dev
= spage
->dev
;
2079 bio
= btrfs_io_bio_alloc(GFP_NOFS
, sctx
->pages_per_rd_bio
);
2085 bio
->bi_private
= sbio
;
2086 bio
->bi_end_io
= scrub_bio_end_io
;
2087 bio
->bi_bdev
= sbio
->dev
->bdev
;
2088 bio
->bi_iter
.bi_sector
= sbio
->physical
>> 9;
2090 } else if (sbio
->physical
+ sbio
->page_count
* PAGE_SIZE
!=
2092 sbio
->logical
+ sbio
->page_count
* PAGE_SIZE
!=
2094 sbio
->dev
!= spage
->dev
) {
2099 sbio
->pagev
[sbio
->page_count
] = spage
;
2100 ret
= bio_add_page(sbio
->bio
, spage
->page
, PAGE_SIZE
, 0);
2101 if (ret
!= PAGE_SIZE
) {
2102 if (sbio
->page_count
< 1) {
2111 scrub_block_get(sblock
); /* one for the page added to the bio */
2112 atomic_inc(&sblock
->outstanding_pages
);
2114 if (sbio
->page_count
== sctx
->pages_per_rd_bio
)
2120 static void scrub_missing_raid56_end_io(struct bio
*bio
)
2122 struct scrub_block
*sblock
= bio
->bi_private
;
2123 struct btrfs_fs_info
*fs_info
= sblock
->sctx
->dev_root
->fs_info
;
2126 sblock
->no_io_error_seen
= 0;
2128 btrfs_queue_work(fs_info
->scrub_workers
, &sblock
->work
);
2131 static void scrub_missing_raid56_worker(struct btrfs_work
*work
)
2133 struct scrub_block
*sblock
= container_of(work
, struct scrub_block
, work
);
2134 struct scrub_ctx
*sctx
= sblock
->sctx
;
2136 struct btrfs_device
*dev
;
2138 logical
= sblock
->pagev
[0]->logical
;
2139 dev
= sblock
->pagev
[0]->dev
;
2141 if (sblock
->no_io_error_seen
)
2142 scrub_recheck_block_checksum(sblock
);
2144 if (!sblock
->no_io_error_seen
) {
2145 spin_lock(&sctx
->stat_lock
);
2146 sctx
->stat
.read_errors
++;
2147 spin_unlock(&sctx
->stat_lock
);
2148 btrfs_err_rl_in_rcu(sctx
->dev_root
->fs_info
,
2149 "IO error rebuilding logical %llu for dev %s",
2150 logical
, rcu_str_deref(dev
->name
));
2151 } else if (sblock
->header_error
|| sblock
->checksum_error
) {
2152 spin_lock(&sctx
->stat_lock
);
2153 sctx
->stat
.uncorrectable_errors
++;
2154 spin_unlock(&sctx
->stat_lock
);
2155 btrfs_err_rl_in_rcu(sctx
->dev_root
->fs_info
,
2156 "failed to rebuild valid logical %llu for dev %s",
2157 logical
, rcu_str_deref(dev
->name
));
2159 scrub_write_block_to_dev_replace(sblock
);
2162 scrub_block_put(sblock
);
2164 if (sctx
->is_dev_replace
&&
2165 atomic_read(&sctx
->wr_ctx
.flush_all_writes
)) {
2166 mutex_lock(&sctx
->wr_ctx
.wr_lock
);
2167 scrub_wr_submit(sctx
);
2168 mutex_unlock(&sctx
->wr_ctx
.wr_lock
);
2171 scrub_pending_bio_dec(sctx
);
2174 static void scrub_missing_raid56_pages(struct scrub_block
*sblock
)
2176 struct scrub_ctx
*sctx
= sblock
->sctx
;
2177 struct btrfs_fs_info
*fs_info
= sctx
->dev_root
->fs_info
;
2178 u64 length
= sblock
->page_count
* PAGE_SIZE
;
2179 u64 logical
= sblock
->pagev
[0]->logical
;
2180 struct btrfs_bio
*bbio
;
2182 struct btrfs_raid_bio
*rbio
;
2186 ret
= btrfs_map_sblock(fs_info
, REQ_GET_READ_MIRRORS
, logical
, &length
,
2188 if (ret
|| !bbio
|| !bbio
->raid_map
)
2191 if (WARN_ON(!sctx
->is_dev_replace
||
2192 !(bbio
->map_type
& BTRFS_BLOCK_GROUP_RAID56_MASK
))) {
2194 * We shouldn't be scrubbing a missing device. Even for dev
2195 * replace, we should only get here for RAID 5/6. We either
2196 * managed to mount something with no mirrors remaining or
2197 * there's a bug in scrub_remap_extent()/btrfs_map_block().
2202 bio
= btrfs_io_bio_alloc(GFP_NOFS
, 0);
2206 bio
->bi_iter
.bi_sector
= logical
>> 9;
2207 bio
->bi_private
= sblock
;
2208 bio
->bi_end_io
= scrub_missing_raid56_end_io
;
2210 rbio
= raid56_alloc_missing_rbio(sctx
->dev_root
, bio
, bbio
, length
);
2214 for (i
= 0; i
< sblock
->page_count
; i
++) {
2215 struct scrub_page
*spage
= sblock
->pagev
[i
];
2217 raid56_add_scrub_pages(rbio
, spage
->page
, spage
->logical
);
2220 btrfs_init_work(&sblock
->work
, btrfs_scrub_helper
,
2221 scrub_missing_raid56_worker
, NULL
, NULL
);
2222 scrub_block_get(sblock
);
2223 scrub_pending_bio_inc(sctx
);
2224 raid56_submit_missing_rbio(rbio
);
2230 btrfs_put_bbio(bbio
);
2231 spin_lock(&sctx
->stat_lock
);
2232 sctx
->stat
.malloc_errors
++;
2233 spin_unlock(&sctx
->stat_lock
);
2236 static int scrub_pages(struct scrub_ctx
*sctx
, u64 logical
, u64 len
,
2237 u64 physical
, struct btrfs_device
*dev
, u64 flags
,
2238 u64 gen
, int mirror_num
, u8
*csum
, int force
,
2239 u64 physical_for_dev_replace
)
2241 struct scrub_block
*sblock
;
2244 sblock
= kzalloc(sizeof(*sblock
), GFP_NOFS
);
2246 spin_lock(&sctx
->stat_lock
);
2247 sctx
->stat
.malloc_errors
++;
2248 spin_unlock(&sctx
->stat_lock
);
2252 /* one ref inside this function, plus one for each page added to
2254 atomic_set(&sblock
->refs
, 1);
2255 sblock
->sctx
= sctx
;
2256 sblock
->no_io_error_seen
= 1;
2258 for (index
= 0; len
> 0; index
++) {
2259 struct scrub_page
*spage
;
2260 u64 l
= min_t(u64
, len
, PAGE_SIZE
);
2262 spage
= kzalloc(sizeof(*spage
), GFP_NOFS
);
2265 spin_lock(&sctx
->stat_lock
);
2266 sctx
->stat
.malloc_errors
++;
2267 spin_unlock(&sctx
->stat_lock
);
2268 scrub_block_put(sblock
);
2271 BUG_ON(index
>= SCRUB_MAX_PAGES_PER_BLOCK
);
2272 scrub_page_get(spage
);
2273 sblock
->pagev
[index
] = spage
;
2274 spage
->sblock
= sblock
;
2276 spage
->flags
= flags
;
2277 spage
->generation
= gen
;
2278 spage
->logical
= logical
;
2279 spage
->physical
= physical
;
2280 spage
->physical_for_dev_replace
= physical_for_dev_replace
;
2281 spage
->mirror_num
= mirror_num
;
2283 spage
->have_csum
= 1;
2284 memcpy(spage
->csum
, csum
, sctx
->csum_size
);
2286 spage
->have_csum
= 0;
2288 sblock
->page_count
++;
2289 spage
->page
= alloc_page(GFP_NOFS
);
2295 physical_for_dev_replace
+= l
;
2298 WARN_ON(sblock
->page_count
== 0);
2301 * This case should only be hit for RAID 5/6 device replace. See
2302 * the comment in scrub_missing_raid56_pages() for details.
2304 scrub_missing_raid56_pages(sblock
);
2306 for (index
= 0; index
< sblock
->page_count
; index
++) {
2307 struct scrub_page
*spage
= sblock
->pagev
[index
];
2310 ret
= scrub_add_page_to_rd_bio(sctx
, spage
);
2312 scrub_block_put(sblock
);
2321 /* last one frees, either here or in bio completion for last page */
2322 scrub_block_put(sblock
);
2326 static void scrub_bio_end_io(struct bio
*bio
)
2328 struct scrub_bio
*sbio
= bio
->bi_private
;
2329 struct btrfs_fs_info
*fs_info
= sbio
->dev
->dev_root
->fs_info
;
2331 sbio
->err
= bio
->bi_error
;
2334 btrfs_queue_work(fs_info
->scrub_workers
, &sbio
->work
);
2337 static void scrub_bio_end_io_worker(struct btrfs_work
*work
)
2339 struct scrub_bio
*sbio
= container_of(work
, struct scrub_bio
, work
);
2340 struct scrub_ctx
*sctx
= sbio
->sctx
;
2343 BUG_ON(sbio
->page_count
> SCRUB_PAGES_PER_RD_BIO
);
2345 for (i
= 0; i
< sbio
->page_count
; i
++) {
2346 struct scrub_page
*spage
= sbio
->pagev
[i
];
2348 spage
->io_error
= 1;
2349 spage
->sblock
->no_io_error_seen
= 0;
2353 /* now complete the scrub_block items that have all pages completed */
2354 for (i
= 0; i
< sbio
->page_count
; i
++) {
2355 struct scrub_page
*spage
= sbio
->pagev
[i
];
2356 struct scrub_block
*sblock
= spage
->sblock
;
2358 if (atomic_dec_and_test(&sblock
->outstanding_pages
))
2359 scrub_block_complete(sblock
);
2360 scrub_block_put(sblock
);
2365 spin_lock(&sctx
->list_lock
);
2366 sbio
->next_free
= sctx
->first_free
;
2367 sctx
->first_free
= sbio
->index
;
2368 spin_unlock(&sctx
->list_lock
);
2370 if (sctx
->is_dev_replace
&&
2371 atomic_read(&sctx
->wr_ctx
.flush_all_writes
)) {
2372 mutex_lock(&sctx
->wr_ctx
.wr_lock
);
2373 scrub_wr_submit(sctx
);
2374 mutex_unlock(&sctx
->wr_ctx
.wr_lock
);
2377 scrub_pending_bio_dec(sctx
);
2380 static inline void __scrub_mark_bitmap(struct scrub_parity
*sparity
,
2381 unsigned long *bitmap
,
2386 int sectorsize
= sparity
->sctx
->dev_root
->sectorsize
;
2388 if (len
>= sparity
->stripe_len
) {
2389 bitmap_set(bitmap
, 0, sparity
->nsectors
);
2393 start
-= sparity
->logic_start
;
2394 start
= div_u64_rem(start
, sparity
->stripe_len
, &offset
);
2395 offset
/= sectorsize
;
2396 nsectors
= (int)len
/ sectorsize
;
2398 if (offset
+ nsectors
<= sparity
->nsectors
) {
2399 bitmap_set(bitmap
, offset
, nsectors
);
2403 bitmap_set(bitmap
, offset
, sparity
->nsectors
- offset
);
2404 bitmap_set(bitmap
, 0, nsectors
- (sparity
->nsectors
- offset
));
2407 static inline void scrub_parity_mark_sectors_error(struct scrub_parity
*sparity
,
2410 __scrub_mark_bitmap(sparity
, sparity
->ebitmap
, start
, len
);
2413 static inline void scrub_parity_mark_sectors_data(struct scrub_parity
*sparity
,
2416 __scrub_mark_bitmap(sparity
, sparity
->dbitmap
, start
, len
);
2419 static void scrub_block_complete(struct scrub_block
*sblock
)
2423 if (!sblock
->no_io_error_seen
) {
2425 scrub_handle_errored_block(sblock
);
2428 * if has checksum error, write via repair mechanism in
2429 * dev replace case, otherwise write here in dev replace
2432 corrupted
= scrub_checksum(sblock
);
2433 if (!corrupted
&& sblock
->sctx
->is_dev_replace
)
2434 scrub_write_block_to_dev_replace(sblock
);
2437 if (sblock
->sparity
&& corrupted
&& !sblock
->data_corrected
) {
2438 u64 start
= sblock
->pagev
[0]->logical
;
2439 u64 end
= sblock
->pagev
[sblock
->page_count
- 1]->logical
+
2442 scrub_parity_mark_sectors_error(sblock
->sparity
,
2443 start
, end
- start
);
2447 static int scrub_find_csum(struct scrub_ctx
*sctx
, u64 logical
, u8
*csum
)
2449 struct btrfs_ordered_sum
*sum
= NULL
;
2450 unsigned long index
;
2451 unsigned long num_sectors
;
2453 while (!list_empty(&sctx
->csum_list
)) {
2454 sum
= list_first_entry(&sctx
->csum_list
,
2455 struct btrfs_ordered_sum
, list
);
2456 if (sum
->bytenr
> logical
)
2458 if (sum
->bytenr
+ sum
->len
> logical
)
2461 ++sctx
->stat
.csum_discards
;
2462 list_del(&sum
->list
);
2469 index
= ((u32
)(logical
- sum
->bytenr
)) / sctx
->sectorsize
;
2470 num_sectors
= sum
->len
/ sctx
->sectorsize
;
2471 memcpy(csum
, sum
->sums
+ index
, sctx
->csum_size
);
2472 if (index
== num_sectors
- 1) {
2473 list_del(&sum
->list
);
2479 /* scrub extent tries to collect up to 64 kB for each bio */
2480 static int scrub_extent(struct scrub_ctx
*sctx
, u64 logical
, u64 len
,
2481 u64 physical
, struct btrfs_device
*dev
, u64 flags
,
2482 u64 gen
, int mirror_num
, u64 physical_for_dev_replace
)
2485 u8 csum
[BTRFS_CSUM_SIZE
];
2488 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
2489 blocksize
= sctx
->sectorsize
;
2490 spin_lock(&sctx
->stat_lock
);
2491 sctx
->stat
.data_extents_scrubbed
++;
2492 sctx
->stat
.data_bytes_scrubbed
+= len
;
2493 spin_unlock(&sctx
->stat_lock
);
2494 } else if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
2495 blocksize
= sctx
->nodesize
;
2496 spin_lock(&sctx
->stat_lock
);
2497 sctx
->stat
.tree_extents_scrubbed
++;
2498 sctx
->stat
.tree_bytes_scrubbed
+= len
;
2499 spin_unlock(&sctx
->stat_lock
);
2501 blocksize
= sctx
->sectorsize
;
2506 u64 l
= min_t(u64
, len
, blocksize
);
2509 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
2510 /* push csums to sbio */
2511 have_csum
= scrub_find_csum(sctx
, logical
, csum
);
2513 ++sctx
->stat
.no_csum
;
2514 if (sctx
->is_dev_replace
&& !have_csum
) {
2515 ret
= copy_nocow_pages(sctx
, logical
, l
,
2517 physical_for_dev_replace
);
2518 goto behind_scrub_pages
;
2521 ret
= scrub_pages(sctx
, logical
, l
, physical
, dev
, flags
, gen
,
2522 mirror_num
, have_csum
? csum
: NULL
, 0,
2523 physical_for_dev_replace
);
2530 physical_for_dev_replace
+= l
;
2535 static int scrub_pages_for_parity(struct scrub_parity
*sparity
,
2536 u64 logical
, u64 len
,
2537 u64 physical
, struct btrfs_device
*dev
,
2538 u64 flags
, u64 gen
, int mirror_num
, u8
*csum
)
2540 struct scrub_ctx
*sctx
= sparity
->sctx
;
2541 struct scrub_block
*sblock
;
2544 sblock
= kzalloc(sizeof(*sblock
), GFP_NOFS
);
2546 spin_lock(&sctx
->stat_lock
);
2547 sctx
->stat
.malloc_errors
++;
2548 spin_unlock(&sctx
->stat_lock
);
2552 /* one ref inside this function, plus one for each page added to
2554 atomic_set(&sblock
->refs
, 1);
2555 sblock
->sctx
= sctx
;
2556 sblock
->no_io_error_seen
= 1;
2557 sblock
->sparity
= sparity
;
2558 scrub_parity_get(sparity
);
2560 for (index
= 0; len
> 0; index
++) {
2561 struct scrub_page
*spage
;
2562 u64 l
= min_t(u64
, len
, PAGE_SIZE
);
2564 spage
= kzalloc(sizeof(*spage
), GFP_NOFS
);
2567 spin_lock(&sctx
->stat_lock
);
2568 sctx
->stat
.malloc_errors
++;
2569 spin_unlock(&sctx
->stat_lock
);
2570 scrub_block_put(sblock
);
2573 BUG_ON(index
>= SCRUB_MAX_PAGES_PER_BLOCK
);
2574 /* For scrub block */
2575 scrub_page_get(spage
);
2576 sblock
->pagev
[index
] = spage
;
2577 /* For scrub parity */
2578 scrub_page_get(spage
);
2579 list_add_tail(&spage
->list
, &sparity
->spages
);
2580 spage
->sblock
= sblock
;
2582 spage
->flags
= flags
;
2583 spage
->generation
= gen
;
2584 spage
->logical
= logical
;
2585 spage
->physical
= physical
;
2586 spage
->mirror_num
= mirror_num
;
2588 spage
->have_csum
= 1;
2589 memcpy(spage
->csum
, csum
, sctx
->csum_size
);
2591 spage
->have_csum
= 0;
2593 sblock
->page_count
++;
2594 spage
->page
= alloc_page(GFP_NOFS
);
2602 WARN_ON(sblock
->page_count
== 0);
2603 for (index
= 0; index
< sblock
->page_count
; index
++) {
2604 struct scrub_page
*spage
= sblock
->pagev
[index
];
2607 ret
= scrub_add_page_to_rd_bio(sctx
, spage
);
2609 scrub_block_put(sblock
);
2614 /* last one frees, either here or in bio completion for last page */
2615 scrub_block_put(sblock
);
2619 static int scrub_extent_for_parity(struct scrub_parity
*sparity
,
2620 u64 logical
, u64 len
,
2621 u64 physical
, struct btrfs_device
*dev
,
2622 u64 flags
, u64 gen
, int mirror_num
)
2624 struct scrub_ctx
*sctx
= sparity
->sctx
;
2626 u8 csum
[BTRFS_CSUM_SIZE
];
2630 scrub_parity_mark_sectors_error(sparity
, logical
, len
);
2634 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
2635 blocksize
= sctx
->sectorsize
;
2636 } else if (flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) {
2637 blocksize
= sctx
->nodesize
;
2639 blocksize
= sctx
->sectorsize
;
2644 u64 l
= min_t(u64
, len
, blocksize
);
2647 if (flags
& BTRFS_EXTENT_FLAG_DATA
) {
2648 /* push csums to sbio */
2649 have_csum
= scrub_find_csum(sctx
, logical
, csum
);
2653 ret
= scrub_pages_for_parity(sparity
, logical
, l
, physical
, dev
,
2654 flags
, gen
, mirror_num
,
2655 have_csum
? csum
: NULL
);
2667 * Given a physical address, this will calculate it's
2668 * logical offset. if this is a parity stripe, it will return
2669 * the most left data stripe's logical offset.
2671 * return 0 if it is a data stripe, 1 means parity stripe.
2673 static int get_raid56_logic_offset(u64 physical
, int num
,
2674 struct map_lookup
*map
, u64
*offset
,
2684 last_offset
= (physical
- map
->stripes
[num
].physical
) *
2685 nr_data_stripes(map
);
2687 *stripe_start
= last_offset
;
2689 *offset
= last_offset
;
2690 for (i
= 0; i
< nr_data_stripes(map
); i
++) {
2691 *offset
= last_offset
+ i
* map
->stripe_len
;
2693 stripe_nr
= div_u64(*offset
, map
->stripe_len
);
2694 stripe_nr
= div_u64(stripe_nr
, nr_data_stripes(map
));
2696 /* Work out the disk rotation on this stripe-set */
2697 stripe_nr
= div_u64_rem(stripe_nr
, map
->num_stripes
, &rot
);
2698 /* calculate which stripe this data locates */
2700 stripe_index
= rot
% map
->num_stripes
;
2701 if (stripe_index
== num
)
2703 if (stripe_index
< num
)
2706 *offset
= last_offset
+ j
* map
->stripe_len
;
2710 static void scrub_free_parity(struct scrub_parity
*sparity
)
2712 struct scrub_ctx
*sctx
= sparity
->sctx
;
2713 struct scrub_page
*curr
, *next
;
2716 nbits
= bitmap_weight(sparity
->ebitmap
, sparity
->nsectors
);
2718 spin_lock(&sctx
->stat_lock
);
2719 sctx
->stat
.read_errors
+= nbits
;
2720 sctx
->stat
.uncorrectable_errors
+= nbits
;
2721 spin_unlock(&sctx
->stat_lock
);
2724 list_for_each_entry_safe(curr
, next
, &sparity
->spages
, list
) {
2725 list_del_init(&curr
->list
);
2726 scrub_page_put(curr
);
2732 static void scrub_parity_bio_endio_worker(struct btrfs_work
*work
)
2734 struct scrub_parity
*sparity
= container_of(work
, struct scrub_parity
,
2736 struct scrub_ctx
*sctx
= sparity
->sctx
;
2738 scrub_free_parity(sparity
);
2739 scrub_pending_bio_dec(sctx
);
2742 static void scrub_parity_bio_endio(struct bio
*bio
)
2744 struct scrub_parity
*sparity
= (struct scrub_parity
*)bio
->bi_private
;
2747 bitmap_or(sparity
->ebitmap
, sparity
->ebitmap
, sparity
->dbitmap
,
2752 btrfs_init_work(&sparity
->work
, btrfs_scrubparity_helper
,
2753 scrub_parity_bio_endio_worker
, NULL
, NULL
);
2754 btrfs_queue_work(sparity
->sctx
->dev_root
->fs_info
->scrub_parity_workers
,
2758 static void scrub_parity_check_and_repair(struct scrub_parity
*sparity
)
2760 struct scrub_ctx
*sctx
= sparity
->sctx
;
2762 struct btrfs_raid_bio
*rbio
;
2763 struct scrub_page
*spage
;
2764 struct btrfs_bio
*bbio
= NULL
;
2768 if (!bitmap_andnot(sparity
->dbitmap
, sparity
->dbitmap
, sparity
->ebitmap
,
2772 length
= sparity
->logic_end
- sparity
->logic_start
;
2773 ret
= btrfs_map_sblock(sctx
->dev_root
->fs_info
, WRITE
,
2774 sparity
->logic_start
,
2775 &length
, &bbio
, 0, 1);
2776 if (ret
|| !bbio
|| !bbio
->raid_map
)
2779 bio
= btrfs_io_bio_alloc(GFP_NOFS
, 0);
2783 bio
->bi_iter
.bi_sector
= sparity
->logic_start
>> 9;
2784 bio
->bi_private
= sparity
;
2785 bio
->bi_end_io
= scrub_parity_bio_endio
;
2787 rbio
= raid56_parity_alloc_scrub_rbio(sctx
->dev_root
, bio
, bbio
,
2788 length
, sparity
->scrub_dev
,
2794 list_for_each_entry(spage
, &sparity
->spages
, list
)
2795 raid56_add_scrub_pages(rbio
, spage
->page
, spage
->logical
);
2797 scrub_pending_bio_inc(sctx
);
2798 raid56_parity_submit_scrub_rbio(rbio
);
2804 btrfs_put_bbio(bbio
);
2805 bitmap_or(sparity
->ebitmap
, sparity
->ebitmap
, sparity
->dbitmap
,
2807 spin_lock(&sctx
->stat_lock
);
2808 sctx
->stat
.malloc_errors
++;
2809 spin_unlock(&sctx
->stat_lock
);
2811 scrub_free_parity(sparity
);
2814 static inline int scrub_calc_parity_bitmap_len(int nsectors
)
2816 return DIV_ROUND_UP(nsectors
, BITS_PER_LONG
) * sizeof(long);
2819 static void scrub_parity_get(struct scrub_parity
*sparity
)
2821 atomic_inc(&sparity
->refs
);
2824 static void scrub_parity_put(struct scrub_parity
*sparity
)
2826 if (!atomic_dec_and_test(&sparity
->refs
))
2829 scrub_parity_check_and_repair(sparity
);
2832 static noinline_for_stack
int scrub_raid56_parity(struct scrub_ctx
*sctx
,
2833 struct map_lookup
*map
,
2834 struct btrfs_device
*sdev
,
2835 struct btrfs_path
*path
,
2839 struct btrfs_fs_info
*fs_info
= sctx
->dev_root
->fs_info
;
2840 struct btrfs_root
*root
= fs_info
->extent_root
;
2841 struct btrfs_root
*csum_root
= fs_info
->csum_root
;
2842 struct btrfs_extent_item
*extent
;
2843 struct btrfs_bio
*bbio
= NULL
;
2847 struct extent_buffer
*l
;
2848 struct btrfs_key key
;
2851 u64 extent_physical
;
2854 struct btrfs_device
*extent_dev
;
2855 struct scrub_parity
*sparity
;
2858 int extent_mirror_num
;
2861 nsectors
= map
->stripe_len
/ root
->sectorsize
;
2862 bitmap_len
= scrub_calc_parity_bitmap_len(nsectors
);
2863 sparity
= kzalloc(sizeof(struct scrub_parity
) + 2 * bitmap_len
,
2866 spin_lock(&sctx
->stat_lock
);
2867 sctx
->stat
.malloc_errors
++;
2868 spin_unlock(&sctx
->stat_lock
);
2872 sparity
->stripe_len
= map
->stripe_len
;
2873 sparity
->nsectors
= nsectors
;
2874 sparity
->sctx
= sctx
;
2875 sparity
->scrub_dev
= sdev
;
2876 sparity
->logic_start
= logic_start
;
2877 sparity
->logic_end
= logic_end
;
2878 atomic_set(&sparity
->refs
, 1);
2879 INIT_LIST_HEAD(&sparity
->spages
);
2880 sparity
->dbitmap
= sparity
->bitmap
;
2881 sparity
->ebitmap
= (void *)sparity
->bitmap
+ bitmap_len
;
2884 while (logic_start
< logic_end
) {
2885 if (btrfs_fs_incompat(fs_info
, SKINNY_METADATA
))
2886 key
.type
= BTRFS_METADATA_ITEM_KEY
;
2888 key
.type
= BTRFS_EXTENT_ITEM_KEY
;
2889 key
.objectid
= logic_start
;
2890 key
.offset
= (u64
)-1;
2892 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
2897 ret
= btrfs_previous_extent_item(root
, path
, 0);
2901 btrfs_release_path(path
);
2902 ret
= btrfs_search_slot(NULL
, root
, &key
,
2914 slot
= path
->slots
[0];
2915 if (slot
>= btrfs_header_nritems(l
)) {
2916 ret
= btrfs_next_leaf(root
, path
);
2925 btrfs_item_key_to_cpu(l
, &key
, slot
);
2927 if (key
.type
!= BTRFS_EXTENT_ITEM_KEY
&&
2928 key
.type
!= BTRFS_METADATA_ITEM_KEY
)
2931 if (key
.type
== BTRFS_METADATA_ITEM_KEY
)
2932 bytes
= root
->nodesize
;
2936 if (key
.objectid
+ bytes
<= logic_start
)
2939 if (key
.objectid
>= logic_end
) {
2944 while (key
.objectid
>= logic_start
+ map
->stripe_len
)
2945 logic_start
+= map
->stripe_len
;
2947 extent
= btrfs_item_ptr(l
, slot
,
2948 struct btrfs_extent_item
);
2949 flags
= btrfs_extent_flags(l
, extent
);
2950 generation
= btrfs_extent_generation(l
, extent
);
2952 if ((flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) &&
2953 (key
.objectid
< logic_start
||
2954 key
.objectid
+ bytes
>
2955 logic_start
+ map
->stripe_len
)) {
2956 btrfs_err(fs_info
, "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
2957 key
.objectid
, logic_start
);
2958 spin_lock(&sctx
->stat_lock
);
2959 sctx
->stat
.uncorrectable_errors
++;
2960 spin_unlock(&sctx
->stat_lock
);
2964 extent_logical
= key
.objectid
;
2967 if (extent_logical
< logic_start
) {
2968 extent_len
-= logic_start
- extent_logical
;
2969 extent_logical
= logic_start
;
2972 if (extent_logical
+ extent_len
>
2973 logic_start
+ map
->stripe_len
)
2974 extent_len
= logic_start
+ map
->stripe_len
-
2977 scrub_parity_mark_sectors_data(sparity
, extent_logical
,
2980 mapped_length
= extent_len
;
2981 ret
= btrfs_map_block(fs_info
, READ
, extent_logical
,
2982 &mapped_length
, &bbio
, 0);
2984 if (!bbio
|| mapped_length
< extent_len
)
2988 btrfs_put_bbio(bbio
);
2991 extent_physical
= bbio
->stripes
[0].physical
;
2992 extent_mirror_num
= bbio
->mirror_num
;
2993 extent_dev
= bbio
->stripes
[0].dev
;
2994 btrfs_put_bbio(bbio
);
2996 ret
= btrfs_lookup_csums_range(csum_root
,
2998 extent_logical
+ extent_len
- 1,
2999 &sctx
->csum_list
, 1);
3003 ret
= scrub_extent_for_parity(sparity
, extent_logical
,
3010 scrub_free_csums(sctx
);
3015 if (extent_logical
+ extent_len
<
3016 key
.objectid
+ bytes
) {
3017 logic_start
+= map
->stripe_len
;
3019 if (logic_start
>= logic_end
) {
3024 if (logic_start
< key
.objectid
+ bytes
) {
3033 btrfs_release_path(path
);
3038 logic_start
+= map
->stripe_len
;
3042 scrub_parity_mark_sectors_error(sparity
, logic_start
,
3043 logic_end
- logic_start
);
3044 scrub_parity_put(sparity
);
3046 mutex_lock(&sctx
->wr_ctx
.wr_lock
);
3047 scrub_wr_submit(sctx
);
3048 mutex_unlock(&sctx
->wr_ctx
.wr_lock
);
3050 btrfs_release_path(path
);
3051 return ret
< 0 ? ret
: 0;
3054 static noinline_for_stack
int scrub_stripe(struct scrub_ctx
*sctx
,
3055 struct map_lookup
*map
,
3056 struct btrfs_device
*scrub_dev
,
3057 int num
, u64 base
, u64 length
,
3060 struct btrfs_path
*path
, *ppath
;
3061 struct btrfs_fs_info
*fs_info
= sctx
->dev_root
->fs_info
;
3062 struct btrfs_root
*root
= fs_info
->extent_root
;
3063 struct btrfs_root
*csum_root
= fs_info
->csum_root
;
3064 struct btrfs_extent_item
*extent
;
3065 struct blk_plug plug
;
3070 struct extent_buffer
*l
;
3071 struct btrfs_key key
;
3078 struct reada_control
*reada1
;
3079 struct reada_control
*reada2
;
3080 struct btrfs_key key_start
;
3081 struct btrfs_key key_end
;
3082 u64 increment
= map
->stripe_len
;
3085 u64 extent_physical
;
3089 struct btrfs_device
*extent_dev
;
3090 int extent_mirror_num
;
3093 physical
= map
->stripes
[num
].physical
;
3095 nstripes
= div_u64(length
, map
->stripe_len
);
3096 if (map
->type
& BTRFS_BLOCK_GROUP_RAID0
) {
3097 offset
= map
->stripe_len
* num
;
3098 increment
= map
->stripe_len
* map
->num_stripes
;
3100 } else if (map
->type
& BTRFS_BLOCK_GROUP_RAID10
) {
3101 int factor
= map
->num_stripes
/ map
->sub_stripes
;
3102 offset
= map
->stripe_len
* (num
/ map
->sub_stripes
);
3103 increment
= map
->stripe_len
* factor
;
3104 mirror_num
= num
% map
->sub_stripes
+ 1;
3105 } else if (map
->type
& BTRFS_BLOCK_GROUP_RAID1
) {
3106 increment
= map
->stripe_len
;
3107 mirror_num
= num
% map
->num_stripes
+ 1;
3108 } else if (map
->type
& BTRFS_BLOCK_GROUP_DUP
) {
3109 increment
= map
->stripe_len
;
3110 mirror_num
= num
% map
->num_stripes
+ 1;
3111 } else if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
3112 get_raid56_logic_offset(physical
, num
, map
, &offset
, NULL
);
3113 increment
= map
->stripe_len
* nr_data_stripes(map
);
3116 increment
= map
->stripe_len
;
3120 path
= btrfs_alloc_path();
3124 ppath
= btrfs_alloc_path();
3126 btrfs_free_path(path
);
3131 * work on commit root. The related disk blocks are static as
3132 * long as COW is applied. This means, it is save to rewrite
3133 * them to repair disk errors without any race conditions
3135 path
->search_commit_root
= 1;
3136 path
->skip_locking
= 1;
3138 ppath
->search_commit_root
= 1;
3139 ppath
->skip_locking
= 1;
3141 * trigger the readahead for extent tree csum tree and wait for
3142 * completion. During readahead, the scrub is officially paused
3143 * to not hold off transaction commits
3145 logical
= base
+ offset
;
3146 physical_end
= physical
+ nstripes
* map
->stripe_len
;
3147 if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
3148 get_raid56_logic_offset(physical_end
, num
,
3149 map
, &logic_end
, NULL
);
3152 logic_end
= logical
+ increment
* nstripes
;
3154 wait_event(sctx
->list_wait
,
3155 atomic_read(&sctx
->bios_in_flight
) == 0);
3156 scrub_blocked_if_needed(fs_info
);
3158 /* FIXME it might be better to start readahead at commit root */
3159 key_start
.objectid
= logical
;
3160 key_start
.type
= BTRFS_EXTENT_ITEM_KEY
;
3161 key_start
.offset
= (u64
)0;
3162 key_end
.objectid
= logic_end
;
3163 key_end
.type
= BTRFS_METADATA_ITEM_KEY
;
3164 key_end
.offset
= (u64
)-1;
3165 reada1
= btrfs_reada_add(root
, &key_start
, &key_end
);
3167 key_start
.objectid
= BTRFS_EXTENT_CSUM_OBJECTID
;
3168 key_start
.type
= BTRFS_EXTENT_CSUM_KEY
;
3169 key_start
.offset
= logical
;
3170 key_end
.objectid
= BTRFS_EXTENT_CSUM_OBJECTID
;
3171 key_end
.type
= BTRFS_EXTENT_CSUM_KEY
;
3172 key_end
.offset
= logic_end
;
3173 reada2
= btrfs_reada_add(csum_root
, &key_start
, &key_end
);
3175 if (!IS_ERR(reada1
))
3176 btrfs_reada_wait(reada1
);
3177 if (!IS_ERR(reada2
))
3178 btrfs_reada_wait(reada2
);
3182 * collect all data csums for the stripe to avoid seeking during
3183 * the scrub. This might currently (crc32) end up to be about 1MB
3185 blk_start_plug(&plug
);
3188 * now find all extents for each stripe and scrub them
3191 while (physical
< physical_end
) {
3195 if (atomic_read(&fs_info
->scrub_cancel_req
) ||
3196 atomic_read(&sctx
->cancel_req
)) {
3201 * check to see if we have to pause
3203 if (atomic_read(&fs_info
->scrub_pause_req
)) {
3204 /* push queued extents */
3205 atomic_set(&sctx
->wr_ctx
.flush_all_writes
, 1);
3207 mutex_lock(&sctx
->wr_ctx
.wr_lock
);
3208 scrub_wr_submit(sctx
);
3209 mutex_unlock(&sctx
->wr_ctx
.wr_lock
);
3210 wait_event(sctx
->list_wait
,
3211 atomic_read(&sctx
->bios_in_flight
) == 0);
3212 atomic_set(&sctx
->wr_ctx
.flush_all_writes
, 0);
3213 scrub_blocked_if_needed(fs_info
);
3216 if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
3217 ret
= get_raid56_logic_offset(physical
, num
, map
,
3222 /* it is parity strip */
3223 stripe_logical
+= base
;
3224 stripe_end
= stripe_logical
+ increment
;
3225 ret
= scrub_raid56_parity(sctx
, map
, scrub_dev
,
3226 ppath
, stripe_logical
,
3234 if (btrfs_fs_incompat(fs_info
, SKINNY_METADATA
))
3235 key
.type
= BTRFS_METADATA_ITEM_KEY
;
3237 key
.type
= BTRFS_EXTENT_ITEM_KEY
;
3238 key
.objectid
= logical
;
3239 key
.offset
= (u64
)-1;
3241 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
3246 ret
= btrfs_previous_extent_item(root
, path
, 0);
3250 /* there's no smaller item, so stick with the
3252 btrfs_release_path(path
);
3253 ret
= btrfs_search_slot(NULL
, root
, &key
,
3265 slot
= path
->slots
[0];
3266 if (slot
>= btrfs_header_nritems(l
)) {
3267 ret
= btrfs_next_leaf(root
, path
);
3276 btrfs_item_key_to_cpu(l
, &key
, slot
);
3278 if (key
.type
!= BTRFS_EXTENT_ITEM_KEY
&&
3279 key
.type
!= BTRFS_METADATA_ITEM_KEY
)
3282 if (key
.type
== BTRFS_METADATA_ITEM_KEY
)
3283 bytes
= root
->nodesize
;
3287 if (key
.objectid
+ bytes
<= logical
)
3290 if (key
.objectid
>= logical
+ map
->stripe_len
) {
3291 /* out of this device extent */
3292 if (key
.objectid
>= logic_end
)
3297 extent
= btrfs_item_ptr(l
, slot
,
3298 struct btrfs_extent_item
);
3299 flags
= btrfs_extent_flags(l
, extent
);
3300 generation
= btrfs_extent_generation(l
, extent
);
3302 if ((flags
& BTRFS_EXTENT_FLAG_TREE_BLOCK
) &&
3303 (key
.objectid
< logical
||
3304 key
.objectid
+ bytes
>
3305 logical
+ map
->stripe_len
)) {
3307 "scrub: tree block %llu spanning "
3308 "stripes, ignored. logical=%llu",
3309 key
.objectid
, logical
);
3310 spin_lock(&sctx
->stat_lock
);
3311 sctx
->stat
.uncorrectable_errors
++;
3312 spin_unlock(&sctx
->stat_lock
);
3317 extent_logical
= key
.objectid
;
3321 * trim extent to this stripe
3323 if (extent_logical
< logical
) {
3324 extent_len
-= logical
- extent_logical
;
3325 extent_logical
= logical
;
3327 if (extent_logical
+ extent_len
>
3328 logical
+ map
->stripe_len
) {
3329 extent_len
= logical
+ map
->stripe_len
-
3333 extent_physical
= extent_logical
- logical
+ physical
;
3334 extent_dev
= scrub_dev
;
3335 extent_mirror_num
= mirror_num
;
3337 scrub_remap_extent(fs_info
, extent_logical
,
3338 extent_len
, &extent_physical
,
3340 &extent_mirror_num
);
3342 ret
= btrfs_lookup_csums_range(csum_root
,
3346 &sctx
->csum_list
, 1);
3350 ret
= scrub_extent(sctx
, extent_logical
, extent_len
,
3351 extent_physical
, extent_dev
, flags
,
3352 generation
, extent_mirror_num
,
3353 extent_logical
- logical
+ physical
);
3355 scrub_free_csums(sctx
);
3360 if (extent_logical
+ extent_len
<
3361 key
.objectid
+ bytes
) {
3362 if (map
->type
& BTRFS_BLOCK_GROUP_RAID56_MASK
) {
3364 * loop until we find next data stripe
3365 * or we have finished all stripes.
3368 physical
+= map
->stripe_len
;
3369 ret
= get_raid56_logic_offset(physical
,
3374 if (ret
&& physical
< physical_end
) {
3375 stripe_logical
+= base
;
3376 stripe_end
= stripe_logical
+
3378 ret
= scrub_raid56_parity(sctx
,
3379 map
, scrub_dev
, ppath
,
3387 physical
+= map
->stripe_len
;
3388 logical
+= increment
;
3390 if (logical
< key
.objectid
+ bytes
) {
3395 if (physical
>= physical_end
) {
3403 btrfs_release_path(path
);
3405 logical
+= increment
;
3406 physical
+= map
->stripe_len
;
3407 spin_lock(&sctx
->stat_lock
);
3409 sctx
->stat
.last_physical
= map
->stripes
[num
].physical
+
3412 sctx
->stat
.last_physical
= physical
;
3413 spin_unlock(&sctx
->stat_lock
);
3418 /* push queued extents */
3420 mutex_lock(&sctx
->wr_ctx
.wr_lock
);
3421 scrub_wr_submit(sctx
);
3422 mutex_unlock(&sctx
->wr_ctx
.wr_lock
);
3424 blk_finish_plug(&plug
);
3425 btrfs_free_path(path
);
3426 btrfs_free_path(ppath
);
3427 return ret
< 0 ? ret
: 0;
3430 static noinline_for_stack
int scrub_chunk(struct scrub_ctx
*sctx
,
3431 struct btrfs_device
*scrub_dev
,
3432 u64 chunk_offset
, u64 length
,
3434 struct btrfs_block_group_cache
*cache
,
3437 struct btrfs_mapping_tree
*map_tree
=
3438 &sctx
->dev_root
->fs_info
->mapping_tree
;
3439 struct map_lookup
*map
;
3440 struct extent_map
*em
;
3444 read_lock(&map_tree
->map_tree
.lock
);
3445 em
= lookup_extent_mapping(&map_tree
->map_tree
, chunk_offset
, 1);
3446 read_unlock(&map_tree
->map_tree
.lock
);
3450 * Might have been an unused block group deleted by the cleaner
3451 * kthread or relocation.
3453 spin_lock(&cache
->lock
);
3454 if (!cache
->removed
)
3456 spin_unlock(&cache
->lock
);
3461 map
= em
->map_lookup
;
3462 if (em
->start
!= chunk_offset
)
3465 if (em
->len
< length
)
3468 for (i
= 0; i
< map
->num_stripes
; ++i
) {
3469 if (map
->stripes
[i
].dev
->bdev
== scrub_dev
->bdev
&&
3470 map
->stripes
[i
].physical
== dev_offset
) {
3471 ret
= scrub_stripe(sctx
, map
, scrub_dev
, i
,
3472 chunk_offset
, length
,
3479 free_extent_map(em
);
3484 static noinline_for_stack
3485 int scrub_enumerate_chunks(struct scrub_ctx
*sctx
,
3486 struct btrfs_device
*scrub_dev
, u64 start
, u64 end
,
3489 struct btrfs_dev_extent
*dev_extent
= NULL
;
3490 struct btrfs_path
*path
;
3491 struct btrfs_root
*root
= sctx
->dev_root
;
3492 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
3498 struct extent_buffer
*l
;
3499 struct btrfs_key key
;
3500 struct btrfs_key found_key
;
3501 struct btrfs_block_group_cache
*cache
;
3502 struct btrfs_dev_replace
*dev_replace
= &fs_info
->dev_replace
;
3504 path
= btrfs_alloc_path();
3508 path
->reada
= READA_FORWARD
;
3509 path
->search_commit_root
= 1;
3510 path
->skip_locking
= 1;
3512 key
.objectid
= scrub_dev
->devid
;
3514 key
.type
= BTRFS_DEV_EXTENT_KEY
;
3517 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
3521 if (path
->slots
[0] >=
3522 btrfs_header_nritems(path
->nodes
[0])) {
3523 ret
= btrfs_next_leaf(root
, path
);
3536 slot
= path
->slots
[0];
3538 btrfs_item_key_to_cpu(l
, &found_key
, slot
);
3540 if (found_key
.objectid
!= scrub_dev
->devid
)
3543 if (found_key
.type
!= BTRFS_DEV_EXTENT_KEY
)
3546 if (found_key
.offset
>= end
)
3549 if (found_key
.offset
< key
.offset
)
3552 dev_extent
= btrfs_item_ptr(l
, slot
, struct btrfs_dev_extent
);
3553 length
= btrfs_dev_extent_length(l
, dev_extent
);
3555 if (found_key
.offset
+ length
<= start
)
3558 chunk_offset
= btrfs_dev_extent_chunk_offset(l
, dev_extent
);
3561 * get a reference on the corresponding block group to prevent
3562 * the chunk from going away while we scrub it
3564 cache
= btrfs_lookup_block_group(fs_info
, chunk_offset
);
3566 /* some chunks are removed but not committed to disk yet,
3567 * continue scrubbing */
3572 * we need call btrfs_inc_block_group_ro() with scrubs_paused,
3573 * to avoid deadlock caused by:
3574 * btrfs_inc_block_group_ro()
3575 * -> btrfs_wait_for_commit()
3576 * -> btrfs_commit_transaction()
3577 * -> btrfs_scrub_pause()
3579 scrub_pause_on(fs_info
);
3580 ret
= btrfs_inc_block_group_ro(root
, cache
);
3581 scrub_pause_off(fs_info
);
3585 } else if (ret
== -ENOSPC
) {
3587 * btrfs_inc_block_group_ro return -ENOSPC when it
3588 * failed in creating new chunk for metadata.
3589 * It is not a problem for scrub/replace, because
3590 * metadata are always cowed, and our scrub paused
3591 * commit_transactions.
3595 btrfs_warn(fs_info
, "failed setting block group ro, ret=%d\n",
3597 btrfs_put_block_group(cache
);
3601 dev_replace
->cursor_right
= found_key
.offset
+ length
;
3602 dev_replace
->cursor_left
= found_key
.offset
;
3603 dev_replace
->item_needs_writeback
= 1;
3604 ret
= scrub_chunk(sctx
, scrub_dev
, chunk_offset
, length
,
3605 found_key
.offset
, cache
, is_dev_replace
);
3608 * flush, submit all pending read and write bios, afterwards
3610 * Note that in the dev replace case, a read request causes
3611 * write requests that are submitted in the read completion
3612 * worker. Therefore in the current situation, it is required
3613 * that all write requests are flushed, so that all read and
3614 * write requests are really completed when bios_in_flight
3617 atomic_set(&sctx
->wr_ctx
.flush_all_writes
, 1);
3619 mutex_lock(&sctx
->wr_ctx
.wr_lock
);
3620 scrub_wr_submit(sctx
);
3621 mutex_unlock(&sctx
->wr_ctx
.wr_lock
);
3623 wait_event(sctx
->list_wait
,
3624 atomic_read(&sctx
->bios_in_flight
) == 0);
3626 scrub_pause_on(fs_info
);
3629 * must be called before we decrease @scrub_paused.
3630 * make sure we don't block transaction commit while
3631 * we are waiting pending workers finished.
3633 wait_event(sctx
->list_wait
,
3634 atomic_read(&sctx
->workers_pending
) == 0);
3635 atomic_set(&sctx
->wr_ctx
.flush_all_writes
, 0);
3637 scrub_pause_off(fs_info
);
3640 btrfs_dec_block_group_ro(root
, cache
);
3643 * We might have prevented the cleaner kthread from deleting
3644 * this block group if it was already unused because we raced
3645 * and set it to RO mode first. So add it back to the unused
3646 * list, otherwise it might not ever be deleted unless a manual
3647 * balance is triggered or it becomes used and unused again.
3649 spin_lock(&cache
->lock
);
3650 if (!cache
->removed
&& !cache
->ro
&& cache
->reserved
== 0 &&
3651 btrfs_block_group_used(&cache
->item
) == 0) {
3652 spin_unlock(&cache
->lock
);
3653 spin_lock(&fs_info
->unused_bgs_lock
);
3654 if (list_empty(&cache
->bg_list
)) {
3655 btrfs_get_block_group(cache
);
3656 list_add_tail(&cache
->bg_list
,
3657 &fs_info
->unused_bgs
);
3659 spin_unlock(&fs_info
->unused_bgs_lock
);
3661 spin_unlock(&cache
->lock
);
3664 btrfs_put_block_group(cache
);
3667 if (is_dev_replace
&&
3668 atomic64_read(&dev_replace
->num_write_errors
) > 0) {
3672 if (sctx
->stat
.malloc_errors
> 0) {
3677 dev_replace
->cursor_left
= dev_replace
->cursor_right
;
3678 dev_replace
->item_needs_writeback
= 1;
3680 key
.offset
= found_key
.offset
+ length
;
3681 btrfs_release_path(path
);
3684 btrfs_free_path(path
);
3689 static noinline_for_stack
int scrub_supers(struct scrub_ctx
*sctx
,
3690 struct btrfs_device
*scrub_dev
)
3696 struct btrfs_root
*root
= sctx
->dev_root
;
3698 if (test_bit(BTRFS_FS_STATE_ERROR
, &root
->fs_info
->fs_state
))
3701 /* Seed devices of a new filesystem has their own generation. */
3702 if (scrub_dev
->fs_devices
!= root
->fs_info
->fs_devices
)
3703 gen
= scrub_dev
->generation
;
3705 gen
= root
->fs_info
->last_trans_committed
;
3707 for (i
= 0; i
< BTRFS_SUPER_MIRROR_MAX
; i
++) {
3708 bytenr
= btrfs_sb_offset(i
);
3709 if (bytenr
+ BTRFS_SUPER_INFO_SIZE
>
3710 scrub_dev
->commit_total_bytes
)
3713 ret
= scrub_pages(sctx
, bytenr
, BTRFS_SUPER_INFO_SIZE
, bytenr
,
3714 scrub_dev
, BTRFS_EXTENT_FLAG_SUPER
, gen
, i
,
3719 wait_event(sctx
->list_wait
, atomic_read(&sctx
->bios_in_flight
) == 0);
3725 * get a reference count on fs_info->scrub_workers. start worker if necessary
3727 static noinline_for_stack
int scrub_workers_get(struct btrfs_fs_info
*fs_info
,
3730 unsigned int flags
= WQ_FREEZABLE
| WQ_UNBOUND
;
3731 int max_active
= fs_info
->thread_pool_size
;
3733 if (fs_info
->scrub_workers_refcnt
== 0) {
3735 fs_info
->scrub_workers
=
3736 btrfs_alloc_workqueue("scrub", flags
,
3739 fs_info
->scrub_workers
=
3740 btrfs_alloc_workqueue("scrub", flags
,
3742 if (!fs_info
->scrub_workers
)
3743 goto fail_scrub_workers
;
3745 fs_info
->scrub_wr_completion_workers
=
3746 btrfs_alloc_workqueue("scrubwrc", flags
,
3748 if (!fs_info
->scrub_wr_completion_workers
)
3749 goto fail_scrub_wr_completion_workers
;
3751 fs_info
->scrub_nocow_workers
=
3752 btrfs_alloc_workqueue("scrubnc", flags
, 1, 0);
3753 if (!fs_info
->scrub_nocow_workers
)
3754 goto fail_scrub_nocow_workers
;
3755 fs_info
->scrub_parity_workers
=
3756 btrfs_alloc_workqueue("scrubparity", flags
,
3758 if (!fs_info
->scrub_parity_workers
)
3759 goto fail_scrub_parity_workers
;
3761 ++fs_info
->scrub_workers_refcnt
;
3764 fail_scrub_parity_workers
:
3765 btrfs_destroy_workqueue(fs_info
->scrub_nocow_workers
);
3766 fail_scrub_nocow_workers
:
3767 btrfs_destroy_workqueue(fs_info
->scrub_wr_completion_workers
);
3768 fail_scrub_wr_completion_workers
:
3769 btrfs_destroy_workqueue(fs_info
->scrub_workers
);
3774 static noinline_for_stack
void scrub_workers_put(struct btrfs_fs_info
*fs_info
)
3776 if (--fs_info
->scrub_workers_refcnt
== 0) {
3777 btrfs_destroy_workqueue(fs_info
->scrub_workers
);
3778 btrfs_destroy_workqueue(fs_info
->scrub_wr_completion_workers
);
3779 btrfs_destroy_workqueue(fs_info
->scrub_nocow_workers
);
3780 btrfs_destroy_workqueue(fs_info
->scrub_parity_workers
);
3782 WARN_ON(fs_info
->scrub_workers_refcnt
< 0);
3785 int btrfs_scrub_dev(struct btrfs_fs_info
*fs_info
, u64 devid
, u64 start
,
3786 u64 end
, struct btrfs_scrub_progress
*progress
,
3787 int readonly
, int is_dev_replace
)
3789 struct scrub_ctx
*sctx
;
3791 struct btrfs_device
*dev
;
3792 struct rcu_string
*name
;
3794 if (btrfs_fs_closing(fs_info
))
3797 if (fs_info
->chunk_root
->nodesize
> BTRFS_STRIPE_LEN
) {
3799 * in this case scrub is unable to calculate the checksum
3800 * the way scrub is implemented. Do not handle this
3801 * situation at all because it won't ever happen.
3804 "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
3805 fs_info
->chunk_root
->nodesize
, BTRFS_STRIPE_LEN
);
3809 if (fs_info
->chunk_root
->sectorsize
!= PAGE_SIZE
) {
3810 /* not supported for data w/o checksums */
3812 "scrub: size assumption sectorsize != PAGE_SIZE "
3813 "(%d != %lu) fails",
3814 fs_info
->chunk_root
->sectorsize
, PAGE_SIZE
);
3818 if (fs_info
->chunk_root
->nodesize
>
3819 PAGE_SIZE
* SCRUB_MAX_PAGES_PER_BLOCK
||
3820 fs_info
->chunk_root
->sectorsize
>
3821 PAGE_SIZE
* SCRUB_MAX_PAGES_PER_BLOCK
) {
3823 * would exhaust the array bounds of pagev member in
3824 * struct scrub_block
3826 btrfs_err(fs_info
, "scrub: size assumption nodesize and sectorsize "
3827 "<= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
3828 fs_info
->chunk_root
->nodesize
,
3829 SCRUB_MAX_PAGES_PER_BLOCK
,
3830 fs_info
->chunk_root
->sectorsize
,
3831 SCRUB_MAX_PAGES_PER_BLOCK
);
3836 mutex_lock(&fs_info
->fs_devices
->device_list_mutex
);
3837 dev
= btrfs_find_device(fs_info
, devid
, NULL
, NULL
);
3838 if (!dev
|| (dev
->missing
&& !is_dev_replace
)) {
3839 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3843 if (!is_dev_replace
&& !readonly
&& !dev
->writeable
) {
3844 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3846 name
= rcu_dereference(dev
->name
);
3847 btrfs_err(fs_info
, "scrub: device %s is not writable",
3853 mutex_lock(&fs_info
->scrub_lock
);
3854 if (!dev
->in_fs_metadata
|| dev
->is_tgtdev_for_dev_replace
) {
3855 mutex_unlock(&fs_info
->scrub_lock
);
3856 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3860 btrfs_dev_replace_lock(&fs_info
->dev_replace
);
3861 if (dev
->scrub_device
||
3863 btrfs_dev_replace_is_ongoing(&fs_info
->dev_replace
))) {
3864 btrfs_dev_replace_unlock(&fs_info
->dev_replace
);
3865 mutex_unlock(&fs_info
->scrub_lock
);
3866 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3867 return -EINPROGRESS
;
3869 btrfs_dev_replace_unlock(&fs_info
->dev_replace
);
3871 ret
= scrub_workers_get(fs_info
, is_dev_replace
);
3873 mutex_unlock(&fs_info
->scrub_lock
);
3874 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3878 sctx
= scrub_setup_ctx(dev
, is_dev_replace
);
3880 mutex_unlock(&fs_info
->scrub_lock
);
3881 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3882 scrub_workers_put(fs_info
);
3883 return PTR_ERR(sctx
);
3885 sctx
->readonly
= readonly
;
3886 dev
->scrub_device
= sctx
;
3887 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3890 * checking @scrub_pause_req here, we can avoid
3891 * race between committing transaction and scrubbing.
3893 __scrub_blocked_if_needed(fs_info
);
3894 atomic_inc(&fs_info
->scrubs_running
);
3895 mutex_unlock(&fs_info
->scrub_lock
);
3897 if (!is_dev_replace
) {
3899 * by holding device list mutex, we can
3900 * kick off writing super in log tree sync.
3902 mutex_lock(&fs_info
->fs_devices
->device_list_mutex
);
3903 ret
= scrub_supers(sctx
, dev
);
3904 mutex_unlock(&fs_info
->fs_devices
->device_list_mutex
);
3908 ret
= scrub_enumerate_chunks(sctx
, dev
, start
, end
,
3911 wait_event(sctx
->list_wait
, atomic_read(&sctx
->bios_in_flight
) == 0);
3912 atomic_dec(&fs_info
->scrubs_running
);
3913 wake_up(&fs_info
->scrub_pause_wait
);
3915 wait_event(sctx
->list_wait
, atomic_read(&sctx
->workers_pending
) == 0);
3918 memcpy(progress
, &sctx
->stat
, sizeof(*progress
));
3920 mutex_lock(&fs_info
->scrub_lock
);
3921 dev
->scrub_device
= NULL
;
3922 scrub_workers_put(fs_info
);
3923 mutex_unlock(&fs_info
->scrub_lock
);
3925 scrub_put_ctx(sctx
);
3930 void btrfs_scrub_pause(struct btrfs_root
*root
)
3932 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
3934 mutex_lock(&fs_info
->scrub_lock
);
3935 atomic_inc(&fs_info
->scrub_pause_req
);
3936 while (atomic_read(&fs_info
->scrubs_paused
) !=
3937 atomic_read(&fs_info
->scrubs_running
)) {
3938 mutex_unlock(&fs_info
->scrub_lock
);
3939 wait_event(fs_info
->scrub_pause_wait
,
3940 atomic_read(&fs_info
->scrubs_paused
) ==
3941 atomic_read(&fs_info
->scrubs_running
));
3942 mutex_lock(&fs_info
->scrub_lock
);
3944 mutex_unlock(&fs_info
->scrub_lock
);
3947 void btrfs_scrub_continue(struct btrfs_root
*root
)
3949 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
3951 atomic_dec(&fs_info
->scrub_pause_req
);
3952 wake_up(&fs_info
->scrub_pause_wait
);
3955 int btrfs_scrub_cancel(struct btrfs_fs_info
*fs_info
)
3957 mutex_lock(&fs_info
->scrub_lock
);
3958 if (!atomic_read(&fs_info
->scrubs_running
)) {
3959 mutex_unlock(&fs_info
->scrub_lock
);
3963 atomic_inc(&fs_info
->scrub_cancel_req
);
3964 while (atomic_read(&fs_info
->scrubs_running
)) {
3965 mutex_unlock(&fs_info
->scrub_lock
);
3966 wait_event(fs_info
->scrub_pause_wait
,
3967 atomic_read(&fs_info
->scrubs_running
) == 0);
3968 mutex_lock(&fs_info
->scrub_lock
);
3970 atomic_dec(&fs_info
->scrub_cancel_req
);
3971 mutex_unlock(&fs_info
->scrub_lock
);
3976 int btrfs_scrub_cancel_dev(struct btrfs_fs_info
*fs_info
,
3977 struct btrfs_device
*dev
)
3979 struct scrub_ctx
*sctx
;
3981 mutex_lock(&fs_info
->scrub_lock
);
3982 sctx
= dev
->scrub_device
;
3984 mutex_unlock(&fs_info
->scrub_lock
);
3987 atomic_inc(&sctx
->cancel_req
);
3988 while (dev
->scrub_device
) {
3989 mutex_unlock(&fs_info
->scrub_lock
);
3990 wait_event(fs_info
->scrub_pause_wait
,
3991 dev
->scrub_device
== NULL
);
3992 mutex_lock(&fs_info
->scrub_lock
);
3994 mutex_unlock(&fs_info
->scrub_lock
);
3999 int btrfs_scrub_progress(struct btrfs_root
*root
, u64 devid
,
4000 struct btrfs_scrub_progress
*progress
)
4002 struct btrfs_device
*dev
;
4003 struct scrub_ctx
*sctx
= NULL
;
4005 mutex_lock(&root
->fs_info
->fs_devices
->device_list_mutex
);
4006 dev
= btrfs_find_device(root
->fs_info
, devid
, NULL
, NULL
);
4008 sctx
= dev
->scrub_device
;
4010 memcpy(progress
, &sctx
->stat
, sizeof(*progress
));
4011 mutex_unlock(&root
->fs_info
->fs_devices
->device_list_mutex
);
4013 return dev
? (sctx
? 0 : -ENOTCONN
) : -ENODEV
;
4016 static void scrub_remap_extent(struct btrfs_fs_info
*fs_info
,
4017 u64 extent_logical
, u64 extent_len
,
4018 u64
*extent_physical
,
4019 struct btrfs_device
**extent_dev
,
4020 int *extent_mirror_num
)
4023 struct btrfs_bio
*bbio
= NULL
;
4026 mapped_length
= extent_len
;
4027 ret
= btrfs_map_block(fs_info
, READ
, extent_logical
,
4028 &mapped_length
, &bbio
, 0);
4029 if (ret
|| !bbio
|| mapped_length
< extent_len
||
4030 !bbio
->stripes
[0].dev
->bdev
) {
4031 btrfs_put_bbio(bbio
);
4035 *extent_physical
= bbio
->stripes
[0].physical
;
4036 *extent_mirror_num
= bbio
->mirror_num
;
4037 *extent_dev
= bbio
->stripes
[0].dev
;
4038 btrfs_put_bbio(bbio
);
4041 static int scrub_setup_wr_ctx(struct scrub_ctx
*sctx
,
4042 struct scrub_wr_ctx
*wr_ctx
,
4043 struct btrfs_fs_info
*fs_info
,
4044 struct btrfs_device
*dev
,
4047 WARN_ON(wr_ctx
->wr_curr_bio
!= NULL
);
4049 mutex_init(&wr_ctx
->wr_lock
);
4050 wr_ctx
->wr_curr_bio
= NULL
;
4051 if (!is_dev_replace
)
4054 WARN_ON(!dev
->bdev
);
4055 wr_ctx
->pages_per_wr_bio
= SCRUB_PAGES_PER_WR_BIO
;
4056 wr_ctx
->tgtdev
= dev
;
4057 atomic_set(&wr_ctx
->flush_all_writes
, 0);
4061 static void scrub_free_wr_ctx(struct scrub_wr_ctx
*wr_ctx
)
4063 mutex_lock(&wr_ctx
->wr_lock
);
4064 kfree(wr_ctx
->wr_curr_bio
);
4065 wr_ctx
->wr_curr_bio
= NULL
;
4066 mutex_unlock(&wr_ctx
->wr_lock
);
4069 static int copy_nocow_pages(struct scrub_ctx
*sctx
, u64 logical
, u64 len
,
4070 int mirror_num
, u64 physical_for_dev_replace
)
4072 struct scrub_copy_nocow_ctx
*nocow_ctx
;
4073 struct btrfs_fs_info
*fs_info
= sctx
->dev_root
->fs_info
;
4075 nocow_ctx
= kzalloc(sizeof(*nocow_ctx
), GFP_NOFS
);
4077 spin_lock(&sctx
->stat_lock
);
4078 sctx
->stat
.malloc_errors
++;
4079 spin_unlock(&sctx
->stat_lock
);
4083 scrub_pending_trans_workers_inc(sctx
);
4085 nocow_ctx
->sctx
= sctx
;
4086 nocow_ctx
->logical
= logical
;
4087 nocow_ctx
->len
= len
;
4088 nocow_ctx
->mirror_num
= mirror_num
;
4089 nocow_ctx
->physical_for_dev_replace
= physical_for_dev_replace
;
4090 btrfs_init_work(&nocow_ctx
->work
, btrfs_scrubnc_helper
,
4091 copy_nocow_pages_worker
, NULL
, NULL
);
4092 INIT_LIST_HEAD(&nocow_ctx
->inodes
);
4093 btrfs_queue_work(fs_info
->scrub_nocow_workers
,
4099 static int record_inode_for_nocow(u64 inum
, u64 offset
, u64 root
, void *ctx
)
4101 struct scrub_copy_nocow_ctx
*nocow_ctx
= ctx
;
4102 struct scrub_nocow_inode
*nocow_inode
;
4104 nocow_inode
= kzalloc(sizeof(*nocow_inode
), GFP_NOFS
);
4107 nocow_inode
->inum
= inum
;
4108 nocow_inode
->offset
= offset
;
4109 nocow_inode
->root
= root
;
4110 list_add_tail(&nocow_inode
->list
, &nocow_ctx
->inodes
);
4114 #define COPY_COMPLETE 1
4116 static void copy_nocow_pages_worker(struct btrfs_work
*work
)
4118 struct scrub_copy_nocow_ctx
*nocow_ctx
=
4119 container_of(work
, struct scrub_copy_nocow_ctx
, work
);
4120 struct scrub_ctx
*sctx
= nocow_ctx
->sctx
;
4121 u64 logical
= nocow_ctx
->logical
;
4122 u64 len
= nocow_ctx
->len
;
4123 int mirror_num
= nocow_ctx
->mirror_num
;
4124 u64 physical_for_dev_replace
= nocow_ctx
->physical_for_dev_replace
;
4126 struct btrfs_trans_handle
*trans
= NULL
;
4127 struct btrfs_fs_info
*fs_info
;
4128 struct btrfs_path
*path
;
4129 struct btrfs_root
*root
;
4130 int not_written
= 0;
4132 fs_info
= sctx
->dev_root
->fs_info
;
4133 root
= fs_info
->extent_root
;
4135 path
= btrfs_alloc_path();
4137 spin_lock(&sctx
->stat_lock
);
4138 sctx
->stat
.malloc_errors
++;
4139 spin_unlock(&sctx
->stat_lock
);
4144 trans
= btrfs_join_transaction(root
);
4145 if (IS_ERR(trans
)) {
4150 ret
= iterate_inodes_from_logical(logical
, fs_info
, path
,
4151 record_inode_for_nocow
, nocow_ctx
);
4152 if (ret
!= 0 && ret
!= -ENOENT
) {
4153 btrfs_warn(fs_info
, "iterate_inodes_from_logical() failed: log %llu, "
4154 "phys %llu, len %llu, mir %u, ret %d",
4155 logical
, physical_for_dev_replace
, len
, mirror_num
,
4161 btrfs_end_transaction(trans
, root
);
4163 while (!list_empty(&nocow_ctx
->inodes
)) {
4164 struct scrub_nocow_inode
*entry
;
4165 entry
= list_first_entry(&nocow_ctx
->inodes
,
4166 struct scrub_nocow_inode
,
4168 list_del_init(&entry
->list
);
4169 ret
= copy_nocow_pages_for_inode(entry
->inum
, entry
->offset
,
4170 entry
->root
, nocow_ctx
);
4172 if (ret
== COPY_COMPLETE
) {
4180 while (!list_empty(&nocow_ctx
->inodes
)) {
4181 struct scrub_nocow_inode
*entry
;
4182 entry
= list_first_entry(&nocow_ctx
->inodes
,
4183 struct scrub_nocow_inode
,
4185 list_del_init(&entry
->list
);
4188 if (trans
&& !IS_ERR(trans
))
4189 btrfs_end_transaction(trans
, root
);
4191 btrfs_dev_replace_stats_inc(&fs_info
->dev_replace
.
4192 num_uncorrectable_read_errors
);
4194 btrfs_free_path(path
);
4197 scrub_pending_trans_workers_dec(sctx
);
4200 static int check_extent_to_block(struct inode
*inode
, u64 start
, u64 len
,
4203 struct extent_state
*cached_state
= NULL
;
4204 struct btrfs_ordered_extent
*ordered
;
4205 struct extent_io_tree
*io_tree
;
4206 struct extent_map
*em
;
4207 u64 lockstart
= start
, lockend
= start
+ len
- 1;
4210 io_tree
= &BTRFS_I(inode
)->io_tree
;
4212 lock_extent_bits(io_tree
, lockstart
, lockend
, &cached_state
);
4213 ordered
= btrfs_lookup_ordered_range(inode
, lockstart
, len
);
4215 btrfs_put_ordered_extent(ordered
);
4220 em
= btrfs_get_extent(inode
, NULL
, 0, start
, len
, 0);
4227 * This extent does not actually cover the logical extent anymore,
4228 * move on to the next inode.
4230 if (em
->block_start
> logical
||
4231 em
->block_start
+ em
->block_len
< logical
+ len
) {
4232 free_extent_map(em
);
4236 free_extent_map(em
);
4239 unlock_extent_cached(io_tree
, lockstart
, lockend
, &cached_state
,
4244 static int copy_nocow_pages_for_inode(u64 inum
, u64 offset
, u64 root
,
4245 struct scrub_copy_nocow_ctx
*nocow_ctx
)
4247 struct btrfs_fs_info
*fs_info
= nocow_ctx
->sctx
->dev_root
->fs_info
;
4248 struct btrfs_key key
;
4249 struct inode
*inode
;
4251 struct btrfs_root
*local_root
;
4252 struct extent_io_tree
*io_tree
;
4253 u64 physical_for_dev_replace
;
4254 u64 nocow_ctx_logical
;
4255 u64 len
= nocow_ctx
->len
;
4256 unsigned long index
;
4261 key
.objectid
= root
;
4262 key
.type
= BTRFS_ROOT_ITEM_KEY
;
4263 key
.offset
= (u64
)-1;
4265 srcu_index
= srcu_read_lock(&fs_info
->subvol_srcu
);
4267 local_root
= btrfs_read_fs_root_no_name(fs_info
, &key
);
4268 if (IS_ERR(local_root
)) {
4269 srcu_read_unlock(&fs_info
->subvol_srcu
, srcu_index
);
4270 return PTR_ERR(local_root
);
4273 key
.type
= BTRFS_INODE_ITEM_KEY
;
4274 key
.objectid
= inum
;
4276 inode
= btrfs_iget(fs_info
->sb
, &key
, local_root
, NULL
);
4277 srcu_read_unlock(&fs_info
->subvol_srcu
, srcu_index
);
4279 return PTR_ERR(inode
);
4281 /* Avoid truncate/dio/punch hole.. */
4283 inode_dio_wait(inode
);
4285 physical_for_dev_replace
= nocow_ctx
->physical_for_dev_replace
;
4286 io_tree
= &BTRFS_I(inode
)->io_tree
;
4287 nocow_ctx_logical
= nocow_ctx
->logical
;
4289 ret
= check_extent_to_block(inode
, offset
, len
, nocow_ctx_logical
);
4291 ret
= ret
> 0 ? 0 : ret
;
4295 while (len
>= PAGE_CACHE_SIZE
) {
4296 index
= offset
>> PAGE_CACHE_SHIFT
;
4298 page
= find_or_create_page(inode
->i_mapping
, index
, GFP_NOFS
);
4300 btrfs_err(fs_info
, "find_or_create_page() failed");
4305 if (PageUptodate(page
)) {
4306 if (PageDirty(page
))
4309 ClearPageError(page
);
4310 err
= extent_read_full_page(io_tree
, page
,
4312 nocow_ctx
->mirror_num
);
4320 * If the page has been remove from the page cache,
4321 * the data on it is meaningless, because it may be
4322 * old one, the new data may be written into the new
4323 * page in the page cache.
4325 if (page
->mapping
!= inode
->i_mapping
) {
4327 page_cache_release(page
);
4330 if (!PageUptodate(page
)) {
4336 ret
= check_extent_to_block(inode
, offset
, len
,
4339 ret
= ret
> 0 ? 0 : ret
;
4343 err
= write_page_nocow(nocow_ctx
->sctx
,
4344 physical_for_dev_replace
, page
);
4349 page_cache_release(page
);
4354 offset
+= PAGE_CACHE_SIZE
;
4355 physical_for_dev_replace
+= PAGE_CACHE_SIZE
;
4356 nocow_ctx_logical
+= PAGE_CACHE_SIZE
;
4357 len
-= PAGE_CACHE_SIZE
;
4359 ret
= COPY_COMPLETE
;
4361 inode_unlock(inode
);
4366 static int write_page_nocow(struct scrub_ctx
*sctx
,
4367 u64 physical_for_dev_replace
, struct page
*page
)
4370 struct btrfs_device
*dev
;
4373 dev
= sctx
->wr_ctx
.tgtdev
;
4377 btrfs_warn_rl(dev
->dev_root
->fs_info
,
4378 "scrub write_page_nocow(bdev == NULL) is unexpected");
4381 bio
= btrfs_io_bio_alloc(GFP_NOFS
, 1);
4383 spin_lock(&sctx
->stat_lock
);
4384 sctx
->stat
.malloc_errors
++;
4385 spin_unlock(&sctx
->stat_lock
);
4388 bio
->bi_iter
.bi_size
= 0;
4389 bio
->bi_iter
.bi_sector
= physical_for_dev_replace
>> 9;
4390 bio
->bi_bdev
= dev
->bdev
;
4391 ret
= bio_add_page(bio
, page
, PAGE_CACHE_SIZE
, 0);
4392 if (ret
!= PAGE_CACHE_SIZE
) {
4395 btrfs_dev_stat_inc_and_print(dev
, BTRFS_DEV_STAT_WRITE_ERRS
);
4399 if (btrfsic_submit_bio_wait(WRITE_SYNC
, bio
))
4400 goto leave_with_eio
;