2 * Copyright (C) 2011 STRATO. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
19 #include <linux/sched.h>
20 #include <linux/pagemap.h>
21 #include <linux/writeback.h>
22 #include <linux/blkdev.h>
23 #include <linux/rbtree.h>
24 #include <linux/slab.h>
25 #include <linux/workqueue.h>
29 #include "transaction.h"
30 #include "dev-replace.h"
35 * This is the implementation for the generic read ahead framework.
37 * To trigger a readahead, btrfs_reada_add must be called. It will start
38 * a read ahead for the given range [start, end) on tree root. The returned
39 * handle can either be used to wait on the readahead to finish
40 * (btrfs_reada_wait), or to send it to the background (btrfs_reada_detach).
42 * The read ahead works as follows:
43 * On btrfs_reada_add, the root of the tree is inserted into a radix_tree.
44 * reada_start_machine will then search for extents to prefetch and trigger
45 * some reads. When a read finishes for a node, all contained node/leaf
46 * pointers that lie in the given range will also be enqueued. The reads will
47 * be triggered in sequential order, thus giving a big win over a naive
48 * enumeration. It will also make use of multi-device layouts. Each disk
49 * will have its on read pointer and all disks will by utilized in parallel.
50 * Also will no two disks read both sides of a mirror simultaneously, as this
51 * would waste seeking capacity. Instead both disks will read different parts
53 * Any number of readaheads can be started in parallel. The read order will be
54 * determined globally, i.e. 2 parallel readaheads will normally finish faster
55 * than the 2 started one after another.
58 #define MAX_IN_FLIGHT 6
61 struct list_head list
;
62 struct reada_control
*rc
;
70 struct list_head extctl
;
73 struct reada_zone
*zones
[BTRFS_MAX_MIRRORS
];
82 struct list_head list
;
85 struct btrfs_device
*device
;
86 struct btrfs_device
*devs
[BTRFS_MAX_MIRRORS
]; /* full list, incl
92 struct reada_machine_work
{
93 struct btrfs_work work
;
94 struct btrfs_fs_info
*fs_info
;
97 static void reada_extent_put(struct btrfs_fs_info
*, struct reada_extent
*);
98 static void reada_control_release(struct kref
*kref
);
99 static void reada_zone_release(struct kref
*kref
);
100 static void reada_start_machine(struct btrfs_fs_info
*fs_info
);
101 static void __reada_start_machine(struct btrfs_fs_info
*fs_info
);
103 static int reada_add_block(struct reada_control
*rc
, u64 logical
,
104 struct btrfs_key
*top
, u64 generation
);
107 /* in case of err, eb might be NULL */
108 static void __readahead_hook(struct btrfs_fs_info
*fs_info
,
109 struct reada_extent
*re
, struct extent_buffer
*eb
,
116 struct list_head list
;
118 spin_lock(&re
->lock
);
120 * just take the full list from the extent. afterwards we
121 * don't need the lock anymore
123 list_replace_init(&re
->extctl
, &list
);
125 spin_unlock(&re
->lock
);
128 * this is the error case, the extent buffer has not been
129 * read correctly. We won't access anything from it and
130 * just cleanup our data structures. Effectively this will
131 * cut the branch below this node from read ahead.
137 * FIXME: currently we just set nritems to 0 if this is a leaf,
138 * effectively ignoring the content. In a next step we could
139 * trigger more readahead depending from the content, e.g.
140 * fetch the checksums for the extents in the leaf.
142 if (!btrfs_header_level(eb
))
145 nritems
= btrfs_header_nritems(eb
);
146 generation
= btrfs_header_generation(eb
);
147 for (i
= 0; i
< nritems
; i
++) {
148 struct reada_extctl
*rec
;
150 struct btrfs_key key
;
151 struct btrfs_key next_key
;
153 btrfs_node_key_to_cpu(eb
, &key
, i
);
155 btrfs_node_key_to_cpu(eb
, &next_key
, i
+ 1);
158 bytenr
= btrfs_node_blockptr(eb
, i
);
159 n_gen
= btrfs_node_ptr_generation(eb
, i
);
161 list_for_each_entry(rec
, &list
, list
) {
162 struct reada_control
*rc
= rec
->rc
;
165 * if the generation doesn't match, just ignore this
166 * extctl. This will probably cut off a branch from
167 * prefetch. Alternatively one could start a new (sub-)
168 * prefetch for this branch, starting again from root.
169 * FIXME: move the generation check out of this loop
172 if (rec
->generation
!= generation
) {
174 "generation mismatch for (%llu,%d,%llu) %llu != %llu",
175 key
.objectid
, key
.type
, key
.offset
,
176 rec
->generation
, generation
);
179 if (rec
->generation
== generation
&&
180 btrfs_comp_cpu_keys(&key
, &rc
->key_end
) < 0 &&
181 btrfs_comp_cpu_keys(&next_key
, &rc
->key_start
) > 0)
182 reada_add_block(rc
, bytenr
, &next_key
, n_gen
);
188 * free extctl records
190 while (!list_empty(&list
)) {
191 struct reada_control
*rc
;
192 struct reada_extctl
*rec
;
194 rec
= list_first_entry(&list
, struct reada_extctl
, list
);
195 list_del(&rec
->list
);
199 kref_get(&rc
->refcnt
);
200 if (atomic_dec_and_test(&rc
->elems
)) {
201 kref_put(&rc
->refcnt
, reada_control_release
);
204 kref_put(&rc
->refcnt
, reada_control_release
);
206 reada_extent_put(fs_info
, re
); /* one ref for each entry */
212 int btree_readahead_hook(struct btrfs_fs_info
*fs_info
,
213 struct extent_buffer
*eb
, int err
)
216 struct reada_extent
*re
;
219 spin_lock(&fs_info
->reada_lock
);
220 re
= radix_tree_lookup(&fs_info
->reada_tree
,
221 eb
->start
>> PAGE_SHIFT
);
224 spin_unlock(&fs_info
->reada_lock
);
230 __readahead_hook(fs_info
, re
, eb
, err
);
231 reada_extent_put(fs_info
, re
); /* our ref */
234 reada_start_machine(fs_info
);
238 static struct reada_zone
*reada_find_zone(struct btrfs_fs_info
*fs_info
,
239 struct btrfs_device
*dev
, u64 logical
,
240 struct btrfs_bio
*bbio
)
243 struct reada_zone
*zone
;
244 struct btrfs_block_group_cache
*cache
= NULL
;
250 spin_lock(&fs_info
->reada_lock
);
251 ret
= radix_tree_gang_lookup(&dev
->reada_zones
, (void **)&zone
,
252 logical
>> PAGE_SHIFT
, 1);
253 if (ret
== 1 && logical
>= zone
->start
&& logical
<= zone
->end
) {
254 kref_get(&zone
->refcnt
);
255 spin_unlock(&fs_info
->reada_lock
);
259 spin_unlock(&fs_info
->reada_lock
);
261 cache
= btrfs_lookup_block_group(fs_info
, logical
);
265 start
= cache
->key
.objectid
;
266 end
= start
+ cache
->key
.offset
- 1;
267 btrfs_put_block_group(cache
);
269 zone
= kzalloc(sizeof(*zone
), GFP_KERNEL
);
273 ret
= radix_tree_preload(GFP_KERNEL
);
281 INIT_LIST_HEAD(&zone
->list
);
282 spin_lock_init(&zone
->lock
);
284 kref_init(&zone
->refcnt
);
286 zone
->device
= dev
; /* our device always sits at index 0 */
287 for (i
= 0; i
< bbio
->num_stripes
; ++i
) {
288 /* bounds have already been checked */
289 zone
->devs
[i
] = bbio
->stripes
[i
].dev
;
291 zone
->ndevs
= bbio
->num_stripes
;
293 spin_lock(&fs_info
->reada_lock
);
294 ret
= radix_tree_insert(&dev
->reada_zones
,
295 (unsigned long)(zone
->end
>> PAGE_SHIFT
),
298 if (ret
== -EEXIST
) {
300 ret
= radix_tree_gang_lookup(&dev
->reada_zones
, (void **)&zone
,
301 logical
>> PAGE_SHIFT
, 1);
302 if (ret
== 1 && logical
>= zone
->start
&& logical
<= zone
->end
)
303 kref_get(&zone
->refcnt
);
307 spin_unlock(&fs_info
->reada_lock
);
308 radix_tree_preload_end();
313 static struct reada_extent
*reada_find_extent(struct btrfs_fs_info
*fs_info
,
315 struct btrfs_key
*top
)
318 struct reada_extent
*re
= NULL
;
319 struct reada_extent
*re_exist
= NULL
;
320 struct btrfs_bio
*bbio
= NULL
;
321 struct btrfs_device
*dev
;
322 struct btrfs_device
*prev_dev
;
327 unsigned long index
= logical
>> PAGE_SHIFT
;
328 int dev_replace_is_ongoing
;
331 spin_lock(&fs_info
->reada_lock
);
332 re
= radix_tree_lookup(&fs_info
->reada_tree
, index
);
335 spin_unlock(&fs_info
->reada_lock
);
340 re
= kzalloc(sizeof(*re
), GFP_KERNEL
);
344 blocksize
= fs_info
->nodesize
;
345 re
->logical
= logical
;
347 INIT_LIST_HEAD(&re
->extctl
);
348 spin_lock_init(&re
->lock
);
355 ret
= btrfs_map_block(fs_info
, BTRFS_MAP_GET_READ_MIRRORS
, logical
,
357 if (ret
|| !bbio
|| length
< blocksize
)
360 if (bbio
->num_stripes
> BTRFS_MAX_MIRRORS
) {
362 "readahead: more than %d copies not supported",
367 real_stripes
= bbio
->num_stripes
- bbio
->num_tgtdevs
;
368 for (nzones
= 0; nzones
< real_stripes
; ++nzones
) {
369 struct reada_zone
*zone
;
371 dev
= bbio
->stripes
[nzones
].dev
;
373 /* cannot read ahead on missing device. */
377 zone
= reada_find_zone(fs_info
, dev
, logical
, bbio
);
381 re
->zones
[re
->nzones
++] = zone
;
382 spin_lock(&zone
->lock
);
384 kref_get(&zone
->refcnt
);
386 spin_unlock(&zone
->lock
);
387 spin_lock(&fs_info
->reada_lock
);
388 kref_put(&zone
->refcnt
, reada_zone_release
);
389 spin_unlock(&fs_info
->reada_lock
);
391 if (re
->nzones
== 0) {
392 /* not a single zone found, error and out */
396 ret
= radix_tree_preload(GFP_KERNEL
);
400 /* insert extent in reada_tree + all per-device trees, all or nothing */
401 btrfs_dev_replace_lock(&fs_info
->dev_replace
, 0);
402 spin_lock(&fs_info
->reada_lock
);
403 ret
= radix_tree_insert(&fs_info
->reada_tree
, index
, re
);
404 if (ret
== -EEXIST
) {
405 re_exist
= radix_tree_lookup(&fs_info
->reada_tree
, index
);
407 spin_unlock(&fs_info
->reada_lock
);
408 btrfs_dev_replace_unlock(&fs_info
->dev_replace
, 0);
409 radix_tree_preload_end();
413 spin_unlock(&fs_info
->reada_lock
);
414 btrfs_dev_replace_unlock(&fs_info
->dev_replace
, 0);
415 radix_tree_preload_end();
418 radix_tree_preload_end();
420 dev_replace_is_ongoing
= btrfs_dev_replace_is_ongoing(
421 &fs_info
->dev_replace
);
422 for (nzones
= 0; nzones
< re
->nzones
; ++nzones
) {
423 dev
= re
->zones
[nzones
]->device
;
425 if (dev
== prev_dev
) {
427 * in case of DUP, just add the first zone. As both
428 * are on the same device, there's nothing to gain
430 * Also, it wouldn't work, as the tree is per device
431 * and adding would fail with EEXIST
438 if (dev_replace_is_ongoing
&&
439 dev
== fs_info
->dev_replace
.tgtdev
) {
441 * as this device is selected for reading only as
442 * a last resort, skip it for read ahead.
447 ret
= radix_tree_insert(&dev
->reada_extents
, index
, re
);
449 while (--nzones
>= 0) {
450 dev
= re
->zones
[nzones
]->device
;
452 /* ignore whether the entry was inserted */
453 radix_tree_delete(&dev
->reada_extents
, index
);
455 radix_tree_delete(&fs_info
->reada_tree
, index
);
456 spin_unlock(&fs_info
->reada_lock
);
457 btrfs_dev_replace_unlock(&fs_info
->dev_replace
, 0);
462 spin_unlock(&fs_info
->reada_lock
);
463 btrfs_dev_replace_unlock(&fs_info
->dev_replace
, 0);
468 btrfs_put_bbio(bbio
);
472 for (nzones
= 0; nzones
< re
->nzones
; ++nzones
) {
473 struct reada_zone
*zone
;
475 zone
= re
->zones
[nzones
];
476 kref_get(&zone
->refcnt
);
477 spin_lock(&zone
->lock
);
479 if (zone
->elems
== 0) {
481 * no fs_info->reada_lock needed, as this can't be
484 kref_put(&zone
->refcnt
, reada_zone_release
);
486 spin_unlock(&zone
->lock
);
488 spin_lock(&fs_info
->reada_lock
);
489 kref_put(&zone
->refcnt
, reada_zone_release
);
490 spin_unlock(&fs_info
->reada_lock
);
492 btrfs_put_bbio(bbio
);
497 static void reada_extent_put(struct btrfs_fs_info
*fs_info
,
498 struct reada_extent
*re
)
501 unsigned long index
= re
->logical
>> PAGE_SHIFT
;
503 spin_lock(&fs_info
->reada_lock
);
505 spin_unlock(&fs_info
->reada_lock
);
509 radix_tree_delete(&fs_info
->reada_tree
, index
);
510 for (i
= 0; i
< re
->nzones
; ++i
) {
511 struct reada_zone
*zone
= re
->zones
[i
];
513 radix_tree_delete(&zone
->device
->reada_extents
, index
);
516 spin_unlock(&fs_info
->reada_lock
);
518 for (i
= 0; i
< re
->nzones
; ++i
) {
519 struct reada_zone
*zone
= re
->zones
[i
];
521 kref_get(&zone
->refcnt
);
522 spin_lock(&zone
->lock
);
524 if (zone
->elems
== 0) {
525 /* no fs_info->reada_lock needed, as this can't be
527 kref_put(&zone
->refcnt
, reada_zone_release
);
529 spin_unlock(&zone
->lock
);
531 spin_lock(&fs_info
->reada_lock
);
532 kref_put(&zone
->refcnt
, reada_zone_release
);
533 spin_unlock(&fs_info
->reada_lock
);
539 static void reada_zone_release(struct kref
*kref
)
541 struct reada_zone
*zone
= container_of(kref
, struct reada_zone
, refcnt
);
543 radix_tree_delete(&zone
->device
->reada_zones
,
544 zone
->end
>> PAGE_SHIFT
);
549 static void reada_control_release(struct kref
*kref
)
551 struct reada_control
*rc
= container_of(kref
, struct reada_control
,
557 static int reada_add_block(struct reada_control
*rc
, u64 logical
,
558 struct btrfs_key
*top
, u64 generation
)
560 struct btrfs_fs_info
*fs_info
= rc
->fs_info
;
561 struct reada_extent
*re
;
562 struct reada_extctl
*rec
;
565 re
= reada_find_extent(fs_info
, logical
, top
);
569 rec
= kzalloc(sizeof(*rec
), GFP_KERNEL
);
571 reada_extent_put(fs_info
, re
);
576 rec
->generation
= generation
;
577 atomic_inc(&rc
->elems
);
579 spin_lock(&re
->lock
);
580 list_add_tail(&rec
->list
, &re
->extctl
);
581 spin_unlock(&re
->lock
);
583 /* leave the ref on the extent */
589 * called with fs_info->reada_lock held
591 static void reada_peer_zones_set_lock(struct reada_zone
*zone
, int lock
)
594 unsigned long index
= zone
->end
>> PAGE_SHIFT
;
596 for (i
= 0; i
< zone
->ndevs
; ++i
) {
597 struct reada_zone
*peer
;
598 peer
= radix_tree_lookup(&zone
->devs
[i
]->reada_zones
, index
);
599 if (peer
&& peer
->device
!= zone
->device
)
605 * called with fs_info->reada_lock held
607 static int reada_pick_zone(struct btrfs_device
*dev
)
609 struct reada_zone
*top_zone
= NULL
;
610 struct reada_zone
*top_locked_zone
= NULL
;
612 u64 top_locked_elems
= 0;
613 unsigned long index
= 0;
616 if (dev
->reada_curr_zone
) {
617 reada_peer_zones_set_lock(dev
->reada_curr_zone
, 0);
618 kref_put(&dev
->reada_curr_zone
->refcnt
, reada_zone_release
);
619 dev
->reada_curr_zone
= NULL
;
621 /* pick the zone with the most elements */
623 struct reada_zone
*zone
;
625 ret
= radix_tree_gang_lookup(&dev
->reada_zones
,
626 (void **)&zone
, index
, 1);
629 index
= (zone
->end
>> PAGE_SHIFT
) + 1;
631 if (zone
->elems
> top_locked_elems
) {
632 top_locked_elems
= zone
->elems
;
633 top_locked_zone
= zone
;
636 if (zone
->elems
> top_elems
) {
637 top_elems
= zone
->elems
;
643 dev
->reada_curr_zone
= top_zone
;
644 else if (top_locked_zone
)
645 dev
->reada_curr_zone
= top_locked_zone
;
649 dev
->reada_next
= dev
->reada_curr_zone
->start
;
650 kref_get(&dev
->reada_curr_zone
->refcnt
);
651 reada_peer_zones_set_lock(dev
->reada_curr_zone
, 1);
656 static int reada_start_machine_dev(struct btrfs_fs_info
*fs_info
,
657 struct btrfs_device
*dev
)
659 struct reada_extent
*re
= NULL
;
661 struct extent_buffer
*eb
= NULL
;
666 spin_lock(&fs_info
->reada_lock
);
667 if (dev
->reada_curr_zone
== NULL
) {
668 ret
= reada_pick_zone(dev
);
670 spin_unlock(&fs_info
->reada_lock
);
675 * FIXME currently we issue the reads one extent at a time. If we have
676 * a contiguous block of extents, we could also coagulate them or use
677 * plugging to speed things up
679 ret
= radix_tree_gang_lookup(&dev
->reada_extents
, (void **)&re
,
680 dev
->reada_next
>> PAGE_SHIFT
, 1);
681 if (ret
== 0 || re
->logical
> dev
->reada_curr_zone
->end
) {
682 ret
= reada_pick_zone(dev
);
684 spin_unlock(&fs_info
->reada_lock
);
688 ret
= radix_tree_gang_lookup(&dev
->reada_extents
, (void **)&re
,
689 dev
->reada_next
>> PAGE_SHIFT
, 1);
692 spin_unlock(&fs_info
->reada_lock
);
695 dev
->reada_next
= re
->logical
+ fs_info
->nodesize
;
698 spin_unlock(&fs_info
->reada_lock
);
700 spin_lock(&re
->lock
);
701 if (re
->scheduled
|| list_empty(&re
->extctl
)) {
702 spin_unlock(&re
->lock
);
703 reada_extent_put(fs_info
, re
);
707 spin_unlock(&re
->lock
);
712 for (i
= 0; i
< re
->nzones
; ++i
) {
713 if (re
->zones
[i
]->device
== dev
) {
718 logical
= re
->logical
;
720 atomic_inc(&dev
->reada_in_flight
);
721 ret
= reada_tree_block_flagged(fs_info
, logical
, mirror_num
, &eb
);
723 __readahead_hook(fs_info
, re
, NULL
, ret
);
725 __readahead_hook(fs_info
, re
, eb
, ret
);
728 free_extent_buffer(eb
);
730 atomic_dec(&dev
->reada_in_flight
);
731 reada_extent_put(fs_info
, re
);
737 static void reada_start_machine_worker(struct btrfs_work
*work
)
739 struct reada_machine_work
*rmw
;
740 struct btrfs_fs_info
*fs_info
;
743 rmw
= container_of(work
, struct reada_machine_work
, work
);
744 fs_info
= rmw
->fs_info
;
748 old_ioprio
= IOPRIO_PRIO_VALUE(task_nice_ioclass(current
),
749 task_nice_ioprio(current
));
750 set_task_ioprio(current
, BTRFS_IOPRIO_READA
);
751 __reada_start_machine(fs_info
);
752 set_task_ioprio(current
, old_ioprio
);
754 atomic_dec(&fs_info
->reada_works_cnt
);
757 static void __reada_start_machine(struct btrfs_fs_info
*fs_info
)
759 struct btrfs_device
*device
;
760 struct btrfs_fs_devices
*fs_devices
= fs_info
->fs_devices
;
767 mutex_lock(&fs_devices
->device_list_mutex
);
768 list_for_each_entry(device
, &fs_devices
->devices
, dev_list
) {
769 if (atomic_read(&device
->reada_in_flight
) <
771 enqueued
+= reada_start_machine_dev(fs_info
,
774 mutex_unlock(&fs_devices
->device_list_mutex
);
776 } while (enqueued
&& total
< 10000);
782 * If everything is already in the cache, this is effectively single
783 * threaded. To a) not hold the caller for too long and b) to utilize
784 * more cores, we broke the loop above after 10000 iterations and now
785 * enqueue to workers to finish it. This will distribute the load to
788 for (i
= 0; i
< 2; ++i
) {
789 reada_start_machine(fs_info
);
790 if (atomic_read(&fs_info
->reada_works_cnt
) >
791 BTRFS_MAX_MIRRORS
* 2)
796 static void reada_start_machine(struct btrfs_fs_info
*fs_info
)
798 struct reada_machine_work
*rmw
;
800 rmw
= kzalloc(sizeof(*rmw
), GFP_KERNEL
);
802 /* FIXME we cannot handle this properly right now */
805 btrfs_init_work(&rmw
->work
, btrfs_readahead_helper
,
806 reada_start_machine_worker
, NULL
, NULL
);
807 rmw
->fs_info
= fs_info
;
809 btrfs_queue_work(fs_info
->readahead_workers
, &rmw
->work
);
810 atomic_inc(&fs_info
->reada_works_cnt
);
814 static void dump_devs(struct btrfs_fs_info
*fs_info
, int all
)
816 struct btrfs_device
*device
;
817 struct btrfs_fs_devices
*fs_devices
= fs_info
->fs_devices
;
824 spin_lock(&fs_info
->reada_lock
);
825 list_for_each_entry(device
, &fs_devices
->devices
, dev_list
) {
826 btrfs_debug(fs_info
, "dev %lld has %d in flight", device
->devid
,
827 atomic_read(&device
->reada_in_flight
));
830 struct reada_zone
*zone
;
831 ret
= radix_tree_gang_lookup(&device
->reada_zones
,
832 (void **)&zone
, index
, 1);
835 pr_debug(" zone %llu-%llu elems %llu locked %d devs",
836 zone
->start
, zone
->end
, zone
->elems
,
838 for (j
= 0; j
< zone
->ndevs
; ++j
) {
840 zone
->devs
[j
]->devid
);
842 if (device
->reada_curr_zone
== zone
)
843 pr_cont(" curr off %llu",
844 device
->reada_next
- zone
->start
);
846 index
= (zone
->end
>> PAGE_SHIFT
) + 1;
851 struct reada_extent
*re
= NULL
;
853 ret
= radix_tree_gang_lookup(&device
->reada_extents
,
854 (void **)&re
, index
, 1);
857 pr_debug(" re: logical %llu size %u empty %d scheduled %d",
858 re
->logical
, fs_info
->nodesize
,
859 list_empty(&re
->extctl
), re
->scheduled
);
861 for (i
= 0; i
< re
->nzones
; ++i
) {
862 pr_cont(" zone %llu-%llu devs",
865 for (j
= 0; j
< re
->zones
[i
]->ndevs
; ++j
) {
867 re
->zones
[i
]->devs
[j
]->devid
);
871 index
= (re
->logical
>> PAGE_SHIFT
) + 1;
880 struct reada_extent
*re
= NULL
;
882 ret
= radix_tree_gang_lookup(&fs_info
->reada_tree
, (void **)&re
,
886 if (!re
->scheduled
) {
887 index
= (re
->logical
>> PAGE_SHIFT
) + 1;
890 pr_debug("re: logical %llu size %u list empty %d scheduled %d",
891 re
->logical
, fs_info
->nodesize
,
892 list_empty(&re
->extctl
), re
->scheduled
);
893 for (i
= 0; i
< re
->nzones
; ++i
) {
894 pr_cont(" zone %llu-%llu devs",
897 for (j
= 0; j
< re
->zones
[i
]->ndevs
; ++j
) {
899 re
->zones
[i
]->devs
[j
]->devid
);
903 index
= (re
->logical
>> PAGE_SHIFT
) + 1;
905 spin_unlock(&fs_info
->reada_lock
);
912 struct reada_control
*btrfs_reada_add(struct btrfs_root
*root
,
913 struct btrfs_key
*key_start
, struct btrfs_key
*key_end
)
915 struct reada_control
*rc
;
919 struct extent_buffer
*node
;
920 static struct btrfs_key max_key
= {
926 rc
= kzalloc(sizeof(*rc
), GFP_KERNEL
);
928 return ERR_PTR(-ENOMEM
);
930 rc
->fs_info
= root
->fs_info
;
931 rc
->key_start
= *key_start
;
932 rc
->key_end
= *key_end
;
933 atomic_set(&rc
->elems
, 0);
934 init_waitqueue_head(&rc
->wait
);
935 kref_init(&rc
->refcnt
);
936 kref_get(&rc
->refcnt
); /* one ref for having elements */
938 node
= btrfs_root_node(root
);
940 generation
= btrfs_header_generation(node
);
941 free_extent_buffer(node
);
943 ret
= reada_add_block(rc
, start
, &max_key
, generation
);
949 reada_start_machine(root
->fs_info
);
955 int btrfs_reada_wait(void *handle
)
957 struct reada_control
*rc
= handle
;
958 struct btrfs_fs_info
*fs_info
= rc
->fs_info
;
960 while (atomic_read(&rc
->elems
)) {
961 if (!atomic_read(&fs_info
->reada_works_cnt
))
962 reada_start_machine(fs_info
);
963 wait_event_timeout(rc
->wait
, atomic_read(&rc
->elems
) == 0,
965 dump_devs(fs_info
, atomic_read(&rc
->elems
) < 10 ? 1 : 0);
968 dump_devs(fs_info
, atomic_read(&rc
->elems
) < 10 ? 1 : 0);
970 kref_put(&rc
->refcnt
, reada_control_release
);
975 int btrfs_reada_wait(void *handle
)
977 struct reada_control
*rc
= handle
;
978 struct btrfs_fs_info
*fs_info
= rc
->fs_info
;
980 while (atomic_read(&rc
->elems
)) {
981 if (!atomic_read(&fs_info
->reada_works_cnt
))
982 reada_start_machine(fs_info
);
983 wait_event_timeout(rc
->wait
, atomic_read(&rc
->elems
) == 0,
987 kref_put(&rc
->refcnt
, reada_control_release
);
993 void btrfs_reada_detach(void *handle
)
995 struct reada_control
*rc
= handle
;
997 kref_put(&rc
->refcnt
, reada_control_release
);