]>
Commit | Line | Data |
---|---|---|
7414a03f AJ |
1 | /* |
2 | * Copyright (C) 2011 STRATO. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public | |
6 | * License v2 as published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public | |
14 | * License along with this program; if not, write to the | |
15 | * Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
16 | * Boston, MA 021110-1307, USA. | |
17 | */ | |
18 | ||
19 | #include <linux/sched.h> | |
20 | #include <linux/pagemap.h> | |
21 | #include <linux/writeback.h> | |
22 | #include <linux/blkdev.h> | |
23 | #include <linux/rbtree.h> | |
24 | #include <linux/slab.h> | |
25 | #include <linux/workqueue.h> | |
26 | #include "ctree.h" | |
27 | #include "volumes.h" | |
28 | #include "disk-io.h" | |
29 | #include "transaction.h" | |
8dabb742 | 30 | #include "dev-replace.h" |
7414a03f AJ |
31 | |
32 | #undef DEBUG | |
33 | ||
34 | /* | |
35 | * This is the implementation for the generic read ahead framework. | |
36 | * | |
37 | * To trigger a readahead, btrfs_reada_add must be called. It will start | |
38 | * a read ahead for the given range [start, end) on tree root. The returned | |
39 | * handle can either be used to wait on the readahead to finish | |
40 | * (btrfs_reada_wait), or to send it to the background (btrfs_reada_detach). | |
41 | * | |
42 | * The read ahead works as follows: | |
43 | * On btrfs_reada_add, the root of the tree is inserted into a radix_tree. | |
44 | * reada_start_machine will then search for extents to prefetch and trigger | |
45 | * some reads. When a read finishes for a node, all contained node/leaf | |
46 | * pointers that lie in the given range will also be enqueued. The reads will | |
47 | * be triggered in sequential order, thus giving a big win over a naive | |
48 | * enumeration. It will also make use of multi-device layouts. Each disk | |
49 | * will have its on read pointer and all disks will by utilized in parallel. | |
50 | * Also will no two disks read both sides of a mirror simultaneously, as this | |
51 | * would waste seeking capacity. Instead both disks will read different parts | |
52 | * of the filesystem. | |
53 | * Any number of readaheads can be started in parallel. The read order will be | |
54 | * determined globally, i.e. 2 parallel readaheads will normally finish faster | |
55 | * than the 2 started one after another. | |
56 | */ | |
57 | ||
7414a03f AJ |
58 | #define MAX_IN_FLIGHT 6 |
59 | ||
60 | struct reada_extctl { | |
61 | struct list_head list; | |
62 | struct reada_control *rc; | |
63 | u64 generation; | |
64 | }; | |
65 | ||
66 | struct reada_extent { | |
67 | u64 logical; | |
68 | struct btrfs_key top; | |
7414a03f AJ |
69 | int err; |
70 | struct list_head extctl; | |
99621b44 | 71 | int refcnt; |
7414a03f | 72 | spinlock_t lock; |
94598ba8 | 73 | struct reada_zone *zones[BTRFS_MAX_MIRRORS]; |
7414a03f | 74 | int nzones; |
895a11b8 | 75 | int scheduled; |
7414a03f AJ |
76 | }; |
77 | ||
78 | struct reada_zone { | |
79 | u64 start; | |
80 | u64 end; | |
81 | u64 elems; | |
82 | struct list_head list; | |
83 | spinlock_t lock; | |
84 | int locked; | |
85 | struct btrfs_device *device; | |
94598ba8 SB |
86 | struct btrfs_device *devs[BTRFS_MAX_MIRRORS]; /* full list, incl |
87 | * self */ | |
7414a03f AJ |
88 | int ndevs; |
89 | struct kref refcnt; | |
90 | }; | |
91 | ||
92 | struct reada_machine_work { | |
d458b054 | 93 | struct btrfs_work work; |
7414a03f AJ |
94 | struct btrfs_fs_info *fs_info; |
95 | }; | |
96 | ||
97 | static void reada_extent_put(struct btrfs_fs_info *, struct reada_extent *); | |
98 | static void reada_control_release(struct kref *kref); | |
99 | static void reada_zone_release(struct kref *kref); | |
100 | static void reada_start_machine(struct btrfs_fs_info *fs_info); | |
101 | static void __reada_start_machine(struct btrfs_fs_info *fs_info); | |
102 | ||
103 | static int reada_add_block(struct reada_control *rc, u64 logical, | |
1e7970c0 | 104 | struct btrfs_key *top, u64 generation); |
7414a03f AJ |
105 | |
106 | /* recurses */ | |
107 | /* in case of err, eb might be NULL */ | |
02873e43 ZL |
108 | static void __readahead_hook(struct btrfs_fs_info *fs_info, |
109 | struct reada_extent *re, struct extent_buffer *eb, | |
110 | u64 start, int err) | |
7414a03f | 111 | { |
7414a03f AJ |
112 | int nritems; |
113 | int i; | |
114 | u64 bytenr; | |
115 | u64 generation; | |
7414a03f | 116 | struct list_head list; |
7414a03f | 117 | |
7414a03f AJ |
118 | spin_lock(&re->lock); |
119 | /* | |
120 | * just take the full list from the extent. afterwards we | |
121 | * don't need the lock anymore | |
122 | */ | |
123 | list_replace_init(&re->extctl, &list); | |
895a11b8 | 124 | re->scheduled = 0; |
7414a03f AJ |
125 | spin_unlock(&re->lock); |
126 | ||
57f16e08 ZL |
127 | /* |
128 | * this is the error case, the extent buffer has not been | |
129 | * read correctly. We won't access anything from it and | |
130 | * just cleanup our data structures. Effectively this will | |
131 | * cut the branch below this node from read ahead. | |
132 | */ | |
133 | if (err) | |
134 | goto cleanup; | |
7414a03f | 135 | |
57f16e08 ZL |
136 | /* |
137 | * FIXME: currently we just set nritems to 0 if this is a leaf, | |
138 | * effectively ignoring the content. In a next step we could | |
139 | * trigger more readahead depending from the content, e.g. | |
140 | * fetch the checksums for the extents in the leaf. | |
141 | */ | |
04998b33 | 142 | if (!btrfs_header_level(eb)) |
57f16e08 ZL |
143 | goto cleanup; |
144 | ||
145 | nritems = btrfs_header_nritems(eb); | |
146 | generation = btrfs_header_generation(eb); | |
7414a03f AJ |
147 | for (i = 0; i < nritems; i++) { |
148 | struct reada_extctl *rec; | |
149 | u64 n_gen; | |
150 | struct btrfs_key key; | |
151 | struct btrfs_key next_key; | |
152 | ||
153 | btrfs_node_key_to_cpu(eb, &key, i); | |
154 | if (i + 1 < nritems) | |
155 | btrfs_node_key_to_cpu(eb, &next_key, i + 1); | |
156 | else | |
157 | next_key = re->top; | |
158 | bytenr = btrfs_node_blockptr(eb, i); | |
159 | n_gen = btrfs_node_ptr_generation(eb, i); | |
160 | ||
161 | list_for_each_entry(rec, &list, list) { | |
162 | struct reada_control *rc = rec->rc; | |
163 | ||
164 | /* | |
165 | * if the generation doesn't match, just ignore this | |
166 | * extctl. This will probably cut off a branch from | |
167 | * prefetch. Alternatively one could start a new (sub-) | |
168 | * prefetch for this branch, starting again from root. | |
169 | * FIXME: move the generation check out of this loop | |
170 | */ | |
171 | #ifdef DEBUG | |
172 | if (rec->generation != generation) { | |
02873e43 ZL |
173 | btrfs_debug(fs_info, |
174 | "generation mismatch for (%llu,%d,%llu) %llu != %llu", | |
175 | key.objectid, key.type, key.offset, | |
176 | rec->generation, generation); | |
7414a03f AJ |
177 | } |
178 | #endif | |
179 | if (rec->generation == generation && | |
180 | btrfs_comp_cpu_keys(&key, &rc->key_end) < 0 && | |
181 | btrfs_comp_cpu_keys(&next_key, &rc->key_start) > 0) | |
1e7970c0 | 182 | reada_add_block(rc, bytenr, &next_key, n_gen); |
7414a03f AJ |
183 | } |
184 | } | |
57f16e08 ZL |
185 | |
186 | cleanup: | |
7414a03f AJ |
187 | /* |
188 | * free extctl records | |
189 | */ | |
190 | while (!list_empty(&list)) { | |
191 | struct reada_control *rc; | |
192 | struct reada_extctl *rec; | |
193 | ||
194 | rec = list_first_entry(&list, struct reada_extctl, list); | |
195 | list_del(&rec->list); | |
196 | rc = rec->rc; | |
197 | kfree(rec); | |
198 | ||
199 | kref_get(&rc->refcnt); | |
200 | if (atomic_dec_and_test(&rc->elems)) { | |
201 | kref_put(&rc->refcnt, reada_control_release); | |
202 | wake_up(&rc->wait); | |
203 | } | |
204 | kref_put(&rc->refcnt, reada_control_release); | |
205 | ||
206 | reada_extent_put(fs_info, re); /* one ref for each entry */ | |
207 | } | |
7414a03f | 208 | |
6e39dbe8 | 209 | return; |
7414a03f AJ |
210 | } |
211 | ||
212 | /* | |
213 | * start is passed separately in case eb in NULL, which may be the case with | |
214 | * failed I/O | |
215 | */ | |
02873e43 ZL |
216 | int btree_readahead_hook(struct btrfs_fs_info *fs_info, |
217 | struct extent_buffer *eb, u64 start, int err) | |
7414a03f | 218 | { |
6e39dbe8 ZL |
219 | int ret = 0; |
220 | struct reada_extent *re; | |
7414a03f | 221 | |
6e39dbe8 ZL |
222 | /* find extent */ |
223 | spin_lock(&fs_info->reada_lock); | |
224 | re = radix_tree_lookup(&fs_info->reada_tree, | |
09cbfeaf | 225 | start >> PAGE_SHIFT); |
6e39dbe8 ZL |
226 | if (re) |
227 | re->refcnt++; | |
228 | spin_unlock(&fs_info->reada_lock); | |
229 | if (!re) { | |
230 | ret = -1; | |
231 | goto start_machine; | |
232 | } | |
7414a03f | 233 | |
6e39dbe8 ZL |
234 | __readahead_hook(fs_info, re, eb, start, err); |
235 | reada_extent_put(fs_info, re); /* our ref */ | |
7414a03f | 236 | |
6e39dbe8 ZL |
237 | start_machine: |
238 | reada_start_machine(fs_info); | |
7414a03f AJ |
239 | return ret; |
240 | } | |
241 | ||
242 | static struct reada_zone *reada_find_zone(struct btrfs_fs_info *fs_info, | |
243 | struct btrfs_device *dev, u64 logical, | |
21ca543e | 244 | struct btrfs_bio *bbio) |
7414a03f AJ |
245 | { |
246 | int ret; | |
7414a03f AJ |
247 | struct reada_zone *zone; |
248 | struct btrfs_block_group_cache *cache = NULL; | |
249 | u64 start; | |
250 | u64 end; | |
251 | int i; | |
252 | ||
7414a03f AJ |
253 | zone = NULL; |
254 | spin_lock(&fs_info->reada_lock); | |
255 | ret = radix_tree_gang_lookup(&dev->reada_zones, (void **)&zone, | |
09cbfeaf | 256 | logical >> PAGE_SHIFT, 1); |
c37f49c7 | 257 | if (ret == 1 && logical >= zone->start && logical <= zone->end) { |
7414a03f | 258 | kref_get(&zone->refcnt); |
7414a03f | 259 | spin_unlock(&fs_info->reada_lock); |
c37f49c7 | 260 | return zone; |
7414a03f AJ |
261 | } |
262 | ||
c37f49c7 ZL |
263 | spin_unlock(&fs_info->reada_lock); |
264 | ||
7414a03f AJ |
265 | cache = btrfs_lookup_block_group(fs_info, logical); |
266 | if (!cache) | |
267 | return NULL; | |
268 | ||
269 | start = cache->key.objectid; | |
270 | end = start + cache->key.offset - 1; | |
271 | btrfs_put_block_group(cache); | |
272 | ||
ed0244fa | 273 | zone = kzalloc(sizeof(*zone), GFP_KERNEL); |
7414a03f AJ |
274 | if (!zone) |
275 | return NULL; | |
276 | ||
277 | zone->start = start; | |
278 | zone->end = end; | |
279 | INIT_LIST_HEAD(&zone->list); | |
280 | spin_lock_init(&zone->lock); | |
281 | zone->locked = 0; | |
282 | kref_init(&zone->refcnt); | |
283 | zone->elems = 0; | |
284 | zone->device = dev; /* our device always sits at index 0 */ | |
21ca543e | 285 | for (i = 0; i < bbio->num_stripes; ++i) { |
7414a03f | 286 | /* bounds have already been checked */ |
21ca543e | 287 | zone->devs[i] = bbio->stripes[i].dev; |
7414a03f | 288 | } |
21ca543e | 289 | zone->ndevs = bbio->num_stripes; |
7414a03f AJ |
290 | |
291 | spin_lock(&fs_info->reada_lock); | |
292 | ret = radix_tree_insert(&dev->reada_zones, | |
09cbfeaf | 293 | (unsigned long)(zone->end >> PAGE_SHIFT), |
7414a03f | 294 | zone); |
7414a03f | 295 | |
8c9c2bf7 | 296 | if (ret == -EEXIST) { |
7414a03f | 297 | kfree(zone); |
8c9c2bf7 | 298 | ret = radix_tree_gang_lookup(&dev->reada_zones, (void **)&zone, |
09cbfeaf | 299 | logical >> PAGE_SHIFT, 1); |
8e9aa51f | 300 | if (ret == 1 && logical >= zone->start && logical <= zone->end) |
8c9c2bf7 | 301 | kref_get(&zone->refcnt); |
8e9aa51f ZL |
302 | else |
303 | zone = NULL; | |
7414a03f | 304 | } |
8c9c2bf7 | 305 | spin_unlock(&fs_info->reada_lock); |
7414a03f AJ |
306 | |
307 | return zone; | |
308 | } | |
309 | ||
310 | static struct reada_extent *reada_find_extent(struct btrfs_root *root, | |
311 | u64 logical, | |
1e7970c0 | 312 | struct btrfs_key *top) |
7414a03f AJ |
313 | { |
314 | int ret; | |
7414a03f | 315 | struct reada_extent *re = NULL; |
8c9c2bf7 | 316 | struct reada_extent *re_exist = NULL; |
7414a03f | 317 | struct btrfs_fs_info *fs_info = root->fs_info; |
21ca543e | 318 | struct btrfs_bio *bbio = NULL; |
7414a03f | 319 | struct btrfs_device *dev; |
207a232c | 320 | struct btrfs_device *prev_dev; |
7414a03f AJ |
321 | u32 blocksize; |
322 | u64 length; | |
7cb2c420 | 323 | int real_stripes; |
7414a03f | 324 | int nzones = 0; |
09cbfeaf | 325 | unsigned long index = logical >> PAGE_SHIFT; |
8dabb742 | 326 | int dev_replace_is_ongoing; |
31945021 | 327 | int have_zone = 0; |
7414a03f | 328 | |
7414a03f AJ |
329 | spin_lock(&fs_info->reada_lock); |
330 | re = radix_tree_lookup(&fs_info->reada_tree, index); | |
331 | if (re) | |
99621b44 | 332 | re->refcnt++; |
7414a03f AJ |
333 | spin_unlock(&fs_info->reada_lock); |
334 | ||
8c9c2bf7 | 335 | if (re) |
7414a03f AJ |
336 | return re; |
337 | ||
ed0244fa | 338 | re = kzalloc(sizeof(*re), GFP_KERNEL); |
7414a03f AJ |
339 | if (!re) |
340 | return NULL; | |
341 | ||
707e8a07 | 342 | blocksize = root->nodesize; |
7414a03f | 343 | re->logical = logical; |
7414a03f AJ |
344 | re->top = *top; |
345 | INIT_LIST_HEAD(&re->extctl); | |
346 | spin_lock_init(&re->lock); | |
99621b44 | 347 | re->refcnt = 1; |
7414a03f AJ |
348 | |
349 | /* | |
350 | * map block | |
351 | */ | |
352 | length = blocksize; | |
cf8cddd3 CH |
353 | ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical, |
354 | &length, &bbio, 0); | |
21ca543e | 355 | if (ret || !bbio || length < blocksize) |
7414a03f AJ |
356 | goto error; |
357 | ||
94598ba8 | 358 | if (bbio->num_stripes > BTRFS_MAX_MIRRORS) { |
efe120a0 FH |
359 | btrfs_err(root->fs_info, |
360 | "readahead: more than %d copies not supported", | |
361 | BTRFS_MAX_MIRRORS); | |
7414a03f AJ |
362 | goto error; |
363 | } | |
364 | ||
7cb2c420 OS |
365 | real_stripes = bbio->num_stripes - bbio->num_tgtdevs; |
366 | for (nzones = 0; nzones < real_stripes; ++nzones) { | |
7414a03f AJ |
367 | struct reada_zone *zone; |
368 | ||
21ca543e | 369 | dev = bbio->stripes[nzones].dev; |
7aff8cf4 ZL |
370 | |
371 | /* cannot read ahead on missing device. */ | |
372 | if (!dev->bdev) | |
373 | continue; | |
374 | ||
21ca543e | 375 | zone = reada_find_zone(fs_info, dev, logical, bbio); |
7414a03f | 376 | if (!zone) |
6a159d2a | 377 | continue; |
7414a03f | 378 | |
6a159d2a | 379 | re->zones[re->nzones++] = zone; |
7414a03f AJ |
380 | spin_lock(&zone->lock); |
381 | if (!zone->elems) | |
382 | kref_get(&zone->refcnt); | |
383 | ++zone->elems; | |
384 | spin_unlock(&zone->lock); | |
385 | spin_lock(&fs_info->reada_lock); | |
386 | kref_put(&zone->refcnt, reada_zone_release); | |
387 | spin_unlock(&fs_info->reada_lock); | |
388 | } | |
6a159d2a | 389 | if (re->nzones == 0) { |
7414a03f AJ |
390 | /* not a single zone found, error and out */ |
391 | goto error; | |
392 | } | |
393 | ||
394 | /* insert extent in reada_tree + all per-device trees, all or nothing */ | |
73beece9 | 395 | btrfs_dev_replace_lock(&fs_info->dev_replace, 0); |
7414a03f AJ |
396 | spin_lock(&fs_info->reada_lock); |
397 | ret = radix_tree_insert(&fs_info->reada_tree, index, re); | |
8c9c2bf7 AJ |
398 | if (ret == -EEXIST) { |
399 | re_exist = radix_tree_lookup(&fs_info->reada_tree, index); | |
400 | BUG_ON(!re_exist); | |
99621b44 | 401 | re_exist->refcnt++; |
8c9c2bf7 | 402 | spin_unlock(&fs_info->reada_lock); |
73beece9 | 403 | btrfs_dev_replace_unlock(&fs_info->dev_replace, 0); |
8c9c2bf7 AJ |
404 | goto error; |
405 | } | |
7414a03f AJ |
406 | if (ret) { |
407 | spin_unlock(&fs_info->reada_lock); | |
73beece9 | 408 | btrfs_dev_replace_unlock(&fs_info->dev_replace, 0); |
7414a03f AJ |
409 | goto error; |
410 | } | |
207a232c | 411 | prev_dev = NULL; |
8dabb742 SB |
412 | dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing( |
413 | &fs_info->dev_replace); | |
6a159d2a ZL |
414 | for (nzones = 0; nzones < re->nzones; ++nzones) { |
415 | dev = re->zones[nzones]->device; | |
416 | ||
207a232c AJ |
417 | if (dev == prev_dev) { |
418 | /* | |
419 | * in case of DUP, just add the first zone. As both | |
420 | * are on the same device, there's nothing to gain | |
421 | * from adding both. | |
422 | * Also, it wouldn't work, as the tree is per device | |
423 | * and adding would fail with EEXIST | |
424 | */ | |
425 | continue; | |
426 | } | |
7aff8cf4 ZL |
427 | if (!dev->bdev) |
428 | continue; | |
429 | ||
8dabb742 SB |
430 | if (dev_replace_is_ongoing && |
431 | dev == fs_info->dev_replace.tgtdev) { | |
432 | /* | |
433 | * as this device is selected for reading only as | |
434 | * a last resort, skip it for read ahead. | |
435 | */ | |
436 | continue; | |
437 | } | |
207a232c | 438 | prev_dev = dev; |
7414a03f AJ |
439 | ret = radix_tree_insert(&dev->reada_extents, index, re); |
440 | if (ret) { | |
6a159d2a ZL |
441 | while (--nzones >= 0) { |
442 | dev = re->zones[nzones]->device; | |
7414a03f | 443 | BUG_ON(dev == NULL); |
ff023aac | 444 | /* ignore whether the entry was inserted */ |
7414a03f AJ |
445 | radix_tree_delete(&dev->reada_extents, index); |
446 | } | |
447 | BUG_ON(fs_info == NULL); | |
448 | radix_tree_delete(&fs_info->reada_tree, index); | |
449 | spin_unlock(&fs_info->reada_lock); | |
73beece9 | 450 | btrfs_dev_replace_unlock(&fs_info->dev_replace, 0); |
7414a03f AJ |
451 | goto error; |
452 | } | |
31945021 | 453 | have_zone = 1; |
7414a03f AJ |
454 | } |
455 | spin_unlock(&fs_info->reada_lock); | |
73beece9 | 456 | btrfs_dev_replace_unlock(&fs_info->dev_replace, 0); |
7414a03f | 457 | |
31945021 ZL |
458 | if (!have_zone) |
459 | goto error; | |
460 | ||
6e9606d2 | 461 | btrfs_put_bbio(bbio); |
7414a03f AJ |
462 | return re; |
463 | ||
464 | error: | |
6a159d2a | 465 | for (nzones = 0; nzones < re->nzones; ++nzones) { |
7414a03f AJ |
466 | struct reada_zone *zone; |
467 | ||
7414a03f AJ |
468 | zone = re->zones[nzones]; |
469 | kref_get(&zone->refcnt); | |
470 | spin_lock(&zone->lock); | |
471 | --zone->elems; | |
472 | if (zone->elems == 0) { | |
473 | /* | |
474 | * no fs_info->reada_lock needed, as this can't be | |
475 | * the last ref | |
476 | */ | |
477 | kref_put(&zone->refcnt, reada_zone_release); | |
478 | } | |
479 | spin_unlock(&zone->lock); | |
480 | ||
481 | spin_lock(&fs_info->reada_lock); | |
482 | kref_put(&zone->refcnt, reada_zone_release); | |
483 | spin_unlock(&fs_info->reada_lock); | |
484 | } | |
6e9606d2 | 485 | btrfs_put_bbio(bbio); |
7414a03f | 486 | kfree(re); |
8c9c2bf7 | 487 | return re_exist; |
7414a03f AJ |
488 | } |
489 | ||
7414a03f AJ |
490 | static void reada_extent_put(struct btrfs_fs_info *fs_info, |
491 | struct reada_extent *re) | |
492 | { | |
493 | int i; | |
09cbfeaf | 494 | unsigned long index = re->logical >> PAGE_SHIFT; |
7414a03f AJ |
495 | |
496 | spin_lock(&fs_info->reada_lock); | |
99621b44 | 497 | if (--re->refcnt) { |
7414a03f AJ |
498 | spin_unlock(&fs_info->reada_lock); |
499 | return; | |
500 | } | |
501 | ||
502 | radix_tree_delete(&fs_info->reada_tree, index); | |
503 | for (i = 0; i < re->nzones; ++i) { | |
504 | struct reada_zone *zone = re->zones[i]; | |
505 | ||
506 | radix_tree_delete(&zone->device->reada_extents, index); | |
507 | } | |
508 | ||
509 | spin_unlock(&fs_info->reada_lock); | |
510 | ||
511 | for (i = 0; i < re->nzones; ++i) { | |
512 | struct reada_zone *zone = re->zones[i]; | |
513 | ||
514 | kref_get(&zone->refcnt); | |
515 | spin_lock(&zone->lock); | |
516 | --zone->elems; | |
517 | if (zone->elems == 0) { | |
518 | /* no fs_info->reada_lock needed, as this can't be | |
519 | * the last ref */ | |
520 | kref_put(&zone->refcnt, reada_zone_release); | |
521 | } | |
522 | spin_unlock(&zone->lock); | |
523 | ||
524 | spin_lock(&fs_info->reada_lock); | |
525 | kref_put(&zone->refcnt, reada_zone_release); | |
526 | spin_unlock(&fs_info->reada_lock); | |
527 | } | |
7414a03f AJ |
528 | |
529 | kfree(re); | |
530 | } | |
531 | ||
532 | static void reada_zone_release(struct kref *kref) | |
533 | { | |
534 | struct reada_zone *zone = container_of(kref, struct reada_zone, refcnt); | |
535 | ||
536 | radix_tree_delete(&zone->device->reada_zones, | |
09cbfeaf | 537 | zone->end >> PAGE_SHIFT); |
7414a03f AJ |
538 | |
539 | kfree(zone); | |
540 | } | |
541 | ||
542 | static void reada_control_release(struct kref *kref) | |
543 | { | |
544 | struct reada_control *rc = container_of(kref, struct reada_control, | |
545 | refcnt); | |
546 | ||
547 | kfree(rc); | |
548 | } | |
549 | ||
550 | static int reada_add_block(struct reada_control *rc, u64 logical, | |
1e7970c0 | 551 | struct btrfs_key *top, u64 generation) |
7414a03f AJ |
552 | { |
553 | struct btrfs_root *root = rc->root; | |
554 | struct reada_extent *re; | |
555 | struct reada_extctl *rec; | |
556 | ||
1e7970c0 | 557 | re = reada_find_extent(root, logical, top); /* takes one ref */ |
7414a03f AJ |
558 | if (!re) |
559 | return -1; | |
560 | ||
ed0244fa | 561 | rec = kzalloc(sizeof(*rec), GFP_KERNEL); |
7414a03f AJ |
562 | if (!rec) { |
563 | reada_extent_put(root->fs_info, re); | |
ddd664f4 | 564 | return -ENOMEM; |
7414a03f AJ |
565 | } |
566 | ||
567 | rec->rc = rc; | |
568 | rec->generation = generation; | |
569 | atomic_inc(&rc->elems); | |
570 | ||
571 | spin_lock(&re->lock); | |
572 | list_add_tail(&rec->list, &re->extctl); | |
573 | spin_unlock(&re->lock); | |
574 | ||
575 | /* leave the ref on the extent */ | |
576 | ||
577 | return 0; | |
578 | } | |
579 | ||
580 | /* | |
581 | * called with fs_info->reada_lock held | |
582 | */ | |
583 | static void reada_peer_zones_set_lock(struct reada_zone *zone, int lock) | |
584 | { | |
585 | int i; | |
09cbfeaf | 586 | unsigned long index = zone->end >> PAGE_SHIFT; |
7414a03f AJ |
587 | |
588 | for (i = 0; i < zone->ndevs; ++i) { | |
589 | struct reada_zone *peer; | |
590 | peer = radix_tree_lookup(&zone->devs[i]->reada_zones, index); | |
591 | if (peer && peer->device != zone->device) | |
592 | peer->locked = lock; | |
593 | } | |
594 | } | |
595 | ||
596 | /* | |
597 | * called with fs_info->reada_lock held | |
598 | */ | |
599 | static int reada_pick_zone(struct btrfs_device *dev) | |
600 | { | |
601 | struct reada_zone *top_zone = NULL; | |
602 | struct reada_zone *top_locked_zone = NULL; | |
603 | u64 top_elems = 0; | |
604 | u64 top_locked_elems = 0; | |
605 | unsigned long index = 0; | |
606 | int ret; | |
607 | ||
608 | if (dev->reada_curr_zone) { | |
609 | reada_peer_zones_set_lock(dev->reada_curr_zone, 0); | |
610 | kref_put(&dev->reada_curr_zone->refcnt, reada_zone_release); | |
611 | dev->reada_curr_zone = NULL; | |
612 | } | |
613 | /* pick the zone with the most elements */ | |
614 | while (1) { | |
615 | struct reada_zone *zone; | |
616 | ||
617 | ret = radix_tree_gang_lookup(&dev->reada_zones, | |
618 | (void **)&zone, index, 1); | |
619 | if (ret == 0) | |
620 | break; | |
09cbfeaf | 621 | index = (zone->end >> PAGE_SHIFT) + 1; |
7414a03f AJ |
622 | if (zone->locked) { |
623 | if (zone->elems > top_locked_elems) { | |
624 | top_locked_elems = zone->elems; | |
625 | top_locked_zone = zone; | |
626 | } | |
627 | } else { | |
628 | if (zone->elems > top_elems) { | |
629 | top_elems = zone->elems; | |
630 | top_zone = zone; | |
631 | } | |
632 | } | |
633 | } | |
634 | if (top_zone) | |
635 | dev->reada_curr_zone = top_zone; | |
636 | else if (top_locked_zone) | |
637 | dev->reada_curr_zone = top_locked_zone; | |
638 | else | |
639 | return 0; | |
640 | ||
641 | dev->reada_next = dev->reada_curr_zone->start; | |
642 | kref_get(&dev->reada_curr_zone->refcnt); | |
643 | reada_peer_zones_set_lock(dev->reada_curr_zone, 1); | |
644 | ||
645 | return 1; | |
646 | } | |
647 | ||
648 | static int reada_start_machine_dev(struct btrfs_fs_info *fs_info, | |
649 | struct btrfs_device *dev) | |
650 | { | |
651 | struct reada_extent *re = NULL; | |
652 | int mirror_num = 0; | |
653 | struct extent_buffer *eb = NULL; | |
654 | u64 logical; | |
7414a03f AJ |
655 | int ret; |
656 | int i; | |
7414a03f AJ |
657 | |
658 | spin_lock(&fs_info->reada_lock); | |
659 | if (dev->reada_curr_zone == NULL) { | |
660 | ret = reada_pick_zone(dev); | |
661 | if (!ret) { | |
662 | spin_unlock(&fs_info->reada_lock); | |
663 | return 0; | |
664 | } | |
665 | } | |
666 | /* | |
667 | * FIXME currently we issue the reads one extent at a time. If we have | |
668 | * a contiguous block of extents, we could also coagulate them or use | |
669 | * plugging to speed things up | |
670 | */ | |
671 | ret = radix_tree_gang_lookup(&dev->reada_extents, (void **)&re, | |
09cbfeaf | 672 | dev->reada_next >> PAGE_SHIFT, 1); |
50378530 | 673 | if (ret == 0 || re->logical > dev->reada_curr_zone->end) { |
7414a03f AJ |
674 | ret = reada_pick_zone(dev); |
675 | if (!ret) { | |
676 | spin_unlock(&fs_info->reada_lock); | |
677 | return 0; | |
678 | } | |
679 | re = NULL; | |
680 | ret = radix_tree_gang_lookup(&dev->reada_extents, (void **)&re, | |
09cbfeaf | 681 | dev->reada_next >> PAGE_SHIFT, 1); |
7414a03f AJ |
682 | } |
683 | if (ret == 0) { | |
684 | spin_unlock(&fs_info->reada_lock); | |
685 | return 0; | |
686 | } | |
b6ae40ec | 687 | dev->reada_next = re->logical + fs_info->tree_root->nodesize; |
99621b44 | 688 | re->refcnt++; |
7414a03f AJ |
689 | |
690 | spin_unlock(&fs_info->reada_lock); | |
691 | ||
a3f7fde2 | 692 | spin_lock(&re->lock); |
895a11b8 | 693 | if (re->scheduled || list_empty(&re->extctl)) { |
a3f7fde2 ZL |
694 | spin_unlock(&re->lock); |
695 | reada_extent_put(fs_info, re); | |
696 | return 0; | |
697 | } | |
895a11b8 | 698 | re->scheduled = 1; |
a3f7fde2 ZL |
699 | spin_unlock(&re->lock); |
700 | ||
7414a03f AJ |
701 | /* |
702 | * find mirror num | |
703 | */ | |
704 | for (i = 0; i < re->nzones; ++i) { | |
705 | if (re->zones[i]->device == dev) { | |
706 | mirror_num = i + 1; | |
707 | break; | |
708 | } | |
709 | } | |
710 | logical = re->logical; | |
7414a03f | 711 | |
7414a03f | 712 | atomic_inc(&dev->reada_in_flight); |
b6ae40ec | 713 | ret = reada_tree_block_flagged(fs_info->extent_root, logical, |
c0dcaa4d | 714 | mirror_num, &eb); |
7414a03f | 715 | if (ret) |
02873e43 | 716 | __readahead_hook(fs_info, re, NULL, logical, ret); |
7414a03f | 717 | else if (eb) |
02873e43 | 718 | __readahead_hook(fs_info, re, eb, eb->start, ret); |
7414a03f AJ |
719 | |
720 | if (eb) | |
721 | free_extent_buffer(eb); | |
722 | ||
895a11b8 | 723 | atomic_dec(&dev->reada_in_flight); |
b257cf50 ZL |
724 | reada_extent_put(fs_info, re); |
725 | ||
7414a03f AJ |
726 | return 1; |
727 | ||
728 | } | |
729 | ||
d458b054 | 730 | static void reada_start_machine_worker(struct btrfs_work *work) |
7414a03f AJ |
731 | { |
732 | struct reada_machine_work *rmw; | |
733 | struct btrfs_fs_info *fs_info; | |
3d136a11 | 734 | int old_ioprio; |
7414a03f AJ |
735 | |
736 | rmw = container_of(work, struct reada_machine_work, work); | |
737 | fs_info = rmw->fs_info; | |
738 | ||
739 | kfree(rmw); | |
740 | ||
3d136a11 SB |
741 | old_ioprio = IOPRIO_PRIO_VALUE(task_nice_ioclass(current), |
742 | task_nice_ioprio(current)); | |
743 | set_task_ioprio(current, BTRFS_IOPRIO_READA); | |
7414a03f | 744 | __reada_start_machine(fs_info); |
3d136a11 | 745 | set_task_ioprio(current, old_ioprio); |
2fefd558 ZL |
746 | |
747 | atomic_dec(&fs_info->reada_works_cnt); | |
7414a03f AJ |
748 | } |
749 | ||
750 | static void __reada_start_machine(struct btrfs_fs_info *fs_info) | |
751 | { | |
752 | struct btrfs_device *device; | |
753 | struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; | |
754 | u64 enqueued; | |
755 | u64 total = 0; | |
756 | int i; | |
757 | ||
758 | do { | |
759 | enqueued = 0; | |
ce7791ff | 760 | mutex_lock(&fs_devices->device_list_mutex); |
7414a03f AJ |
761 | list_for_each_entry(device, &fs_devices->devices, dev_list) { |
762 | if (atomic_read(&device->reada_in_flight) < | |
763 | MAX_IN_FLIGHT) | |
764 | enqueued += reada_start_machine_dev(fs_info, | |
765 | device); | |
766 | } | |
ce7791ff | 767 | mutex_unlock(&fs_devices->device_list_mutex); |
7414a03f AJ |
768 | total += enqueued; |
769 | } while (enqueued && total < 10000); | |
770 | ||
771 | if (enqueued == 0) | |
772 | return; | |
773 | ||
774 | /* | |
775 | * If everything is already in the cache, this is effectively single | |
776 | * threaded. To a) not hold the caller for too long and b) to utilize | |
777 | * more cores, we broke the loop above after 10000 iterations and now | |
778 | * enqueue to workers to finish it. This will distribute the load to | |
779 | * the cores. | |
780 | */ | |
2fefd558 | 781 | for (i = 0; i < 2; ++i) { |
7414a03f | 782 | reada_start_machine(fs_info); |
2fefd558 ZL |
783 | if (atomic_read(&fs_info->reada_works_cnt) > |
784 | BTRFS_MAX_MIRRORS * 2) | |
785 | break; | |
786 | } | |
7414a03f AJ |
787 | } |
788 | ||
789 | static void reada_start_machine(struct btrfs_fs_info *fs_info) | |
790 | { | |
791 | struct reada_machine_work *rmw; | |
792 | ||
ed0244fa | 793 | rmw = kzalloc(sizeof(*rmw), GFP_KERNEL); |
7414a03f AJ |
794 | if (!rmw) { |
795 | /* FIXME we cannot handle this properly right now */ | |
796 | BUG(); | |
797 | } | |
9e0af237 LB |
798 | btrfs_init_work(&rmw->work, btrfs_readahead_helper, |
799 | reada_start_machine_worker, NULL, NULL); | |
7414a03f AJ |
800 | rmw->fs_info = fs_info; |
801 | ||
736cfa15 | 802 | btrfs_queue_work(fs_info->readahead_workers, &rmw->work); |
2fefd558 | 803 | atomic_inc(&fs_info->reada_works_cnt); |
7414a03f AJ |
804 | } |
805 | ||
806 | #ifdef DEBUG | |
807 | static void dump_devs(struct btrfs_fs_info *fs_info, int all) | |
808 | { | |
809 | struct btrfs_device *device; | |
810 | struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; | |
811 | unsigned long index; | |
812 | int ret; | |
813 | int i; | |
814 | int j; | |
815 | int cnt; | |
816 | ||
817 | spin_lock(&fs_info->reada_lock); | |
818 | list_for_each_entry(device, &fs_devices->devices, dev_list) { | |
ab8d0fc4 | 819 | btrfs_debug(fs_info, "dev %lld has %d in flight", device->devid, |
7414a03f AJ |
820 | atomic_read(&device->reada_in_flight)); |
821 | index = 0; | |
822 | while (1) { | |
823 | struct reada_zone *zone; | |
824 | ret = radix_tree_gang_lookup(&device->reada_zones, | |
825 | (void **)&zone, index, 1); | |
826 | if (ret == 0) | |
827 | break; | |
62e85577 | 828 | pr_debug(" zone %llu-%llu elems %llu locked %d devs", |
ab8d0fc4 JM |
829 | zone->start, zone->end, zone->elems, |
830 | zone->locked); | |
7414a03f | 831 | for (j = 0; j < zone->ndevs; ++j) { |
62e85577 | 832 | pr_cont(" %lld", |
7414a03f AJ |
833 | zone->devs[j]->devid); |
834 | } | |
835 | if (device->reada_curr_zone == zone) | |
62e85577 | 836 | pr_cont(" curr off %llu", |
7414a03f | 837 | device->reada_next - zone->start); |
62e85577 | 838 | pr_cont("\n"); |
09cbfeaf | 839 | index = (zone->end >> PAGE_SHIFT) + 1; |
7414a03f AJ |
840 | } |
841 | cnt = 0; | |
842 | index = 0; | |
843 | while (all) { | |
844 | struct reada_extent *re = NULL; | |
845 | ||
846 | ret = radix_tree_gang_lookup(&device->reada_extents, | |
847 | (void **)&re, index, 1); | |
848 | if (ret == 0) | |
849 | break; | |
62e85577 | 850 | pr_debug(" re: logical %llu size %u empty %d scheduled %d", |
b6ae40ec | 851 | re->logical, fs_info->tree_root->nodesize, |
895a11b8 | 852 | list_empty(&re->extctl), re->scheduled); |
7414a03f AJ |
853 | |
854 | for (i = 0; i < re->nzones; ++i) { | |
62e85577 | 855 | pr_cont(" zone %llu-%llu devs", |
7414a03f AJ |
856 | re->zones[i]->start, |
857 | re->zones[i]->end); | |
858 | for (j = 0; j < re->zones[i]->ndevs; ++j) { | |
62e85577 | 859 | pr_cont(" %lld", |
7414a03f AJ |
860 | re->zones[i]->devs[j]->devid); |
861 | } | |
862 | } | |
62e85577 | 863 | pr_cont("\n"); |
09cbfeaf | 864 | index = (re->logical >> PAGE_SHIFT) + 1; |
7414a03f AJ |
865 | if (++cnt > 15) |
866 | break; | |
867 | } | |
868 | } | |
869 | ||
870 | index = 0; | |
871 | cnt = 0; | |
872 | while (all) { | |
873 | struct reada_extent *re = NULL; | |
874 | ||
875 | ret = radix_tree_gang_lookup(&fs_info->reada_tree, (void **)&re, | |
876 | index, 1); | |
877 | if (ret == 0) | |
878 | break; | |
895a11b8 | 879 | if (!re->scheduled) { |
09cbfeaf | 880 | index = (re->logical >> PAGE_SHIFT) + 1; |
7414a03f AJ |
881 | continue; |
882 | } | |
62e85577 | 883 | pr_debug("re: logical %llu size %u list empty %d scheduled %d", |
b6ae40ec | 884 | re->logical, fs_info->tree_root->nodesize, |
895a11b8 | 885 | list_empty(&re->extctl), re->scheduled); |
7414a03f | 886 | for (i = 0; i < re->nzones; ++i) { |
62e85577 | 887 | pr_cont(" zone %llu-%llu devs", |
7414a03f AJ |
888 | re->zones[i]->start, |
889 | re->zones[i]->end); | |
8afd6841 | 890 | for (j = 0; j < re->zones[i]->ndevs; ++j) { |
62e85577 | 891 | pr_cont(" %lld", |
8afd6841 | 892 | re->zones[i]->devs[j]->devid); |
7414a03f AJ |
893 | } |
894 | } | |
62e85577 | 895 | pr_cont("\n"); |
09cbfeaf | 896 | index = (re->logical >> PAGE_SHIFT) + 1; |
7414a03f AJ |
897 | } |
898 | spin_unlock(&fs_info->reada_lock); | |
899 | } | |
900 | #endif | |
901 | ||
902 | /* | |
903 | * interface | |
904 | */ | |
905 | struct reada_control *btrfs_reada_add(struct btrfs_root *root, | |
906 | struct btrfs_key *key_start, struct btrfs_key *key_end) | |
907 | { | |
908 | struct reada_control *rc; | |
909 | u64 start; | |
910 | u64 generation; | |
ddd664f4 | 911 | int ret; |
7414a03f AJ |
912 | struct extent_buffer *node; |
913 | static struct btrfs_key max_key = { | |
914 | .objectid = (u64)-1, | |
915 | .type = (u8)-1, | |
916 | .offset = (u64)-1 | |
917 | }; | |
918 | ||
ed0244fa | 919 | rc = kzalloc(sizeof(*rc), GFP_KERNEL); |
7414a03f AJ |
920 | if (!rc) |
921 | return ERR_PTR(-ENOMEM); | |
922 | ||
923 | rc->root = root; | |
924 | rc->key_start = *key_start; | |
925 | rc->key_end = *key_end; | |
926 | atomic_set(&rc->elems, 0); | |
927 | init_waitqueue_head(&rc->wait); | |
928 | kref_init(&rc->refcnt); | |
929 | kref_get(&rc->refcnt); /* one ref for having elements */ | |
930 | ||
931 | node = btrfs_root_node(root); | |
932 | start = node->start; | |
7414a03f AJ |
933 | generation = btrfs_header_generation(node); |
934 | free_extent_buffer(node); | |
935 | ||
1e7970c0 | 936 | ret = reada_add_block(rc, start, &max_key, generation); |
ddd664f4 | 937 | if (ret) { |
ff023aac | 938 | kfree(rc); |
ddd664f4 | 939 | return ERR_PTR(ret); |
ff023aac | 940 | } |
7414a03f AJ |
941 | |
942 | reada_start_machine(root->fs_info); | |
943 | ||
944 | return rc; | |
945 | } | |
946 | ||
947 | #ifdef DEBUG | |
948 | int btrfs_reada_wait(void *handle) | |
949 | { | |
950 | struct reada_control *rc = handle; | |
4fe7a0e1 | 951 | struct btrfs_fs_info *fs_info = rc->root->fs_info; |
7414a03f AJ |
952 | |
953 | while (atomic_read(&rc->elems)) { | |
4fe7a0e1 ZL |
954 | if (!atomic_read(&fs_info->reada_works_cnt)) |
955 | reada_start_machine(fs_info); | |
7414a03f AJ |
956 | wait_event_timeout(rc->wait, atomic_read(&rc->elems) == 0, |
957 | 5 * HZ); | |
3c59ccd3 V |
958 | dump_devs(rc->root->fs_info, |
959 | atomic_read(&rc->elems) < 10 ? 1 : 0); | |
7414a03f AJ |
960 | } |
961 | ||
3c59ccd3 | 962 | dump_devs(rc->root->fs_info, atomic_read(&rc->elems) < 10 ? 1 : 0); |
7414a03f AJ |
963 | |
964 | kref_put(&rc->refcnt, reada_control_release); | |
965 | ||
966 | return 0; | |
967 | } | |
968 | #else | |
969 | int btrfs_reada_wait(void *handle) | |
970 | { | |
971 | struct reada_control *rc = handle; | |
4fe7a0e1 | 972 | struct btrfs_fs_info *fs_info = rc->root->fs_info; |
7414a03f AJ |
973 | |
974 | while (atomic_read(&rc->elems)) { | |
4fe7a0e1 ZL |
975 | if (!atomic_read(&fs_info->reada_works_cnt)) |
976 | reada_start_machine(fs_info); | |
977 | wait_event_timeout(rc->wait, atomic_read(&rc->elems) == 0, | |
978 | (HZ + 9) / 10); | |
7414a03f AJ |
979 | } |
980 | ||
981 | kref_put(&rc->refcnt, reada_control_release); | |
982 | ||
983 | return 0; | |
984 | } | |
985 | #endif | |
986 | ||
987 | void btrfs_reada_detach(void *handle) | |
988 | { | |
989 | struct reada_control *rc = handle; | |
990 | ||
991 | kref_put(&rc->refcnt, reada_control_release); | |
992 | } |