]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - fs/btrfs/extent_io.c
fs/btrfs: use WARN
[mirror_ubuntu-bionic-kernel.git] / fs / btrfs / extent_io.c
CommitLineData
d1310b2e
CM
1#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
d1310b2e
CM
5#include <linux/pagemap.h>
6#include <linux/page-flags.h>
7#include <linux/module.h>
8#include <linux/spinlock.h>
9#include <linux/blkdev.h>
10#include <linux/swap.h>
d1310b2e
CM
11#include <linux/writeback.h>
12#include <linux/pagevec.h>
268bb0ce 13#include <linux/prefetch.h>
90a887c9 14#include <linux/cleancache.h>
d1310b2e
CM
15#include "extent_io.h"
16#include "extent_map.h"
2db04966 17#include "compat.h"
902b22f3
DW
18#include "ctree.h"
19#include "btrfs_inode.h"
4a54c8c1 20#include "volumes.h"
21adbd5c 21#include "check-integrity.h"
0b32f4bb 22#include "locking.h"
606686ee 23#include "rcu-string.h"
d1310b2e 24
d1310b2e
CM
25static struct kmem_cache *extent_state_cache;
26static struct kmem_cache *extent_buffer_cache;
27
28static LIST_HEAD(buffers);
29static LIST_HEAD(states);
4bef0848 30
b47eda86 31#define LEAK_DEBUG 0
3935127c 32#if LEAK_DEBUG
d397712b 33static DEFINE_SPINLOCK(leak_lock);
4bef0848 34#endif
d1310b2e 35
d1310b2e
CM
36#define BUFFER_LRU_MAX 64
37
38struct tree_entry {
39 u64 start;
40 u64 end;
d1310b2e
CM
41 struct rb_node rb_node;
42};
43
44struct extent_page_data {
45 struct bio *bio;
46 struct extent_io_tree *tree;
47 get_extent_t *get_extent;
de0022b9 48 unsigned long bio_flags;
771ed689
CM
49
50 /* tells writepage not to lock the state bits for this range
51 * it still does the unlocking
52 */
ffbd517d
CM
53 unsigned int extent_locked:1;
54
55 /* tells the submit_bio code to use a WRITE_SYNC */
56 unsigned int sync_io:1;
d1310b2e
CM
57};
58
0b32f4bb 59static noinline void flush_write_bio(void *data);
c2d904e0
JM
60static inline struct btrfs_fs_info *
61tree_fs_info(struct extent_io_tree *tree)
62{
63 return btrfs_sb(tree->mapping->host->i_sb);
64}
0b32f4bb 65
d1310b2e
CM
66int __init extent_io_init(void)
67{
837e1972 68 extent_state_cache = kmem_cache_create("btrfs_extent_state",
9601e3f6
CH
69 sizeof(struct extent_state), 0,
70 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
d1310b2e
CM
71 if (!extent_state_cache)
72 return -ENOMEM;
73
837e1972 74 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
9601e3f6
CH
75 sizeof(struct extent_buffer), 0,
76 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
d1310b2e
CM
77 if (!extent_buffer_cache)
78 goto free_state_cache;
79 return 0;
80
81free_state_cache:
82 kmem_cache_destroy(extent_state_cache);
83 return -ENOMEM;
84}
85
86void extent_io_exit(void)
87{
88 struct extent_state *state;
2d2ae547 89 struct extent_buffer *eb;
d1310b2e
CM
90
91 while (!list_empty(&states)) {
2d2ae547 92 state = list_entry(states.next, struct extent_state, leak_list);
d397712b
CM
93 printk(KERN_ERR "btrfs state leak: start %llu end %llu "
94 "state %lu in tree %p refs %d\n",
95 (unsigned long long)state->start,
96 (unsigned long long)state->end,
97 state->state, state->tree, atomic_read(&state->refs));
2d2ae547 98 list_del(&state->leak_list);
d1310b2e
CM
99 kmem_cache_free(extent_state_cache, state);
100
101 }
102
2d2ae547
CM
103 while (!list_empty(&buffers)) {
104 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
d397712b
CM
105 printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
106 "refs %d\n", (unsigned long long)eb->start,
107 eb->len, atomic_read(&eb->refs));
2d2ae547
CM
108 list_del(&eb->leak_list);
109 kmem_cache_free(extent_buffer_cache, eb);
110 }
8c0a8537
KS
111
112 /*
113 * Make sure all delayed rcu free are flushed before we
114 * destroy caches.
115 */
116 rcu_barrier();
d1310b2e
CM
117 if (extent_state_cache)
118 kmem_cache_destroy(extent_state_cache);
119 if (extent_buffer_cache)
120 kmem_cache_destroy(extent_buffer_cache);
121}
122
123void extent_io_tree_init(struct extent_io_tree *tree,
f993c883 124 struct address_space *mapping)
d1310b2e 125{
6bef4d31 126 tree->state = RB_ROOT;
19fe0a8b 127 INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
d1310b2e
CM
128 tree->ops = NULL;
129 tree->dirty_bytes = 0;
70dec807 130 spin_lock_init(&tree->lock);
6af118ce 131 spin_lock_init(&tree->buffer_lock);
d1310b2e 132 tree->mapping = mapping;
d1310b2e 133}
d1310b2e 134
b2950863 135static struct extent_state *alloc_extent_state(gfp_t mask)
d1310b2e
CM
136{
137 struct extent_state *state;
3935127c 138#if LEAK_DEBUG
2d2ae547 139 unsigned long flags;
4bef0848 140#endif
d1310b2e
CM
141
142 state = kmem_cache_alloc(extent_state_cache, mask);
2b114d1d 143 if (!state)
d1310b2e
CM
144 return state;
145 state->state = 0;
d1310b2e 146 state->private = 0;
70dec807 147 state->tree = NULL;
3935127c 148#if LEAK_DEBUG
2d2ae547
CM
149 spin_lock_irqsave(&leak_lock, flags);
150 list_add(&state->leak_list, &states);
151 spin_unlock_irqrestore(&leak_lock, flags);
4bef0848 152#endif
d1310b2e
CM
153 atomic_set(&state->refs, 1);
154 init_waitqueue_head(&state->wq);
143bede5 155 trace_alloc_extent_state(state, mask, _RET_IP_);
d1310b2e
CM
156 return state;
157}
d1310b2e 158
4845e44f 159void free_extent_state(struct extent_state *state)
d1310b2e 160{
d1310b2e
CM
161 if (!state)
162 return;
163 if (atomic_dec_and_test(&state->refs)) {
3935127c 164#if LEAK_DEBUG
2d2ae547 165 unsigned long flags;
4bef0848 166#endif
70dec807 167 WARN_ON(state->tree);
3935127c 168#if LEAK_DEBUG
2d2ae547
CM
169 spin_lock_irqsave(&leak_lock, flags);
170 list_del(&state->leak_list);
171 spin_unlock_irqrestore(&leak_lock, flags);
4bef0848 172#endif
143bede5 173 trace_free_extent_state(state, _RET_IP_);
d1310b2e
CM
174 kmem_cache_free(extent_state_cache, state);
175 }
176}
d1310b2e
CM
177
178static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
179 struct rb_node *node)
180{
d397712b
CM
181 struct rb_node **p = &root->rb_node;
182 struct rb_node *parent = NULL;
d1310b2e
CM
183 struct tree_entry *entry;
184
d397712b 185 while (*p) {
d1310b2e
CM
186 parent = *p;
187 entry = rb_entry(parent, struct tree_entry, rb_node);
188
189 if (offset < entry->start)
190 p = &(*p)->rb_left;
191 else if (offset > entry->end)
192 p = &(*p)->rb_right;
193 else
194 return parent;
195 }
196
d1310b2e
CM
197 rb_link_node(node, parent, p);
198 rb_insert_color(node, root);
199 return NULL;
200}
201
80ea96b1 202static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
d1310b2e
CM
203 struct rb_node **prev_ret,
204 struct rb_node **next_ret)
205{
80ea96b1 206 struct rb_root *root = &tree->state;
d397712b 207 struct rb_node *n = root->rb_node;
d1310b2e
CM
208 struct rb_node *prev = NULL;
209 struct rb_node *orig_prev = NULL;
210 struct tree_entry *entry;
211 struct tree_entry *prev_entry = NULL;
212
d397712b 213 while (n) {
d1310b2e
CM
214 entry = rb_entry(n, struct tree_entry, rb_node);
215 prev = n;
216 prev_entry = entry;
217
218 if (offset < entry->start)
219 n = n->rb_left;
220 else if (offset > entry->end)
221 n = n->rb_right;
d397712b 222 else
d1310b2e
CM
223 return n;
224 }
225
226 if (prev_ret) {
227 orig_prev = prev;
d397712b 228 while (prev && offset > prev_entry->end) {
d1310b2e
CM
229 prev = rb_next(prev);
230 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
231 }
232 *prev_ret = prev;
233 prev = orig_prev;
234 }
235
236 if (next_ret) {
237 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
d397712b 238 while (prev && offset < prev_entry->start) {
d1310b2e
CM
239 prev = rb_prev(prev);
240 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
241 }
242 *next_ret = prev;
243 }
244 return NULL;
245}
246
80ea96b1
CM
247static inline struct rb_node *tree_search(struct extent_io_tree *tree,
248 u64 offset)
d1310b2e 249{
70dec807 250 struct rb_node *prev = NULL;
d1310b2e 251 struct rb_node *ret;
70dec807 252
80ea96b1 253 ret = __etree_search(tree, offset, &prev, NULL);
d397712b 254 if (!ret)
d1310b2e
CM
255 return prev;
256 return ret;
257}
258
9ed74f2d
JB
259static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
260 struct extent_state *other)
261{
262 if (tree->ops && tree->ops->merge_extent_hook)
263 tree->ops->merge_extent_hook(tree->mapping->host, new,
264 other);
265}
266
d1310b2e
CM
267/*
268 * utility function to look for merge candidates inside a given range.
269 * Any extents with matching state are merged together into a single
270 * extent in the tree. Extents with EXTENT_IO in their state field
271 * are not merged because the end_io handlers need to be able to do
272 * operations on them without sleeping (or doing allocations/splits).
273 *
274 * This should be called with the tree lock held.
275 */
1bf85046
JM
276static void merge_state(struct extent_io_tree *tree,
277 struct extent_state *state)
d1310b2e
CM
278{
279 struct extent_state *other;
280 struct rb_node *other_node;
281
5b21f2ed 282 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
1bf85046 283 return;
d1310b2e
CM
284
285 other_node = rb_prev(&state->rb_node);
286 if (other_node) {
287 other = rb_entry(other_node, struct extent_state, rb_node);
288 if (other->end == state->start - 1 &&
289 other->state == state->state) {
9ed74f2d 290 merge_cb(tree, state, other);
d1310b2e 291 state->start = other->start;
70dec807 292 other->tree = NULL;
d1310b2e
CM
293 rb_erase(&other->rb_node, &tree->state);
294 free_extent_state(other);
295 }
296 }
297 other_node = rb_next(&state->rb_node);
298 if (other_node) {
299 other = rb_entry(other_node, struct extent_state, rb_node);
300 if (other->start == state->end + 1 &&
301 other->state == state->state) {
9ed74f2d 302 merge_cb(tree, state, other);
df98b6e2
JB
303 state->end = other->end;
304 other->tree = NULL;
305 rb_erase(&other->rb_node, &tree->state);
306 free_extent_state(other);
d1310b2e
CM
307 }
308 }
d1310b2e
CM
309}
310
1bf85046 311static void set_state_cb(struct extent_io_tree *tree,
0ca1f7ce 312 struct extent_state *state, int *bits)
291d673e 313{
1bf85046
JM
314 if (tree->ops && tree->ops->set_bit_hook)
315 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
291d673e
CM
316}
317
318static void clear_state_cb(struct extent_io_tree *tree,
0ca1f7ce 319 struct extent_state *state, int *bits)
291d673e 320{
9ed74f2d
JB
321 if (tree->ops && tree->ops->clear_bit_hook)
322 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
291d673e
CM
323}
324
3150b699
XG
325static void set_state_bits(struct extent_io_tree *tree,
326 struct extent_state *state, int *bits);
327
d1310b2e
CM
328/*
329 * insert an extent_state struct into the tree. 'bits' are set on the
330 * struct before it is inserted.
331 *
332 * This may return -EEXIST if the extent is already there, in which case the
333 * state struct is freed.
334 *
335 * The tree lock is not taken internally. This is a utility function and
336 * probably isn't what you want to call (see set/clear_extent_bit).
337 */
338static int insert_state(struct extent_io_tree *tree,
339 struct extent_state *state, u64 start, u64 end,
0ca1f7ce 340 int *bits)
d1310b2e
CM
341{
342 struct rb_node *node;
343
31b1a2bd
JL
344 if (end < start)
345 WARN(1, KERN_ERR "btrfs end < start %llu %llu\n",
d397712b
CM
346 (unsigned long long)end,
347 (unsigned long long)start);
d1310b2e
CM
348 state->start = start;
349 state->end = end;
9ed74f2d 350
3150b699
XG
351 set_state_bits(tree, state, bits);
352
d1310b2e
CM
353 node = tree_insert(&tree->state, end, &state->rb_node);
354 if (node) {
355 struct extent_state *found;
356 found = rb_entry(node, struct extent_state, rb_node);
d397712b
CM
357 printk(KERN_ERR "btrfs found node %llu %llu on insert of "
358 "%llu %llu\n", (unsigned long long)found->start,
359 (unsigned long long)found->end,
360 (unsigned long long)start, (unsigned long long)end);
d1310b2e
CM
361 return -EEXIST;
362 }
70dec807 363 state->tree = tree;
d1310b2e
CM
364 merge_state(tree, state);
365 return 0;
366}
367
1bf85046 368static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
9ed74f2d
JB
369 u64 split)
370{
371 if (tree->ops && tree->ops->split_extent_hook)
1bf85046 372 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
9ed74f2d
JB
373}
374
d1310b2e
CM
375/*
376 * split a given extent state struct in two, inserting the preallocated
377 * struct 'prealloc' as the newly created second half. 'split' indicates an
378 * offset inside 'orig' where it should be split.
379 *
380 * Before calling,
381 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
382 * are two extent state structs in the tree:
383 * prealloc: [orig->start, split - 1]
384 * orig: [ split, orig->end ]
385 *
386 * The tree locks are not taken by this function. They need to be held
387 * by the caller.
388 */
389static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
390 struct extent_state *prealloc, u64 split)
391{
392 struct rb_node *node;
9ed74f2d
JB
393
394 split_cb(tree, orig, split);
395
d1310b2e
CM
396 prealloc->start = orig->start;
397 prealloc->end = split - 1;
398 prealloc->state = orig->state;
399 orig->start = split;
400
401 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
402 if (node) {
d1310b2e
CM
403 free_extent_state(prealloc);
404 return -EEXIST;
405 }
70dec807 406 prealloc->tree = tree;
d1310b2e
CM
407 return 0;
408}
409
cdc6a395
LZ
410static struct extent_state *next_state(struct extent_state *state)
411{
412 struct rb_node *next = rb_next(&state->rb_node);
413 if (next)
414 return rb_entry(next, struct extent_state, rb_node);
415 else
416 return NULL;
417}
418
d1310b2e
CM
419/*
420 * utility function to clear some bits in an extent state struct.
1b303fc0 421 * it will optionally wake up any one waiting on this state (wake == 1).
d1310b2e
CM
422 *
423 * If no bits are set on the state struct after clearing things, the
424 * struct is freed and removed from the tree
425 */
cdc6a395
LZ
426static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
427 struct extent_state *state,
428 int *bits, int wake)
d1310b2e 429{
cdc6a395 430 struct extent_state *next;
0ca1f7ce 431 int bits_to_clear = *bits & ~EXTENT_CTLBITS;
d1310b2e 432
0ca1f7ce 433 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
d1310b2e
CM
434 u64 range = state->end - state->start + 1;
435 WARN_ON(range > tree->dirty_bytes);
436 tree->dirty_bytes -= range;
437 }
291d673e 438 clear_state_cb(tree, state, bits);
32c00aff 439 state->state &= ~bits_to_clear;
d1310b2e
CM
440 if (wake)
441 wake_up(&state->wq);
0ca1f7ce 442 if (state->state == 0) {
cdc6a395 443 next = next_state(state);
70dec807 444 if (state->tree) {
d1310b2e 445 rb_erase(&state->rb_node, &tree->state);
70dec807 446 state->tree = NULL;
d1310b2e
CM
447 free_extent_state(state);
448 } else {
449 WARN_ON(1);
450 }
451 } else {
452 merge_state(tree, state);
cdc6a395 453 next = next_state(state);
d1310b2e 454 }
cdc6a395 455 return next;
d1310b2e
CM
456}
457
8233767a
XG
458static struct extent_state *
459alloc_extent_state_atomic(struct extent_state *prealloc)
460{
461 if (!prealloc)
462 prealloc = alloc_extent_state(GFP_ATOMIC);
463
464 return prealloc;
465}
466
c2d904e0
JM
467void extent_io_tree_panic(struct extent_io_tree *tree, int err)
468{
469 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
470 "Extent tree was modified by another "
471 "thread while locked.");
472}
473
d1310b2e
CM
474/*
475 * clear some bits on a range in the tree. This may require splitting
476 * or inserting elements in the tree, so the gfp mask is used to
477 * indicate which allocations or sleeping are allowed.
478 *
479 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
480 * the given range from the tree regardless of state (ie for truncate).
481 *
482 * the range [start, end] is inclusive.
483 *
6763af84 484 * This takes the tree lock, and returns 0 on success and < 0 on error.
d1310b2e
CM
485 */
486int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
2c64c53d
CM
487 int bits, int wake, int delete,
488 struct extent_state **cached_state,
489 gfp_t mask)
d1310b2e
CM
490{
491 struct extent_state *state;
2c64c53d 492 struct extent_state *cached;
d1310b2e
CM
493 struct extent_state *prealloc = NULL;
494 struct rb_node *node;
5c939df5 495 u64 last_end;
d1310b2e 496 int err;
2ac55d41 497 int clear = 0;
d1310b2e 498
0ca1f7ce
YZ
499 if (delete)
500 bits |= ~EXTENT_CTLBITS;
501 bits |= EXTENT_FIRST_DELALLOC;
502
2ac55d41
JB
503 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
504 clear = 1;
d1310b2e
CM
505again:
506 if (!prealloc && (mask & __GFP_WAIT)) {
507 prealloc = alloc_extent_state(mask);
508 if (!prealloc)
509 return -ENOMEM;
510 }
511
cad321ad 512 spin_lock(&tree->lock);
2c64c53d
CM
513 if (cached_state) {
514 cached = *cached_state;
2ac55d41
JB
515
516 if (clear) {
517 *cached_state = NULL;
518 cached_state = NULL;
519 }
520
df98b6e2
JB
521 if (cached && cached->tree && cached->start <= start &&
522 cached->end > start) {
2ac55d41
JB
523 if (clear)
524 atomic_dec(&cached->refs);
2c64c53d 525 state = cached;
42daec29 526 goto hit_next;
2c64c53d 527 }
2ac55d41
JB
528 if (clear)
529 free_extent_state(cached);
2c64c53d 530 }
d1310b2e
CM
531 /*
532 * this search will find the extents that end after
533 * our range starts
534 */
80ea96b1 535 node = tree_search(tree, start);
d1310b2e
CM
536 if (!node)
537 goto out;
538 state = rb_entry(node, struct extent_state, rb_node);
2c64c53d 539hit_next:
d1310b2e
CM
540 if (state->start > end)
541 goto out;
542 WARN_ON(state->end < start);
5c939df5 543 last_end = state->end;
d1310b2e 544
0449314a 545 /* the state doesn't have the wanted bits, go ahead */
cdc6a395
LZ
546 if (!(state->state & bits)) {
547 state = next_state(state);
0449314a 548 goto next;
cdc6a395 549 }
0449314a 550
d1310b2e
CM
551 /*
552 * | ---- desired range ---- |
553 * | state | or
554 * | ------------- state -------------- |
555 *
556 * We need to split the extent we found, and may flip
557 * bits on second half.
558 *
559 * If the extent we found extends past our range, we
560 * just split and search again. It'll get split again
561 * the next time though.
562 *
563 * If the extent we found is inside our range, we clear
564 * the desired bit on it.
565 */
566
567 if (state->start < start) {
8233767a
XG
568 prealloc = alloc_extent_state_atomic(prealloc);
569 BUG_ON(!prealloc);
d1310b2e 570 err = split_state(tree, state, prealloc, start);
c2d904e0
JM
571 if (err)
572 extent_io_tree_panic(tree, err);
573
d1310b2e
CM
574 prealloc = NULL;
575 if (err)
576 goto out;
577 if (state->end <= end) {
d1ac6e41
LB
578 state = clear_state_bit(tree, state, &bits, wake);
579 goto next;
d1310b2e
CM
580 }
581 goto search_again;
582 }
583 /*
584 * | ---- desired range ---- |
585 * | state |
586 * We need to split the extent, and clear the bit
587 * on the first half
588 */
589 if (state->start <= end && state->end > end) {
8233767a
XG
590 prealloc = alloc_extent_state_atomic(prealloc);
591 BUG_ON(!prealloc);
d1310b2e 592 err = split_state(tree, state, prealloc, end + 1);
c2d904e0
JM
593 if (err)
594 extent_io_tree_panic(tree, err);
595
d1310b2e
CM
596 if (wake)
597 wake_up(&state->wq);
42daec29 598
6763af84 599 clear_state_bit(tree, prealloc, &bits, wake);
9ed74f2d 600
d1310b2e
CM
601 prealloc = NULL;
602 goto out;
603 }
42daec29 604
cdc6a395 605 state = clear_state_bit(tree, state, &bits, wake);
0449314a 606next:
5c939df5
YZ
607 if (last_end == (u64)-1)
608 goto out;
609 start = last_end + 1;
cdc6a395 610 if (start <= end && state && !need_resched())
692e5759 611 goto hit_next;
d1310b2e
CM
612 goto search_again;
613
614out:
cad321ad 615 spin_unlock(&tree->lock);
d1310b2e
CM
616 if (prealloc)
617 free_extent_state(prealloc);
618
6763af84 619 return 0;
d1310b2e
CM
620
621search_again:
622 if (start > end)
623 goto out;
cad321ad 624 spin_unlock(&tree->lock);
d1310b2e
CM
625 if (mask & __GFP_WAIT)
626 cond_resched();
627 goto again;
628}
d1310b2e 629
143bede5
JM
630static void wait_on_state(struct extent_io_tree *tree,
631 struct extent_state *state)
641f5219
CH
632 __releases(tree->lock)
633 __acquires(tree->lock)
d1310b2e
CM
634{
635 DEFINE_WAIT(wait);
636 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
cad321ad 637 spin_unlock(&tree->lock);
d1310b2e 638 schedule();
cad321ad 639 spin_lock(&tree->lock);
d1310b2e 640 finish_wait(&state->wq, &wait);
d1310b2e
CM
641}
642
643/*
644 * waits for one or more bits to clear on a range in the state tree.
645 * The range [start, end] is inclusive.
646 * The tree lock is taken by this function
647 */
143bede5 648void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
d1310b2e
CM
649{
650 struct extent_state *state;
651 struct rb_node *node;
652
cad321ad 653 spin_lock(&tree->lock);
d1310b2e
CM
654again:
655 while (1) {
656 /*
657 * this search will find all the extents that end after
658 * our range starts
659 */
80ea96b1 660 node = tree_search(tree, start);
d1310b2e
CM
661 if (!node)
662 break;
663
664 state = rb_entry(node, struct extent_state, rb_node);
665
666 if (state->start > end)
667 goto out;
668
669 if (state->state & bits) {
670 start = state->start;
671 atomic_inc(&state->refs);
672 wait_on_state(tree, state);
673 free_extent_state(state);
674 goto again;
675 }
676 start = state->end + 1;
677
678 if (start > end)
679 break;
680
ded91f08 681 cond_resched_lock(&tree->lock);
d1310b2e
CM
682 }
683out:
cad321ad 684 spin_unlock(&tree->lock);
d1310b2e 685}
d1310b2e 686
1bf85046 687static void set_state_bits(struct extent_io_tree *tree,
d1310b2e 688 struct extent_state *state,
0ca1f7ce 689 int *bits)
d1310b2e 690{
0ca1f7ce 691 int bits_to_set = *bits & ~EXTENT_CTLBITS;
9ed74f2d 692
1bf85046 693 set_state_cb(tree, state, bits);
0ca1f7ce 694 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
d1310b2e
CM
695 u64 range = state->end - state->start + 1;
696 tree->dirty_bytes += range;
697 }
0ca1f7ce 698 state->state |= bits_to_set;
d1310b2e
CM
699}
700
2c64c53d
CM
701static void cache_state(struct extent_state *state,
702 struct extent_state **cached_ptr)
703{
704 if (cached_ptr && !(*cached_ptr)) {
705 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
706 *cached_ptr = state;
707 atomic_inc(&state->refs);
708 }
709 }
710}
711
507903b8
AJ
712static void uncache_state(struct extent_state **cached_ptr)
713{
714 if (cached_ptr && (*cached_ptr)) {
715 struct extent_state *state = *cached_ptr;
109b36a2
CM
716 *cached_ptr = NULL;
717 free_extent_state(state);
507903b8
AJ
718 }
719}
720
d1310b2e 721/*
1edbb734
CM
722 * set some bits on a range in the tree. This may require allocations or
723 * sleeping, so the gfp mask is used to indicate what is allowed.
d1310b2e 724 *
1edbb734
CM
725 * If any of the exclusive bits are set, this will fail with -EEXIST if some
726 * part of the range already has the desired bits set. The start of the
727 * existing range is returned in failed_start in this case.
d1310b2e 728 *
1edbb734 729 * [start, end] is inclusive This takes the tree lock.
d1310b2e 730 */
1edbb734 731
3fbe5c02
JM
732static int __must_check
733__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
734 int bits, int exclusive_bits, u64 *failed_start,
735 struct extent_state **cached_state, gfp_t mask)
d1310b2e
CM
736{
737 struct extent_state *state;
738 struct extent_state *prealloc = NULL;
739 struct rb_node *node;
d1310b2e 740 int err = 0;
d1310b2e
CM
741 u64 last_start;
742 u64 last_end;
42daec29 743
0ca1f7ce 744 bits |= EXTENT_FIRST_DELALLOC;
d1310b2e
CM
745again:
746 if (!prealloc && (mask & __GFP_WAIT)) {
747 prealloc = alloc_extent_state(mask);
8233767a 748 BUG_ON(!prealloc);
d1310b2e
CM
749 }
750
cad321ad 751 spin_lock(&tree->lock);
9655d298
CM
752 if (cached_state && *cached_state) {
753 state = *cached_state;
df98b6e2
JB
754 if (state->start <= start && state->end > start &&
755 state->tree) {
9655d298
CM
756 node = &state->rb_node;
757 goto hit_next;
758 }
759 }
d1310b2e
CM
760 /*
761 * this search will find all the extents that end after
762 * our range starts.
763 */
80ea96b1 764 node = tree_search(tree, start);
d1310b2e 765 if (!node) {
8233767a
XG
766 prealloc = alloc_extent_state_atomic(prealloc);
767 BUG_ON(!prealloc);
0ca1f7ce 768 err = insert_state(tree, prealloc, start, end, &bits);
c2d904e0
JM
769 if (err)
770 extent_io_tree_panic(tree, err);
771
d1310b2e 772 prealloc = NULL;
d1310b2e
CM
773 goto out;
774 }
d1310b2e 775 state = rb_entry(node, struct extent_state, rb_node);
40431d6c 776hit_next:
d1310b2e
CM
777 last_start = state->start;
778 last_end = state->end;
779
780 /*
781 * | ---- desired range ---- |
782 * | state |
783 *
784 * Just lock what we found and keep going
785 */
786 if (state->start == start && state->end <= end) {
1edbb734 787 if (state->state & exclusive_bits) {
d1310b2e
CM
788 *failed_start = state->start;
789 err = -EEXIST;
790 goto out;
791 }
42daec29 792
1bf85046 793 set_state_bits(tree, state, &bits);
2c64c53d 794 cache_state(state, cached_state);
d1310b2e 795 merge_state(tree, state);
5c939df5
YZ
796 if (last_end == (u64)-1)
797 goto out;
798 start = last_end + 1;
d1ac6e41
LB
799 state = next_state(state);
800 if (start < end && state && state->start == start &&
801 !need_resched())
802 goto hit_next;
d1310b2e
CM
803 goto search_again;
804 }
805
806 /*
807 * | ---- desired range ---- |
808 * | state |
809 * or
810 * | ------------- state -------------- |
811 *
812 * We need to split the extent we found, and may flip bits on
813 * second half.
814 *
815 * If the extent we found extends past our
816 * range, we just split and search again. It'll get split
817 * again the next time though.
818 *
819 * If the extent we found is inside our range, we set the
820 * desired bit on it.
821 */
822 if (state->start < start) {
1edbb734 823 if (state->state & exclusive_bits) {
d1310b2e
CM
824 *failed_start = start;
825 err = -EEXIST;
826 goto out;
827 }
8233767a
XG
828
829 prealloc = alloc_extent_state_atomic(prealloc);
830 BUG_ON(!prealloc);
d1310b2e 831 err = split_state(tree, state, prealloc, start);
c2d904e0
JM
832 if (err)
833 extent_io_tree_panic(tree, err);
834
d1310b2e
CM
835 prealloc = NULL;
836 if (err)
837 goto out;
838 if (state->end <= end) {
1bf85046 839 set_state_bits(tree, state, &bits);
2c64c53d 840 cache_state(state, cached_state);
d1310b2e 841 merge_state(tree, state);
5c939df5
YZ
842 if (last_end == (u64)-1)
843 goto out;
844 start = last_end + 1;
d1ac6e41
LB
845 state = next_state(state);
846 if (start < end && state && state->start == start &&
847 !need_resched())
848 goto hit_next;
d1310b2e
CM
849 }
850 goto search_again;
851 }
852 /*
853 * | ---- desired range ---- |
854 * | state | or | state |
855 *
856 * There's a hole, we need to insert something in it and
857 * ignore the extent we found.
858 */
859 if (state->start > start) {
860 u64 this_end;
861 if (end < last_start)
862 this_end = end;
863 else
d397712b 864 this_end = last_start - 1;
8233767a
XG
865
866 prealloc = alloc_extent_state_atomic(prealloc);
867 BUG_ON(!prealloc);
c7f895a2
XG
868
869 /*
870 * Avoid to free 'prealloc' if it can be merged with
871 * the later extent.
872 */
d1310b2e 873 err = insert_state(tree, prealloc, start, this_end,
0ca1f7ce 874 &bits);
c2d904e0
JM
875 if (err)
876 extent_io_tree_panic(tree, err);
877
9ed74f2d
JB
878 cache_state(prealloc, cached_state);
879 prealloc = NULL;
d1310b2e
CM
880 start = this_end + 1;
881 goto search_again;
882 }
883 /*
884 * | ---- desired range ---- |
885 * | state |
886 * We need to split the extent, and set the bit
887 * on the first half
888 */
889 if (state->start <= end && state->end > end) {
1edbb734 890 if (state->state & exclusive_bits) {
d1310b2e
CM
891 *failed_start = start;
892 err = -EEXIST;
893 goto out;
894 }
8233767a
XG
895
896 prealloc = alloc_extent_state_atomic(prealloc);
897 BUG_ON(!prealloc);
d1310b2e 898 err = split_state(tree, state, prealloc, end + 1);
c2d904e0
JM
899 if (err)
900 extent_io_tree_panic(tree, err);
d1310b2e 901
1bf85046 902 set_state_bits(tree, prealloc, &bits);
2c64c53d 903 cache_state(prealloc, cached_state);
d1310b2e
CM
904 merge_state(tree, prealloc);
905 prealloc = NULL;
906 goto out;
907 }
908
909 goto search_again;
910
911out:
cad321ad 912 spin_unlock(&tree->lock);
d1310b2e
CM
913 if (prealloc)
914 free_extent_state(prealloc);
915
916 return err;
917
918search_again:
919 if (start > end)
920 goto out;
cad321ad 921 spin_unlock(&tree->lock);
d1310b2e
CM
922 if (mask & __GFP_WAIT)
923 cond_resched();
924 goto again;
925}
d1310b2e 926
3fbe5c02
JM
927int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits,
928 u64 *failed_start, struct extent_state **cached_state,
929 gfp_t mask)
930{
931 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
932 cached_state, mask);
933}
934
935
462d6fac 936/**
10983f2e
LB
937 * convert_extent_bit - convert all bits in a given range from one bit to
938 * another
462d6fac
JB
939 * @tree: the io tree to search
940 * @start: the start offset in bytes
941 * @end: the end offset in bytes (inclusive)
942 * @bits: the bits to set in this range
943 * @clear_bits: the bits to clear in this range
e6138876 944 * @cached_state: state that we're going to cache
462d6fac
JB
945 * @mask: the allocation mask
946 *
947 * This will go through and set bits for the given range. If any states exist
948 * already in this range they are set with the given bit and cleared of the
949 * clear_bits. This is only meant to be used by things that are mergeable, ie
950 * converting from say DELALLOC to DIRTY. This is not meant to be used with
951 * boundary bits like LOCK.
952 */
953int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
e6138876
JB
954 int bits, int clear_bits,
955 struct extent_state **cached_state, gfp_t mask)
462d6fac
JB
956{
957 struct extent_state *state;
958 struct extent_state *prealloc = NULL;
959 struct rb_node *node;
960 int err = 0;
961 u64 last_start;
962 u64 last_end;
963
964again:
965 if (!prealloc && (mask & __GFP_WAIT)) {
966 prealloc = alloc_extent_state(mask);
967 if (!prealloc)
968 return -ENOMEM;
969 }
970
971 spin_lock(&tree->lock);
e6138876
JB
972 if (cached_state && *cached_state) {
973 state = *cached_state;
974 if (state->start <= start && state->end > start &&
975 state->tree) {
976 node = &state->rb_node;
977 goto hit_next;
978 }
979 }
980
462d6fac
JB
981 /*
982 * this search will find all the extents that end after
983 * our range starts.
984 */
985 node = tree_search(tree, start);
986 if (!node) {
987 prealloc = alloc_extent_state_atomic(prealloc);
1cf4ffdb
LB
988 if (!prealloc) {
989 err = -ENOMEM;
990 goto out;
991 }
462d6fac
JB
992 err = insert_state(tree, prealloc, start, end, &bits);
993 prealloc = NULL;
c2d904e0
JM
994 if (err)
995 extent_io_tree_panic(tree, err);
462d6fac
JB
996 goto out;
997 }
998 state = rb_entry(node, struct extent_state, rb_node);
999hit_next:
1000 last_start = state->start;
1001 last_end = state->end;
1002
1003 /*
1004 * | ---- desired range ---- |
1005 * | state |
1006 *
1007 * Just lock what we found and keep going
1008 */
1009 if (state->start == start && state->end <= end) {
462d6fac 1010 set_state_bits(tree, state, &bits);
e6138876 1011 cache_state(state, cached_state);
d1ac6e41 1012 state = clear_state_bit(tree, state, &clear_bits, 0);
462d6fac
JB
1013 if (last_end == (u64)-1)
1014 goto out;
462d6fac 1015 start = last_end + 1;
d1ac6e41
LB
1016 if (start < end && state && state->start == start &&
1017 !need_resched())
1018 goto hit_next;
462d6fac
JB
1019 goto search_again;
1020 }
1021
1022 /*
1023 * | ---- desired range ---- |
1024 * | state |
1025 * or
1026 * | ------------- state -------------- |
1027 *
1028 * We need to split the extent we found, and may flip bits on
1029 * second half.
1030 *
1031 * If the extent we found extends past our
1032 * range, we just split and search again. It'll get split
1033 * again the next time though.
1034 *
1035 * If the extent we found is inside our range, we set the
1036 * desired bit on it.
1037 */
1038 if (state->start < start) {
1039 prealloc = alloc_extent_state_atomic(prealloc);
1cf4ffdb
LB
1040 if (!prealloc) {
1041 err = -ENOMEM;
1042 goto out;
1043 }
462d6fac 1044 err = split_state(tree, state, prealloc, start);
c2d904e0
JM
1045 if (err)
1046 extent_io_tree_panic(tree, err);
462d6fac
JB
1047 prealloc = NULL;
1048 if (err)
1049 goto out;
1050 if (state->end <= end) {
1051 set_state_bits(tree, state, &bits);
e6138876 1052 cache_state(state, cached_state);
d1ac6e41 1053 state = clear_state_bit(tree, state, &clear_bits, 0);
462d6fac
JB
1054 if (last_end == (u64)-1)
1055 goto out;
1056 start = last_end + 1;
d1ac6e41
LB
1057 if (start < end && state && state->start == start &&
1058 !need_resched())
1059 goto hit_next;
462d6fac
JB
1060 }
1061 goto search_again;
1062 }
1063 /*
1064 * | ---- desired range ---- |
1065 * | state | or | state |
1066 *
1067 * There's a hole, we need to insert something in it and
1068 * ignore the extent we found.
1069 */
1070 if (state->start > start) {
1071 u64 this_end;
1072 if (end < last_start)
1073 this_end = end;
1074 else
1075 this_end = last_start - 1;
1076
1077 prealloc = alloc_extent_state_atomic(prealloc);
1cf4ffdb
LB
1078 if (!prealloc) {
1079 err = -ENOMEM;
1080 goto out;
1081 }
462d6fac
JB
1082
1083 /*
1084 * Avoid to free 'prealloc' if it can be merged with
1085 * the later extent.
1086 */
1087 err = insert_state(tree, prealloc, start, this_end,
1088 &bits);
c2d904e0
JM
1089 if (err)
1090 extent_io_tree_panic(tree, err);
e6138876 1091 cache_state(prealloc, cached_state);
462d6fac
JB
1092 prealloc = NULL;
1093 start = this_end + 1;
1094 goto search_again;
1095 }
1096 /*
1097 * | ---- desired range ---- |
1098 * | state |
1099 * We need to split the extent, and set the bit
1100 * on the first half
1101 */
1102 if (state->start <= end && state->end > end) {
1103 prealloc = alloc_extent_state_atomic(prealloc);
1cf4ffdb
LB
1104 if (!prealloc) {
1105 err = -ENOMEM;
1106 goto out;
1107 }
462d6fac
JB
1108
1109 err = split_state(tree, state, prealloc, end + 1);
c2d904e0
JM
1110 if (err)
1111 extent_io_tree_panic(tree, err);
462d6fac
JB
1112
1113 set_state_bits(tree, prealloc, &bits);
e6138876 1114 cache_state(prealloc, cached_state);
462d6fac 1115 clear_state_bit(tree, prealloc, &clear_bits, 0);
462d6fac
JB
1116 prealloc = NULL;
1117 goto out;
1118 }
1119
1120 goto search_again;
1121
1122out:
1123 spin_unlock(&tree->lock);
1124 if (prealloc)
1125 free_extent_state(prealloc);
1126
1127 return err;
1128
1129search_again:
1130 if (start > end)
1131 goto out;
1132 spin_unlock(&tree->lock);
1133 if (mask & __GFP_WAIT)
1134 cond_resched();
1135 goto again;
1136}
1137
d1310b2e
CM
1138/* wrappers around set/clear extent bit */
1139int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1140 gfp_t mask)
1141{
3fbe5c02 1142 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
2c64c53d 1143 NULL, mask);
d1310b2e 1144}
d1310b2e
CM
1145
1146int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1147 int bits, gfp_t mask)
1148{
3fbe5c02 1149 return set_extent_bit(tree, start, end, bits, NULL,
2c64c53d 1150 NULL, mask);
d1310b2e 1151}
d1310b2e
CM
1152
1153int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1154 int bits, gfp_t mask)
1155{
2c64c53d 1156 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
d1310b2e 1157}
d1310b2e
CM
1158
1159int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
2ac55d41 1160 struct extent_state **cached_state, gfp_t mask)
d1310b2e
CM
1161{
1162 return set_extent_bit(tree, start, end,
fee187d9 1163 EXTENT_DELALLOC | EXTENT_UPTODATE,
3fbe5c02 1164 NULL, cached_state, mask);
d1310b2e 1165}
d1310b2e 1166
9e8a4a8b
LB
1167int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1168 struct extent_state **cached_state, gfp_t mask)
1169{
1170 return set_extent_bit(tree, start, end,
1171 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1172 NULL, cached_state, mask);
1173}
1174
d1310b2e
CM
1175int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1176 gfp_t mask)
1177{
1178 return clear_extent_bit(tree, start, end,
32c00aff 1179 EXTENT_DIRTY | EXTENT_DELALLOC |
0ca1f7ce 1180 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
d1310b2e 1181}
d1310b2e
CM
1182
1183int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1184 gfp_t mask)
1185{
3fbe5c02 1186 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
2c64c53d 1187 NULL, mask);
d1310b2e 1188}
d1310b2e 1189
d1310b2e 1190int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
507903b8 1191 struct extent_state **cached_state, gfp_t mask)
d1310b2e 1192{
507903b8 1193 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0,
3fbe5c02 1194 cached_state, mask);
d1310b2e 1195}
d1310b2e 1196
5fd02043
JB
1197int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1198 struct extent_state **cached_state, gfp_t mask)
d1310b2e 1199{
2c64c53d 1200 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
2ac55d41 1201 cached_state, mask);
d1310b2e 1202}
d1310b2e 1203
d352ac68
CM
1204/*
1205 * either insert or lock state struct between start and end use mask to tell
1206 * us if waiting is desired.
1207 */
1edbb734 1208int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
d0082371 1209 int bits, struct extent_state **cached_state)
d1310b2e
CM
1210{
1211 int err;
1212 u64 failed_start;
1213 while (1) {
3fbe5c02
JM
1214 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1215 EXTENT_LOCKED, &failed_start,
1216 cached_state, GFP_NOFS);
d0082371 1217 if (err == -EEXIST) {
d1310b2e
CM
1218 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1219 start = failed_start;
d0082371 1220 } else
d1310b2e 1221 break;
d1310b2e
CM
1222 WARN_ON(start > end);
1223 }
1224 return err;
1225}
d1310b2e 1226
d0082371 1227int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1edbb734 1228{
d0082371 1229 return lock_extent_bits(tree, start, end, 0, NULL);
1edbb734
CM
1230}
1231
d0082371 1232int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
25179201
JB
1233{
1234 int err;
1235 u64 failed_start;
1236
3fbe5c02
JM
1237 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1238 &failed_start, NULL, GFP_NOFS);
6643558d
YZ
1239 if (err == -EEXIST) {
1240 if (failed_start > start)
1241 clear_extent_bit(tree, start, failed_start - 1,
d0082371 1242 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
25179201 1243 return 0;
6643558d 1244 }
25179201
JB
1245 return 1;
1246}
25179201 1247
2c64c53d
CM
1248int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1249 struct extent_state **cached, gfp_t mask)
1250{
1251 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1252 mask);
1253}
1254
d0082371 1255int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
d1310b2e 1256{
2c64c53d 1257 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
d0082371 1258 GFP_NOFS);
d1310b2e 1259}
d1310b2e 1260
d1310b2e
CM
1261/*
1262 * helper function to set both pages and extents in the tree writeback
1263 */
b2950863 1264static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
d1310b2e
CM
1265{
1266 unsigned long index = start >> PAGE_CACHE_SHIFT;
1267 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1268 struct page *page;
1269
1270 while (index <= end_index) {
1271 page = find_get_page(tree->mapping, index);
79787eaa 1272 BUG_ON(!page); /* Pages should be in the extent_io_tree */
d1310b2e
CM
1273 set_page_writeback(page);
1274 page_cache_release(page);
1275 index++;
1276 }
d1310b2e
CM
1277 return 0;
1278}
d1310b2e 1279
d352ac68
CM
1280/* find the first state struct with 'bits' set after 'start', and
1281 * return it. tree->lock must be held. NULL will returned if
1282 * nothing was found after 'start'
1283 */
d7fc640e
CM
1284struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree,
1285 u64 start, int bits)
1286{
1287 struct rb_node *node;
1288 struct extent_state *state;
1289
1290 /*
1291 * this search will find all the extents that end after
1292 * our range starts.
1293 */
1294 node = tree_search(tree, start);
d397712b 1295 if (!node)
d7fc640e 1296 goto out;
d7fc640e 1297
d397712b 1298 while (1) {
d7fc640e 1299 state = rb_entry(node, struct extent_state, rb_node);
d397712b 1300 if (state->end >= start && (state->state & bits))
d7fc640e 1301 return state;
d397712b 1302
d7fc640e
CM
1303 node = rb_next(node);
1304 if (!node)
1305 break;
1306 }
1307out:
1308 return NULL;
1309}
d7fc640e 1310
69261c4b
XG
1311/*
1312 * find the first offset in the io tree with 'bits' set. zero is
1313 * returned if we find something, and *start_ret and *end_ret are
1314 * set to reflect the state struct that was found.
1315 *
477d7eaf 1316 * If nothing was found, 1 is returned. If found something, return 0.
69261c4b
XG
1317 */
1318int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
e6138876
JB
1319 u64 *start_ret, u64 *end_ret, int bits,
1320 struct extent_state **cached_state)
69261c4b
XG
1321{
1322 struct extent_state *state;
e6138876 1323 struct rb_node *n;
69261c4b
XG
1324 int ret = 1;
1325
1326 spin_lock(&tree->lock);
e6138876
JB
1327 if (cached_state && *cached_state) {
1328 state = *cached_state;
1329 if (state->end == start - 1 && state->tree) {
1330 n = rb_next(&state->rb_node);
1331 while (n) {
1332 state = rb_entry(n, struct extent_state,
1333 rb_node);
1334 if (state->state & bits)
1335 goto got_it;
1336 n = rb_next(n);
1337 }
1338 free_extent_state(*cached_state);
1339 *cached_state = NULL;
1340 goto out;
1341 }
1342 free_extent_state(*cached_state);
1343 *cached_state = NULL;
1344 }
1345
69261c4b 1346 state = find_first_extent_bit_state(tree, start, bits);
e6138876 1347got_it:
69261c4b 1348 if (state) {
e6138876 1349 cache_state(state, cached_state);
69261c4b
XG
1350 *start_ret = state->start;
1351 *end_ret = state->end;
1352 ret = 0;
1353 }
e6138876 1354out:
69261c4b
XG
1355 spin_unlock(&tree->lock);
1356 return ret;
1357}
1358
d352ac68
CM
1359/*
1360 * find a contiguous range of bytes in the file marked as delalloc, not
1361 * more than 'max_bytes'. start and end are used to return the range,
1362 *
1363 * 1 is returned if we find something, 0 if nothing was in the tree
1364 */
c8b97818 1365static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
c2a128d2
JB
1366 u64 *start, u64 *end, u64 max_bytes,
1367 struct extent_state **cached_state)
d1310b2e
CM
1368{
1369 struct rb_node *node;
1370 struct extent_state *state;
1371 u64 cur_start = *start;
1372 u64 found = 0;
1373 u64 total_bytes = 0;
1374
cad321ad 1375 spin_lock(&tree->lock);
c8b97818 1376
d1310b2e
CM
1377 /*
1378 * this search will find all the extents that end after
1379 * our range starts.
1380 */
80ea96b1 1381 node = tree_search(tree, cur_start);
2b114d1d 1382 if (!node) {
3b951516
CM
1383 if (!found)
1384 *end = (u64)-1;
d1310b2e
CM
1385 goto out;
1386 }
1387
d397712b 1388 while (1) {
d1310b2e 1389 state = rb_entry(node, struct extent_state, rb_node);
5b21f2ed
ZY
1390 if (found && (state->start != cur_start ||
1391 (state->state & EXTENT_BOUNDARY))) {
d1310b2e
CM
1392 goto out;
1393 }
1394 if (!(state->state & EXTENT_DELALLOC)) {
1395 if (!found)
1396 *end = state->end;
1397 goto out;
1398 }
c2a128d2 1399 if (!found) {
d1310b2e 1400 *start = state->start;
c2a128d2
JB
1401 *cached_state = state;
1402 atomic_inc(&state->refs);
1403 }
d1310b2e
CM
1404 found++;
1405 *end = state->end;
1406 cur_start = state->end + 1;
1407 node = rb_next(node);
1408 if (!node)
1409 break;
1410 total_bytes += state->end - state->start + 1;
1411 if (total_bytes >= max_bytes)
1412 break;
1413 }
1414out:
cad321ad 1415 spin_unlock(&tree->lock);
d1310b2e
CM
1416 return found;
1417}
1418
143bede5
JM
1419static noinline void __unlock_for_delalloc(struct inode *inode,
1420 struct page *locked_page,
1421 u64 start, u64 end)
c8b97818
CM
1422{
1423 int ret;
1424 struct page *pages[16];
1425 unsigned long index = start >> PAGE_CACHE_SHIFT;
1426 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1427 unsigned long nr_pages = end_index - index + 1;
1428 int i;
1429
1430 if (index == locked_page->index && end_index == index)
143bede5 1431 return;
c8b97818 1432
d397712b 1433 while (nr_pages > 0) {
c8b97818 1434 ret = find_get_pages_contig(inode->i_mapping, index,
5b050f04
CM
1435 min_t(unsigned long, nr_pages,
1436 ARRAY_SIZE(pages)), pages);
c8b97818
CM
1437 for (i = 0; i < ret; i++) {
1438 if (pages[i] != locked_page)
1439 unlock_page(pages[i]);
1440 page_cache_release(pages[i]);
1441 }
1442 nr_pages -= ret;
1443 index += ret;
1444 cond_resched();
1445 }
c8b97818
CM
1446}
1447
1448static noinline int lock_delalloc_pages(struct inode *inode,
1449 struct page *locked_page,
1450 u64 delalloc_start,
1451 u64 delalloc_end)
1452{
1453 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1454 unsigned long start_index = index;
1455 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1456 unsigned long pages_locked = 0;
1457 struct page *pages[16];
1458 unsigned long nrpages;
1459 int ret;
1460 int i;
1461
1462 /* the caller is responsible for locking the start index */
1463 if (index == locked_page->index && index == end_index)
1464 return 0;
1465
1466 /* skip the page at the start index */
1467 nrpages = end_index - index + 1;
d397712b 1468 while (nrpages > 0) {
c8b97818 1469 ret = find_get_pages_contig(inode->i_mapping, index,
5b050f04
CM
1470 min_t(unsigned long,
1471 nrpages, ARRAY_SIZE(pages)), pages);
c8b97818
CM
1472 if (ret == 0) {
1473 ret = -EAGAIN;
1474 goto done;
1475 }
1476 /* now we have an array of pages, lock them all */
1477 for (i = 0; i < ret; i++) {
1478 /*
1479 * the caller is taking responsibility for
1480 * locked_page
1481 */
771ed689 1482 if (pages[i] != locked_page) {
c8b97818 1483 lock_page(pages[i]);
f2b1c41c
CM
1484 if (!PageDirty(pages[i]) ||
1485 pages[i]->mapping != inode->i_mapping) {
771ed689
CM
1486 ret = -EAGAIN;
1487 unlock_page(pages[i]);
1488 page_cache_release(pages[i]);
1489 goto done;
1490 }
1491 }
c8b97818 1492 page_cache_release(pages[i]);
771ed689 1493 pages_locked++;
c8b97818 1494 }
c8b97818
CM
1495 nrpages -= ret;
1496 index += ret;
1497 cond_resched();
1498 }
1499 ret = 0;
1500done:
1501 if (ret && pages_locked) {
1502 __unlock_for_delalloc(inode, locked_page,
1503 delalloc_start,
1504 ((u64)(start_index + pages_locked - 1)) <<
1505 PAGE_CACHE_SHIFT);
1506 }
1507 return ret;
1508}
1509
1510/*
1511 * find a contiguous range of bytes in the file marked as delalloc, not
1512 * more than 'max_bytes'. start and end are used to return the range,
1513 *
1514 * 1 is returned if we find something, 0 if nothing was in the tree
1515 */
1516static noinline u64 find_lock_delalloc_range(struct inode *inode,
1517 struct extent_io_tree *tree,
1518 struct page *locked_page,
1519 u64 *start, u64 *end,
1520 u64 max_bytes)
1521{
1522 u64 delalloc_start;
1523 u64 delalloc_end;
1524 u64 found;
9655d298 1525 struct extent_state *cached_state = NULL;
c8b97818
CM
1526 int ret;
1527 int loops = 0;
1528
1529again:
1530 /* step one, find a bunch of delalloc bytes starting at start */
1531 delalloc_start = *start;
1532 delalloc_end = 0;
1533 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
c2a128d2 1534 max_bytes, &cached_state);
70b99e69 1535 if (!found || delalloc_end <= *start) {
c8b97818
CM
1536 *start = delalloc_start;
1537 *end = delalloc_end;
c2a128d2 1538 free_extent_state(cached_state);
c8b97818
CM
1539 return found;
1540 }
1541
70b99e69
CM
1542 /*
1543 * start comes from the offset of locked_page. We have to lock
1544 * pages in order, so we can't process delalloc bytes before
1545 * locked_page
1546 */
d397712b 1547 if (delalloc_start < *start)
70b99e69 1548 delalloc_start = *start;
70b99e69 1549
c8b97818
CM
1550 /*
1551 * make sure to limit the number of pages we try to lock down
1552 * if we're looping.
1553 */
d397712b 1554 if (delalloc_end + 1 - delalloc_start > max_bytes && loops)
771ed689 1555 delalloc_end = delalloc_start + PAGE_CACHE_SIZE - 1;
d397712b 1556
c8b97818
CM
1557 /* step two, lock all the pages after the page that has start */
1558 ret = lock_delalloc_pages(inode, locked_page,
1559 delalloc_start, delalloc_end);
1560 if (ret == -EAGAIN) {
1561 /* some of the pages are gone, lets avoid looping by
1562 * shortening the size of the delalloc range we're searching
1563 */
9655d298 1564 free_extent_state(cached_state);
c8b97818
CM
1565 if (!loops) {
1566 unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1);
1567 max_bytes = PAGE_CACHE_SIZE - offset;
1568 loops = 1;
1569 goto again;
1570 } else {
1571 found = 0;
1572 goto out_failed;
1573 }
1574 }
79787eaa 1575 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
c8b97818
CM
1576
1577 /* step three, lock the state bits for the whole range */
d0082371 1578 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
c8b97818
CM
1579
1580 /* then test to make sure it is all still delalloc */
1581 ret = test_range_bit(tree, delalloc_start, delalloc_end,
9655d298 1582 EXTENT_DELALLOC, 1, cached_state);
c8b97818 1583 if (!ret) {
9655d298
CM
1584 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1585 &cached_state, GFP_NOFS);
c8b97818
CM
1586 __unlock_for_delalloc(inode, locked_page,
1587 delalloc_start, delalloc_end);
1588 cond_resched();
1589 goto again;
1590 }
9655d298 1591 free_extent_state(cached_state);
c8b97818
CM
1592 *start = delalloc_start;
1593 *end = delalloc_end;
1594out_failed:
1595 return found;
1596}
1597
1598int extent_clear_unlock_delalloc(struct inode *inode,
1599 struct extent_io_tree *tree,
1600 u64 start, u64 end, struct page *locked_page,
a791e35e 1601 unsigned long op)
c8b97818
CM
1602{
1603 int ret;
1604 struct page *pages[16];
1605 unsigned long index = start >> PAGE_CACHE_SHIFT;
1606 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1607 unsigned long nr_pages = end_index - index + 1;
1608 int i;
771ed689 1609 int clear_bits = 0;
c8b97818 1610
a791e35e 1611 if (op & EXTENT_CLEAR_UNLOCK)
771ed689 1612 clear_bits |= EXTENT_LOCKED;
a791e35e 1613 if (op & EXTENT_CLEAR_DIRTY)
c8b97818
CM
1614 clear_bits |= EXTENT_DIRTY;
1615
a791e35e 1616 if (op & EXTENT_CLEAR_DELALLOC)
771ed689
CM
1617 clear_bits |= EXTENT_DELALLOC;
1618
2c64c53d 1619 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
32c00aff
JB
1620 if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY |
1621 EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK |
1622 EXTENT_SET_PRIVATE2)))
771ed689 1623 return 0;
c8b97818 1624
d397712b 1625 while (nr_pages > 0) {
c8b97818 1626 ret = find_get_pages_contig(inode->i_mapping, index,
5b050f04
CM
1627 min_t(unsigned long,
1628 nr_pages, ARRAY_SIZE(pages)), pages);
c8b97818 1629 for (i = 0; i < ret; i++) {
8b62b72b 1630
a791e35e 1631 if (op & EXTENT_SET_PRIVATE2)
8b62b72b
CM
1632 SetPagePrivate2(pages[i]);
1633
c8b97818
CM
1634 if (pages[i] == locked_page) {
1635 page_cache_release(pages[i]);
1636 continue;
1637 }
a791e35e 1638 if (op & EXTENT_CLEAR_DIRTY)
c8b97818 1639 clear_page_dirty_for_io(pages[i]);
a791e35e 1640 if (op & EXTENT_SET_WRITEBACK)
c8b97818 1641 set_page_writeback(pages[i]);
a791e35e 1642 if (op & EXTENT_END_WRITEBACK)
c8b97818 1643 end_page_writeback(pages[i]);
a791e35e 1644 if (op & EXTENT_CLEAR_UNLOCK_PAGE)
771ed689 1645 unlock_page(pages[i]);
c8b97818
CM
1646 page_cache_release(pages[i]);
1647 }
1648 nr_pages -= ret;
1649 index += ret;
1650 cond_resched();
1651 }
1652 return 0;
1653}
c8b97818 1654
d352ac68
CM
1655/*
1656 * count the number of bytes in the tree that have a given bit(s)
1657 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1658 * cached. The total number found is returned.
1659 */
d1310b2e
CM
1660u64 count_range_bits(struct extent_io_tree *tree,
1661 u64 *start, u64 search_end, u64 max_bytes,
ec29ed5b 1662 unsigned long bits, int contig)
d1310b2e
CM
1663{
1664 struct rb_node *node;
1665 struct extent_state *state;
1666 u64 cur_start = *start;
1667 u64 total_bytes = 0;
ec29ed5b 1668 u64 last = 0;
d1310b2e
CM
1669 int found = 0;
1670
1671 if (search_end <= cur_start) {
d1310b2e
CM
1672 WARN_ON(1);
1673 return 0;
1674 }
1675
cad321ad 1676 spin_lock(&tree->lock);
d1310b2e
CM
1677 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1678 total_bytes = tree->dirty_bytes;
1679 goto out;
1680 }
1681 /*
1682 * this search will find all the extents that end after
1683 * our range starts.
1684 */
80ea96b1 1685 node = tree_search(tree, cur_start);
d397712b 1686 if (!node)
d1310b2e 1687 goto out;
d1310b2e 1688
d397712b 1689 while (1) {
d1310b2e
CM
1690 state = rb_entry(node, struct extent_state, rb_node);
1691 if (state->start > search_end)
1692 break;
ec29ed5b
CM
1693 if (contig && found && state->start > last + 1)
1694 break;
1695 if (state->end >= cur_start && (state->state & bits) == bits) {
d1310b2e
CM
1696 total_bytes += min(search_end, state->end) + 1 -
1697 max(cur_start, state->start);
1698 if (total_bytes >= max_bytes)
1699 break;
1700 if (!found) {
af60bed2 1701 *start = max(cur_start, state->start);
d1310b2e
CM
1702 found = 1;
1703 }
ec29ed5b
CM
1704 last = state->end;
1705 } else if (contig && found) {
1706 break;
d1310b2e
CM
1707 }
1708 node = rb_next(node);
1709 if (!node)
1710 break;
1711 }
1712out:
cad321ad 1713 spin_unlock(&tree->lock);
d1310b2e
CM
1714 return total_bytes;
1715}
b2950863 1716
d352ac68
CM
1717/*
1718 * set the private field for a given byte offset in the tree. If there isn't
1719 * an extent_state there already, this does nothing.
1720 */
d1310b2e
CM
1721int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1722{
1723 struct rb_node *node;
1724 struct extent_state *state;
1725 int ret = 0;
1726
cad321ad 1727 spin_lock(&tree->lock);
d1310b2e
CM
1728 /*
1729 * this search will find all the extents that end after
1730 * our range starts.
1731 */
80ea96b1 1732 node = tree_search(tree, start);
2b114d1d 1733 if (!node) {
d1310b2e
CM
1734 ret = -ENOENT;
1735 goto out;
1736 }
1737 state = rb_entry(node, struct extent_state, rb_node);
1738 if (state->start != start) {
1739 ret = -ENOENT;
1740 goto out;
1741 }
1742 state->private = private;
1743out:
cad321ad 1744 spin_unlock(&tree->lock);
d1310b2e
CM
1745 return ret;
1746}
1747
1748int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1749{
1750 struct rb_node *node;
1751 struct extent_state *state;
1752 int ret = 0;
1753
cad321ad 1754 spin_lock(&tree->lock);
d1310b2e
CM
1755 /*
1756 * this search will find all the extents that end after
1757 * our range starts.
1758 */
80ea96b1 1759 node = tree_search(tree, start);
2b114d1d 1760 if (!node) {
d1310b2e
CM
1761 ret = -ENOENT;
1762 goto out;
1763 }
1764 state = rb_entry(node, struct extent_state, rb_node);
1765 if (state->start != start) {
1766 ret = -ENOENT;
1767 goto out;
1768 }
1769 *private = state->private;
1770out:
cad321ad 1771 spin_unlock(&tree->lock);
d1310b2e
CM
1772 return ret;
1773}
1774
1775/*
1776 * searches a range in the state tree for a given mask.
70dec807 1777 * If 'filled' == 1, this returns 1 only if every extent in the tree
d1310b2e
CM
1778 * has the bits set. Otherwise, 1 is returned if any bit in the
1779 * range is found set.
1780 */
1781int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
9655d298 1782 int bits, int filled, struct extent_state *cached)
d1310b2e
CM
1783{
1784 struct extent_state *state = NULL;
1785 struct rb_node *node;
1786 int bitset = 0;
d1310b2e 1787
cad321ad 1788 spin_lock(&tree->lock);
df98b6e2
JB
1789 if (cached && cached->tree && cached->start <= start &&
1790 cached->end > start)
9655d298
CM
1791 node = &cached->rb_node;
1792 else
1793 node = tree_search(tree, start);
d1310b2e
CM
1794 while (node && start <= end) {
1795 state = rb_entry(node, struct extent_state, rb_node);
1796
1797 if (filled && state->start > start) {
1798 bitset = 0;
1799 break;
1800 }
1801
1802 if (state->start > end)
1803 break;
1804
1805 if (state->state & bits) {
1806 bitset = 1;
1807 if (!filled)
1808 break;
1809 } else if (filled) {
1810 bitset = 0;
1811 break;
1812 }
46562cec
CM
1813
1814 if (state->end == (u64)-1)
1815 break;
1816
d1310b2e
CM
1817 start = state->end + 1;
1818 if (start > end)
1819 break;
1820 node = rb_next(node);
1821 if (!node) {
1822 if (filled)
1823 bitset = 0;
1824 break;
1825 }
1826 }
cad321ad 1827 spin_unlock(&tree->lock);
d1310b2e
CM
1828 return bitset;
1829}
d1310b2e
CM
1830
1831/*
1832 * helper function to set a given page up to date if all the
1833 * extents in the tree for that page are up to date
1834 */
143bede5 1835static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
d1310b2e
CM
1836{
1837 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1838 u64 end = start + PAGE_CACHE_SIZE - 1;
9655d298 1839 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
d1310b2e 1840 SetPageUptodate(page);
d1310b2e
CM
1841}
1842
1843/*
1844 * helper function to unlock a page if all the extents in the tree
1845 * for that page are unlocked
1846 */
143bede5 1847static void check_page_locked(struct extent_io_tree *tree, struct page *page)
d1310b2e
CM
1848{
1849 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1850 u64 end = start + PAGE_CACHE_SIZE - 1;
9655d298 1851 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL))
d1310b2e 1852 unlock_page(page);
d1310b2e
CM
1853}
1854
1855/*
1856 * helper function to end page writeback if all the extents
1857 * in the tree for that page are done with writeback
1858 */
143bede5
JM
1859static void check_page_writeback(struct extent_io_tree *tree,
1860 struct page *page)
d1310b2e 1861{
1edbb734 1862 end_page_writeback(page);
d1310b2e
CM
1863}
1864
4a54c8c1
JS
1865/*
1866 * When IO fails, either with EIO or csum verification fails, we
1867 * try other mirrors that might have a good copy of the data. This
1868 * io_failure_record is used to record state as we go through all the
1869 * mirrors. If another mirror has good data, the page is set up to date
1870 * and things continue. If a good mirror can't be found, the original
1871 * bio end_io callback is called to indicate things have failed.
1872 */
1873struct io_failure_record {
1874 struct page *page;
1875 u64 start;
1876 u64 len;
1877 u64 logical;
1878 unsigned long bio_flags;
1879 int this_mirror;
1880 int failed_mirror;
1881 int in_validation;
1882};
1883
1884static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
1885 int did_repair)
1886{
1887 int ret;
1888 int err = 0;
1889 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1890
1891 set_state_private(failure_tree, rec->start, 0);
1892 ret = clear_extent_bits(failure_tree, rec->start,
1893 rec->start + rec->len - 1,
1894 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1895 if (ret)
1896 err = ret;
1897
1898 if (did_repair) {
1899 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1900 rec->start + rec->len - 1,
1901 EXTENT_DAMAGED, GFP_NOFS);
1902 if (ret && !err)
1903 err = ret;
1904 }
1905
1906 kfree(rec);
1907 return err;
1908}
1909
1910static void repair_io_failure_callback(struct bio *bio, int err)
1911{
1912 complete(bio->bi_private);
1913}
1914
1915/*
1916 * this bypasses the standard btrfs submit functions deliberately, as
1917 * the standard behavior is to write all copies in a raid setup. here we only
1918 * want to write the one bad copy. so we do the mapping for ourselves and issue
1919 * submit_bio directly.
1920 * to avoid any synchonization issues, wait for the data after writing, which
1921 * actually prevents the read that triggered the error from finishing.
1922 * currently, there can be no more than two copies of every data bit. thus,
1923 * exactly one rewrite is required.
1924 */
1925int repair_io_failure(struct btrfs_mapping_tree *map_tree, u64 start,
1926 u64 length, u64 logical, struct page *page,
1927 int mirror_num)
1928{
1929 struct bio *bio;
1930 struct btrfs_device *dev;
1931 DECLARE_COMPLETION_ONSTACK(compl);
1932 u64 map_length = 0;
1933 u64 sector;
1934 struct btrfs_bio *bbio = NULL;
1935 int ret;
1936
1937 BUG_ON(!mirror_num);
1938
1939 bio = bio_alloc(GFP_NOFS, 1);
1940 if (!bio)
1941 return -EIO;
1942 bio->bi_private = &compl;
1943 bio->bi_end_io = repair_io_failure_callback;
1944 bio->bi_size = 0;
1945 map_length = length;
1946
1947 ret = btrfs_map_block(map_tree, WRITE, logical,
1948 &map_length, &bbio, mirror_num);
1949 if (ret) {
1950 bio_put(bio);
1951 return -EIO;
1952 }
1953 BUG_ON(mirror_num != bbio->mirror_num);
1954 sector = bbio->stripes[mirror_num-1].physical >> 9;
1955 bio->bi_sector = sector;
1956 dev = bbio->stripes[mirror_num-1].dev;
1957 kfree(bbio);
1958 if (!dev || !dev->bdev || !dev->writeable) {
1959 bio_put(bio);
1960 return -EIO;
1961 }
1962 bio->bi_bdev = dev->bdev;
1963 bio_add_page(bio, page, length, start-page_offset(page));
21adbd5c 1964 btrfsic_submit_bio(WRITE_SYNC, bio);
4a54c8c1
JS
1965 wait_for_completion(&compl);
1966
1967 if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
1968 /* try to remap that extent elsewhere? */
1969 bio_put(bio);
442a4f63 1970 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
4a54c8c1
JS
1971 return -EIO;
1972 }
1973
d5b025d5 1974 printk_ratelimited_in_rcu(KERN_INFO "btrfs read error corrected: ino %lu off %llu "
606686ee
JB
1975 "(dev %s sector %llu)\n", page->mapping->host->i_ino,
1976 start, rcu_str_deref(dev->name), sector);
4a54c8c1
JS
1977
1978 bio_put(bio);
1979 return 0;
1980}
1981
ea466794
JB
1982int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
1983 int mirror_num)
1984{
1985 struct btrfs_mapping_tree *map_tree = &root->fs_info->mapping_tree;
1986 u64 start = eb->start;
1987 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
d95603b2 1988 int ret = 0;
ea466794
JB
1989
1990 for (i = 0; i < num_pages; i++) {
1991 struct page *p = extent_buffer_page(eb, i);
1992 ret = repair_io_failure(map_tree, start, PAGE_CACHE_SIZE,
1993 start, p, mirror_num);
1994 if (ret)
1995 break;
1996 start += PAGE_CACHE_SIZE;
1997 }
1998
1999 return ret;
2000}
2001
4a54c8c1
JS
2002/*
2003 * each time an IO finishes, we do a fast check in the IO failure tree
2004 * to see if we need to process or clean up an io_failure_record
2005 */
2006static int clean_io_failure(u64 start, struct page *page)
2007{
2008 u64 private;
2009 u64 private_failure;
2010 struct io_failure_record *failrec;
2011 struct btrfs_mapping_tree *map_tree;
2012 struct extent_state *state;
2013 int num_copies;
2014 int did_repair = 0;
2015 int ret;
2016 struct inode *inode = page->mapping->host;
2017
2018 private = 0;
2019 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2020 (u64)-1, 1, EXTENT_DIRTY, 0);
2021 if (!ret)
2022 return 0;
2023
2024 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2025 &private_failure);
2026 if (ret)
2027 return 0;
2028
2029 failrec = (struct io_failure_record *)(unsigned long) private_failure;
2030 BUG_ON(!failrec->this_mirror);
2031
2032 if (failrec->in_validation) {
2033 /* there was no real error, just free the record */
2034 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2035 failrec->start);
2036 did_repair = 1;
2037 goto out;
2038 }
2039
2040 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2041 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2042 failrec->start,
2043 EXTENT_LOCKED);
2044 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2045
2046 if (state && state->start == failrec->start) {
2047 map_tree = &BTRFS_I(inode)->root->fs_info->mapping_tree;
2048 num_copies = btrfs_num_copies(map_tree, failrec->logical,
2049 failrec->len);
2050 if (num_copies > 1) {
2051 ret = repair_io_failure(map_tree, start, failrec->len,
2052 failrec->logical, page,
2053 failrec->failed_mirror);
2054 did_repair = !ret;
2055 }
2056 }
2057
2058out:
2059 if (!ret)
2060 ret = free_io_failure(inode, failrec, did_repair);
2061
2062 return ret;
2063}
2064
2065/*
2066 * this is a generic handler for readpage errors (default
2067 * readpage_io_failed_hook). if other copies exist, read those and write back
2068 * good data to the failed position. does not investigate in remapping the
2069 * failed extent elsewhere, hoping the device will be smart enough to do this as
2070 * needed
2071 */
2072
2073static int bio_readpage_error(struct bio *failed_bio, struct page *page,
2074 u64 start, u64 end, int failed_mirror,
2075 struct extent_state *state)
2076{
2077 struct io_failure_record *failrec = NULL;
2078 u64 private;
2079 struct extent_map *em;
2080 struct inode *inode = page->mapping->host;
2081 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2082 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2083 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2084 struct bio *bio;
2085 int num_copies;
2086 int ret;
2087 int read_mode;
2088 u64 logical;
2089
2090 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2091
2092 ret = get_state_private(failure_tree, start, &private);
2093 if (ret) {
2094 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2095 if (!failrec)
2096 return -ENOMEM;
2097 failrec->start = start;
2098 failrec->len = end - start + 1;
2099 failrec->this_mirror = 0;
2100 failrec->bio_flags = 0;
2101 failrec->in_validation = 0;
2102
2103 read_lock(&em_tree->lock);
2104 em = lookup_extent_mapping(em_tree, start, failrec->len);
2105 if (!em) {
2106 read_unlock(&em_tree->lock);
2107 kfree(failrec);
2108 return -EIO;
2109 }
2110
2111 if (em->start > start || em->start + em->len < start) {
2112 free_extent_map(em);
2113 em = NULL;
2114 }
2115 read_unlock(&em_tree->lock);
2116
7a2d6a64 2117 if (!em) {
4a54c8c1
JS
2118 kfree(failrec);
2119 return -EIO;
2120 }
2121 logical = start - em->start;
2122 logical = em->block_start + logical;
2123 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2124 logical = em->block_start;
2125 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2126 extent_set_compress_type(&failrec->bio_flags,
2127 em->compress_type);
2128 }
2129 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2130 "len=%llu\n", logical, start, failrec->len);
2131 failrec->logical = logical;
2132 free_extent_map(em);
2133
2134 /* set the bits in the private failure tree */
2135 ret = set_extent_bits(failure_tree, start, end,
2136 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2137 if (ret >= 0)
2138 ret = set_state_private(failure_tree, start,
2139 (u64)(unsigned long)failrec);
2140 /* set the bits in the inode's tree */
2141 if (ret >= 0)
2142 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2143 GFP_NOFS);
2144 if (ret < 0) {
2145 kfree(failrec);
2146 return ret;
2147 }
2148 } else {
2149 failrec = (struct io_failure_record *)(unsigned long)private;
2150 pr_debug("bio_readpage_error: (found) logical=%llu, "
2151 "start=%llu, len=%llu, validation=%d\n",
2152 failrec->logical, failrec->start, failrec->len,
2153 failrec->in_validation);
2154 /*
2155 * when data can be on disk more than twice, add to failrec here
2156 * (e.g. with a list for failed_mirror) to make
2157 * clean_io_failure() clean all those errors at once.
2158 */
2159 }
2160 num_copies = btrfs_num_copies(
2161 &BTRFS_I(inode)->root->fs_info->mapping_tree,
2162 failrec->logical, failrec->len);
2163 if (num_copies == 1) {
2164 /*
2165 * we only have a single copy of the data, so don't bother with
2166 * all the retry and error correction code that follows. no
2167 * matter what the error is, it is very likely to persist.
2168 */
2169 pr_debug("bio_readpage_error: cannot repair, num_copies == 1. "
2170 "state=%p, num_copies=%d, next_mirror %d, "
2171 "failed_mirror %d\n", state, num_copies,
2172 failrec->this_mirror, failed_mirror);
2173 free_io_failure(inode, failrec, 0);
2174 return -EIO;
2175 }
2176
2177 if (!state) {
2178 spin_lock(&tree->lock);
2179 state = find_first_extent_bit_state(tree, failrec->start,
2180 EXTENT_LOCKED);
2181 if (state && state->start != failrec->start)
2182 state = NULL;
2183 spin_unlock(&tree->lock);
2184 }
2185
2186 /*
2187 * there are two premises:
2188 * a) deliver good data to the caller
2189 * b) correct the bad sectors on disk
2190 */
2191 if (failed_bio->bi_vcnt > 1) {
2192 /*
2193 * to fulfill b), we need to know the exact failing sectors, as
2194 * we don't want to rewrite any more than the failed ones. thus,
2195 * we need separate read requests for the failed bio
2196 *
2197 * if the following BUG_ON triggers, our validation request got
2198 * merged. we need separate requests for our algorithm to work.
2199 */
2200 BUG_ON(failrec->in_validation);
2201 failrec->in_validation = 1;
2202 failrec->this_mirror = failed_mirror;
2203 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2204 } else {
2205 /*
2206 * we're ready to fulfill a) and b) alongside. get a good copy
2207 * of the failed sector and if we succeed, we have setup
2208 * everything for repair_io_failure to do the rest for us.
2209 */
2210 if (failrec->in_validation) {
2211 BUG_ON(failrec->this_mirror != failed_mirror);
2212 failrec->in_validation = 0;
2213 failrec->this_mirror = 0;
2214 }
2215 failrec->failed_mirror = failed_mirror;
2216 failrec->this_mirror++;
2217 if (failrec->this_mirror == failed_mirror)
2218 failrec->this_mirror++;
2219 read_mode = READ_SYNC;
2220 }
2221
2222 if (!state || failrec->this_mirror > num_copies) {
2223 pr_debug("bio_readpage_error: (fail) state=%p, num_copies=%d, "
2224 "next_mirror %d, failed_mirror %d\n", state,
2225 num_copies, failrec->this_mirror, failed_mirror);
2226 free_io_failure(inode, failrec, 0);
2227 return -EIO;
2228 }
2229
2230 bio = bio_alloc(GFP_NOFS, 1);
e627ee7b
TI
2231 if (!bio) {
2232 free_io_failure(inode, failrec, 0);
2233 return -EIO;
2234 }
4a54c8c1
JS
2235 bio->bi_private = state;
2236 bio->bi_end_io = failed_bio->bi_end_io;
2237 bio->bi_sector = failrec->logical >> 9;
2238 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2239 bio->bi_size = 0;
2240
2241 bio_add_page(bio, page, failrec->len, start - page_offset(page));
2242
2243 pr_debug("bio_readpage_error: submitting new read[%#x] to "
2244 "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2245 failrec->this_mirror, num_copies, failrec->in_validation);
2246
013bd4c3
TI
2247 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2248 failrec->this_mirror,
2249 failrec->bio_flags, 0);
2250 return ret;
4a54c8c1
JS
2251}
2252
d1310b2e
CM
2253/* lots and lots of room for performance fixes in the end_bio funcs */
2254
87826df0
JM
2255int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2256{
2257 int uptodate = (err == 0);
2258 struct extent_io_tree *tree;
2259 int ret;
2260
2261 tree = &BTRFS_I(page->mapping->host)->io_tree;
2262
2263 if (tree->ops && tree->ops->writepage_end_io_hook) {
2264 ret = tree->ops->writepage_end_io_hook(page, start,
2265 end, NULL, uptodate);
2266 if (ret)
2267 uptodate = 0;
2268 }
2269
87826df0 2270 if (!uptodate) {
87826df0
JM
2271 ClearPageUptodate(page);
2272 SetPageError(page);
2273 }
2274 return 0;
2275}
2276
d1310b2e
CM
2277/*
2278 * after a writepage IO is done, we need to:
2279 * clear the uptodate bits on error
2280 * clear the writeback bits in the extent tree for this IO
2281 * end_page_writeback if the page has no more pending IO
2282 *
2283 * Scheduling is not allowed, so the extent state tree is expected
2284 * to have one and only one object corresponding to this IO.
2285 */
d1310b2e 2286static void end_bio_extent_writepage(struct bio *bio, int err)
d1310b2e 2287{
d1310b2e 2288 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
902b22f3 2289 struct extent_io_tree *tree;
d1310b2e
CM
2290 u64 start;
2291 u64 end;
2292 int whole_page;
2293
d1310b2e
CM
2294 do {
2295 struct page *page = bvec->bv_page;
902b22f3
DW
2296 tree = &BTRFS_I(page->mapping->host)->io_tree;
2297
d1310b2e
CM
2298 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
2299 bvec->bv_offset;
2300 end = start + bvec->bv_len - 1;
2301
2302 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
2303 whole_page = 1;
2304 else
2305 whole_page = 0;
2306
2307 if (--bvec >= bio->bi_io_vec)
2308 prefetchw(&bvec->bv_page->flags);
1259ab75 2309
87826df0
JM
2310 if (end_extent_writepage(page, err, start, end))
2311 continue;
70dec807 2312
d1310b2e
CM
2313 if (whole_page)
2314 end_page_writeback(page);
2315 else
2316 check_page_writeback(tree, page);
d1310b2e 2317 } while (bvec >= bio->bi_io_vec);
2b1f55b0 2318
d1310b2e 2319 bio_put(bio);
d1310b2e
CM
2320}
2321
2322/*
2323 * after a readpage IO is done, we need to:
2324 * clear the uptodate bits on error
2325 * set the uptodate bits if things worked
2326 * set the page up to date if all extents in the tree are uptodate
2327 * clear the lock bit in the extent tree
2328 * unlock the page if there are no other extents locked for it
2329 *
2330 * Scheduling is not allowed, so the extent state tree is expected
2331 * to have one and only one object corresponding to this IO.
2332 */
d1310b2e 2333static void end_bio_extent_readpage(struct bio *bio, int err)
d1310b2e
CM
2334{
2335 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
4125bf76
CM
2336 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
2337 struct bio_vec *bvec = bio->bi_io_vec;
902b22f3 2338 struct extent_io_tree *tree;
d1310b2e
CM
2339 u64 start;
2340 u64 end;
2341 int whole_page;
5cf1ab56 2342 int mirror;
d1310b2e
CM
2343 int ret;
2344
d20f7043
CM
2345 if (err)
2346 uptodate = 0;
2347
d1310b2e
CM
2348 do {
2349 struct page *page = bvec->bv_page;
507903b8
AJ
2350 struct extent_state *cached = NULL;
2351 struct extent_state *state;
2352
be3940c0
KO
2353 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
2354 "mirror=%ld\n", (u64)bio->bi_sector, err,
4a54c8c1 2355 (long int)bio->bi_bdev);
902b22f3
DW
2356 tree = &BTRFS_I(page->mapping->host)->io_tree;
2357
d1310b2e
CM
2358 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
2359 bvec->bv_offset;
2360 end = start + bvec->bv_len - 1;
2361
2362 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
2363 whole_page = 1;
2364 else
2365 whole_page = 0;
2366
4125bf76 2367 if (++bvec <= bvec_end)
d1310b2e
CM
2368 prefetchw(&bvec->bv_page->flags);
2369
507903b8 2370 spin_lock(&tree->lock);
0d399205 2371 state = find_first_extent_bit_state(tree, start, EXTENT_LOCKED);
109b36a2 2372 if (state && state->start == start) {
507903b8
AJ
2373 /*
2374 * take a reference on the state, unlock will drop
2375 * the ref
2376 */
2377 cache_state(state, &cached);
2378 }
2379 spin_unlock(&tree->lock);
2380
5cf1ab56 2381 mirror = (int)(unsigned long)bio->bi_bdev;
d1310b2e 2382 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
70dec807 2383 ret = tree->ops->readpage_end_io_hook(page, start, end,
5cf1ab56 2384 state, mirror);
5ee0844d 2385 if (ret)
d1310b2e 2386 uptodate = 0;
5ee0844d 2387 else
4a54c8c1 2388 clean_io_failure(start, page);
d1310b2e 2389 }
ea466794 2390
ea466794 2391 if (!uptodate && tree->ops && tree->ops->readpage_io_failed_hook) {
5cf1ab56 2392 ret = tree->ops->readpage_io_failed_hook(page, mirror);
ea466794
JB
2393 if (!ret && !err &&
2394 test_bit(BIO_UPTODATE, &bio->bi_flags))
2395 uptodate = 1;
2396 } else if (!uptodate) {
f4a8e656
JS
2397 /*
2398 * The generic bio_readpage_error handles errors the
2399 * following way: If possible, new read requests are
2400 * created and submitted and will end up in
2401 * end_bio_extent_readpage as well (if we're lucky, not
2402 * in the !uptodate case). In that case it returns 0 and
2403 * we just go on with the next page in our bio. If it
2404 * can't handle the error it will return -EIO and we
2405 * remain responsible for that page.
2406 */
5cf1ab56 2407 ret = bio_readpage_error(bio, page, start, end, mirror, NULL);
7e38326f 2408 if (ret == 0) {
3b951516
CM
2409 uptodate =
2410 test_bit(BIO_UPTODATE, &bio->bi_flags);
d20f7043
CM
2411 if (err)
2412 uptodate = 0;
507903b8 2413 uncache_state(&cached);
7e38326f
CM
2414 continue;
2415 }
2416 }
d1310b2e 2417
0b32f4bb 2418 if (uptodate && tree->track_uptodate) {
507903b8 2419 set_extent_uptodate(tree, start, end, &cached,
902b22f3 2420 GFP_ATOMIC);
771ed689 2421 }
507903b8 2422 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
d1310b2e 2423
70dec807
CM
2424 if (whole_page) {
2425 if (uptodate) {
2426 SetPageUptodate(page);
2427 } else {
2428 ClearPageUptodate(page);
2429 SetPageError(page);
2430 }
d1310b2e 2431 unlock_page(page);
70dec807
CM
2432 } else {
2433 if (uptodate) {
2434 check_page_uptodate(tree, page);
2435 } else {
2436 ClearPageUptodate(page);
2437 SetPageError(page);
2438 }
d1310b2e 2439 check_page_locked(tree, page);
70dec807 2440 }
4125bf76 2441 } while (bvec <= bvec_end);
d1310b2e
CM
2442
2443 bio_put(bio);
d1310b2e
CM
2444}
2445
88f794ed
MX
2446struct bio *
2447btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2448 gfp_t gfp_flags)
d1310b2e
CM
2449{
2450 struct bio *bio;
2451
2452 bio = bio_alloc(gfp_flags, nr_vecs);
2453
2454 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
2455 while (!bio && (nr_vecs /= 2))
2456 bio = bio_alloc(gfp_flags, nr_vecs);
2457 }
2458
2459 if (bio) {
e1c4b745 2460 bio->bi_size = 0;
d1310b2e
CM
2461 bio->bi_bdev = bdev;
2462 bio->bi_sector = first_sector;
2463 }
2464 return bio;
2465}
2466
79787eaa
JM
2467/*
2468 * Since writes are async, they will only return -ENOMEM.
2469 * Reads can return the full range of I/O error conditions.
2470 */
355808c2
JM
2471static int __must_check submit_one_bio(int rw, struct bio *bio,
2472 int mirror_num, unsigned long bio_flags)
d1310b2e 2473{
d1310b2e 2474 int ret = 0;
70dec807
CM
2475 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2476 struct page *page = bvec->bv_page;
2477 struct extent_io_tree *tree = bio->bi_private;
70dec807 2478 u64 start;
70dec807
CM
2479
2480 start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
70dec807 2481
902b22f3 2482 bio->bi_private = NULL;
d1310b2e
CM
2483
2484 bio_get(bio);
2485
065631f6 2486 if (tree->ops && tree->ops->submit_bio_hook)
6b82ce8d 2487 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
eaf25d93 2488 mirror_num, bio_flags, start);
0b86a832 2489 else
21adbd5c 2490 btrfsic_submit_bio(rw, bio);
4a54c8c1 2491
d1310b2e
CM
2492 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2493 ret = -EOPNOTSUPP;
2494 bio_put(bio);
2495 return ret;
2496}
2497
3444a972
JM
2498static int merge_bio(struct extent_io_tree *tree, struct page *page,
2499 unsigned long offset, size_t size, struct bio *bio,
2500 unsigned long bio_flags)
2501{
2502 int ret = 0;
2503 if (tree->ops && tree->ops->merge_bio_hook)
2504 ret = tree->ops->merge_bio_hook(page, offset, size, bio,
2505 bio_flags);
2506 BUG_ON(ret < 0);
2507 return ret;
2508
2509}
2510
d1310b2e
CM
2511static int submit_extent_page(int rw, struct extent_io_tree *tree,
2512 struct page *page, sector_t sector,
2513 size_t size, unsigned long offset,
2514 struct block_device *bdev,
2515 struct bio **bio_ret,
2516 unsigned long max_pages,
f188591e 2517 bio_end_io_t end_io_func,
c8b97818
CM
2518 int mirror_num,
2519 unsigned long prev_bio_flags,
2520 unsigned long bio_flags)
d1310b2e
CM
2521{
2522 int ret = 0;
2523 struct bio *bio;
2524 int nr;
c8b97818
CM
2525 int contig = 0;
2526 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2527 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
5b050f04 2528 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
d1310b2e
CM
2529
2530 if (bio_ret && *bio_ret) {
2531 bio = *bio_ret;
c8b97818
CM
2532 if (old_compressed)
2533 contig = bio->bi_sector == sector;
2534 else
2535 contig = bio->bi_sector + (bio->bi_size >> 9) ==
2536 sector;
2537
2538 if (prev_bio_flags != bio_flags || !contig ||
3444a972 2539 merge_bio(tree, page, offset, page_size, bio, bio_flags) ||
c8b97818
CM
2540 bio_add_page(bio, page, page_size, offset) < page_size) {
2541 ret = submit_one_bio(rw, bio, mirror_num,
2542 prev_bio_flags);
79787eaa
JM
2543 if (ret < 0)
2544 return ret;
d1310b2e
CM
2545 bio = NULL;
2546 } else {
2547 return 0;
2548 }
2549 }
c8b97818
CM
2550 if (this_compressed)
2551 nr = BIO_MAX_PAGES;
2552 else
2553 nr = bio_get_nr_vecs(bdev);
2554
88f794ed 2555 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
5df67083
TI
2556 if (!bio)
2557 return -ENOMEM;
70dec807 2558
c8b97818 2559 bio_add_page(bio, page, page_size, offset);
d1310b2e
CM
2560 bio->bi_end_io = end_io_func;
2561 bio->bi_private = tree;
70dec807 2562
d397712b 2563 if (bio_ret)
d1310b2e 2564 *bio_ret = bio;
d397712b 2565 else
c8b97818 2566 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
d1310b2e
CM
2567
2568 return ret;
2569}
2570
4f2de97a 2571void attach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
d1310b2e
CM
2572{
2573 if (!PagePrivate(page)) {
2574 SetPagePrivate(page);
d1310b2e 2575 page_cache_get(page);
4f2de97a
JB
2576 set_page_private(page, (unsigned long)eb);
2577 } else {
2578 WARN_ON(page->private != (unsigned long)eb);
d1310b2e
CM
2579 }
2580}
2581
4f2de97a 2582void set_page_extent_mapped(struct page *page)
d1310b2e 2583{
4f2de97a
JB
2584 if (!PagePrivate(page)) {
2585 SetPagePrivate(page);
2586 page_cache_get(page);
2587 set_page_private(page, EXTENT_PAGE_PRIVATE);
2588 }
d1310b2e
CM
2589}
2590
2591/*
2592 * basic readpage implementation. Locked extent state structs are inserted
2593 * into the tree that are removed when the IO is done (by the end_io
2594 * handlers)
79787eaa 2595 * XXX JDM: This needs looking at to ensure proper page locking
d1310b2e
CM
2596 */
2597static int __extent_read_full_page(struct extent_io_tree *tree,
2598 struct page *page,
2599 get_extent_t *get_extent,
c8b97818
CM
2600 struct bio **bio, int mirror_num,
2601 unsigned long *bio_flags)
d1310b2e
CM
2602{
2603 struct inode *inode = page->mapping->host;
2604 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2605 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2606 u64 end;
2607 u64 cur = start;
2608 u64 extent_offset;
2609 u64 last_byte = i_size_read(inode);
2610 u64 block_start;
2611 u64 cur_end;
2612 sector_t sector;
2613 struct extent_map *em;
2614 struct block_device *bdev;
11c65dcc 2615 struct btrfs_ordered_extent *ordered;
d1310b2e
CM
2616 int ret;
2617 int nr = 0;
306e16ce 2618 size_t pg_offset = 0;
d1310b2e 2619 size_t iosize;
c8b97818 2620 size_t disk_io_size;
d1310b2e 2621 size_t blocksize = inode->i_sb->s_blocksize;
c8b97818 2622 unsigned long this_bio_flag = 0;
d1310b2e
CM
2623
2624 set_page_extent_mapped(page);
2625
90a887c9
DM
2626 if (!PageUptodate(page)) {
2627 if (cleancache_get_page(page) == 0) {
2628 BUG_ON(blocksize != PAGE_SIZE);
2629 goto out;
2630 }
2631 }
2632
d1310b2e 2633 end = page_end;
11c65dcc 2634 while (1) {
d0082371 2635 lock_extent(tree, start, end);
11c65dcc
JB
2636 ordered = btrfs_lookup_ordered_extent(inode, start);
2637 if (!ordered)
2638 break;
d0082371 2639 unlock_extent(tree, start, end);
11c65dcc
JB
2640 btrfs_start_ordered_extent(inode, ordered, 1);
2641 btrfs_put_ordered_extent(ordered);
2642 }
d1310b2e 2643
c8b97818
CM
2644 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2645 char *userpage;
2646 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2647
2648 if (zero_offset) {
2649 iosize = PAGE_CACHE_SIZE - zero_offset;
7ac687d9 2650 userpage = kmap_atomic(page);
c8b97818
CM
2651 memset(userpage + zero_offset, 0, iosize);
2652 flush_dcache_page(page);
7ac687d9 2653 kunmap_atomic(userpage);
c8b97818
CM
2654 }
2655 }
d1310b2e
CM
2656 while (cur <= end) {
2657 if (cur >= last_byte) {
2658 char *userpage;
507903b8
AJ
2659 struct extent_state *cached = NULL;
2660
306e16ce 2661 iosize = PAGE_CACHE_SIZE - pg_offset;
7ac687d9 2662 userpage = kmap_atomic(page);
306e16ce 2663 memset(userpage + pg_offset, 0, iosize);
d1310b2e 2664 flush_dcache_page(page);
7ac687d9 2665 kunmap_atomic(userpage);
d1310b2e 2666 set_extent_uptodate(tree, cur, cur + iosize - 1,
507903b8
AJ
2667 &cached, GFP_NOFS);
2668 unlock_extent_cached(tree, cur, cur + iosize - 1,
2669 &cached, GFP_NOFS);
d1310b2e
CM
2670 break;
2671 }
306e16ce 2672 em = get_extent(inode, page, pg_offset, cur,
d1310b2e 2673 end - cur + 1, 0);
c704005d 2674 if (IS_ERR_OR_NULL(em)) {
d1310b2e 2675 SetPageError(page);
d0082371 2676 unlock_extent(tree, cur, end);
d1310b2e
CM
2677 break;
2678 }
d1310b2e
CM
2679 extent_offset = cur - em->start;
2680 BUG_ON(extent_map_end(em) <= cur);
2681 BUG_ON(end < cur);
2682
261507a0 2683 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
c8b97818 2684 this_bio_flag = EXTENT_BIO_COMPRESSED;
261507a0
LZ
2685 extent_set_compress_type(&this_bio_flag,
2686 em->compress_type);
2687 }
c8b97818 2688
d1310b2e
CM
2689 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2690 cur_end = min(extent_map_end(em) - 1, end);
2691 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
c8b97818
CM
2692 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2693 disk_io_size = em->block_len;
2694 sector = em->block_start >> 9;
2695 } else {
2696 sector = (em->block_start + extent_offset) >> 9;
2697 disk_io_size = iosize;
2698 }
d1310b2e
CM
2699 bdev = em->bdev;
2700 block_start = em->block_start;
d899e052
YZ
2701 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2702 block_start = EXTENT_MAP_HOLE;
d1310b2e
CM
2703 free_extent_map(em);
2704 em = NULL;
2705
2706 /* we've found a hole, just zero and go on */
2707 if (block_start == EXTENT_MAP_HOLE) {
2708 char *userpage;
507903b8
AJ
2709 struct extent_state *cached = NULL;
2710
7ac687d9 2711 userpage = kmap_atomic(page);
306e16ce 2712 memset(userpage + pg_offset, 0, iosize);
d1310b2e 2713 flush_dcache_page(page);
7ac687d9 2714 kunmap_atomic(userpage);
d1310b2e
CM
2715
2716 set_extent_uptodate(tree, cur, cur + iosize - 1,
507903b8
AJ
2717 &cached, GFP_NOFS);
2718 unlock_extent_cached(tree, cur, cur + iosize - 1,
2719 &cached, GFP_NOFS);
d1310b2e 2720 cur = cur + iosize;
306e16ce 2721 pg_offset += iosize;
d1310b2e
CM
2722 continue;
2723 }
2724 /* the get_extent function already copied into the page */
9655d298
CM
2725 if (test_range_bit(tree, cur, cur_end,
2726 EXTENT_UPTODATE, 1, NULL)) {
a1b32a59 2727 check_page_uptodate(tree, page);
d0082371 2728 unlock_extent(tree, cur, cur + iosize - 1);
d1310b2e 2729 cur = cur + iosize;
306e16ce 2730 pg_offset += iosize;
d1310b2e
CM
2731 continue;
2732 }
70dec807
CM
2733 /* we have an inline extent but it didn't get marked up
2734 * to date. Error out
2735 */
2736 if (block_start == EXTENT_MAP_INLINE) {
2737 SetPageError(page);
d0082371 2738 unlock_extent(tree, cur, cur + iosize - 1);
70dec807 2739 cur = cur + iosize;
306e16ce 2740 pg_offset += iosize;
70dec807
CM
2741 continue;
2742 }
d1310b2e
CM
2743
2744 ret = 0;
2745 if (tree->ops && tree->ops->readpage_io_hook) {
2746 ret = tree->ops->readpage_io_hook(page, cur,
2747 cur + iosize - 1);
2748 }
2749 if (!ret) {
89642229
CM
2750 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2751 pnr -= page->index;
d1310b2e 2752 ret = submit_extent_page(READ, tree, page,
306e16ce 2753 sector, disk_io_size, pg_offset,
89642229 2754 bdev, bio, pnr,
c8b97818
CM
2755 end_bio_extent_readpage, mirror_num,
2756 *bio_flags,
2757 this_bio_flag);
edd33c99
JB
2758 if (!ret) {
2759 nr++;
2760 *bio_flags = this_bio_flag;
2761 }
d1310b2e 2762 }
edd33c99 2763 if (ret) {
d1310b2e 2764 SetPageError(page);
edd33c99
JB
2765 unlock_extent(tree, cur, cur + iosize - 1);
2766 }
d1310b2e 2767 cur = cur + iosize;
306e16ce 2768 pg_offset += iosize;
d1310b2e 2769 }
90a887c9 2770out:
d1310b2e
CM
2771 if (!nr) {
2772 if (!PageError(page))
2773 SetPageUptodate(page);
2774 unlock_page(page);
2775 }
2776 return 0;
2777}
2778
2779int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
8ddc7d9c 2780 get_extent_t *get_extent, int mirror_num)
d1310b2e
CM
2781{
2782 struct bio *bio = NULL;
c8b97818 2783 unsigned long bio_flags = 0;
d1310b2e
CM
2784 int ret;
2785
8ddc7d9c 2786 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
c8b97818 2787 &bio_flags);
d1310b2e 2788 if (bio)
8ddc7d9c 2789 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
d1310b2e
CM
2790 return ret;
2791}
d1310b2e 2792
11c8349b
CM
2793static noinline void update_nr_written(struct page *page,
2794 struct writeback_control *wbc,
2795 unsigned long nr_written)
2796{
2797 wbc->nr_to_write -= nr_written;
2798 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
2799 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
2800 page->mapping->writeback_index = page->index + nr_written;
2801}
2802
d1310b2e
CM
2803/*
2804 * the writepage semantics are similar to regular writepage. extent
2805 * records are inserted to lock ranges in the tree, and as dirty areas
2806 * are found, they are marked writeback. Then the lock bits are removed
2807 * and the end_io handler clears the writeback ranges
2808 */
2809static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2810 void *data)
2811{
2812 struct inode *inode = page->mapping->host;
2813 struct extent_page_data *epd = data;
2814 struct extent_io_tree *tree = epd->tree;
2815 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2816 u64 delalloc_start;
2817 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2818 u64 end;
2819 u64 cur = start;
2820 u64 extent_offset;
2821 u64 last_byte = i_size_read(inode);
2822 u64 block_start;
2823 u64 iosize;
2824 sector_t sector;
2c64c53d 2825 struct extent_state *cached_state = NULL;
d1310b2e
CM
2826 struct extent_map *em;
2827 struct block_device *bdev;
2828 int ret;
2829 int nr = 0;
7f3c74fb 2830 size_t pg_offset = 0;
d1310b2e
CM
2831 size_t blocksize;
2832 loff_t i_size = i_size_read(inode);
2833 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
2834 u64 nr_delalloc;
2835 u64 delalloc_end;
c8b97818
CM
2836 int page_started;
2837 int compressed;
ffbd517d 2838 int write_flags;
771ed689 2839 unsigned long nr_written = 0;
9e487107 2840 bool fill_delalloc = true;
d1310b2e 2841
ffbd517d 2842 if (wbc->sync_mode == WB_SYNC_ALL)
721a9602 2843 write_flags = WRITE_SYNC;
ffbd517d
CM
2844 else
2845 write_flags = WRITE;
2846
1abe9b8a 2847 trace___extent_writepage(page, inode, wbc);
2848
d1310b2e 2849 WARN_ON(!PageLocked(page));
bf0da8c1
CM
2850
2851 ClearPageError(page);
2852
7f3c74fb 2853 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
211c17f5 2854 if (page->index > end_index ||
7f3c74fb 2855 (page->index == end_index && !pg_offset)) {
39be25cd 2856 page->mapping->a_ops->invalidatepage(page, 0);
d1310b2e
CM
2857 unlock_page(page);
2858 return 0;
2859 }
2860
2861 if (page->index == end_index) {
2862 char *userpage;
2863
7ac687d9 2864 userpage = kmap_atomic(page);
7f3c74fb
CM
2865 memset(userpage + pg_offset, 0,
2866 PAGE_CACHE_SIZE - pg_offset);
7ac687d9 2867 kunmap_atomic(userpage);
211c17f5 2868 flush_dcache_page(page);
d1310b2e 2869 }
7f3c74fb 2870 pg_offset = 0;
d1310b2e
CM
2871
2872 set_page_extent_mapped(page);
2873
9e487107
JB
2874 if (!tree->ops || !tree->ops->fill_delalloc)
2875 fill_delalloc = false;
2876
d1310b2e
CM
2877 delalloc_start = start;
2878 delalloc_end = 0;
c8b97818 2879 page_started = 0;
9e487107 2880 if (!epd->extent_locked && fill_delalloc) {
f85d7d6c 2881 u64 delalloc_to_write = 0;
11c8349b
CM
2882 /*
2883 * make sure the wbc mapping index is at least updated
2884 * to this page.
2885 */
2886 update_nr_written(page, wbc, 0);
2887
d397712b 2888 while (delalloc_end < page_end) {
771ed689 2889 nr_delalloc = find_lock_delalloc_range(inode, tree,
c8b97818
CM
2890 page,
2891 &delalloc_start,
d1310b2e
CM
2892 &delalloc_end,
2893 128 * 1024 * 1024);
771ed689
CM
2894 if (nr_delalloc == 0) {
2895 delalloc_start = delalloc_end + 1;
2896 continue;
2897 }
013bd4c3
TI
2898 ret = tree->ops->fill_delalloc(inode, page,
2899 delalloc_start,
2900 delalloc_end,
2901 &page_started,
2902 &nr_written);
79787eaa
JM
2903 /* File system has been set read-only */
2904 if (ret) {
2905 SetPageError(page);
2906 goto done;
2907 }
f85d7d6c
CM
2908 /*
2909 * delalloc_end is already one less than the total
2910 * length, so we don't subtract one from
2911 * PAGE_CACHE_SIZE
2912 */
2913 delalloc_to_write += (delalloc_end - delalloc_start +
2914 PAGE_CACHE_SIZE) >>
2915 PAGE_CACHE_SHIFT;
d1310b2e 2916 delalloc_start = delalloc_end + 1;
d1310b2e 2917 }
f85d7d6c
CM
2918 if (wbc->nr_to_write < delalloc_to_write) {
2919 int thresh = 8192;
2920
2921 if (delalloc_to_write < thresh * 2)
2922 thresh = delalloc_to_write;
2923 wbc->nr_to_write = min_t(u64, delalloc_to_write,
2924 thresh);
2925 }
c8b97818 2926
771ed689
CM
2927 /* did the fill delalloc function already unlock and start
2928 * the IO?
2929 */
2930 if (page_started) {
2931 ret = 0;
11c8349b
CM
2932 /*
2933 * we've unlocked the page, so we can't update
2934 * the mapping's writeback index, just update
2935 * nr_to_write.
2936 */
2937 wbc->nr_to_write -= nr_written;
2938 goto done_unlocked;
771ed689 2939 }
c8b97818 2940 }
247e743c 2941 if (tree->ops && tree->ops->writepage_start_hook) {
c8b97818
CM
2942 ret = tree->ops->writepage_start_hook(page, start,
2943 page_end);
87826df0
JM
2944 if (ret) {
2945 /* Fixup worker will requeue */
2946 if (ret == -EBUSY)
2947 wbc->pages_skipped++;
2948 else
2949 redirty_page_for_writepage(wbc, page);
11c8349b 2950 update_nr_written(page, wbc, nr_written);
247e743c 2951 unlock_page(page);
771ed689 2952 ret = 0;
11c8349b 2953 goto done_unlocked;
247e743c
CM
2954 }
2955 }
2956
11c8349b
CM
2957 /*
2958 * we don't want to touch the inode after unlocking the page,
2959 * so we update the mapping writeback index now
2960 */
2961 update_nr_written(page, wbc, nr_written + 1);
771ed689 2962
d1310b2e 2963 end = page_end;
d1310b2e 2964 if (last_byte <= start) {
e6dcd2dc
CM
2965 if (tree->ops && tree->ops->writepage_end_io_hook)
2966 tree->ops->writepage_end_io_hook(page, start,
2967 page_end, NULL, 1);
d1310b2e
CM
2968 goto done;
2969 }
2970
d1310b2e
CM
2971 blocksize = inode->i_sb->s_blocksize;
2972
2973 while (cur <= end) {
2974 if (cur >= last_byte) {
e6dcd2dc
CM
2975 if (tree->ops && tree->ops->writepage_end_io_hook)
2976 tree->ops->writepage_end_io_hook(page, cur,
2977 page_end, NULL, 1);
d1310b2e
CM
2978 break;
2979 }
7f3c74fb 2980 em = epd->get_extent(inode, page, pg_offset, cur,
d1310b2e 2981 end - cur + 1, 1);
c704005d 2982 if (IS_ERR_OR_NULL(em)) {
d1310b2e
CM
2983 SetPageError(page);
2984 break;
2985 }
2986
2987 extent_offset = cur - em->start;
2988 BUG_ON(extent_map_end(em) <= cur);
2989 BUG_ON(end < cur);
2990 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2991 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2992 sector = (em->block_start + extent_offset) >> 9;
2993 bdev = em->bdev;
2994 block_start = em->block_start;
c8b97818 2995 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
d1310b2e
CM
2996 free_extent_map(em);
2997 em = NULL;
2998
c8b97818
CM
2999 /*
3000 * compressed and inline extents are written through other
3001 * paths in the FS
3002 */
3003 if (compressed || block_start == EXTENT_MAP_HOLE ||
d1310b2e 3004 block_start == EXTENT_MAP_INLINE) {
c8b97818
CM
3005 /*
3006 * end_io notification does not happen here for
3007 * compressed extents
3008 */
3009 if (!compressed && tree->ops &&
3010 tree->ops->writepage_end_io_hook)
e6dcd2dc
CM
3011 tree->ops->writepage_end_io_hook(page, cur,
3012 cur + iosize - 1,
3013 NULL, 1);
c8b97818
CM
3014 else if (compressed) {
3015 /* we don't want to end_page_writeback on
3016 * a compressed extent. this happens
3017 * elsewhere
3018 */
3019 nr++;
3020 }
3021
3022 cur += iosize;
7f3c74fb 3023 pg_offset += iosize;
d1310b2e
CM
3024 continue;
3025 }
d1310b2e
CM
3026 /* leave this out until we have a page_mkwrite call */
3027 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
9655d298 3028 EXTENT_DIRTY, 0, NULL)) {
d1310b2e 3029 cur = cur + iosize;
7f3c74fb 3030 pg_offset += iosize;
d1310b2e
CM
3031 continue;
3032 }
c8b97818 3033
d1310b2e
CM
3034 if (tree->ops && tree->ops->writepage_io_hook) {
3035 ret = tree->ops->writepage_io_hook(page, cur,
3036 cur + iosize - 1);
3037 } else {
3038 ret = 0;
3039 }
1259ab75 3040 if (ret) {
d1310b2e 3041 SetPageError(page);
1259ab75 3042 } else {
d1310b2e 3043 unsigned long max_nr = end_index + 1;
7f3c74fb 3044
d1310b2e
CM
3045 set_range_writeback(tree, cur, cur + iosize - 1);
3046 if (!PageWriteback(page)) {
d397712b
CM
3047 printk(KERN_ERR "btrfs warning page %lu not "
3048 "writeback, cur %llu end %llu\n",
3049 page->index, (unsigned long long)cur,
d1310b2e
CM
3050 (unsigned long long)end);
3051 }
3052
ffbd517d
CM
3053 ret = submit_extent_page(write_flags, tree, page,
3054 sector, iosize, pg_offset,
3055 bdev, &epd->bio, max_nr,
c8b97818
CM
3056 end_bio_extent_writepage,
3057 0, 0, 0);
d1310b2e
CM
3058 if (ret)
3059 SetPageError(page);
3060 }
3061 cur = cur + iosize;
7f3c74fb 3062 pg_offset += iosize;
d1310b2e
CM
3063 nr++;
3064 }
3065done:
3066 if (nr == 0) {
3067 /* make sure the mapping tag for page dirty gets cleared */
3068 set_page_writeback(page);
3069 end_page_writeback(page);
3070 }
d1310b2e 3071 unlock_page(page);
771ed689 3072
11c8349b
CM
3073done_unlocked:
3074
2c64c53d
CM
3075 /* drop our reference on any cached states */
3076 free_extent_state(cached_state);
d1310b2e
CM
3077 return 0;
3078}
3079
0b32f4bb
JB
3080static int eb_wait(void *word)
3081{
3082 io_schedule();
3083 return 0;
3084}
3085
3086static void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3087{
3088 wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3089 TASK_UNINTERRUPTIBLE);
3090}
3091
3092static int lock_extent_buffer_for_io(struct extent_buffer *eb,
3093 struct btrfs_fs_info *fs_info,
3094 struct extent_page_data *epd)
3095{
3096 unsigned long i, num_pages;
3097 int flush = 0;
3098 int ret = 0;
3099
3100 if (!btrfs_try_tree_write_lock(eb)) {
3101 flush = 1;
3102 flush_write_bio(epd);
3103 btrfs_tree_lock(eb);
3104 }
3105
3106 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3107 btrfs_tree_unlock(eb);
3108 if (!epd->sync_io)
3109 return 0;
3110 if (!flush) {
3111 flush_write_bio(epd);
3112 flush = 1;
3113 }
a098d8e8
CM
3114 while (1) {
3115 wait_on_extent_buffer_writeback(eb);
3116 btrfs_tree_lock(eb);
3117 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3118 break;
0b32f4bb 3119 btrfs_tree_unlock(eb);
0b32f4bb
JB
3120 }
3121 }
3122
51561ffe
JB
3123 /*
3124 * We need to do this to prevent races in people who check if the eb is
3125 * under IO since we can end up having no IO bits set for a short period
3126 * of time.
3127 */
3128 spin_lock(&eb->refs_lock);
0b32f4bb
JB
3129 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3130 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
51561ffe 3131 spin_unlock(&eb->refs_lock);
0b32f4bb
JB
3132 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3133 spin_lock(&fs_info->delalloc_lock);
3134 if (fs_info->dirty_metadata_bytes >= eb->len)
3135 fs_info->dirty_metadata_bytes -= eb->len;
3136 else
3137 WARN_ON(1);
3138 spin_unlock(&fs_info->delalloc_lock);
3139 ret = 1;
51561ffe
JB
3140 } else {
3141 spin_unlock(&eb->refs_lock);
0b32f4bb
JB
3142 }
3143
3144 btrfs_tree_unlock(eb);
3145
3146 if (!ret)
3147 return ret;
3148
3149 num_pages = num_extent_pages(eb->start, eb->len);
3150 for (i = 0; i < num_pages; i++) {
3151 struct page *p = extent_buffer_page(eb, i);
3152
3153 if (!trylock_page(p)) {
3154 if (!flush) {
3155 flush_write_bio(epd);
3156 flush = 1;
3157 }
3158 lock_page(p);
3159 }
3160 }
3161
3162 return ret;
3163}
3164
3165static void end_extent_buffer_writeback(struct extent_buffer *eb)
3166{
3167 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3168 smp_mb__after_clear_bit();
3169 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3170}
3171
3172static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3173{
3174 int uptodate = err == 0;
3175 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
3176 struct extent_buffer *eb;
3177 int done;
3178
3179 do {
3180 struct page *page = bvec->bv_page;
3181
3182 bvec--;
3183 eb = (struct extent_buffer *)page->private;
3184 BUG_ON(!eb);
3185 done = atomic_dec_and_test(&eb->io_pages);
3186
3187 if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3188 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3189 ClearPageUptodate(page);
3190 SetPageError(page);
3191 }
3192
3193 end_page_writeback(page);
3194
3195 if (!done)
3196 continue;
3197
3198 end_extent_buffer_writeback(eb);
3199 } while (bvec >= bio->bi_io_vec);
3200
3201 bio_put(bio);
3202
3203}
3204
3205static int write_one_eb(struct extent_buffer *eb,
3206 struct btrfs_fs_info *fs_info,
3207 struct writeback_control *wbc,
3208 struct extent_page_data *epd)
3209{
3210 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3211 u64 offset = eb->start;
3212 unsigned long i, num_pages;
de0022b9 3213 unsigned long bio_flags = 0;
0b32f4bb 3214 int rw = (epd->sync_io ? WRITE_SYNC : WRITE);
d7dbe9e7 3215 int ret = 0;
0b32f4bb
JB
3216
3217 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3218 num_pages = num_extent_pages(eb->start, eb->len);
3219 atomic_set(&eb->io_pages, num_pages);
de0022b9
JB
3220 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3221 bio_flags = EXTENT_BIO_TREE_LOG;
3222
0b32f4bb
JB
3223 for (i = 0; i < num_pages; i++) {
3224 struct page *p = extent_buffer_page(eb, i);
3225
3226 clear_page_dirty_for_io(p);
3227 set_page_writeback(p);
3228 ret = submit_extent_page(rw, eb->tree, p, offset >> 9,
3229 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3230 -1, end_bio_extent_buffer_writepage,
de0022b9
JB
3231 0, epd->bio_flags, bio_flags);
3232 epd->bio_flags = bio_flags;
0b32f4bb
JB
3233 if (ret) {
3234 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3235 SetPageError(p);
3236 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3237 end_extent_buffer_writeback(eb);
3238 ret = -EIO;
3239 break;
3240 }
3241 offset += PAGE_CACHE_SIZE;
3242 update_nr_written(p, wbc, 1);
3243 unlock_page(p);
3244 }
3245
3246 if (unlikely(ret)) {
3247 for (; i < num_pages; i++) {
3248 struct page *p = extent_buffer_page(eb, i);
3249 unlock_page(p);
3250 }
3251 }
3252
3253 return ret;
3254}
3255
3256int btree_write_cache_pages(struct address_space *mapping,
3257 struct writeback_control *wbc)
3258{
3259 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3260 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3261 struct extent_buffer *eb, *prev_eb = NULL;
3262 struct extent_page_data epd = {
3263 .bio = NULL,
3264 .tree = tree,
3265 .extent_locked = 0,
3266 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
de0022b9 3267 .bio_flags = 0,
0b32f4bb
JB
3268 };
3269 int ret = 0;
3270 int done = 0;
3271 int nr_to_write_done = 0;
3272 struct pagevec pvec;
3273 int nr_pages;
3274 pgoff_t index;
3275 pgoff_t end; /* Inclusive */
3276 int scanned = 0;
3277 int tag;
3278
3279 pagevec_init(&pvec, 0);
3280 if (wbc->range_cyclic) {
3281 index = mapping->writeback_index; /* Start from prev offset */
3282 end = -1;
3283 } else {
3284 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3285 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3286 scanned = 1;
3287 }
3288 if (wbc->sync_mode == WB_SYNC_ALL)
3289 tag = PAGECACHE_TAG_TOWRITE;
3290 else
3291 tag = PAGECACHE_TAG_DIRTY;
3292retry:
3293 if (wbc->sync_mode == WB_SYNC_ALL)
3294 tag_pages_for_writeback(mapping, index, end);
3295 while (!done && !nr_to_write_done && (index <= end) &&
3296 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3297 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3298 unsigned i;
3299
3300 scanned = 1;
3301 for (i = 0; i < nr_pages; i++) {
3302 struct page *page = pvec.pages[i];
3303
3304 if (!PagePrivate(page))
3305 continue;
3306
3307 if (!wbc->range_cyclic && page->index > end) {
3308 done = 1;
3309 break;
3310 }
3311
b5bae261
JB
3312 spin_lock(&mapping->private_lock);
3313 if (!PagePrivate(page)) {
3314 spin_unlock(&mapping->private_lock);
3315 continue;
3316 }
3317
0b32f4bb 3318 eb = (struct extent_buffer *)page->private;
b5bae261
JB
3319
3320 /*
3321 * Shouldn't happen and normally this would be a BUG_ON
3322 * but no sense in crashing the users box for something
3323 * we can survive anyway.
3324 */
0b32f4bb 3325 if (!eb) {
b5bae261 3326 spin_unlock(&mapping->private_lock);
0b32f4bb
JB
3327 WARN_ON(1);
3328 continue;
3329 }
3330
b5bae261
JB
3331 if (eb == prev_eb) {
3332 spin_unlock(&mapping->private_lock);
0b32f4bb 3333 continue;
b5bae261 3334 }
0b32f4bb 3335
b5bae261
JB
3336 ret = atomic_inc_not_zero(&eb->refs);
3337 spin_unlock(&mapping->private_lock);
3338 if (!ret)
0b32f4bb 3339 continue;
0b32f4bb
JB
3340
3341 prev_eb = eb;
3342 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3343 if (!ret) {
3344 free_extent_buffer(eb);
3345 continue;
3346 }
3347
3348 ret = write_one_eb(eb, fs_info, wbc, &epd);
3349 if (ret) {
3350 done = 1;
3351 free_extent_buffer(eb);
3352 break;
3353 }
3354 free_extent_buffer(eb);
3355
3356 /*
3357 * the filesystem may choose to bump up nr_to_write.
3358 * We have to make sure to honor the new nr_to_write
3359 * at any time
3360 */
3361 nr_to_write_done = wbc->nr_to_write <= 0;
3362 }
3363 pagevec_release(&pvec);
3364 cond_resched();
3365 }
3366 if (!scanned && !done) {
3367 /*
3368 * We hit the last page and there is more work to be done: wrap
3369 * back to the start of the file
3370 */
3371 scanned = 1;
3372 index = 0;
3373 goto retry;
3374 }
3375 flush_write_bio(&epd);
3376 return ret;
3377}
3378
d1310b2e 3379/**
4bef0848 3380 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
d1310b2e
CM
3381 * @mapping: address space structure to write
3382 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3383 * @writepage: function called for each page
3384 * @data: data passed to writepage function
3385 *
3386 * If a page is already under I/O, write_cache_pages() skips it, even
3387 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3388 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3389 * and msync() need to guarantee that all the data which was dirty at the time
3390 * the call was made get new I/O started against them. If wbc->sync_mode is
3391 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3392 * existing IO to complete.
3393 */
b2950863 3394static int extent_write_cache_pages(struct extent_io_tree *tree,
4bef0848
CM
3395 struct address_space *mapping,
3396 struct writeback_control *wbc,
d2c3f4f6
CM
3397 writepage_t writepage, void *data,
3398 void (*flush_fn)(void *))
d1310b2e 3399{
7fd1a3f7 3400 struct inode *inode = mapping->host;
d1310b2e
CM
3401 int ret = 0;
3402 int done = 0;
f85d7d6c 3403 int nr_to_write_done = 0;
d1310b2e
CM
3404 struct pagevec pvec;
3405 int nr_pages;
3406 pgoff_t index;
3407 pgoff_t end; /* Inclusive */
3408 int scanned = 0;
f7aaa06b 3409 int tag;
d1310b2e 3410
7fd1a3f7
JB
3411 /*
3412 * We have to hold onto the inode so that ordered extents can do their
3413 * work when the IO finishes. The alternative to this is failing to add
3414 * an ordered extent if the igrab() fails there and that is a huge pain
3415 * to deal with, so instead just hold onto the inode throughout the
3416 * writepages operation. If it fails here we are freeing up the inode
3417 * anyway and we'd rather not waste our time writing out stuff that is
3418 * going to be truncated anyway.
3419 */
3420 if (!igrab(inode))
3421 return 0;
3422
d1310b2e
CM
3423 pagevec_init(&pvec, 0);
3424 if (wbc->range_cyclic) {
3425 index = mapping->writeback_index; /* Start from prev offset */
3426 end = -1;
3427 } else {
3428 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3429 end = wbc->range_end >> PAGE_CACHE_SHIFT;
d1310b2e
CM
3430 scanned = 1;
3431 }
f7aaa06b
JB
3432 if (wbc->sync_mode == WB_SYNC_ALL)
3433 tag = PAGECACHE_TAG_TOWRITE;
3434 else
3435 tag = PAGECACHE_TAG_DIRTY;
d1310b2e 3436retry:
f7aaa06b
JB
3437 if (wbc->sync_mode == WB_SYNC_ALL)
3438 tag_pages_for_writeback(mapping, index, end);
f85d7d6c 3439 while (!done && !nr_to_write_done && (index <= end) &&
f7aaa06b
JB
3440 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3441 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
d1310b2e
CM
3442 unsigned i;
3443
3444 scanned = 1;
3445 for (i = 0; i < nr_pages; i++) {
3446 struct page *page = pvec.pages[i];
3447
3448 /*
3449 * At this point we hold neither mapping->tree_lock nor
3450 * lock on the page itself: the page may be truncated or
3451 * invalidated (changing page->mapping to NULL), or even
3452 * swizzled back from swapper_space to tmpfs file
3453 * mapping
3454 */
01d658f2
CM
3455 if (tree->ops &&
3456 tree->ops->write_cache_pages_lock_hook) {
3457 tree->ops->write_cache_pages_lock_hook(page,
3458 data, flush_fn);
3459 } else {
3460 if (!trylock_page(page)) {
3461 flush_fn(data);
3462 lock_page(page);
3463 }
3464 }
d1310b2e
CM
3465
3466 if (unlikely(page->mapping != mapping)) {
3467 unlock_page(page);
3468 continue;
3469 }
3470
3471 if (!wbc->range_cyclic && page->index > end) {
3472 done = 1;
3473 unlock_page(page);
3474 continue;
3475 }
3476
d2c3f4f6 3477 if (wbc->sync_mode != WB_SYNC_NONE) {
0e6bd956
CM
3478 if (PageWriteback(page))
3479 flush_fn(data);
d1310b2e 3480 wait_on_page_writeback(page);
d2c3f4f6 3481 }
d1310b2e
CM
3482
3483 if (PageWriteback(page) ||
3484 !clear_page_dirty_for_io(page)) {
3485 unlock_page(page);
3486 continue;
3487 }
3488
3489 ret = (*writepage)(page, wbc, data);
3490
3491 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3492 unlock_page(page);
3493 ret = 0;
3494 }
f85d7d6c 3495 if (ret)
d1310b2e 3496 done = 1;
f85d7d6c
CM
3497
3498 /*
3499 * the filesystem may choose to bump up nr_to_write.
3500 * We have to make sure to honor the new nr_to_write
3501 * at any time
3502 */
3503 nr_to_write_done = wbc->nr_to_write <= 0;
d1310b2e
CM
3504 }
3505 pagevec_release(&pvec);
3506 cond_resched();
3507 }
3508 if (!scanned && !done) {
3509 /*
3510 * We hit the last page and there is more work to be done: wrap
3511 * back to the start of the file
3512 */
3513 scanned = 1;
3514 index = 0;
3515 goto retry;
3516 }
7fd1a3f7 3517 btrfs_add_delayed_iput(inode);
d1310b2e
CM
3518 return ret;
3519}
d1310b2e 3520
ffbd517d 3521static void flush_epd_write_bio(struct extent_page_data *epd)
d2c3f4f6 3522{
d2c3f4f6 3523 if (epd->bio) {
355808c2
JM
3524 int rw = WRITE;
3525 int ret;
3526
ffbd517d 3527 if (epd->sync_io)
355808c2
JM
3528 rw = WRITE_SYNC;
3529
de0022b9 3530 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
79787eaa 3531 BUG_ON(ret < 0); /* -ENOMEM */
d2c3f4f6
CM
3532 epd->bio = NULL;
3533 }
3534}
3535
ffbd517d
CM
3536static noinline void flush_write_bio(void *data)
3537{
3538 struct extent_page_data *epd = data;
3539 flush_epd_write_bio(epd);
3540}
3541
d1310b2e
CM
3542int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3543 get_extent_t *get_extent,
3544 struct writeback_control *wbc)
3545{
3546 int ret;
d1310b2e
CM
3547 struct extent_page_data epd = {
3548 .bio = NULL,
3549 .tree = tree,
3550 .get_extent = get_extent,
771ed689 3551 .extent_locked = 0,
ffbd517d 3552 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
de0022b9 3553 .bio_flags = 0,
d1310b2e 3554 };
d1310b2e 3555
d1310b2e
CM
3556 ret = __extent_writepage(page, wbc, &epd);
3557
ffbd517d 3558 flush_epd_write_bio(&epd);
d1310b2e
CM
3559 return ret;
3560}
d1310b2e 3561
771ed689
CM
3562int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3563 u64 start, u64 end, get_extent_t *get_extent,
3564 int mode)
3565{
3566 int ret = 0;
3567 struct address_space *mapping = inode->i_mapping;
3568 struct page *page;
3569 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3570 PAGE_CACHE_SHIFT;
3571
3572 struct extent_page_data epd = {
3573 .bio = NULL,
3574 .tree = tree,
3575 .get_extent = get_extent,
3576 .extent_locked = 1,
ffbd517d 3577 .sync_io = mode == WB_SYNC_ALL,
de0022b9 3578 .bio_flags = 0,
771ed689
CM
3579 };
3580 struct writeback_control wbc_writepages = {
771ed689 3581 .sync_mode = mode,
771ed689
CM
3582 .nr_to_write = nr_pages * 2,
3583 .range_start = start,
3584 .range_end = end + 1,
3585 };
3586
d397712b 3587 while (start <= end) {
771ed689
CM
3588 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3589 if (clear_page_dirty_for_io(page))
3590 ret = __extent_writepage(page, &wbc_writepages, &epd);
3591 else {
3592 if (tree->ops && tree->ops->writepage_end_io_hook)
3593 tree->ops->writepage_end_io_hook(page, start,
3594 start + PAGE_CACHE_SIZE - 1,
3595 NULL, 1);
3596 unlock_page(page);
3597 }
3598 page_cache_release(page);
3599 start += PAGE_CACHE_SIZE;
3600 }
3601
ffbd517d 3602 flush_epd_write_bio(&epd);
771ed689
CM
3603 return ret;
3604}
d1310b2e
CM
3605
3606int extent_writepages(struct extent_io_tree *tree,
3607 struct address_space *mapping,
3608 get_extent_t *get_extent,
3609 struct writeback_control *wbc)
3610{
3611 int ret = 0;
3612 struct extent_page_data epd = {
3613 .bio = NULL,
3614 .tree = tree,
3615 .get_extent = get_extent,
771ed689 3616 .extent_locked = 0,
ffbd517d 3617 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
de0022b9 3618 .bio_flags = 0,
d1310b2e
CM
3619 };
3620
4bef0848 3621 ret = extent_write_cache_pages(tree, mapping, wbc,
d2c3f4f6
CM
3622 __extent_writepage, &epd,
3623 flush_write_bio);
ffbd517d 3624 flush_epd_write_bio(&epd);
d1310b2e
CM
3625 return ret;
3626}
d1310b2e
CM
3627
3628int extent_readpages(struct extent_io_tree *tree,
3629 struct address_space *mapping,
3630 struct list_head *pages, unsigned nr_pages,
3631 get_extent_t get_extent)
3632{
3633 struct bio *bio = NULL;
3634 unsigned page_idx;
c8b97818 3635 unsigned long bio_flags = 0;
67c9684f
LB
3636 struct page *pagepool[16];
3637 struct page *page;
3638 int i = 0;
3639 int nr = 0;
d1310b2e 3640
d1310b2e 3641 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
67c9684f 3642 page = list_entry(pages->prev, struct page, lru);
d1310b2e
CM
3643
3644 prefetchw(&page->flags);
3645 list_del(&page->lru);
67c9684f 3646 if (add_to_page_cache_lru(page, mapping,
43e817a1 3647 page->index, GFP_NOFS)) {
67c9684f
LB
3648 page_cache_release(page);
3649 continue;
d1310b2e 3650 }
67c9684f
LB
3651
3652 pagepool[nr++] = page;
3653 if (nr < ARRAY_SIZE(pagepool))
3654 continue;
3655 for (i = 0; i < nr; i++) {
3656 __extent_read_full_page(tree, pagepool[i], get_extent,
3657 &bio, 0, &bio_flags);
3658 page_cache_release(pagepool[i]);
3659 }
3660 nr = 0;
d1310b2e 3661 }
67c9684f
LB
3662 for (i = 0; i < nr; i++) {
3663 __extent_read_full_page(tree, pagepool[i], get_extent,
3664 &bio, 0, &bio_flags);
3665 page_cache_release(pagepool[i]);
d1310b2e 3666 }
67c9684f 3667
d1310b2e
CM
3668 BUG_ON(!list_empty(pages));
3669 if (bio)
79787eaa 3670 return submit_one_bio(READ, bio, 0, bio_flags);
d1310b2e
CM
3671 return 0;
3672}
d1310b2e
CM
3673
3674/*
3675 * basic invalidatepage code, this waits on any locked or writeback
3676 * ranges corresponding to the page, and then deletes any extent state
3677 * records from the tree
3678 */
3679int extent_invalidatepage(struct extent_io_tree *tree,
3680 struct page *page, unsigned long offset)
3681{
2ac55d41 3682 struct extent_state *cached_state = NULL;
d1310b2e
CM
3683 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
3684 u64 end = start + PAGE_CACHE_SIZE - 1;
3685 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
3686
d397712b 3687 start += (offset + blocksize - 1) & ~(blocksize - 1);
d1310b2e
CM
3688 if (start > end)
3689 return 0;
3690
d0082371 3691 lock_extent_bits(tree, start, end, 0, &cached_state);
1edbb734 3692 wait_on_page_writeback(page);
d1310b2e 3693 clear_extent_bit(tree, start, end,
32c00aff
JB
3694 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
3695 EXTENT_DO_ACCOUNTING,
2ac55d41 3696 1, 1, &cached_state, GFP_NOFS);
d1310b2e
CM
3697 return 0;
3698}
d1310b2e 3699
7b13b7b1
CM
3700/*
3701 * a helper for releasepage, this tests for areas of the page that
3702 * are locked or under IO and drops the related state bits if it is safe
3703 * to drop the page.
3704 */
3705int try_release_extent_state(struct extent_map_tree *map,
3706 struct extent_io_tree *tree, struct page *page,
3707 gfp_t mask)
3708{
3709 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
3710 u64 end = start + PAGE_CACHE_SIZE - 1;
3711 int ret = 1;
3712
211f90e6 3713 if (test_range_bit(tree, start, end,
8b62b72b 3714 EXTENT_IOBITS, 0, NULL))
7b13b7b1
CM
3715 ret = 0;
3716 else {
3717 if ((mask & GFP_NOFS) == GFP_NOFS)
3718 mask = GFP_NOFS;
11ef160f
CM
3719 /*
3720 * at this point we can safely clear everything except the
3721 * locked bit and the nodatasum bit
3722 */
e3f24cc5 3723 ret = clear_extent_bit(tree, start, end,
11ef160f
CM
3724 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
3725 0, 0, NULL, mask);
e3f24cc5
CM
3726
3727 /* if clear_extent_bit failed for enomem reasons,
3728 * we can't allow the release to continue.
3729 */
3730 if (ret < 0)
3731 ret = 0;
3732 else
3733 ret = 1;
7b13b7b1
CM
3734 }
3735 return ret;
3736}
7b13b7b1 3737
d1310b2e
CM
3738/*
3739 * a helper for releasepage. As long as there are no locked extents
3740 * in the range corresponding to the page, both state records and extent
3741 * map records are removed
3742 */
3743int try_release_extent_mapping(struct extent_map_tree *map,
70dec807
CM
3744 struct extent_io_tree *tree, struct page *page,
3745 gfp_t mask)
d1310b2e
CM
3746{
3747 struct extent_map *em;
3748 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
3749 u64 end = start + PAGE_CACHE_SIZE - 1;
7b13b7b1 3750
70dec807
CM
3751 if ((mask & __GFP_WAIT) &&
3752 page->mapping->host->i_size > 16 * 1024 * 1024) {
39b5637f 3753 u64 len;
70dec807 3754 while (start <= end) {
39b5637f 3755 len = end - start + 1;
890871be 3756 write_lock(&map->lock);
39b5637f 3757 em = lookup_extent_mapping(map, start, len);
285190d9 3758 if (!em) {
890871be 3759 write_unlock(&map->lock);
70dec807
CM
3760 break;
3761 }
7f3c74fb
CM
3762 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
3763 em->start != start) {
890871be 3764 write_unlock(&map->lock);
70dec807
CM
3765 free_extent_map(em);
3766 break;
3767 }
3768 if (!test_range_bit(tree, em->start,
3769 extent_map_end(em) - 1,
8b62b72b 3770 EXTENT_LOCKED | EXTENT_WRITEBACK,
9655d298 3771 0, NULL)) {
70dec807
CM
3772 remove_extent_mapping(map, em);
3773 /* once for the rb tree */
3774 free_extent_map(em);
3775 }
3776 start = extent_map_end(em);
890871be 3777 write_unlock(&map->lock);
70dec807
CM
3778
3779 /* once for us */
d1310b2e
CM
3780 free_extent_map(em);
3781 }
d1310b2e 3782 }
7b13b7b1 3783 return try_release_extent_state(map, tree, page, mask);
d1310b2e 3784}
d1310b2e 3785
ec29ed5b
CM
3786/*
3787 * helper function for fiemap, which doesn't want to see any holes.
3788 * This maps until we find something past 'last'
3789 */
3790static struct extent_map *get_extent_skip_holes(struct inode *inode,
3791 u64 offset,
3792 u64 last,
3793 get_extent_t *get_extent)
3794{
3795 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
3796 struct extent_map *em;
3797 u64 len;
3798
3799 if (offset >= last)
3800 return NULL;
3801
3802 while(1) {
3803 len = last - offset;
3804 if (len == 0)
3805 break;
3806 len = (len + sectorsize - 1) & ~(sectorsize - 1);
3807 em = get_extent(inode, NULL, 0, offset, len, 0);
c704005d 3808 if (IS_ERR_OR_NULL(em))
ec29ed5b
CM
3809 return em;
3810
3811 /* if this isn't a hole return it */
3812 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
3813 em->block_start != EXTENT_MAP_HOLE) {
3814 return em;
3815 }
3816
3817 /* this is a hole, advance to the next extent */
3818 offset = extent_map_end(em);
3819 free_extent_map(em);
3820 if (offset >= last)
3821 break;
3822 }
3823 return NULL;
3824}
3825
1506fcc8
YS
3826int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3827 __u64 start, __u64 len, get_extent_t *get_extent)
3828{
975f84fe 3829 int ret = 0;
1506fcc8
YS
3830 u64 off = start;
3831 u64 max = start + len;
3832 u32 flags = 0;
975f84fe
JB
3833 u32 found_type;
3834 u64 last;
ec29ed5b 3835 u64 last_for_get_extent = 0;
1506fcc8 3836 u64 disko = 0;
ec29ed5b 3837 u64 isize = i_size_read(inode);
975f84fe 3838 struct btrfs_key found_key;
1506fcc8 3839 struct extent_map *em = NULL;
2ac55d41 3840 struct extent_state *cached_state = NULL;
975f84fe
JB
3841 struct btrfs_path *path;
3842 struct btrfs_file_extent_item *item;
1506fcc8 3843 int end = 0;
ec29ed5b
CM
3844 u64 em_start = 0;
3845 u64 em_len = 0;
3846 u64 em_end = 0;
1506fcc8 3847 unsigned long emflags;
1506fcc8
YS
3848
3849 if (len == 0)
3850 return -EINVAL;
3851
975f84fe
JB
3852 path = btrfs_alloc_path();
3853 if (!path)
3854 return -ENOMEM;
3855 path->leave_spinning = 1;
3856
4d479cf0
JB
3857 start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
3858 len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
3859
ec29ed5b
CM
3860 /*
3861 * lookup the last file extent. We're not using i_size here
3862 * because there might be preallocation past i_size
3863 */
975f84fe 3864 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
33345d01 3865 path, btrfs_ino(inode), -1, 0);
975f84fe
JB
3866 if (ret < 0) {
3867 btrfs_free_path(path);
3868 return ret;
3869 }
3870 WARN_ON(!ret);
3871 path->slots[0]--;
3872 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
3873 struct btrfs_file_extent_item);
3874 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
3875 found_type = btrfs_key_type(&found_key);
3876
ec29ed5b 3877 /* No extents, but there might be delalloc bits */
33345d01 3878 if (found_key.objectid != btrfs_ino(inode) ||
975f84fe 3879 found_type != BTRFS_EXTENT_DATA_KEY) {
ec29ed5b
CM
3880 /* have to trust i_size as the end */
3881 last = (u64)-1;
3882 last_for_get_extent = isize;
3883 } else {
3884 /*
3885 * remember the start of the last extent. There are a
3886 * bunch of different factors that go into the length of the
3887 * extent, so its much less complex to remember where it started
3888 */
3889 last = found_key.offset;
3890 last_for_get_extent = last + 1;
975f84fe 3891 }
975f84fe
JB
3892 btrfs_free_path(path);
3893
ec29ed5b
CM
3894 /*
3895 * we might have some extents allocated but more delalloc past those
3896 * extents. so, we trust isize unless the start of the last extent is
3897 * beyond isize
3898 */
3899 if (last < isize) {
3900 last = (u64)-1;
3901 last_for_get_extent = isize;
3902 }
3903
2ac55d41 3904 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len, 0,
d0082371 3905 &cached_state);
ec29ed5b 3906
4d479cf0 3907 em = get_extent_skip_holes(inode, start, last_for_get_extent,
ec29ed5b 3908 get_extent);
1506fcc8
YS
3909 if (!em)
3910 goto out;
3911 if (IS_ERR(em)) {
3912 ret = PTR_ERR(em);
3913 goto out;
3914 }
975f84fe 3915
1506fcc8 3916 while (!end) {
ea8efc74
CM
3917 u64 offset_in_extent;
3918
3919 /* break if the extent we found is outside the range */
3920 if (em->start >= max || extent_map_end(em) < off)
3921 break;
3922
3923 /*
3924 * get_extent may return an extent that starts before our
3925 * requested range. We have to make sure the ranges
3926 * we return to fiemap always move forward and don't
3927 * overlap, so adjust the offsets here
3928 */
3929 em_start = max(em->start, off);
1506fcc8 3930
ea8efc74
CM
3931 /*
3932 * record the offset from the start of the extent
3933 * for adjusting the disk offset below
3934 */
3935 offset_in_extent = em_start - em->start;
ec29ed5b 3936 em_end = extent_map_end(em);
ea8efc74 3937 em_len = em_end - em_start;
ec29ed5b 3938 emflags = em->flags;
1506fcc8
YS
3939 disko = 0;
3940 flags = 0;
3941
ea8efc74
CM
3942 /*
3943 * bump off for our next call to get_extent
3944 */
3945 off = extent_map_end(em);
3946 if (off >= max)
3947 end = 1;
3948
93dbfad7 3949 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
1506fcc8
YS
3950 end = 1;
3951 flags |= FIEMAP_EXTENT_LAST;
93dbfad7 3952 } else if (em->block_start == EXTENT_MAP_INLINE) {
1506fcc8
YS
3953 flags |= (FIEMAP_EXTENT_DATA_INLINE |
3954 FIEMAP_EXTENT_NOT_ALIGNED);
93dbfad7 3955 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
1506fcc8
YS
3956 flags |= (FIEMAP_EXTENT_DELALLOC |
3957 FIEMAP_EXTENT_UNKNOWN);
93dbfad7 3958 } else {
ea8efc74 3959 disko = em->block_start + offset_in_extent;
1506fcc8
YS
3960 }
3961 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
3962 flags |= FIEMAP_EXTENT_ENCODED;
3963
1506fcc8
YS
3964 free_extent_map(em);
3965 em = NULL;
ec29ed5b
CM
3966 if ((em_start >= last) || em_len == (u64)-1 ||
3967 (last == (u64)-1 && isize <= em_end)) {
1506fcc8
YS
3968 flags |= FIEMAP_EXTENT_LAST;
3969 end = 1;
3970 }
3971
ec29ed5b
CM
3972 /* now scan forward to see if this is really the last extent. */
3973 em = get_extent_skip_holes(inode, off, last_for_get_extent,
3974 get_extent);
3975 if (IS_ERR(em)) {
3976 ret = PTR_ERR(em);
3977 goto out;
3978 }
3979 if (!em) {
975f84fe
JB
3980 flags |= FIEMAP_EXTENT_LAST;
3981 end = 1;
3982 }
ec29ed5b
CM
3983 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
3984 em_len, flags);
3985 if (ret)
3986 goto out_free;
1506fcc8
YS
3987 }
3988out_free:
3989 free_extent_map(em);
3990out:
2ac55d41
JB
3991 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len,
3992 &cached_state, GFP_NOFS);
1506fcc8
YS
3993 return ret;
3994}
3995
727011e0
CM
3996static void __free_extent_buffer(struct extent_buffer *eb)
3997{
3998#if LEAK_DEBUG
3999 unsigned long flags;
4000 spin_lock_irqsave(&leak_lock, flags);
4001 list_del(&eb->leak_list);
4002 spin_unlock_irqrestore(&leak_lock, flags);
4003#endif
4004 if (eb->pages && eb->pages != eb->inline_pages)
4005 kfree(eb->pages);
4006 kmem_cache_free(extent_buffer_cache, eb);
4007}
4008
d1310b2e
CM
4009static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
4010 u64 start,
4011 unsigned long len,
4012 gfp_t mask)
4013{
4014 struct extent_buffer *eb = NULL;
3935127c 4015#if LEAK_DEBUG
2d2ae547 4016 unsigned long flags;
4bef0848 4017#endif
d1310b2e 4018
d1310b2e 4019 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
91ca338d
TI
4020 if (eb == NULL)
4021 return NULL;
d1310b2e
CM
4022 eb->start = start;
4023 eb->len = len;
4f2de97a 4024 eb->tree = tree;
815a51c7 4025 eb->bflags = 0;
bd681513
CM
4026 rwlock_init(&eb->lock);
4027 atomic_set(&eb->write_locks, 0);
4028 atomic_set(&eb->read_locks, 0);
4029 atomic_set(&eb->blocking_readers, 0);
4030 atomic_set(&eb->blocking_writers, 0);
4031 atomic_set(&eb->spinning_readers, 0);
4032 atomic_set(&eb->spinning_writers, 0);
5b25f70f 4033 eb->lock_nested = 0;
bd681513
CM
4034 init_waitqueue_head(&eb->write_lock_wq);
4035 init_waitqueue_head(&eb->read_lock_wq);
b4ce94de 4036
3935127c 4037#if LEAK_DEBUG
2d2ae547
CM
4038 spin_lock_irqsave(&leak_lock, flags);
4039 list_add(&eb->leak_list, &buffers);
4040 spin_unlock_irqrestore(&leak_lock, flags);
4bef0848 4041#endif
3083ee2e 4042 spin_lock_init(&eb->refs_lock);
d1310b2e 4043 atomic_set(&eb->refs, 1);
0b32f4bb 4044 atomic_set(&eb->io_pages, 0);
727011e0
CM
4045
4046 if (len > MAX_INLINE_EXTENT_BUFFER_SIZE) {
4047 struct page **pages;
4048 int num_pages = (len + PAGE_CACHE_SIZE - 1) >>
4049 PAGE_CACHE_SHIFT;
4050 pages = kzalloc(num_pages, mask);
4051 if (!pages) {
4052 __free_extent_buffer(eb);
4053 return NULL;
4054 }
4055 eb->pages = pages;
4056 } else {
4057 eb->pages = eb->inline_pages;
4058 }
d1310b2e
CM
4059
4060 return eb;
4061}
4062
815a51c7
JS
4063struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4064{
4065 unsigned long i;
4066 struct page *p;
4067 struct extent_buffer *new;
4068 unsigned long num_pages = num_extent_pages(src->start, src->len);
4069
4070 new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_ATOMIC);
4071 if (new == NULL)
4072 return NULL;
4073
4074 for (i = 0; i < num_pages; i++) {
4075 p = alloc_page(GFP_ATOMIC);
4076 BUG_ON(!p);
4077 attach_extent_buffer_page(new, p);
4078 WARN_ON(PageDirty(p));
4079 SetPageUptodate(p);
4080 new->pages[i] = p;
4081 }
4082
4083 copy_extent_buffer(new, src, 0, 0, src->len);
4084 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4085 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4086
4087 return new;
4088}
4089
4090struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
4091{
4092 struct extent_buffer *eb;
4093 unsigned long num_pages = num_extent_pages(0, len);
4094 unsigned long i;
4095
4096 eb = __alloc_extent_buffer(NULL, start, len, GFP_ATOMIC);
4097 if (!eb)
4098 return NULL;
4099
4100 for (i = 0; i < num_pages; i++) {
4101 eb->pages[i] = alloc_page(GFP_ATOMIC);
4102 if (!eb->pages[i])
4103 goto err;
4104 }
4105 set_extent_buffer_uptodate(eb);
4106 btrfs_set_header_nritems(eb, 0);
4107 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4108
4109 return eb;
4110err:
84167d19
SB
4111 for (; i > 0; i--)
4112 __free_page(eb->pages[i - 1]);
815a51c7
JS
4113 __free_extent_buffer(eb);
4114 return NULL;
4115}
4116
0b32f4bb 4117static int extent_buffer_under_io(struct extent_buffer *eb)
d1310b2e 4118{
0b32f4bb
JB
4119 return (atomic_read(&eb->io_pages) ||
4120 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4121 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
d1310b2e
CM
4122}
4123
897ca6e9
MX
4124/*
4125 * Helper for releasing extent buffer page.
4126 */
4127static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
4128 unsigned long start_idx)
4129{
4130 unsigned long index;
39bab87b 4131 unsigned long num_pages;
897ca6e9 4132 struct page *page;
815a51c7 4133 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
897ca6e9 4134
0b32f4bb 4135 BUG_ON(extent_buffer_under_io(eb));
897ca6e9 4136
39bab87b
WSH
4137 num_pages = num_extent_pages(eb->start, eb->len);
4138 index = start_idx + num_pages;
897ca6e9
MX
4139 if (start_idx >= index)
4140 return;
4141
4142 do {
4143 index--;
4144 page = extent_buffer_page(eb, index);
815a51c7 4145 if (page && mapped) {
4f2de97a
JB
4146 spin_lock(&page->mapping->private_lock);
4147 /*
4148 * We do this since we'll remove the pages after we've
4149 * removed the eb from the radix tree, so we could race
4150 * and have this page now attached to the new eb. So
4151 * only clear page_private if it's still connected to
4152 * this eb.
4153 */
4154 if (PagePrivate(page) &&
4155 page->private == (unsigned long)eb) {
0b32f4bb 4156 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
3083ee2e
JB
4157 BUG_ON(PageDirty(page));
4158 BUG_ON(PageWriteback(page));
4f2de97a
JB
4159 /*
4160 * We need to make sure we haven't be attached
4161 * to a new eb.
4162 */
4163 ClearPagePrivate(page);
4164 set_page_private(page, 0);
4165 /* One for the page private */
4166 page_cache_release(page);
4167 }
4168 spin_unlock(&page->mapping->private_lock);
4169
815a51c7
JS
4170 }
4171 if (page) {
4f2de97a 4172 /* One for when we alloced the page */
897ca6e9 4173 page_cache_release(page);
4f2de97a 4174 }
897ca6e9
MX
4175 } while (index != start_idx);
4176}
4177
4178/*
4179 * Helper for releasing the extent buffer.
4180 */
4181static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4182{
4183 btrfs_release_extent_buffer_page(eb, 0);
4184 __free_extent_buffer(eb);
4185}
4186
0b32f4bb
JB
4187static void check_buffer_tree_ref(struct extent_buffer *eb)
4188{
4189 /* the ref bit is tricky. We have to make sure it is set
4190 * if we have the buffer dirty. Otherwise the
4191 * code to free a buffer can end up dropping a dirty
4192 * page
4193 *
4194 * Once the ref bit is set, it won't go away while the
4195 * buffer is dirty or in writeback, and it also won't
4196 * go away while we have the reference count on the
4197 * eb bumped.
4198 *
4199 * We can't just set the ref bit without bumping the
4200 * ref on the eb because free_extent_buffer might
4201 * see the ref bit and try to clear it. If this happens
4202 * free_extent_buffer might end up dropping our original
4203 * ref by mistake and freeing the page before we are able
4204 * to add one more ref.
4205 *
4206 * So bump the ref count first, then set the bit. If someone
4207 * beat us to it, drop the ref we added.
4208 */
594831c4
JB
4209 spin_lock(&eb->refs_lock);
4210 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
0b32f4bb 4211 atomic_inc(&eb->refs);
594831c4 4212 spin_unlock(&eb->refs_lock);
0b32f4bb
JB
4213}
4214
5df4235e
JB
4215static void mark_extent_buffer_accessed(struct extent_buffer *eb)
4216{
4217 unsigned long num_pages, i;
4218
0b32f4bb
JB
4219 check_buffer_tree_ref(eb);
4220
5df4235e
JB
4221 num_pages = num_extent_pages(eb->start, eb->len);
4222 for (i = 0; i < num_pages; i++) {
4223 struct page *p = extent_buffer_page(eb, i);
4224 mark_page_accessed(p);
4225 }
4226}
4227
d1310b2e 4228struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
727011e0 4229 u64 start, unsigned long len)
d1310b2e
CM
4230{
4231 unsigned long num_pages = num_extent_pages(start, len);
4232 unsigned long i;
4233 unsigned long index = start >> PAGE_CACHE_SHIFT;
4234 struct extent_buffer *eb;
6af118ce 4235 struct extent_buffer *exists = NULL;
d1310b2e
CM
4236 struct page *p;
4237 struct address_space *mapping = tree->mapping;
4238 int uptodate = 1;
19fe0a8b 4239 int ret;
d1310b2e 4240
19fe0a8b
MX
4241 rcu_read_lock();
4242 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4243 if (eb && atomic_inc_not_zero(&eb->refs)) {
4244 rcu_read_unlock();
5df4235e 4245 mark_extent_buffer_accessed(eb);
6af118ce
CM
4246 return eb;
4247 }
19fe0a8b 4248 rcu_read_unlock();
6af118ce 4249
ba144192 4250 eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
2b114d1d 4251 if (!eb)
d1310b2e
CM
4252 return NULL;
4253
727011e0 4254 for (i = 0; i < num_pages; i++, index++) {
a6591715 4255 p = find_or_create_page(mapping, index, GFP_NOFS);
4804b382 4256 if (!p)
6af118ce 4257 goto free_eb;
4f2de97a
JB
4258
4259 spin_lock(&mapping->private_lock);
4260 if (PagePrivate(p)) {
4261 /*
4262 * We could have already allocated an eb for this page
4263 * and attached one so lets see if we can get a ref on
4264 * the existing eb, and if we can we know it's good and
4265 * we can just return that one, else we know we can just
4266 * overwrite page->private.
4267 */
4268 exists = (struct extent_buffer *)p->private;
4269 if (atomic_inc_not_zero(&exists->refs)) {
4270 spin_unlock(&mapping->private_lock);
4271 unlock_page(p);
17de39ac 4272 page_cache_release(p);
5df4235e 4273 mark_extent_buffer_accessed(exists);
4f2de97a
JB
4274 goto free_eb;
4275 }
4276
0b32f4bb 4277 /*
4f2de97a
JB
4278 * Do this so attach doesn't complain and we need to
4279 * drop the ref the old guy had.
4280 */
4281 ClearPagePrivate(p);
0b32f4bb 4282 WARN_ON(PageDirty(p));
4f2de97a 4283 page_cache_release(p);
d1310b2e 4284 }
4f2de97a
JB
4285 attach_extent_buffer_page(eb, p);
4286 spin_unlock(&mapping->private_lock);
0b32f4bb 4287 WARN_ON(PageDirty(p));
d1310b2e 4288 mark_page_accessed(p);
727011e0 4289 eb->pages[i] = p;
d1310b2e
CM
4290 if (!PageUptodate(p))
4291 uptodate = 0;
eb14ab8e
CM
4292
4293 /*
4294 * see below about how we avoid a nasty race with release page
4295 * and why we unlock later
4296 */
d1310b2e
CM
4297 }
4298 if (uptodate)
b4ce94de 4299 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
115391d2 4300again:
19fe0a8b
MX
4301 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4302 if (ret)
4303 goto free_eb;
4304
6af118ce 4305 spin_lock(&tree->buffer_lock);
19fe0a8b
MX
4306 ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
4307 if (ret == -EEXIST) {
4308 exists = radix_tree_lookup(&tree->buffer,
4309 start >> PAGE_CACHE_SHIFT);
115391d2
JB
4310 if (!atomic_inc_not_zero(&exists->refs)) {
4311 spin_unlock(&tree->buffer_lock);
4312 radix_tree_preload_end();
115391d2
JB
4313 exists = NULL;
4314 goto again;
4315 }
6af118ce 4316 spin_unlock(&tree->buffer_lock);
19fe0a8b 4317 radix_tree_preload_end();
5df4235e 4318 mark_extent_buffer_accessed(exists);
6af118ce
CM
4319 goto free_eb;
4320 }
6af118ce 4321 /* add one reference for the tree */
0b32f4bb 4322 check_buffer_tree_ref(eb);
f044ba78 4323 spin_unlock(&tree->buffer_lock);
19fe0a8b 4324 radix_tree_preload_end();
eb14ab8e
CM
4325
4326 /*
4327 * there is a race where release page may have
4328 * tried to find this extent buffer in the radix
4329 * but failed. It will tell the VM it is safe to
4330 * reclaim the, and it will clear the page private bit.
4331 * We must make sure to set the page private bit properly
4332 * after the extent buffer is in the radix tree so
4333 * it doesn't get lost
4334 */
727011e0
CM
4335 SetPageChecked(eb->pages[0]);
4336 for (i = 1; i < num_pages; i++) {
4337 p = extent_buffer_page(eb, i);
727011e0
CM
4338 ClearPageChecked(p);
4339 unlock_page(p);
4340 }
4341 unlock_page(eb->pages[0]);
d1310b2e
CM
4342 return eb;
4343
6af118ce 4344free_eb:
727011e0
CM
4345 for (i = 0; i < num_pages; i++) {
4346 if (eb->pages[i])
4347 unlock_page(eb->pages[i]);
4348 }
eb14ab8e 4349
17de39ac 4350 WARN_ON(!atomic_dec_and_test(&eb->refs));
897ca6e9 4351 btrfs_release_extent_buffer(eb);
6af118ce 4352 return exists;
d1310b2e 4353}
d1310b2e
CM
4354
4355struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
f09d1f60 4356 u64 start, unsigned long len)
d1310b2e 4357{
d1310b2e 4358 struct extent_buffer *eb;
d1310b2e 4359
19fe0a8b
MX
4360 rcu_read_lock();
4361 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4362 if (eb && atomic_inc_not_zero(&eb->refs)) {
4363 rcu_read_unlock();
5df4235e 4364 mark_extent_buffer_accessed(eb);
19fe0a8b
MX
4365 return eb;
4366 }
4367 rcu_read_unlock();
0f9dd46c 4368
19fe0a8b 4369 return NULL;
d1310b2e 4370}
d1310b2e 4371
3083ee2e
JB
4372static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4373{
4374 struct extent_buffer *eb =
4375 container_of(head, struct extent_buffer, rcu_head);
4376
4377 __free_extent_buffer(eb);
4378}
4379
3083ee2e 4380/* Expects to have eb->eb_lock already held */
e64860aa 4381static int release_extent_buffer(struct extent_buffer *eb, gfp_t mask)
3083ee2e
JB
4382{
4383 WARN_ON(atomic_read(&eb->refs) == 0);
4384 if (atomic_dec_and_test(&eb->refs)) {
815a51c7
JS
4385 if (test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags)) {
4386 spin_unlock(&eb->refs_lock);
4387 } else {
4388 struct extent_io_tree *tree = eb->tree;
3083ee2e 4389
815a51c7 4390 spin_unlock(&eb->refs_lock);
3083ee2e 4391
815a51c7
JS
4392 spin_lock(&tree->buffer_lock);
4393 radix_tree_delete(&tree->buffer,
4394 eb->start >> PAGE_CACHE_SHIFT);
4395 spin_unlock(&tree->buffer_lock);
4396 }
3083ee2e
JB
4397
4398 /* Should be safe to release our pages at this point */
4399 btrfs_release_extent_buffer_page(eb, 0);
3083ee2e 4400 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
e64860aa 4401 return 1;
3083ee2e
JB
4402 }
4403 spin_unlock(&eb->refs_lock);
e64860aa
JB
4404
4405 return 0;
3083ee2e
JB
4406}
4407
d1310b2e
CM
4408void free_extent_buffer(struct extent_buffer *eb)
4409{
d1310b2e
CM
4410 if (!eb)
4411 return;
4412
3083ee2e 4413 spin_lock(&eb->refs_lock);
815a51c7
JS
4414 if (atomic_read(&eb->refs) == 2 &&
4415 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
4416 atomic_dec(&eb->refs);
4417
3083ee2e
JB
4418 if (atomic_read(&eb->refs) == 2 &&
4419 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
0b32f4bb 4420 !extent_buffer_under_io(eb) &&
3083ee2e
JB
4421 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4422 atomic_dec(&eb->refs);
4423
4424 /*
4425 * I know this is terrible, but it's temporary until we stop tracking
4426 * the uptodate bits and such for the extent buffers.
4427 */
4428 release_extent_buffer(eb, GFP_ATOMIC);
4429}
4430
4431void free_extent_buffer_stale(struct extent_buffer *eb)
4432{
4433 if (!eb)
d1310b2e
CM
4434 return;
4435
3083ee2e
JB
4436 spin_lock(&eb->refs_lock);
4437 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4438
0b32f4bb 4439 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3083ee2e
JB
4440 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4441 atomic_dec(&eb->refs);
4442 release_extent_buffer(eb, GFP_NOFS);
d1310b2e 4443}
d1310b2e 4444
1d4284bd 4445void clear_extent_buffer_dirty(struct extent_buffer *eb)
d1310b2e 4446{
d1310b2e
CM
4447 unsigned long i;
4448 unsigned long num_pages;
4449 struct page *page;
4450
d1310b2e
CM
4451 num_pages = num_extent_pages(eb->start, eb->len);
4452
4453 for (i = 0; i < num_pages; i++) {
4454 page = extent_buffer_page(eb, i);
b9473439 4455 if (!PageDirty(page))
d2c3f4f6
CM
4456 continue;
4457
a61e6f29 4458 lock_page(page);
eb14ab8e
CM
4459 WARN_ON(!PagePrivate(page));
4460
d1310b2e 4461 clear_page_dirty_for_io(page);
0ee0fda0 4462 spin_lock_irq(&page->mapping->tree_lock);
d1310b2e
CM
4463 if (!PageDirty(page)) {
4464 radix_tree_tag_clear(&page->mapping->page_tree,
4465 page_index(page),
4466 PAGECACHE_TAG_DIRTY);
4467 }
0ee0fda0 4468 spin_unlock_irq(&page->mapping->tree_lock);
bf0da8c1 4469 ClearPageError(page);
a61e6f29 4470 unlock_page(page);
d1310b2e 4471 }
0b32f4bb 4472 WARN_ON(atomic_read(&eb->refs) == 0);
d1310b2e 4473}
d1310b2e 4474
0b32f4bb 4475int set_extent_buffer_dirty(struct extent_buffer *eb)
d1310b2e
CM
4476{
4477 unsigned long i;
4478 unsigned long num_pages;
b9473439 4479 int was_dirty = 0;
d1310b2e 4480
0b32f4bb
JB
4481 check_buffer_tree_ref(eb);
4482
b9473439 4483 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
0b32f4bb 4484
d1310b2e 4485 num_pages = num_extent_pages(eb->start, eb->len);
3083ee2e 4486 WARN_ON(atomic_read(&eb->refs) == 0);
0b32f4bb
JB
4487 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4488
b9473439 4489 for (i = 0; i < num_pages; i++)
0b32f4bb 4490 set_page_dirty(extent_buffer_page(eb, i));
b9473439 4491 return was_dirty;
d1310b2e 4492}
d1310b2e 4493
0b32f4bb 4494static int range_straddles_pages(u64 start, u64 len)
19b6caf4
CM
4495{
4496 if (len < PAGE_CACHE_SIZE)
4497 return 1;
4498 if (start & (PAGE_CACHE_SIZE - 1))
4499 return 1;
4500 if ((start + len) & (PAGE_CACHE_SIZE - 1))
4501 return 1;
4502 return 0;
4503}
4504
0b32f4bb 4505int clear_extent_buffer_uptodate(struct extent_buffer *eb)
1259ab75
CM
4506{
4507 unsigned long i;
4508 struct page *page;
4509 unsigned long num_pages;
4510
b4ce94de 4511 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
0b32f4bb 4512 num_pages = num_extent_pages(eb->start, eb->len);
1259ab75
CM
4513 for (i = 0; i < num_pages; i++) {
4514 page = extent_buffer_page(eb, i);
33958dc6
CM
4515 if (page)
4516 ClearPageUptodate(page);
1259ab75
CM
4517 }
4518 return 0;
4519}
4520
0b32f4bb 4521int set_extent_buffer_uptodate(struct extent_buffer *eb)
d1310b2e
CM
4522{
4523 unsigned long i;
4524 struct page *page;
4525 unsigned long num_pages;
4526
0b32f4bb 4527 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
d1310b2e 4528 num_pages = num_extent_pages(eb->start, eb->len);
d1310b2e
CM
4529 for (i = 0; i < num_pages; i++) {
4530 page = extent_buffer_page(eb, i);
d1310b2e
CM
4531 SetPageUptodate(page);
4532 }
4533 return 0;
4534}
d1310b2e 4535
ce9adaa5
CM
4536int extent_range_uptodate(struct extent_io_tree *tree,
4537 u64 start, u64 end)
4538{
4539 struct page *page;
4540 int ret;
4541 int pg_uptodate = 1;
4542 int uptodate;
4543 unsigned long index;
4544
0b32f4bb 4545 if (range_straddles_pages(start, end - start + 1)) {
19b6caf4
CM
4546 ret = test_range_bit(tree, start, end,
4547 EXTENT_UPTODATE, 1, NULL);
4548 if (ret)
4549 return 1;
4550 }
d397712b 4551 while (start <= end) {
ce9adaa5
CM
4552 index = start >> PAGE_CACHE_SHIFT;
4553 page = find_get_page(tree->mapping, index);
8bedd51b
MH
4554 if (!page)
4555 return 1;
ce9adaa5
CM
4556 uptodate = PageUptodate(page);
4557 page_cache_release(page);
4558 if (!uptodate) {
4559 pg_uptodate = 0;
4560 break;
4561 }
4562 start += PAGE_CACHE_SIZE;
4563 }
4564 return pg_uptodate;
4565}
4566
0b32f4bb 4567int extent_buffer_uptodate(struct extent_buffer *eb)
d1310b2e 4568{
0b32f4bb 4569 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
d1310b2e 4570}
d1310b2e
CM
4571
4572int read_extent_buffer_pages(struct extent_io_tree *tree,
bb82ab88 4573 struct extent_buffer *eb, u64 start, int wait,
f188591e 4574 get_extent_t *get_extent, int mirror_num)
d1310b2e
CM
4575{
4576 unsigned long i;
4577 unsigned long start_i;
4578 struct page *page;
4579 int err;
4580 int ret = 0;
ce9adaa5
CM
4581 int locked_pages = 0;
4582 int all_uptodate = 1;
d1310b2e 4583 unsigned long num_pages;
727011e0 4584 unsigned long num_reads = 0;
a86c12c7 4585 struct bio *bio = NULL;
c8b97818 4586 unsigned long bio_flags = 0;
a86c12c7 4587
b4ce94de 4588 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
d1310b2e
CM
4589 return 0;
4590
d1310b2e
CM
4591 if (start) {
4592 WARN_ON(start < eb->start);
4593 start_i = (start >> PAGE_CACHE_SHIFT) -
4594 (eb->start >> PAGE_CACHE_SHIFT);
4595 } else {
4596 start_i = 0;
4597 }
4598
4599 num_pages = num_extent_pages(eb->start, eb->len);
4600 for (i = start_i; i < num_pages; i++) {
4601 page = extent_buffer_page(eb, i);
bb82ab88 4602 if (wait == WAIT_NONE) {
2db04966 4603 if (!trylock_page(page))
ce9adaa5 4604 goto unlock_exit;
d1310b2e
CM
4605 } else {
4606 lock_page(page);
4607 }
ce9adaa5 4608 locked_pages++;
727011e0
CM
4609 if (!PageUptodate(page)) {
4610 num_reads++;
ce9adaa5 4611 all_uptodate = 0;
727011e0 4612 }
ce9adaa5
CM
4613 }
4614 if (all_uptodate) {
4615 if (start_i == 0)
b4ce94de 4616 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
ce9adaa5
CM
4617 goto unlock_exit;
4618 }
4619
ea466794 4620 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
5cf1ab56 4621 eb->read_mirror = 0;
0b32f4bb 4622 atomic_set(&eb->io_pages, num_reads);
ce9adaa5
CM
4623 for (i = start_i; i < num_pages; i++) {
4624 page = extent_buffer_page(eb, i);
ce9adaa5 4625 if (!PageUptodate(page)) {
f188591e 4626 ClearPageError(page);
a86c12c7 4627 err = __extent_read_full_page(tree, page,
f188591e 4628 get_extent, &bio,
c8b97818 4629 mirror_num, &bio_flags);
d397712b 4630 if (err)
d1310b2e 4631 ret = err;
d1310b2e
CM
4632 } else {
4633 unlock_page(page);
4634 }
4635 }
4636
355808c2
JM
4637 if (bio) {
4638 err = submit_one_bio(READ, bio, mirror_num, bio_flags);
79787eaa
JM
4639 if (err)
4640 return err;
355808c2 4641 }
a86c12c7 4642
bb82ab88 4643 if (ret || wait != WAIT_COMPLETE)
d1310b2e 4644 return ret;
d397712b 4645
d1310b2e
CM
4646 for (i = start_i; i < num_pages; i++) {
4647 page = extent_buffer_page(eb, i);
4648 wait_on_page_locked(page);
d397712b 4649 if (!PageUptodate(page))
d1310b2e 4650 ret = -EIO;
d1310b2e 4651 }
d397712b 4652
d1310b2e 4653 return ret;
ce9adaa5
CM
4654
4655unlock_exit:
4656 i = start_i;
d397712b 4657 while (locked_pages > 0) {
ce9adaa5
CM
4658 page = extent_buffer_page(eb, i);
4659 i++;
4660 unlock_page(page);
4661 locked_pages--;
4662 }
4663 return ret;
d1310b2e 4664}
d1310b2e
CM
4665
4666void read_extent_buffer(struct extent_buffer *eb, void *dstv,
4667 unsigned long start,
4668 unsigned long len)
4669{
4670 size_t cur;
4671 size_t offset;
4672 struct page *page;
4673 char *kaddr;
4674 char *dst = (char *)dstv;
4675 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4676 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
d1310b2e
CM
4677
4678 WARN_ON(start > eb->len);
4679 WARN_ON(start + len > eb->start + eb->len);
4680
4681 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4682
d397712b 4683 while (len > 0) {
d1310b2e 4684 page = extent_buffer_page(eb, i);
d1310b2e
CM
4685
4686 cur = min(len, (PAGE_CACHE_SIZE - offset));
a6591715 4687 kaddr = page_address(page);
d1310b2e 4688 memcpy(dst, kaddr + offset, cur);
d1310b2e
CM
4689
4690 dst += cur;
4691 len -= cur;
4692 offset = 0;
4693 i++;
4694 }
4695}
d1310b2e
CM
4696
4697int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
a6591715 4698 unsigned long min_len, char **map,
d1310b2e 4699 unsigned long *map_start,
a6591715 4700 unsigned long *map_len)
d1310b2e
CM
4701{
4702 size_t offset = start & (PAGE_CACHE_SIZE - 1);
4703 char *kaddr;
4704 struct page *p;
4705 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4706 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4707 unsigned long end_i = (start_offset + start + min_len - 1) >>
4708 PAGE_CACHE_SHIFT;
4709
4710 if (i != end_i)
4711 return -EINVAL;
4712
4713 if (i == 0) {
4714 offset = start_offset;
4715 *map_start = 0;
4716 } else {
4717 offset = 0;
4718 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
4719 }
d397712b 4720
d1310b2e 4721 if (start + min_len > eb->len) {
31b1a2bd 4722 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
d397712b
CM
4723 "wanted %lu %lu\n", (unsigned long long)eb->start,
4724 eb->len, start, min_len);
85026533 4725 return -EINVAL;
d1310b2e
CM
4726 }
4727
4728 p = extent_buffer_page(eb, i);
a6591715 4729 kaddr = page_address(p);
d1310b2e
CM
4730 *map = kaddr + offset;
4731 *map_len = PAGE_CACHE_SIZE - offset;
4732 return 0;
4733}
d1310b2e 4734
d1310b2e
CM
4735int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
4736 unsigned long start,
4737 unsigned long len)
4738{
4739 size_t cur;
4740 size_t offset;
4741 struct page *page;
4742 char *kaddr;
4743 char *ptr = (char *)ptrv;
4744 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4745 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4746 int ret = 0;
4747
4748 WARN_ON(start > eb->len);
4749 WARN_ON(start + len > eb->start + eb->len);
4750
4751 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4752
d397712b 4753 while (len > 0) {
d1310b2e 4754 page = extent_buffer_page(eb, i);
d1310b2e
CM
4755
4756 cur = min(len, (PAGE_CACHE_SIZE - offset));
4757
a6591715 4758 kaddr = page_address(page);
d1310b2e 4759 ret = memcmp(ptr, kaddr + offset, cur);
d1310b2e
CM
4760 if (ret)
4761 break;
4762
4763 ptr += cur;
4764 len -= cur;
4765 offset = 0;
4766 i++;
4767 }
4768 return ret;
4769}
d1310b2e
CM
4770
4771void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
4772 unsigned long start, unsigned long len)
4773{
4774 size_t cur;
4775 size_t offset;
4776 struct page *page;
4777 char *kaddr;
4778 char *src = (char *)srcv;
4779 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4780 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4781
4782 WARN_ON(start > eb->len);
4783 WARN_ON(start + len > eb->start + eb->len);
4784
4785 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4786
d397712b 4787 while (len > 0) {
d1310b2e
CM
4788 page = extent_buffer_page(eb, i);
4789 WARN_ON(!PageUptodate(page));
4790
4791 cur = min(len, PAGE_CACHE_SIZE - offset);
a6591715 4792 kaddr = page_address(page);
d1310b2e 4793 memcpy(kaddr + offset, src, cur);
d1310b2e
CM
4794
4795 src += cur;
4796 len -= cur;
4797 offset = 0;
4798 i++;
4799 }
4800}
d1310b2e
CM
4801
4802void memset_extent_buffer(struct extent_buffer *eb, char c,
4803 unsigned long start, unsigned long len)
4804{
4805 size_t cur;
4806 size_t offset;
4807 struct page *page;
4808 char *kaddr;
4809 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4810 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4811
4812 WARN_ON(start > eb->len);
4813 WARN_ON(start + len > eb->start + eb->len);
4814
4815 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4816
d397712b 4817 while (len > 0) {
d1310b2e
CM
4818 page = extent_buffer_page(eb, i);
4819 WARN_ON(!PageUptodate(page));
4820
4821 cur = min(len, PAGE_CACHE_SIZE - offset);
a6591715 4822 kaddr = page_address(page);
d1310b2e 4823 memset(kaddr + offset, c, cur);
d1310b2e
CM
4824
4825 len -= cur;
4826 offset = 0;
4827 i++;
4828 }
4829}
d1310b2e
CM
4830
4831void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
4832 unsigned long dst_offset, unsigned long src_offset,
4833 unsigned long len)
4834{
4835 u64 dst_len = dst->len;
4836 size_t cur;
4837 size_t offset;
4838 struct page *page;
4839 char *kaddr;
4840 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4841 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4842
4843 WARN_ON(src->len != dst_len);
4844
4845 offset = (start_offset + dst_offset) &
4846 ((unsigned long)PAGE_CACHE_SIZE - 1);
4847
d397712b 4848 while (len > 0) {
d1310b2e
CM
4849 page = extent_buffer_page(dst, i);
4850 WARN_ON(!PageUptodate(page));
4851
4852 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
4853
a6591715 4854 kaddr = page_address(page);
d1310b2e 4855 read_extent_buffer(src, kaddr + offset, src_offset, cur);
d1310b2e
CM
4856
4857 src_offset += cur;
4858 len -= cur;
4859 offset = 0;
4860 i++;
4861 }
4862}
d1310b2e
CM
4863
4864static void move_pages(struct page *dst_page, struct page *src_page,
4865 unsigned long dst_off, unsigned long src_off,
4866 unsigned long len)
4867{
a6591715 4868 char *dst_kaddr = page_address(dst_page);
d1310b2e
CM
4869 if (dst_page == src_page) {
4870 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
4871 } else {
a6591715 4872 char *src_kaddr = page_address(src_page);
d1310b2e
CM
4873 char *p = dst_kaddr + dst_off + len;
4874 char *s = src_kaddr + src_off + len;
4875
4876 while (len--)
4877 *--p = *--s;
d1310b2e 4878 }
d1310b2e
CM
4879}
4880
3387206f
ST
4881static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4882{
4883 unsigned long distance = (src > dst) ? src - dst : dst - src;
4884 return distance < len;
4885}
4886
d1310b2e
CM
4887static void copy_pages(struct page *dst_page, struct page *src_page,
4888 unsigned long dst_off, unsigned long src_off,
4889 unsigned long len)
4890{
a6591715 4891 char *dst_kaddr = page_address(dst_page);
d1310b2e 4892 char *src_kaddr;
727011e0 4893 int must_memmove = 0;
d1310b2e 4894
3387206f 4895 if (dst_page != src_page) {
a6591715 4896 src_kaddr = page_address(src_page);
3387206f 4897 } else {
d1310b2e 4898 src_kaddr = dst_kaddr;
727011e0
CM
4899 if (areas_overlap(src_off, dst_off, len))
4900 must_memmove = 1;
3387206f 4901 }
d1310b2e 4902
727011e0
CM
4903 if (must_memmove)
4904 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
4905 else
4906 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
d1310b2e
CM
4907}
4908
4909void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
4910 unsigned long src_offset, unsigned long len)
4911{
4912 size_t cur;
4913 size_t dst_off_in_page;
4914 size_t src_off_in_page;
4915 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4916 unsigned long dst_i;
4917 unsigned long src_i;
4918
4919 if (src_offset + len > dst->len) {
d397712b
CM
4920 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
4921 "len %lu dst len %lu\n", src_offset, len, dst->len);
d1310b2e
CM
4922 BUG_ON(1);
4923 }
4924 if (dst_offset + len > dst->len) {
d397712b
CM
4925 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
4926 "len %lu dst len %lu\n", dst_offset, len, dst->len);
d1310b2e
CM
4927 BUG_ON(1);
4928 }
4929
d397712b 4930 while (len > 0) {
d1310b2e
CM
4931 dst_off_in_page = (start_offset + dst_offset) &
4932 ((unsigned long)PAGE_CACHE_SIZE - 1);
4933 src_off_in_page = (start_offset + src_offset) &
4934 ((unsigned long)PAGE_CACHE_SIZE - 1);
4935
4936 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4937 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
4938
4939 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
4940 src_off_in_page));
4941 cur = min_t(unsigned long, cur,
4942 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
4943
4944 copy_pages(extent_buffer_page(dst, dst_i),
4945 extent_buffer_page(dst, src_i),
4946 dst_off_in_page, src_off_in_page, cur);
4947
4948 src_offset += cur;
4949 dst_offset += cur;
4950 len -= cur;
4951 }
4952}
d1310b2e
CM
4953
4954void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
4955 unsigned long src_offset, unsigned long len)
4956{
4957 size_t cur;
4958 size_t dst_off_in_page;
4959 size_t src_off_in_page;
4960 unsigned long dst_end = dst_offset + len - 1;
4961 unsigned long src_end = src_offset + len - 1;
4962 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4963 unsigned long dst_i;
4964 unsigned long src_i;
4965
4966 if (src_offset + len > dst->len) {
d397712b
CM
4967 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
4968 "len %lu len %lu\n", src_offset, len, dst->len);
d1310b2e
CM
4969 BUG_ON(1);
4970 }
4971 if (dst_offset + len > dst->len) {
d397712b
CM
4972 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
4973 "len %lu len %lu\n", dst_offset, len, dst->len);
d1310b2e
CM
4974 BUG_ON(1);
4975 }
727011e0 4976 if (dst_offset < src_offset) {
d1310b2e
CM
4977 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4978 return;
4979 }
d397712b 4980 while (len > 0) {
d1310b2e
CM
4981 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
4982 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
4983
4984 dst_off_in_page = (start_offset + dst_end) &
4985 ((unsigned long)PAGE_CACHE_SIZE - 1);
4986 src_off_in_page = (start_offset + src_end) &
4987 ((unsigned long)PAGE_CACHE_SIZE - 1);
4988
4989 cur = min_t(unsigned long, len, src_off_in_page + 1);
4990 cur = min(cur, dst_off_in_page + 1);
4991 move_pages(extent_buffer_page(dst, dst_i),
4992 extent_buffer_page(dst, src_i),
4993 dst_off_in_page - cur + 1,
4994 src_off_in_page - cur + 1, cur);
4995
4996 dst_end -= cur;
4997 src_end -= cur;
4998 len -= cur;
4999 }
5000}
6af118ce 5001
3083ee2e 5002int try_release_extent_buffer(struct page *page, gfp_t mask)
19fe0a8b 5003{
6af118ce 5004 struct extent_buffer *eb;
6af118ce 5005
3083ee2e
JB
5006 /*
5007 * We need to make sure noboody is attaching this page to an eb right
5008 * now.
5009 */
5010 spin_lock(&page->mapping->private_lock);
5011 if (!PagePrivate(page)) {
5012 spin_unlock(&page->mapping->private_lock);
4f2de97a 5013 return 1;
45f49bce 5014 }
6af118ce 5015
3083ee2e
JB
5016 eb = (struct extent_buffer *)page->private;
5017 BUG_ON(!eb);
19fe0a8b
MX
5018
5019 /*
3083ee2e
JB
5020 * This is a little awful but should be ok, we need to make sure that
5021 * the eb doesn't disappear out from under us while we're looking at
5022 * this page.
19fe0a8b 5023 */
3083ee2e 5024 spin_lock(&eb->refs_lock);
0b32f4bb 5025 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
3083ee2e
JB
5026 spin_unlock(&eb->refs_lock);
5027 spin_unlock(&page->mapping->private_lock);
5028 return 0;
b9473439 5029 }
3083ee2e 5030 spin_unlock(&page->mapping->private_lock);
897ca6e9 5031
3083ee2e
JB
5032 if ((mask & GFP_NOFS) == GFP_NOFS)
5033 mask = GFP_NOFS;
19fe0a8b 5034
19fe0a8b 5035 /*
3083ee2e
JB
5036 * If tree ref isn't set then we know the ref on this eb is a real ref,
5037 * so just return, this page will likely be freed soon anyway.
19fe0a8b 5038 */
3083ee2e
JB
5039 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5040 spin_unlock(&eb->refs_lock);
5041 return 0;
b9473439 5042 }
19fe0a8b 5043
e64860aa 5044 return release_extent_buffer(eb, mask);
6af118ce 5045}