2 * Copyright (C) 2007 Oracle. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/buffer_head.h>
22 #include <linux/file.h>
24 #include <linux/pagemap.h>
25 #include <linux/highmem.h>
26 #include <linux/time.h>
27 #include <linux/init.h>
28 #include <linux/string.h>
29 #include <linux/backing-dev.h>
30 #include <linux/mpage.h>
31 #include <linux/swap.h>
32 #include <linux/writeback.h>
33 #include <linux/compat.h>
34 #include <linux/bit_spinlock.h>
35 #include <linux/xattr.h>
36 #include <linux/posix_acl.h>
37 #include <linux/falloc.h>
38 #include <linux/slab.h>
39 #include <linux/ratelimit.h>
40 #include <linux/mount.h>
41 #include <linux/btrfs.h>
42 #include <linux/blkdev.h>
43 #include <linux/posix_acl_xattr.h>
44 #include <linux/uio.h>
47 #include "transaction.h"
48 #include "btrfs_inode.h"
49 #include "print-tree.h"
50 #include "ordered-data.h"
54 #include "compression.h"
56 #include "free-space-cache.h"
57 #include "inode-map.h"
64 struct btrfs_iget_args
{
65 struct btrfs_key
*location
;
66 struct btrfs_root
*root
;
69 struct btrfs_dio_data
{
70 u64 outstanding_extents
;
72 u64 unsubmitted_oe_range_start
;
73 u64 unsubmitted_oe_range_end
;
77 static const struct inode_operations btrfs_dir_inode_operations
;
78 static const struct inode_operations btrfs_symlink_inode_operations
;
79 static const struct inode_operations btrfs_dir_ro_inode_operations
;
80 static const struct inode_operations btrfs_special_inode_operations
;
81 static const struct inode_operations btrfs_file_inode_operations
;
82 static const struct address_space_operations btrfs_aops
;
83 static const struct address_space_operations btrfs_symlink_aops
;
84 static const struct file_operations btrfs_dir_file_operations
;
85 static const struct extent_io_ops btrfs_extent_io_ops
;
87 static struct kmem_cache
*btrfs_inode_cachep
;
88 struct kmem_cache
*btrfs_trans_handle_cachep
;
89 struct kmem_cache
*btrfs_transaction_cachep
;
90 struct kmem_cache
*btrfs_path_cachep
;
91 struct kmem_cache
*btrfs_free_space_cachep
;
94 static const unsigned char btrfs_type_by_mode
[S_IFMT
>> S_SHIFT
] = {
95 [S_IFREG
>> S_SHIFT
] = BTRFS_FT_REG_FILE
,
96 [S_IFDIR
>> S_SHIFT
] = BTRFS_FT_DIR
,
97 [S_IFCHR
>> S_SHIFT
] = BTRFS_FT_CHRDEV
,
98 [S_IFBLK
>> S_SHIFT
] = BTRFS_FT_BLKDEV
,
99 [S_IFIFO
>> S_SHIFT
] = BTRFS_FT_FIFO
,
100 [S_IFSOCK
>> S_SHIFT
] = BTRFS_FT_SOCK
,
101 [S_IFLNK
>> S_SHIFT
] = BTRFS_FT_SYMLINK
,
104 static int btrfs_setsize(struct inode
*inode
, struct iattr
*attr
);
105 static int btrfs_truncate(struct inode
*inode
);
106 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent
*ordered_extent
);
107 static noinline
int cow_file_range(struct inode
*inode
,
108 struct page
*locked_page
,
109 u64 start
, u64 end
, u64 delalloc_end
,
110 int *page_started
, unsigned long *nr_written
,
111 int unlock
, struct btrfs_dedupe_hash
*hash
);
112 static struct extent_map
*create_pinned_em(struct inode
*inode
, u64 start
,
113 u64 len
, u64 orig_start
,
114 u64 block_start
, u64 block_len
,
115 u64 orig_block_len
, u64 ram_bytes
,
118 static int btrfs_dirty_inode(struct inode
*inode
);
120 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
121 void btrfs_test_inode_set_ops(struct inode
*inode
)
123 BTRFS_I(inode
)->io_tree
.ops
= &btrfs_extent_io_ops
;
127 static int btrfs_init_inode_security(struct btrfs_trans_handle
*trans
,
128 struct inode
*inode
, struct inode
*dir
,
129 const struct qstr
*qstr
)
133 err
= btrfs_init_acl(trans
, inode
, dir
);
135 err
= btrfs_xattr_security_init(trans
, inode
, dir
, qstr
);
140 * this does all the hard work for inserting an inline extent into
141 * the btree. The caller should have done a btrfs_drop_extents so that
142 * no overlapping inline items exist in the btree
144 static int insert_inline_extent(struct btrfs_trans_handle
*trans
,
145 struct btrfs_path
*path
, int extent_inserted
,
146 struct btrfs_root
*root
, struct inode
*inode
,
147 u64 start
, size_t size
, size_t compressed_size
,
149 struct page
**compressed_pages
)
151 struct extent_buffer
*leaf
;
152 struct page
*page
= NULL
;
155 struct btrfs_file_extent_item
*ei
;
158 size_t cur_size
= size
;
159 unsigned long offset
;
161 if (compressed_size
&& compressed_pages
)
162 cur_size
= compressed_size
;
164 inode_add_bytes(inode
, size
);
166 if (!extent_inserted
) {
167 struct btrfs_key key
;
170 key
.objectid
= btrfs_ino(BTRFS_I(inode
));
172 key
.type
= BTRFS_EXTENT_DATA_KEY
;
174 datasize
= btrfs_file_extent_calc_inline_size(cur_size
);
175 path
->leave_spinning
= 1;
176 ret
= btrfs_insert_empty_item(trans
, root
, path
, &key
,
183 leaf
= path
->nodes
[0];
184 ei
= btrfs_item_ptr(leaf
, path
->slots
[0],
185 struct btrfs_file_extent_item
);
186 btrfs_set_file_extent_generation(leaf
, ei
, trans
->transid
);
187 btrfs_set_file_extent_type(leaf
, ei
, BTRFS_FILE_EXTENT_INLINE
);
188 btrfs_set_file_extent_encryption(leaf
, ei
, 0);
189 btrfs_set_file_extent_other_encoding(leaf
, ei
, 0);
190 btrfs_set_file_extent_ram_bytes(leaf
, ei
, size
);
191 ptr
= btrfs_file_extent_inline_start(ei
);
193 if (compress_type
!= BTRFS_COMPRESS_NONE
) {
196 while (compressed_size
> 0) {
197 cpage
= compressed_pages
[i
];
198 cur_size
= min_t(unsigned long, compressed_size
,
201 kaddr
= kmap_atomic(cpage
);
202 write_extent_buffer(leaf
, kaddr
, ptr
, cur_size
);
203 kunmap_atomic(kaddr
);
207 compressed_size
-= cur_size
;
209 btrfs_set_file_extent_compression(leaf
, ei
,
212 page
= find_get_page(inode
->i_mapping
,
213 start
>> PAGE_SHIFT
);
214 btrfs_set_file_extent_compression(leaf
, ei
, 0);
215 kaddr
= kmap_atomic(page
);
216 offset
= start
& (PAGE_SIZE
- 1);
217 write_extent_buffer(leaf
, kaddr
+ offset
, ptr
, size
);
218 kunmap_atomic(kaddr
);
221 btrfs_mark_buffer_dirty(leaf
);
222 btrfs_release_path(path
);
225 * we're an inline extent, so nobody can
226 * extend the file past i_size without locking
227 * a page we already have locked.
229 * We must do any isize and inode updates
230 * before we unlock the pages. Otherwise we
231 * could end up racing with unlink.
233 BTRFS_I(inode
)->disk_i_size
= inode
->i_size
;
234 ret
= btrfs_update_inode(trans
, root
, inode
);
243 * conditionally insert an inline extent into the file. This
244 * does the checks required to make sure the data is small enough
245 * to fit as an inline extent.
247 static noinline
int cow_file_range_inline(struct btrfs_root
*root
,
248 struct inode
*inode
, u64 start
,
249 u64 end
, size_t compressed_size
,
251 struct page
**compressed_pages
)
253 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
254 struct btrfs_trans_handle
*trans
;
255 u64 isize
= i_size_read(inode
);
256 u64 actual_end
= min(end
+ 1, isize
);
257 u64 inline_len
= actual_end
- start
;
258 u64 aligned_end
= ALIGN(end
, fs_info
->sectorsize
);
259 u64 data_len
= inline_len
;
261 struct btrfs_path
*path
;
262 int extent_inserted
= 0;
263 u32 extent_item_size
;
266 data_len
= compressed_size
;
269 actual_end
> fs_info
->sectorsize
||
270 data_len
> BTRFS_MAX_INLINE_DATA_SIZE(fs_info
) ||
272 (actual_end
& (fs_info
->sectorsize
- 1)) == 0) ||
274 data_len
> fs_info
->max_inline
) {
278 path
= btrfs_alloc_path();
282 trans
= btrfs_join_transaction(root
);
284 btrfs_free_path(path
);
285 return PTR_ERR(trans
);
287 trans
->block_rsv
= &fs_info
->delalloc_block_rsv
;
289 if (compressed_size
&& compressed_pages
)
290 extent_item_size
= btrfs_file_extent_calc_inline_size(
293 extent_item_size
= btrfs_file_extent_calc_inline_size(
296 ret
= __btrfs_drop_extents(trans
, root
, inode
, path
,
297 start
, aligned_end
, NULL
,
298 1, 1, extent_item_size
, &extent_inserted
);
300 btrfs_abort_transaction(trans
, ret
);
304 if (isize
> actual_end
)
305 inline_len
= min_t(u64
, isize
, actual_end
);
306 ret
= insert_inline_extent(trans
, path
, extent_inserted
,
308 inline_len
, compressed_size
,
309 compress_type
, compressed_pages
);
310 if (ret
&& ret
!= -ENOSPC
) {
311 btrfs_abort_transaction(trans
, ret
);
313 } else if (ret
== -ENOSPC
) {
318 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC
, &BTRFS_I(inode
)->runtime_flags
);
319 btrfs_delalloc_release_metadata(inode
, end
+ 1 - start
);
320 btrfs_drop_extent_cache(inode
, start
, aligned_end
- 1, 0);
323 * Don't forget to free the reserved space, as for inlined extent
324 * it won't count as data extent, free them directly here.
325 * And at reserve time, it's always aligned to page size, so
326 * just free one page here.
328 btrfs_qgroup_free_data(inode
, 0, PAGE_SIZE
);
329 btrfs_free_path(path
);
330 btrfs_end_transaction(trans
);
334 struct async_extent
{
339 unsigned long nr_pages
;
341 struct list_head list
;
346 struct btrfs_root
*root
;
347 struct page
*locked_page
;
350 struct list_head extents
;
351 struct btrfs_work work
;
354 static noinline
int add_async_extent(struct async_cow
*cow
,
355 u64 start
, u64 ram_size
,
358 unsigned long nr_pages
,
361 struct async_extent
*async_extent
;
363 async_extent
= kmalloc(sizeof(*async_extent
), GFP_NOFS
);
364 BUG_ON(!async_extent
); /* -ENOMEM */
365 async_extent
->start
= start
;
366 async_extent
->ram_size
= ram_size
;
367 async_extent
->compressed_size
= compressed_size
;
368 async_extent
->pages
= pages
;
369 async_extent
->nr_pages
= nr_pages
;
370 async_extent
->compress_type
= compress_type
;
371 list_add_tail(&async_extent
->list
, &cow
->extents
);
375 static inline int inode_need_compress(struct inode
*inode
)
377 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
380 if (btrfs_test_opt(fs_info
, FORCE_COMPRESS
))
382 /* bad compression ratios */
383 if (BTRFS_I(inode
)->flags
& BTRFS_INODE_NOCOMPRESS
)
385 if (btrfs_test_opt(fs_info
, COMPRESS
) ||
386 BTRFS_I(inode
)->flags
& BTRFS_INODE_COMPRESS
||
387 BTRFS_I(inode
)->force_compress
)
392 static inline void inode_should_defrag(struct inode
*inode
,
393 u64 start
, u64 end
, u64 num_bytes
, u64 small_write
)
395 /* If this is a small write inside eof, kick off a defrag */
396 if (num_bytes
< small_write
&&
397 (start
> 0 || end
+ 1 < BTRFS_I(inode
)->disk_i_size
))
398 btrfs_add_inode_defrag(NULL
, inode
);
402 * we create compressed extents in two phases. The first
403 * phase compresses a range of pages that have already been
404 * locked (both pages and state bits are locked).
406 * This is done inside an ordered work queue, and the compression
407 * is spread across many cpus. The actual IO submission is step
408 * two, and the ordered work queue takes care of making sure that
409 * happens in the same order things were put onto the queue by
410 * writepages and friends.
412 * If this code finds it can't get good compression, it puts an
413 * entry onto the work queue to write the uncompressed bytes. This
414 * makes sure that both compressed inodes and uncompressed inodes
415 * are written in the same order that the flusher thread sent them
418 static noinline
void compress_file_range(struct inode
*inode
,
419 struct page
*locked_page
,
421 struct async_cow
*async_cow
,
424 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
425 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
427 u64 blocksize
= fs_info
->sectorsize
;
429 u64 isize
= i_size_read(inode
);
431 struct page
**pages
= NULL
;
432 unsigned long nr_pages
;
433 unsigned long nr_pages_ret
= 0;
434 unsigned long total_compressed
= 0;
435 unsigned long total_in
= 0;
436 unsigned long max_compressed
= SZ_128K
;
437 unsigned long max_uncompressed
= SZ_128K
;
440 int compress_type
= fs_info
->compress_type
;
443 inode_should_defrag(inode
, start
, end
, end
- start
+ 1, SZ_16K
);
445 actual_end
= min_t(u64
, isize
, end
+ 1);
448 nr_pages
= (end
>> PAGE_SHIFT
) - (start
>> PAGE_SHIFT
) + 1;
449 nr_pages
= min_t(unsigned long, nr_pages
, SZ_128K
/ PAGE_SIZE
);
452 * we don't want to send crud past the end of i_size through
453 * compression, that's just a waste of CPU time. So, if the
454 * end of the file is before the start of our current
455 * requested range of bytes, we bail out to the uncompressed
456 * cleanup code that can deal with all of this.
458 * It isn't really the fastest way to fix things, but this is a
459 * very uncommon corner.
461 if (actual_end
<= start
)
462 goto cleanup_and_bail_uncompressed
;
464 total_compressed
= actual_end
- start
;
467 * skip compression for a small file range(<=blocksize) that
468 * isn't an inline extent, since it doesn't save disk space at all.
470 if (total_compressed
<= blocksize
&&
471 (start
> 0 || end
+ 1 < BTRFS_I(inode
)->disk_i_size
))
472 goto cleanup_and_bail_uncompressed
;
474 /* we want to make sure that amount of ram required to uncompress
475 * an extent is reasonable, so we limit the total size in ram
476 * of a compressed extent to 128k. This is a crucial number
477 * because it also controls how easily we can spread reads across
478 * cpus for decompression.
480 * We also want to make sure the amount of IO required to do
481 * a random read is reasonably small, so we limit the size of
482 * a compressed extent to 128k.
484 total_compressed
= min(total_compressed
, max_uncompressed
);
485 num_bytes
= ALIGN(end
- start
+ 1, blocksize
);
486 num_bytes
= max(blocksize
, num_bytes
);
491 * we do compression for mount -o compress and when the
492 * inode has not been flagged as nocompress. This flag can
493 * change at any time if we discover bad compression ratios.
495 if (inode_need_compress(inode
)) {
497 pages
= kcalloc(nr_pages
, sizeof(struct page
*), GFP_NOFS
);
499 /* just bail out to the uncompressed code */
503 if (BTRFS_I(inode
)->force_compress
)
504 compress_type
= BTRFS_I(inode
)->force_compress
;
507 * we need to call clear_page_dirty_for_io on each
508 * page in the range. Otherwise applications with the file
509 * mmap'd can wander in and change the page contents while
510 * we are compressing them.
512 * If the compression fails for any reason, we set the pages
513 * dirty again later on.
515 extent_range_clear_dirty_for_io(inode
, start
, end
);
517 ret
= btrfs_compress_pages(compress_type
,
518 inode
->i_mapping
, start
,
519 total_compressed
, pages
,
520 nr_pages
, &nr_pages_ret
,
526 unsigned long offset
= total_compressed
&
528 struct page
*page
= pages
[nr_pages_ret
- 1];
531 /* zero the tail end of the last page, we might be
532 * sending it down to disk
535 kaddr
= kmap_atomic(page
);
536 memset(kaddr
+ offset
, 0,
538 kunmap_atomic(kaddr
);
545 /* lets try to make an inline extent */
546 if (ret
|| total_in
< (actual_end
- start
)) {
547 /* we didn't compress the entire range, try
548 * to make an uncompressed inline extent.
550 ret
= cow_file_range_inline(root
, inode
, start
, end
,
551 0, BTRFS_COMPRESS_NONE
, NULL
);
553 /* try making a compressed inline extent */
554 ret
= cow_file_range_inline(root
, inode
, start
, end
,
556 compress_type
, pages
);
559 unsigned long clear_flags
= EXTENT_DELALLOC
|
561 unsigned long page_error_op
;
563 clear_flags
|= (ret
< 0) ? EXTENT_DO_ACCOUNTING
: 0;
564 page_error_op
= ret
< 0 ? PAGE_SET_ERROR
: 0;
567 * inline extent creation worked or returned error,
568 * we don't need to create any more async work items.
569 * Unlock and free up our temp pages.
571 extent_clear_unlock_delalloc(inode
, start
, end
, end
,
578 btrfs_free_reserved_data_space_noquota(inode
, start
,
586 * we aren't doing an inline extent round the compressed size
587 * up to a block size boundary so the allocator does sane
590 total_compressed
= ALIGN(total_compressed
, blocksize
);
593 * one last check to make sure the compression is really a
594 * win, compare the page count read with the blocks on disk
596 total_in
= ALIGN(total_in
, PAGE_SIZE
);
597 if (total_compressed
>= total_in
) {
600 num_bytes
= total_in
;
604 * The async work queues will take care of doing actual
605 * allocation on disk for these compressed pages, and
606 * will submit them to the elevator.
608 add_async_extent(async_cow
, start
, num_bytes
,
609 total_compressed
, pages
, nr_pages_ret
,
612 if (start
+ num_bytes
< end
) {
623 * the compression code ran but failed to make things smaller,
624 * free any pages it allocated and our page pointer array
626 for (i
= 0; i
< nr_pages_ret
; i
++) {
627 WARN_ON(pages
[i
]->mapping
);
632 total_compressed
= 0;
635 /* flag the file so we don't compress in the future */
636 if (!btrfs_test_opt(fs_info
, FORCE_COMPRESS
) &&
637 !(BTRFS_I(inode
)->force_compress
)) {
638 BTRFS_I(inode
)->flags
|= BTRFS_INODE_NOCOMPRESS
;
641 cleanup_and_bail_uncompressed
:
643 * No compression, but we still need to write the pages in the file
644 * we've been given so far. redirty the locked page if it corresponds
645 * to our extent and set things up for the async work queue to run
646 * cow_file_range to do the normal delalloc dance.
648 if (page_offset(locked_page
) >= start
&&
649 page_offset(locked_page
) <= end
)
650 __set_page_dirty_nobuffers(locked_page
);
651 /* unlocked later on in the async handlers */
654 extent_range_redirty_for_io(inode
, start
, end
);
655 add_async_extent(async_cow
, start
, end
- start
+ 1, 0, NULL
, 0,
656 BTRFS_COMPRESS_NONE
);
662 for (i
= 0; i
< nr_pages_ret
; i
++) {
663 WARN_ON(pages
[i
]->mapping
);
669 static void free_async_extent_pages(struct async_extent
*async_extent
)
673 if (!async_extent
->pages
)
676 for (i
= 0; i
< async_extent
->nr_pages
; i
++) {
677 WARN_ON(async_extent
->pages
[i
]->mapping
);
678 put_page(async_extent
->pages
[i
]);
680 kfree(async_extent
->pages
);
681 async_extent
->nr_pages
= 0;
682 async_extent
->pages
= NULL
;
686 * phase two of compressed writeback. This is the ordered portion
687 * of the code, which only gets called in the order the work was
688 * queued. We walk all the async extents created by compress_file_range
689 * and send them down to the disk.
691 static noinline
void submit_compressed_extents(struct inode
*inode
,
692 struct async_cow
*async_cow
)
694 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
695 struct async_extent
*async_extent
;
697 struct btrfs_key ins
;
698 struct extent_map
*em
;
699 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
700 struct extent_map_tree
*em_tree
= &BTRFS_I(inode
)->extent_tree
;
701 struct extent_io_tree
*io_tree
;
705 while (!list_empty(&async_cow
->extents
)) {
706 async_extent
= list_entry(async_cow
->extents
.next
,
707 struct async_extent
, list
);
708 list_del(&async_extent
->list
);
710 io_tree
= &BTRFS_I(inode
)->io_tree
;
713 /* did the compression code fall back to uncompressed IO? */
714 if (!async_extent
->pages
) {
715 int page_started
= 0;
716 unsigned long nr_written
= 0;
718 lock_extent(io_tree
, async_extent
->start
,
719 async_extent
->start
+
720 async_extent
->ram_size
- 1);
722 /* allocate blocks */
723 ret
= cow_file_range(inode
, async_cow
->locked_page
,
725 async_extent
->start
+
726 async_extent
->ram_size
- 1,
727 async_extent
->start
+
728 async_extent
->ram_size
- 1,
729 &page_started
, &nr_written
, 0,
735 * if page_started, cow_file_range inserted an
736 * inline extent and took care of all the unlocking
737 * and IO for us. Otherwise, we need to submit
738 * all those pages down to the drive.
740 if (!page_started
&& !ret
)
741 extent_write_locked_range(io_tree
,
742 inode
, async_extent
->start
,
743 async_extent
->start
+
744 async_extent
->ram_size
- 1,
748 unlock_page(async_cow
->locked_page
);
754 lock_extent(io_tree
, async_extent
->start
,
755 async_extent
->start
+ async_extent
->ram_size
- 1);
757 ret
= btrfs_reserve_extent(root
, async_extent
->ram_size
,
758 async_extent
->compressed_size
,
759 async_extent
->compressed_size
,
760 0, alloc_hint
, &ins
, 1, 1);
762 free_async_extent_pages(async_extent
);
764 if (ret
== -ENOSPC
) {
765 unlock_extent(io_tree
, async_extent
->start
,
766 async_extent
->start
+
767 async_extent
->ram_size
- 1);
770 * we need to redirty the pages if we decide to
771 * fallback to uncompressed IO, otherwise we
772 * will not submit these pages down to lower
775 extent_range_redirty_for_io(inode
,
777 async_extent
->start
+
778 async_extent
->ram_size
- 1);
785 * here we're doing allocation and writeback of the
788 btrfs_drop_extent_cache(inode
, async_extent
->start
,
789 async_extent
->start
+
790 async_extent
->ram_size
- 1, 0);
792 em
= alloc_extent_map();
795 goto out_free_reserve
;
797 em
->start
= async_extent
->start
;
798 em
->len
= async_extent
->ram_size
;
799 em
->orig_start
= em
->start
;
800 em
->mod_start
= em
->start
;
801 em
->mod_len
= em
->len
;
803 em
->block_start
= ins
.objectid
;
804 em
->block_len
= ins
.offset
;
805 em
->orig_block_len
= ins
.offset
;
806 em
->ram_bytes
= async_extent
->ram_size
;
807 em
->bdev
= fs_info
->fs_devices
->latest_bdev
;
808 em
->compress_type
= async_extent
->compress_type
;
809 set_bit(EXTENT_FLAG_PINNED
, &em
->flags
);
810 set_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
);
814 write_lock(&em_tree
->lock
);
815 ret
= add_extent_mapping(em_tree
, em
, 1);
816 write_unlock(&em_tree
->lock
);
817 if (ret
!= -EEXIST
) {
821 btrfs_drop_extent_cache(inode
, async_extent
->start
,
822 async_extent
->start
+
823 async_extent
->ram_size
- 1, 0);
827 goto out_free_reserve
;
829 ret
= btrfs_add_ordered_extent_compress(inode
,
832 async_extent
->ram_size
,
834 BTRFS_ORDERED_COMPRESSED
,
835 async_extent
->compress_type
);
837 btrfs_drop_extent_cache(inode
, async_extent
->start
,
838 async_extent
->start
+
839 async_extent
->ram_size
- 1, 0);
840 goto out_free_reserve
;
842 btrfs_dec_block_group_reservations(fs_info
, ins
.objectid
);
845 * clear dirty, set writeback and unlock the pages.
847 extent_clear_unlock_delalloc(inode
, async_extent
->start
,
848 async_extent
->start
+
849 async_extent
->ram_size
- 1,
850 async_extent
->start
+
851 async_extent
->ram_size
- 1,
852 NULL
, EXTENT_LOCKED
| EXTENT_DELALLOC
,
853 PAGE_UNLOCK
| PAGE_CLEAR_DIRTY
|
855 ret
= btrfs_submit_compressed_write(inode
,
857 async_extent
->ram_size
,
859 ins
.offset
, async_extent
->pages
,
860 async_extent
->nr_pages
);
862 struct extent_io_tree
*tree
= &BTRFS_I(inode
)->io_tree
;
863 struct page
*p
= async_extent
->pages
[0];
864 const u64 start
= async_extent
->start
;
865 const u64 end
= start
+ async_extent
->ram_size
- 1;
867 p
->mapping
= inode
->i_mapping
;
868 tree
->ops
->writepage_end_io_hook(p
, start
, end
,
871 extent_clear_unlock_delalloc(inode
, start
, end
, end
,
875 free_async_extent_pages(async_extent
);
877 alloc_hint
= ins
.objectid
+ ins
.offset
;
883 btrfs_dec_block_group_reservations(fs_info
, ins
.objectid
);
884 btrfs_free_reserved_extent(fs_info
, ins
.objectid
, ins
.offset
, 1);
886 extent_clear_unlock_delalloc(inode
, async_extent
->start
,
887 async_extent
->start
+
888 async_extent
->ram_size
- 1,
889 async_extent
->start
+
890 async_extent
->ram_size
- 1,
891 NULL
, EXTENT_LOCKED
| EXTENT_DELALLOC
|
892 EXTENT_DEFRAG
| EXTENT_DO_ACCOUNTING
,
893 PAGE_UNLOCK
| PAGE_CLEAR_DIRTY
|
894 PAGE_SET_WRITEBACK
| PAGE_END_WRITEBACK
|
896 free_async_extent_pages(async_extent
);
901 static u64
get_extent_allocation_hint(struct inode
*inode
, u64 start
,
904 struct extent_map_tree
*em_tree
= &BTRFS_I(inode
)->extent_tree
;
905 struct extent_map
*em
;
908 read_lock(&em_tree
->lock
);
909 em
= search_extent_mapping(em_tree
, start
, num_bytes
);
912 * if block start isn't an actual block number then find the
913 * first block in this inode and use that as a hint. If that
914 * block is also bogus then just don't worry about it.
916 if (em
->block_start
>= EXTENT_MAP_LAST_BYTE
) {
918 em
= search_extent_mapping(em_tree
, 0, 0);
919 if (em
&& em
->block_start
< EXTENT_MAP_LAST_BYTE
)
920 alloc_hint
= em
->block_start
;
924 alloc_hint
= em
->block_start
;
928 read_unlock(&em_tree
->lock
);
934 * when extent_io.c finds a delayed allocation range in the file,
935 * the call backs end up in this code. The basic idea is to
936 * allocate extents on disk for the range, and create ordered data structs
937 * in ram to track those extents.
939 * locked_page is the page that writepage had locked already. We use
940 * it to make sure we don't do extra locks or unlocks.
942 * *page_started is set to one if we unlock locked_page and do everything
943 * required to start IO on it. It may be clean and already done with
946 static noinline
int cow_file_range(struct inode
*inode
,
947 struct page
*locked_page
,
948 u64 start
, u64 end
, u64 delalloc_end
,
949 int *page_started
, unsigned long *nr_written
,
950 int unlock
, struct btrfs_dedupe_hash
*hash
)
952 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
953 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
956 unsigned long ram_size
;
959 u64 blocksize
= fs_info
->sectorsize
;
960 struct btrfs_key ins
;
961 struct extent_map
*em
;
962 struct extent_map_tree
*em_tree
= &BTRFS_I(inode
)->extent_tree
;
965 if (btrfs_is_free_space_inode(inode
)) {
971 num_bytes
= ALIGN(end
- start
+ 1, blocksize
);
972 num_bytes
= max(blocksize
, num_bytes
);
973 disk_num_bytes
= num_bytes
;
975 inode_should_defrag(inode
, start
, end
, num_bytes
, SZ_64K
);
978 /* lets try to make an inline extent */
979 ret
= cow_file_range_inline(root
, inode
, start
, end
, 0,
980 BTRFS_COMPRESS_NONE
, NULL
);
982 extent_clear_unlock_delalloc(inode
, start
, end
,
984 EXTENT_LOCKED
| EXTENT_DELALLOC
|
985 EXTENT_DEFRAG
, PAGE_UNLOCK
|
986 PAGE_CLEAR_DIRTY
| PAGE_SET_WRITEBACK
|
988 btrfs_free_reserved_data_space_noquota(inode
, start
,
990 *nr_written
= *nr_written
+
991 (end
- start
+ PAGE_SIZE
) / PAGE_SIZE
;
994 } else if (ret
< 0) {
999 BUG_ON(disk_num_bytes
>
1000 btrfs_super_total_bytes(fs_info
->super_copy
));
1002 alloc_hint
= get_extent_allocation_hint(inode
, start
, num_bytes
);
1003 btrfs_drop_extent_cache(inode
, start
, start
+ num_bytes
- 1, 0);
1005 while (disk_num_bytes
> 0) {
1008 cur_alloc_size
= disk_num_bytes
;
1009 ret
= btrfs_reserve_extent(root
, cur_alloc_size
, cur_alloc_size
,
1010 fs_info
->sectorsize
, 0, alloc_hint
,
1015 em
= alloc_extent_map();
1021 em
->orig_start
= em
->start
;
1022 ram_size
= ins
.offset
;
1023 em
->len
= ins
.offset
;
1024 em
->mod_start
= em
->start
;
1025 em
->mod_len
= em
->len
;
1027 em
->block_start
= ins
.objectid
;
1028 em
->block_len
= ins
.offset
;
1029 em
->orig_block_len
= ins
.offset
;
1030 em
->ram_bytes
= ram_size
;
1031 em
->bdev
= fs_info
->fs_devices
->latest_bdev
;
1032 set_bit(EXTENT_FLAG_PINNED
, &em
->flags
);
1033 em
->generation
= -1;
1036 write_lock(&em_tree
->lock
);
1037 ret
= add_extent_mapping(em_tree
, em
, 1);
1038 write_unlock(&em_tree
->lock
);
1039 if (ret
!= -EEXIST
) {
1040 free_extent_map(em
);
1043 btrfs_drop_extent_cache(inode
, start
,
1044 start
+ ram_size
- 1, 0);
1049 cur_alloc_size
= ins
.offset
;
1050 ret
= btrfs_add_ordered_extent(inode
, start
, ins
.objectid
,
1051 ram_size
, cur_alloc_size
, 0);
1053 goto out_drop_extent_cache
;
1055 if (root
->root_key
.objectid
==
1056 BTRFS_DATA_RELOC_TREE_OBJECTID
) {
1057 ret
= btrfs_reloc_clone_csums(inode
, start
,
1060 goto out_drop_extent_cache
;
1063 btrfs_dec_block_group_reservations(fs_info
, ins
.objectid
);
1065 if (disk_num_bytes
< cur_alloc_size
)
1068 /* we're not doing compressed IO, don't unlock the first
1069 * page (which the caller expects to stay locked), don't
1070 * clear any dirty bits and don't set any writeback bits
1072 * Do set the Private2 bit so we know this page was properly
1073 * setup for writepage
1075 op
= unlock
? PAGE_UNLOCK
: 0;
1076 op
|= PAGE_SET_PRIVATE2
;
1078 extent_clear_unlock_delalloc(inode
, start
,
1079 start
+ ram_size
- 1,
1080 delalloc_end
, locked_page
,
1081 EXTENT_LOCKED
| EXTENT_DELALLOC
,
1083 disk_num_bytes
-= cur_alloc_size
;
1084 num_bytes
-= cur_alloc_size
;
1085 alloc_hint
= ins
.objectid
+ ins
.offset
;
1086 start
+= cur_alloc_size
;
1091 out_drop_extent_cache
:
1092 btrfs_drop_extent_cache(inode
, start
, start
+ ram_size
- 1, 0);
1094 btrfs_dec_block_group_reservations(fs_info
, ins
.objectid
);
1095 btrfs_free_reserved_extent(fs_info
, ins
.objectid
, ins
.offset
, 1);
1097 extent_clear_unlock_delalloc(inode
, start
, end
, delalloc_end
,
1099 EXTENT_LOCKED
| EXTENT_DO_ACCOUNTING
|
1100 EXTENT_DELALLOC
| EXTENT_DEFRAG
,
1101 PAGE_UNLOCK
| PAGE_CLEAR_DIRTY
|
1102 PAGE_SET_WRITEBACK
| PAGE_END_WRITEBACK
);
1107 * work queue call back to started compression on a file and pages
1109 static noinline
void async_cow_start(struct btrfs_work
*work
)
1111 struct async_cow
*async_cow
;
1113 async_cow
= container_of(work
, struct async_cow
, work
);
1115 compress_file_range(async_cow
->inode
, async_cow
->locked_page
,
1116 async_cow
->start
, async_cow
->end
, async_cow
,
1118 if (num_added
== 0) {
1119 btrfs_add_delayed_iput(async_cow
->inode
);
1120 async_cow
->inode
= NULL
;
1125 * work queue call back to submit previously compressed pages
1127 static noinline
void async_cow_submit(struct btrfs_work
*work
)
1129 struct btrfs_fs_info
*fs_info
;
1130 struct async_cow
*async_cow
;
1131 struct btrfs_root
*root
;
1132 unsigned long nr_pages
;
1134 async_cow
= container_of(work
, struct async_cow
, work
);
1136 root
= async_cow
->root
;
1137 fs_info
= root
->fs_info
;
1138 nr_pages
= (async_cow
->end
- async_cow
->start
+ PAGE_SIZE
) >>
1142 * atomic_sub_return implies a barrier for waitqueue_active
1144 if (atomic_sub_return(nr_pages
, &fs_info
->async_delalloc_pages
) <
1146 waitqueue_active(&fs_info
->async_submit_wait
))
1147 wake_up(&fs_info
->async_submit_wait
);
1149 if (async_cow
->inode
)
1150 submit_compressed_extents(async_cow
->inode
, async_cow
);
1153 static noinline
void async_cow_free(struct btrfs_work
*work
)
1155 struct async_cow
*async_cow
;
1156 async_cow
= container_of(work
, struct async_cow
, work
);
1157 if (async_cow
->inode
)
1158 btrfs_add_delayed_iput(async_cow
->inode
);
1162 static int cow_file_range_async(struct inode
*inode
, struct page
*locked_page
,
1163 u64 start
, u64 end
, int *page_started
,
1164 unsigned long *nr_written
)
1166 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1167 struct async_cow
*async_cow
;
1168 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
1169 unsigned long nr_pages
;
1172 clear_extent_bit(&BTRFS_I(inode
)->io_tree
, start
, end
, EXTENT_LOCKED
,
1173 1, 0, NULL
, GFP_NOFS
);
1174 while (start
< end
) {
1175 async_cow
= kmalloc(sizeof(*async_cow
), GFP_NOFS
);
1176 BUG_ON(!async_cow
); /* -ENOMEM */
1177 async_cow
->inode
= igrab(inode
);
1178 async_cow
->root
= root
;
1179 async_cow
->locked_page
= locked_page
;
1180 async_cow
->start
= start
;
1182 if (BTRFS_I(inode
)->flags
& BTRFS_INODE_NOCOMPRESS
&&
1183 !btrfs_test_opt(fs_info
, FORCE_COMPRESS
))
1186 cur_end
= min(end
, start
+ SZ_512K
- 1);
1188 async_cow
->end
= cur_end
;
1189 INIT_LIST_HEAD(&async_cow
->extents
);
1191 btrfs_init_work(&async_cow
->work
,
1192 btrfs_delalloc_helper
,
1193 async_cow_start
, async_cow_submit
,
1196 nr_pages
= (cur_end
- start
+ PAGE_SIZE
) >>
1198 atomic_add(nr_pages
, &fs_info
->async_delalloc_pages
);
1200 btrfs_queue_work(fs_info
->delalloc_workers
, &async_cow
->work
);
1202 while (atomic_read(&fs_info
->async_submit_draining
) &&
1203 atomic_read(&fs_info
->async_delalloc_pages
)) {
1204 wait_event(fs_info
->async_submit_wait
,
1205 (atomic_read(&fs_info
->async_delalloc_pages
) ==
1209 *nr_written
+= nr_pages
;
1210 start
= cur_end
+ 1;
1216 static noinline
int csum_exist_in_range(struct btrfs_fs_info
*fs_info
,
1217 u64 bytenr
, u64 num_bytes
)
1220 struct btrfs_ordered_sum
*sums
;
1223 ret
= btrfs_lookup_csums_range(fs_info
->csum_root
, bytenr
,
1224 bytenr
+ num_bytes
- 1, &list
, 0);
1225 if (ret
== 0 && list_empty(&list
))
1228 while (!list_empty(&list
)) {
1229 sums
= list_entry(list
.next
, struct btrfs_ordered_sum
, list
);
1230 list_del(&sums
->list
);
1237 * when nowcow writeback call back. This checks for snapshots or COW copies
1238 * of the extents that exist in the file, and COWs the file as required.
1240 * If no cow copies or snapshots exist, we write directly to the existing
1243 static noinline
int run_delalloc_nocow(struct inode
*inode
,
1244 struct page
*locked_page
,
1245 u64 start
, u64 end
, int *page_started
, int force
,
1246 unsigned long *nr_written
)
1248 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1249 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
1250 struct extent_buffer
*leaf
;
1251 struct btrfs_path
*path
;
1252 struct btrfs_file_extent_item
*fi
;
1253 struct btrfs_key found_key
;
1268 u64 ino
= btrfs_ino(BTRFS_I(inode
));
1270 path
= btrfs_alloc_path();
1272 extent_clear_unlock_delalloc(inode
, start
, end
, end
,
1274 EXTENT_LOCKED
| EXTENT_DELALLOC
|
1275 EXTENT_DO_ACCOUNTING
|
1276 EXTENT_DEFRAG
, PAGE_UNLOCK
|
1278 PAGE_SET_WRITEBACK
|
1279 PAGE_END_WRITEBACK
);
1283 nolock
= btrfs_is_free_space_inode(inode
);
1285 cow_start
= (u64
)-1;
1288 ret
= btrfs_lookup_file_extent(NULL
, root
, path
, ino
,
1292 if (ret
> 0 && path
->slots
[0] > 0 && check_prev
) {
1293 leaf
= path
->nodes
[0];
1294 btrfs_item_key_to_cpu(leaf
, &found_key
,
1295 path
->slots
[0] - 1);
1296 if (found_key
.objectid
== ino
&&
1297 found_key
.type
== BTRFS_EXTENT_DATA_KEY
)
1302 leaf
= path
->nodes
[0];
1303 if (path
->slots
[0] >= btrfs_header_nritems(leaf
)) {
1304 ret
= btrfs_next_leaf(root
, path
);
1309 leaf
= path
->nodes
[0];
1315 btrfs_item_key_to_cpu(leaf
, &found_key
, path
->slots
[0]);
1317 if (found_key
.objectid
> ino
)
1319 if (WARN_ON_ONCE(found_key
.objectid
< ino
) ||
1320 found_key
.type
< BTRFS_EXTENT_DATA_KEY
) {
1324 if (found_key
.type
> BTRFS_EXTENT_DATA_KEY
||
1325 found_key
.offset
> end
)
1328 if (found_key
.offset
> cur_offset
) {
1329 extent_end
= found_key
.offset
;
1334 fi
= btrfs_item_ptr(leaf
, path
->slots
[0],
1335 struct btrfs_file_extent_item
);
1336 extent_type
= btrfs_file_extent_type(leaf
, fi
);
1338 ram_bytes
= btrfs_file_extent_ram_bytes(leaf
, fi
);
1339 if (extent_type
== BTRFS_FILE_EXTENT_REG
||
1340 extent_type
== BTRFS_FILE_EXTENT_PREALLOC
) {
1341 disk_bytenr
= btrfs_file_extent_disk_bytenr(leaf
, fi
);
1342 extent_offset
= btrfs_file_extent_offset(leaf
, fi
);
1343 extent_end
= found_key
.offset
+
1344 btrfs_file_extent_num_bytes(leaf
, fi
);
1346 btrfs_file_extent_disk_num_bytes(leaf
, fi
);
1347 if (extent_end
<= start
) {
1351 if (disk_bytenr
== 0)
1353 if (btrfs_file_extent_compression(leaf
, fi
) ||
1354 btrfs_file_extent_encryption(leaf
, fi
) ||
1355 btrfs_file_extent_other_encoding(leaf
, fi
))
1357 if (extent_type
== BTRFS_FILE_EXTENT_REG
&& !force
)
1359 if (btrfs_extent_readonly(fs_info
, disk_bytenr
))
1361 if (btrfs_cross_ref_exist(root
, ino
,
1363 extent_offset
, disk_bytenr
))
1365 disk_bytenr
+= extent_offset
;
1366 disk_bytenr
+= cur_offset
- found_key
.offset
;
1367 num_bytes
= min(end
+ 1, extent_end
) - cur_offset
;
1369 * if there are pending snapshots for this root,
1370 * we fall into common COW way.
1373 err
= btrfs_start_write_no_snapshoting(root
);
1378 * force cow if csum exists in the range.
1379 * this ensure that csum for a given extent are
1380 * either valid or do not exist.
1382 if (csum_exist_in_range(fs_info
, disk_bytenr
,
1385 if (!btrfs_inc_nocow_writers(fs_info
, disk_bytenr
))
1388 } else if (extent_type
== BTRFS_FILE_EXTENT_INLINE
) {
1389 extent_end
= found_key
.offset
+
1390 btrfs_file_extent_inline_len(leaf
,
1391 path
->slots
[0], fi
);
1392 extent_end
= ALIGN(extent_end
,
1393 fs_info
->sectorsize
);
1398 if (extent_end
<= start
) {
1400 if (!nolock
&& nocow
)
1401 btrfs_end_write_no_snapshoting(root
);
1403 btrfs_dec_nocow_writers(fs_info
, disk_bytenr
);
1407 if (cow_start
== (u64
)-1)
1408 cow_start
= cur_offset
;
1409 cur_offset
= extent_end
;
1410 if (cur_offset
> end
)
1416 btrfs_release_path(path
);
1417 if (cow_start
!= (u64
)-1) {
1418 ret
= cow_file_range(inode
, locked_page
,
1419 cow_start
, found_key
.offset
- 1,
1420 end
, page_started
, nr_written
, 1,
1423 if (!nolock
&& nocow
)
1424 btrfs_end_write_no_snapshoting(root
);
1426 btrfs_dec_nocow_writers(fs_info
,
1430 cow_start
= (u64
)-1;
1433 if (extent_type
== BTRFS_FILE_EXTENT_PREALLOC
) {
1434 struct extent_map
*em
;
1435 struct extent_map_tree
*em_tree
;
1436 em_tree
= &BTRFS_I(inode
)->extent_tree
;
1437 em
= alloc_extent_map();
1438 BUG_ON(!em
); /* -ENOMEM */
1439 em
->start
= cur_offset
;
1440 em
->orig_start
= found_key
.offset
- extent_offset
;
1441 em
->len
= num_bytes
;
1442 em
->block_len
= num_bytes
;
1443 em
->block_start
= disk_bytenr
;
1444 em
->orig_block_len
= disk_num_bytes
;
1445 em
->ram_bytes
= ram_bytes
;
1446 em
->bdev
= fs_info
->fs_devices
->latest_bdev
;
1447 em
->mod_start
= em
->start
;
1448 em
->mod_len
= em
->len
;
1449 set_bit(EXTENT_FLAG_PINNED
, &em
->flags
);
1450 set_bit(EXTENT_FLAG_FILLING
, &em
->flags
);
1451 em
->generation
= -1;
1453 write_lock(&em_tree
->lock
);
1454 ret
= add_extent_mapping(em_tree
, em
, 1);
1455 write_unlock(&em_tree
->lock
);
1456 if (ret
!= -EEXIST
) {
1457 free_extent_map(em
);
1460 btrfs_drop_extent_cache(inode
, em
->start
,
1461 em
->start
+ em
->len
- 1, 0);
1463 type
= BTRFS_ORDERED_PREALLOC
;
1465 type
= BTRFS_ORDERED_NOCOW
;
1468 ret
= btrfs_add_ordered_extent(inode
, cur_offset
, disk_bytenr
,
1469 num_bytes
, num_bytes
, type
);
1471 btrfs_dec_nocow_writers(fs_info
, disk_bytenr
);
1472 BUG_ON(ret
); /* -ENOMEM */
1474 if (root
->root_key
.objectid
==
1475 BTRFS_DATA_RELOC_TREE_OBJECTID
) {
1476 ret
= btrfs_reloc_clone_csums(inode
, cur_offset
,
1479 if (!nolock
&& nocow
)
1480 btrfs_end_write_no_snapshoting(root
);
1485 extent_clear_unlock_delalloc(inode
, cur_offset
,
1486 cur_offset
+ num_bytes
- 1, end
,
1487 locked_page
, EXTENT_LOCKED
|
1489 EXTENT_CLEAR_DATA_RESV
,
1490 PAGE_UNLOCK
| PAGE_SET_PRIVATE2
);
1492 if (!nolock
&& nocow
)
1493 btrfs_end_write_no_snapshoting(root
);
1494 cur_offset
= extent_end
;
1495 if (cur_offset
> end
)
1498 btrfs_release_path(path
);
1500 if (cur_offset
<= end
&& cow_start
== (u64
)-1) {
1501 cow_start
= cur_offset
;
1505 if (cow_start
!= (u64
)-1) {
1506 ret
= cow_file_range(inode
, locked_page
, cow_start
, end
, end
,
1507 page_started
, nr_written
, 1, NULL
);
1513 if (ret
&& cur_offset
< end
)
1514 extent_clear_unlock_delalloc(inode
, cur_offset
, end
, end
,
1515 locked_page
, EXTENT_LOCKED
|
1516 EXTENT_DELALLOC
| EXTENT_DEFRAG
|
1517 EXTENT_DO_ACCOUNTING
, PAGE_UNLOCK
|
1519 PAGE_SET_WRITEBACK
|
1520 PAGE_END_WRITEBACK
);
1521 btrfs_free_path(path
);
1525 static inline int need_force_cow(struct inode
*inode
, u64 start
, u64 end
)
1528 if (!(BTRFS_I(inode
)->flags
& BTRFS_INODE_NODATACOW
) &&
1529 !(BTRFS_I(inode
)->flags
& BTRFS_INODE_PREALLOC
))
1533 * @defrag_bytes is a hint value, no spinlock held here,
1534 * if is not zero, it means the file is defragging.
1535 * Force cow if given extent needs to be defragged.
1537 if (BTRFS_I(inode
)->defrag_bytes
&&
1538 test_range_bit(&BTRFS_I(inode
)->io_tree
, start
, end
,
1539 EXTENT_DEFRAG
, 0, NULL
))
1546 * extent_io.c call back to do delayed allocation processing
1548 static int run_delalloc_range(struct inode
*inode
, struct page
*locked_page
,
1549 u64 start
, u64 end
, int *page_started
,
1550 unsigned long *nr_written
)
1553 int force_cow
= need_force_cow(inode
, start
, end
);
1555 if (BTRFS_I(inode
)->flags
& BTRFS_INODE_NODATACOW
&& !force_cow
) {
1556 ret
= run_delalloc_nocow(inode
, locked_page
, start
, end
,
1557 page_started
, 1, nr_written
);
1558 } else if (BTRFS_I(inode
)->flags
& BTRFS_INODE_PREALLOC
&& !force_cow
) {
1559 ret
= run_delalloc_nocow(inode
, locked_page
, start
, end
,
1560 page_started
, 0, nr_written
);
1561 } else if (!inode_need_compress(inode
)) {
1562 ret
= cow_file_range(inode
, locked_page
, start
, end
, end
,
1563 page_started
, nr_written
, 1, NULL
);
1565 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT
,
1566 &BTRFS_I(inode
)->runtime_flags
);
1567 ret
= cow_file_range_async(inode
, locked_page
, start
, end
,
1568 page_started
, nr_written
);
1573 static void btrfs_split_extent_hook(struct inode
*inode
,
1574 struct extent_state
*orig
, u64 split
)
1578 /* not delalloc, ignore it */
1579 if (!(orig
->state
& EXTENT_DELALLOC
))
1582 size
= orig
->end
- orig
->start
+ 1;
1583 if (size
> BTRFS_MAX_EXTENT_SIZE
) {
1588 * See the explanation in btrfs_merge_extent_hook, the same
1589 * applies here, just in reverse.
1591 new_size
= orig
->end
- split
+ 1;
1592 num_extents
= count_max_extents(new_size
);
1593 new_size
= split
- orig
->start
;
1594 num_extents
+= count_max_extents(new_size
);
1595 if (count_max_extents(size
) >= num_extents
)
1599 spin_lock(&BTRFS_I(inode
)->lock
);
1600 BTRFS_I(inode
)->outstanding_extents
++;
1601 spin_unlock(&BTRFS_I(inode
)->lock
);
1605 * extent_io.c merge_extent_hook, used to track merged delayed allocation
1606 * extents so we can keep track of new extents that are just merged onto old
1607 * extents, such as when we are doing sequential writes, so we can properly
1608 * account for the metadata space we'll need.
1610 static void btrfs_merge_extent_hook(struct inode
*inode
,
1611 struct extent_state
*new,
1612 struct extent_state
*other
)
1614 u64 new_size
, old_size
;
1617 /* not delalloc, ignore it */
1618 if (!(other
->state
& EXTENT_DELALLOC
))
1621 if (new->start
> other
->start
)
1622 new_size
= new->end
- other
->start
+ 1;
1624 new_size
= other
->end
- new->start
+ 1;
1626 /* we're not bigger than the max, unreserve the space and go */
1627 if (new_size
<= BTRFS_MAX_EXTENT_SIZE
) {
1628 spin_lock(&BTRFS_I(inode
)->lock
);
1629 BTRFS_I(inode
)->outstanding_extents
--;
1630 spin_unlock(&BTRFS_I(inode
)->lock
);
1635 * We have to add up either side to figure out how many extents were
1636 * accounted for before we merged into one big extent. If the number of
1637 * extents we accounted for is <= the amount we need for the new range
1638 * then we can return, otherwise drop. Think of it like this
1642 * So we've grown the extent by a MAX_SIZE extent, this would mean we
1643 * need 2 outstanding extents, on one side we have 1 and the other side
1644 * we have 1 so they are == and we can return. But in this case
1646 * [MAX_SIZE+4k][MAX_SIZE+4k]
1648 * Each range on their own accounts for 2 extents, but merged together
1649 * they are only 3 extents worth of accounting, so we need to drop in
1652 old_size
= other
->end
- other
->start
+ 1;
1653 num_extents
= count_max_extents(old_size
);
1654 old_size
= new->end
- new->start
+ 1;
1655 num_extents
+= count_max_extents(old_size
);
1656 if (count_max_extents(new_size
) >= num_extents
)
1659 spin_lock(&BTRFS_I(inode
)->lock
);
1660 BTRFS_I(inode
)->outstanding_extents
--;
1661 spin_unlock(&BTRFS_I(inode
)->lock
);
1664 static void btrfs_add_delalloc_inodes(struct btrfs_root
*root
,
1665 struct inode
*inode
)
1667 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1669 spin_lock(&root
->delalloc_lock
);
1670 if (list_empty(&BTRFS_I(inode
)->delalloc_inodes
)) {
1671 list_add_tail(&BTRFS_I(inode
)->delalloc_inodes
,
1672 &root
->delalloc_inodes
);
1673 set_bit(BTRFS_INODE_IN_DELALLOC_LIST
,
1674 &BTRFS_I(inode
)->runtime_flags
);
1675 root
->nr_delalloc_inodes
++;
1676 if (root
->nr_delalloc_inodes
== 1) {
1677 spin_lock(&fs_info
->delalloc_root_lock
);
1678 BUG_ON(!list_empty(&root
->delalloc_root
));
1679 list_add_tail(&root
->delalloc_root
,
1680 &fs_info
->delalloc_roots
);
1681 spin_unlock(&fs_info
->delalloc_root_lock
);
1684 spin_unlock(&root
->delalloc_lock
);
1687 static void btrfs_del_delalloc_inode(struct btrfs_root
*root
,
1688 struct inode
*inode
)
1690 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1692 spin_lock(&root
->delalloc_lock
);
1693 if (!list_empty(&BTRFS_I(inode
)->delalloc_inodes
)) {
1694 list_del_init(&BTRFS_I(inode
)->delalloc_inodes
);
1695 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST
,
1696 &BTRFS_I(inode
)->runtime_flags
);
1697 root
->nr_delalloc_inodes
--;
1698 if (!root
->nr_delalloc_inodes
) {
1699 spin_lock(&fs_info
->delalloc_root_lock
);
1700 BUG_ON(list_empty(&root
->delalloc_root
));
1701 list_del_init(&root
->delalloc_root
);
1702 spin_unlock(&fs_info
->delalloc_root_lock
);
1705 spin_unlock(&root
->delalloc_lock
);
1709 * extent_io.c set_bit_hook, used to track delayed allocation
1710 * bytes in this file, and to maintain the list of inodes that
1711 * have pending delalloc work to be done.
1713 static void btrfs_set_bit_hook(struct inode
*inode
,
1714 struct extent_state
*state
, unsigned *bits
)
1717 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1719 if ((*bits
& EXTENT_DEFRAG
) && !(*bits
& EXTENT_DELALLOC
))
1722 * set_bit and clear bit hooks normally require _irqsave/restore
1723 * but in this case, we are only testing for the DELALLOC
1724 * bit, which is only set or cleared with irqs on
1726 if (!(state
->state
& EXTENT_DELALLOC
) && (*bits
& EXTENT_DELALLOC
)) {
1727 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
1728 u64 len
= state
->end
+ 1 - state
->start
;
1729 bool do_list
= !btrfs_is_free_space_inode(inode
);
1731 if (*bits
& EXTENT_FIRST_DELALLOC
) {
1732 *bits
&= ~EXTENT_FIRST_DELALLOC
;
1734 spin_lock(&BTRFS_I(inode
)->lock
);
1735 BTRFS_I(inode
)->outstanding_extents
++;
1736 spin_unlock(&BTRFS_I(inode
)->lock
);
1739 /* For sanity tests */
1740 if (btrfs_is_testing(fs_info
))
1743 __percpu_counter_add(&fs_info
->delalloc_bytes
, len
,
1744 fs_info
->delalloc_batch
);
1745 spin_lock(&BTRFS_I(inode
)->lock
);
1746 BTRFS_I(inode
)->delalloc_bytes
+= len
;
1747 if (*bits
& EXTENT_DEFRAG
)
1748 BTRFS_I(inode
)->defrag_bytes
+= len
;
1749 if (do_list
&& !test_bit(BTRFS_INODE_IN_DELALLOC_LIST
,
1750 &BTRFS_I(inode
)->runtime_flags
))
1751 btrfs_add_delalloc_inodes(root
, inode
);
1752 spin_unlock(&BTRFS_I(inode
)->lock
);
1757 * extent_io.c clear_bit_hook, see set_bit_hook for why
1759 static void btrfs_clear_bit_hook(struct inode
*inode
,
1760 struct extent_state
*state
,
1763 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1764 u64 len
= state
->end
+ 1 - state
->start
;
1765 u32 num_extents
= count_max_extents(len
);
1767 spin_lock(&BTRFS_I(inode
)->lock
);
1768 if ((state
->state
& EXTENT_DEFRAG
) && (*bits
& EXTENT_DEFRAG
))
1769 BTRFS_I(inode
)->defrag_bytes
-= len
;
1770 spin_unlock(&BTRFS_I(inode
)->lock
);
1773 * set_bit and clear bit hooks normally require _irqsave/restore
1774 * but in this case, we are only testing for the DELALLOC
1775 * bit, which is only set or cleared with irqs on
1777 if ((state
->state
& EXTENT_DELALLOC
) && (*bits
& EXTENT_DELALLOC
)) {
1778 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
1779 bool do_list
= !btrfs_is_free_space_inode(inode
);
1781 if (*bits
& EXTENT_FIRST_DELALLOC
) {
1782 *bits
&= ~EXTENT_FIRST_DELALLOC
;
1783 } else if (!(*bits
& EXTENT_DO_ACCOUNTING
)) {
1784 spin_lock(&BTRFS_I(inode
)->lock
);
1785 BTRFS_I(inode
)->outstanding_extents
-= num_extents
;
1786 spin_unlock(&BTRFS_I(inode
)->lock
);
1790 * We don't reserve metadata space for space cache inodes so we
1791 * don't need to call dellalloc_release_metadata if there is an
1794 if (*bits
& EXTENT_DO_ACCOUNTING
&&
1795 root
!= fs_info
->tree_root
)
1796 btrfs_delalloc_release_metadata(inode
, len
);
1798 /* For sanity tests. */
1799 if (btrfs_is_testing(fs_info
))
1802 if (root
->root_key
.objectid
!= BTRFS_DATA_RELOC_TREE_OBJECTID
1803 && do_list
&& !(state
->state
& EXTENT_NORESERVE
)
1804 && (*bits
& (EXTENT_DO_ACCOUNTING
|
1805 EXTENT_CLEAR_DATA_RESV
)))
1806 btrfs_free_reserved_data_space_noquota(inode
,
1809 __percpu_counter_add(&fs_info
->delalloc_bytes
, -len
,
1810 fs_info
->delalloc_batch
);
1811 spin_lock(&BTRFS_I(inode
)->lock
);
1812 BTRFS_I(inode
)->delalloc_bytes
-= len
;
1813 if (do_list
&& BTRFS_I(inode
)->delalloc_bytes
== 0 &&
1814 test_bit(BTRFS_INODE_IN_DELALLOC_LIST
,
1815 &BTRFS_I(inode
)->runtime_flags
))
1816 btrfs_del_delalloc_inode(root
, inode
);
1817 spin_unlock(&BTRFS_I(inode
)->lock
);
1822 * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
1823 * we don't create bios that span stripes or chunks
1825 * return 1 if page cannot be merged to bio
1826 * return 0 if page can be merged to bio
1827 * return error otherwise
1829 int btrfs_merge_bio_hook(struct page
*page
, unsigned long offset
,
1830 size_t size
, struct bio
*bio
,
1831 unsigned long bio_flags
)
1833 struct inode
*inode
= page
->mapping
->host
;
1834 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1835 u64 logical
= (u64
)bio
->bi_iter
.bi_sector
<< 9;
1840 if (bio_flags
& EXTENT_BIO_COMPRESSED
)
1843 length
= bio
->bi_iter
.bi_size
;
1844 map_length
= length
;
1845 ret
= btrfs_map_block(fs_info
, btrfs_op(bio
), logical
, &map_length
,
1849 if (map_length
< length
+ size
)
1855 * in order to insert checksums into the metadata in large chunks,
1856 * we wait until bio submission time. All the pages in the bio are
1857 * checksummed and sums are attached onto the ordered extent record.
1859 * At IO completion time the cums attached on the ordered extent record
1860 * are inserted into the btree
1862 static int __btrfs_submit_bio_start(struct inode
*inode
, struct bio
*bio
,
1863 int mirror_num
, unsigned long bio_flags
,
1868 ret
= btrfs_csum_one_bio(inode
, bio
, 0, 0);
1869 BUG_ON(ret
); /* -ENOMEM */
1874 * in order to insert checksums into the metadata in large chunks,
1875 * we wait until bio submission time. All the pages in the bio are
1876 * checksummed and sums are attached onto the ordered extent record.
1878 * At IO completion time the cums attached on the ordered extent record
1879 * are inserted into the btree
1881 static int __btrfs_submit_bio_done(struct inode
*inode
, struct bio
*bio
,
1882 int mirror_num
, unsigned long bio_flags
,
1885 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1888 ret
= btrfs_map_bio(fs_info
, bio
, mirror_num
, 1);
1890 bio
->bi_error
= ret
;
1897 * extent_io.c submission hook. This does the right thing for csum calculation
1898 * on write, or reading the csums from the tree before a read
1900 static int btrfs_submit_bio_hook(struct inode
*inode
, struct bio
*bio
,
1901 int mirror_num
, unsigned long bio_flags
,
1904 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
1905 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
1906 enum btrfs_wq_endio_type metadata
= BTRFS_WQ_ENDIO_DATA
;
1909 int async
= !atomic_read(&BTRFS_I(inode
)->sync_writers
);
1911 skip_sum
= BTRFS_I(inode
)->flags
& BTRFS_INODE_NODATASUM
;
1913 if (btrfs_is_free_space_inode(inode
))
1914 metadata
= BTRFS_WQ_ENDIO_FREE_SPACE
;
1916 if (bio_op(bio
) != REQ_OP_WRITE
) {
1917 ret
= btrfs_bio_wq_end_io(fs_info
, bio
, metadata
);
1921 if (bio_flags
& EXTENT_BIO_COMPRESSED
) {
1922 ret
= btrfs_submit_compressed_read(inode
, bio
,
1926 } else if (!skip_sum
) {
1927 ret
= btrfs_lookup_bio_sums(inode
, bio
, NULL
);
1932 } else if (async
&& !skip_sum
) {
1933 /* csum items have already been cloned */
1934 if (root
->root_key
.objectid
== BTRFS_DATA_RELOC_TREE_OBJECTID
)
1936 /* we're doing a write, do the async checksumming */
1937 ret
= btrfs_wq_submit_bio(fs_info
, inode
, bio
, mirror_num
,
1938 bio_flags
, bio_offset
,
1939 __btrfs_submit_bio_start
,
1940 __btrfs_submit_bio_done
);
1942 } else if (!skip_sum
) {
1943 ret
= btrfs_csum_one_bio(inode
, bio
, 0, 0);
1949 ret
= btrfs_map_bio(fs_info
, bio
, mirror_num
, 0);
1953 bio
->bi_error
= ret
;
1960 * given a list of ordered sums record them in the inode. This happens
1961 * at IO completion time based on sums calculated at bio submission time.
1963 static noinline
int add_pending_csums(struct btrfs_trans_handle
*trans
,
1964 struct inode
*inode
, u64 file_offset
,
1965 struct list_head
*list
)
1967 struct btrfs_ordered_sum
*sum
;
1969 list_for_each_entry(sum
, list
, list
) {
1970 trans
->adding_csums
= 1;
1971 btrfs_csum_file_blocks(trans
,
1972 BTRFS_I(inode
)->root
->fs_info
->csum_root
, sum
);
1973 trans
->adding_csums
= 0;
1978 int btrfs_set_extent_delalloc(struct inode
*inode
, u64 start
, u64 end
,
1979 struct extent_state
**cached_state
, int dedupe
)
1981 WARN_ON((end
& (PAGE_SIZE
- 1)) == 0);
1982 return set_extent_delalloc(&BTRFS_I(inode
)->io_tree
, start
, end
,
1986 /* see btrfs_writepage_start_hook for details on why this is required */
1987 struct btrfs_writepage_fixup
{
1989 struct btrfs_work work
;
1992 static void btrfs_writepage_fixup_worker(struct btrfs_work
*work
)
1994 struct btrfs_writepage_fixup
*fixup
;
1995 struct btrfs_ordered_extent
*ordered
;
1996 struct extent_state
*cached_state
= NULL
;
1998 struct inode
*inode
;
2003 fixup
= container_of(work
, struct btrfs_writepage_fixup
, work
);
2007 if (!page
->mapping
|| !PageDirty(page
) || !PageChecked(page
)) {
2008 ClearPageChecked(page
);
2012 inode
= page
->mapping
->host
;
2013 page_start
= page_offset(page
);
2014 page_end
= page_offset(page
) + PAGE_SIZE
- 1;
2016 lock_extent_bits(&BTRFS_I(inode
)->io_tree
, page_start
, page_end
,
2019 /* already ordered? We're done */
2020 if (PagePrivate2(page
))
2023 ordered
= btrfs_lookup_ordered_range(inode
, page_start
,
2026 unlock_extent_cached(&BTRFS_I(inode
)->io_tree
, page_start
,
2027 page_end
, &cached_state
, GFP_NOFS
);
2029 btrfs_start_ordered_extent(inode
, ordered
, 1);
2030 btrfs_put_ordered_extent(ordered
);
2034 ret
= btrfs_delalloc_reserve_space(inode
, page_start
,
2037 mapping_set_error(page
->mapping
, ret
);
2038 end_extent_writepage(page
, ret
, page_start
, page_end
);
2039 ClearPageChecked(page
);
2043 btrfs_set_extent_delalloc(inode
, page_start
, page_end
, &cached_state
,
2045 ClearPageChecked(page
);
2046 set_page_dirty(page
);
2048 unlock_extent_cached(&BTRFS_I(inode
)->io_tree
, page_start
, page_end
,
2049 &cached_state
, GFP_NOFS
);
2057 * There are a few paths in the higher layers of the kernel that directly
2058 * set the page dirty bit without asking the filesystem if it is a
2059 * good idea. This causes problems because we want to make sure COW
2060 * properly happens and the data=ordered rules are followed.
2062 * In our case any range that doesn't have the ORDERED bit set
2063 * hasn't been properly setup for IO. We kick off an async process
2064 * to fix it up. The async helper will wait for ordered extents, set
2065 * the delalloc bit and make it safe to write the page.
2067 static int btrfs_writepage_start_hook(struct page
*page
, u64 start
, u64 end
)
2069 struct inode
*inode
= page
->mapping
->host
;
2070 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2071 struct btrfs_writepage_fixup
*fixup
;
2073 /* this page is properly in the ordered list */
2074 if (TestClearPagePrivate2(page
))
2077 if (PageChecked(page
))
2080 fixup
= kzalloc(sizeof(*fixup
), GFP_NOFS
);
2084 SetPageChecked(page
);
2086 btrfs_init_work(&fixup
->work
, btrfs_fixup_helper
,
2087 btrfs_writepage_fixup_worker
, NULL
, NULL
);
2089 btrfs_queue_work(fs_info
->fixup_workers
, &fixup
->work
);
2093 static int insert_reserved_file_extent(struct btrfs_trans_handle
*trans
,
2094 struct inode
*inode
, u64 file_pos
,
2095 u64 disk_bytenr
, u64 disk_num_bytes
,
2096 u64 num_bytes
, u64 ram_bytes
,
2097 u8 compression
, u8 encryption
,
2098 u16 other_encoding
, int extent_type
)
2100 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
2101 struct btrfs_file_extent_item
*fi
;
2102 struct btrfs_path
*path
;
2103 struct extent_buffer
*leaf
;
2104 struct btrfs_key ins
;
2105 int extent_inserted
= 0;
2108 path
= btrfs_alloc_path();
2113 * we may be replacing one extent in the tree with another.
2114 * The new extent is pinned in the extent map, and we don't want
2115 * to drop it from the cache until it is completely in the btree.
2117 * So, tell btrfs_drop_extents to leave this extent in the cache.
2118 * the caller is expected to unpin it and allow it to be merged
2121 ret
= __btrfs_drop_extents(trans
, root
, inode
, path
, file_pos
,
2122 file_pos
+ num_bytes
, NULL
, 0,
2123 1, sizeof(*fi
), &extent_inserted
);
2127 if (!extent_inserted
) {
2128 ins
.objectid
= btrfs_ino(BTRFS_I(inode
));
2129 ins
.offset
= file_pos
;
2130 ins
.type
= BTRFS_EXTENT_DATA_KEY
;
2132 path
->leave_spinning
= 1;
2133 ret
= btrfs_insert_empty_item(trans
, root
, path
, &ins
,
2138 leaf
= path
->nodes
[0];
2139 fi
= btrfs_item_ptr(leaf
, path
->slots
[0],
2140 struct btrfs_file_extent_item
);
2141 btrfs_set_file_extent_generation(leaf
, fi
, trans
->transid
);
2142 btrfs_set_file_extent_type(leaf
, fi
, extent_type
);
2143 btrfs_set_file_extent_disk_bytenr(leaf
, fi
, disk_bytenr
);
2144 btrfs_set_file_extent_disk_num_bytes(leaf
, fi
, disk_num_bytes
);
2145 btrfs_set_file_extent_offset(leaf
, fi
, 0);
2146 btrfs_set_file_extent_num_bytes(leaf
, fi
, num_bytes
);
2147 btrfs_set_file_extent_ram_bytes(leaf
, fi
, ram_bytes
);
2148 btrfs_set_file_extent_compression(leaf
, fi
, compression
);
2149 btrfs_set_file_extent_encryption(leaf
, fi
, encryption
);
2150 btrfs_set_file_extent_other_encoding(leaf
, fi
, other_encoding
);
2152 btrfs_mark_buffer_dirty(leaf
);
2153 btrfs_release_path(path
);
2155 inode_add_bytes(inode
, num_bytes
);
2157 ins
.objectid
= disk_bytenr
;
2158 ins
.offset
= disk_num_bytes
;
2159 ins
.type
= BTRFS_EXTENT_ITEM_KEY
;
2160 ret
= btrfs_alloc_reserved_file_extent(trans
, root
->root_key
.objectid
,
2161 btrfs_ino(BTRFS_I(inode
)), file_pos
, ram_bytes
, &ins
);
2163 * Release the reserved range from inode dirty range map, as it is
2164 * already moved into delayed_ref_head
2166 btrfs_qgroup_release_data(inode
, file_pos
, ram_bytes
);
2168 btrfs_free_path(path
);
2173 /* snapshot-aware defrag */
2174 struct sa_defrag_extent_backref
{
2175 struct rb_node node
;
2176 struct old_sa_defrag_extent
*old
;
2185 struct old_sa_defrag_extent
{
2186 struct list_head list
;
2187 struct new_sa_defrag_extent
*new;
2196 struct new_sa_defrag_extent
{
2197 struct rb_root root
;
2198 struct list_head head
;
2199 struct btrfs_path
*path
;
2200 struct inode
*inode
;
2208 static int backref_comp(struct sa_defrag_extent_backref
*b1
,
2209 struct sa_defrag_extent_backref
*b2
)
2211 if (b1
->root_id
< b2
->root_id
)
2213 else if (b1
->root_id
> b2
->root_id
)
2216 if (b1
->inum
< b2
->inum
)
2218 else if (b1
->inum
> b2
->inum
)
2221 if (b1
->file_pos
< b2
->file_pos
)
2223 else if (b1
->file_pos
> b2
->file_pos
)
2227 * [------------------------------] ===> (a range of space)
2228 * |<--->| |<---->| =============> (fs/file tree A)
2229 * |<---------------------------->| ===> (fs/file tree B)
2231 * A range of space can refer to two file extents in one tree while
2232 * refer to only one file extent in another tree.
2234 * So we may process a disk offset more than one time(two extents in A)
2235 * and locate at the same extent(one extent in B), then insert two same
2236 * backrefs(both refer to the extent in B).
2241 static void backref_insert(struct rb_root
*root
,
2242 struct sa_defrag_extent_backref
*backref
)
2244 struct rb_node
**p
= &root
->rb_node
;
2245 struct rb_node
*parent
= NULL
;
2246 struct sa_defrag_extent_backref
*entry
;
2251 entry
= rb_entry(parent
, struct sa_defrag_extent_backref
, node
);
2253 ret
= backref_comp(backref
, entry
);
2257 p
= &(*p
)->rb_right
;
2260 rb_link_node(&backref
->node
, parent
, p
);
2261 rb_insert_color(&backref
->node
, root
);
2265 * Note the backref might has changed, and in this case we just return 0.
2267 static noinline
int record_one_backref(u64 inum
, u64 offset
, u64 root_id
,
2270 struct btrfs_file_extent_item
*extent
;
2271 struct old_sa_defrag_extent
*old
= ctx
;
2272 struct new_sa_defrag_extent
*new = old
->new;
2273 struct btrfs_path
*path
= new->path
;
2274 struct btrfs_key key
;
2275 struct btrfs_root
*root
;
2276 struct sa_defrag_extent_backref
*backref
;
2277 struct extent_buffer
*leaf
;
2278 struct inode
*inode
= new->inode
;
2279 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2285 if (BTRFS_I(inode
)->root
->root_key
.objectid
== root_id
&&
2286 inum
== btrfs_ino(BTRFS_I(inode
)))
2289 key
.objectid
= root_id
;
2290 key
.type
= BTRFS_ROOT_ITEM_KEY
;
2291 key
.offset
= (u64
)-1;
2293 root
= btrfs_read_fs_root_no_name(fs_info
, &key
);
2295 if (PTR_ERR(root
) == -ENOENT
)
2298 btrfs_debug(fs_info
, "inum=%llu, offset=%llu, root_id=%llu",
2299 inum
, offset
, root_id
);
2300 return PTR_ERR(root
);
2303 key
.objectid
= inum
;
2304 key
.type
= BTRFS_EXTENT_DATA_KEY
;
2305 if (offset
> (u64
)-1 << 32)
2308 key
.offset
= offset
;
2310 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
2311 if (WARN_ON(ret
< 0))
2318 leaf
= path
->nodes
[0];
2319 slot
= path
->slots
[0];
2321 if (slot
>= btrfs_header_nritems(leaf
)) {
2322 ret
= btrfs_next_leaf(root
, path
);
2325 } else if (ret
> 0) {
2334 btrfs_item_key_to_cpu(leaf
, &key
, slot
);
2336 if (key
.objectid
> inum
)
2339 if (key
.objectid
< inum
|| key
.type
!= BTRFS_EXTENT_DATA_KEY
)
2342 extent
= btrfs_item_ptr(leaf
, slot
,
2343 struct btrfs_file_extent_item
);
2345 if (btrfs_file_extent_disk_bytenr(leaf
, extent
) != old
->bytenr
)
2349 * 'offset' refers to the exact key.offset,
2350 * NOT the 'offset' field in btrfs_extent_data_ref, ie.
2351 * (key.offset - extent_offset).
2353 if (key
.offset
!= offset
)
2356 extent_offset
= btrfs_file_extent_offset(leaf
, extent
);
2357 num_bytes
= btrfs_file_extent_num_bytes(leaf
, extent
);
2359 if (extent_offset
>= old
->extent_offset
+ old
->offset
+
2360 old
->len
|| extent_offset
+ num_bytes
<=
2361 old
->extent_offset
+ old
->offset
)
2366 backref
= kmalloc(sizeof(*backref
), GFP_NOFS
);
2372 backref
->root_id
= root_id
;
2373 backref
->inum
= inum
;
2374 backref
->file_pos
= offset
;
2375 backref
->num_bytes
= num_bytes
;
2376 backref
->extent_offset
= extent_offset
;
2377 backref
->generation
= btrfs_file_extent_generation(leaf
, extent
);
2379 backref_insert(&new->root
, backref
);
2382 btrfs_release_path(path
);
2387 static noinline
bool record_extent_backrefs(struct btrfs_path
*path
,
2388 struct new_sa_defrag_extent
*new)
2390 struct btrfs_fs_info
*fs_info
= btrfs_sb(new->inode
->i_sb
);
2391 struct old_sa_defrag_extent
*old
, *tmp
;
2396 list_for_each_entry_safe(old
, tmp
, &new->head
, list
) {
2397 ret
= iterate_inodes_from_logical(old
->bytenr
+
2398 old
->extent_offset
, fs_info
,
2399 path
, record_one_backref
,
2401 if (ret
< 0 && ret
!= -ENOENT
)
2404 /* no backref to be processed for this extent */
2406 list_del(&old
->list
);
2411 if (list_empty(&new->head
))
2417 static int relink_is_mergable(struct extent_buffer
*leaf
,
2418 struct btrfs_file_extent_item
*fi
,
2419 struct new_sa_defrag_extent
*new)
2421 if (btrfs_file_extent_disk_bytenr(leaf
, fi
) != new->bytenr
)
2424 if (btrfs_file_extent_type(leaf
, fi
) != BTRFS_FILE_EXTENT_REG
)
2427 if (btrfs_file_extent_compression(leaf
, fi
) != new->compress_type
)
2430 if (btrfs_file_extent_encryption(leaf
, fi
) ||
2431 btrfs_file_extent_other_encoding(leaf
, fi
))
2438 * Note the backref might has changed, and in this case we just return 0.
2440 static noinline
int relink_extent_backref(struct btrfs_path
*path
,
2441 struct sa_defrag_extent_backref
*prev
,
2442 struct sa_defrag_extent_backref
*backref
)
2444 struct btrfs_file_extent_item
*extent
;
2445 struct btrfs_file_extent_item
*item
;
2446 struct btrfs_ordered_extent
*ordered
;
2447 struct btrfs_trans_handle
*trans
;
2448 struct btrfs_root
*root
;
2449 struct btrfs_key key
;
2450 struct extent_buffer
*leaf
;
2451 struct old_sa_defrag_extent
*old
= backref
->old
;
2452 struct new_sa_defrag_extent
*new = old
->new;
2453 struct btrfs_fs_info
*fs_info
= btrfs_sb(new->inode
->i_sb
);
2454 struct inode
*inode
;
2455 struct extent_state
*cached
= NULL
;
2464 if (prev
&& prev
->root_id
== backref
->root_id
&&
2465 prev
->inum
== backref
->inum
&&
2466 prev
->file_pos
+ prev
->num_bytes
== backref
->file_pos
)
2469 /* step 1: get root */
2470 key
.objectid
= backref
->root_id
;
2471 key
.type
= BTRFS_ROOT_ITEM_KEY
;
2472 key
.offset
= (u64
)-1;
2474 index
= srcu_read_lock(&fs_info
->subvol_srcu
);
2476 root
= btrfs_read_fs_root_no_name(fs_info
, &key
);
2478 srcu_read_unlock(&fs_info
->subvol_srcu
, index
);
2479 if (PTR_ERR(root
) == -ENOENT
)
2481 return PTR_ERR(root
);
2484 if (btrfs_root_readonly(root
)) {
2485 srcu_read_unlock(&fs_info
->subvol_srcu
, index
);
2489 /* step 2: get inode */
2490 key
.objectid
= backref
->inum
;
2491 key
.type
= BTRFS_INODE_ITEM_KEY
;
2494 inode
= btrfs_iget(fs_info
->sb
, &key
, root
, NULL
);
2495 if (IS_ERR(inode
)) {
2496 srcu_read_unlock(&fs_info
->subvol_srcu
, index
);
2500 srcu_read_unlock(&fs_info
->subvol_srcu
, index
);
2502 /* step 3: relink backref */
2503 lock_start
= backref
->file_pos
;
2504 lock_end
= backref
->file_pos
+ backref
->num_bytes
- 1;
2505 lock_extent_bits(&BTRFS_I(inode
)->io_tree
, lock_start
, lock_end
,
2508 ordered
= btrfs_lookup_first_ordered_extent(inode
, lock_end
);
2510 btrfs_put_ordered_extent(ordered
);
2514 trans
= btrfs_join_transaction(root
);
2515 if (IS_ERR(trans
)) {
2516 ret
= PTR_ERR(trans
);
2520 key
.objectid
= backref
->inum
;
2521 key
.type
= BTRFS_EXTENT_DATA_KEY
;
2522 key
.offset
= backref
->file_pos
;
2524 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
2527 } else if (ret
> 0) {
2532 extent
= btrfs_item_ptr(path
->nodes
[0], path
->slots
[0],
2533 struct btrfs_file_extent_item
);
2535 if (btrfs_file_extent_generation(path
->nodes
[0], extent
) !=
2536 backref
->generation
)
2539 btrfs_release_path(path
);
2541 start
= backref
->file_pos
;
2542 if (backref
->extent_offset
< old
->extent_offset
+ old
->offset
)
2543 start
+= old
->extent_offset
+ old
->offset
-
2544 backref
->extent_offset
;
2546 len
= min(backref
->extent_offset
+ backref
->num_bytes
,
2547 old
->extent_offset
+ old
->offset
+ old
->len
);
2548 len
-= max(backref
->extent_offset
, old
->extent_offset
+ old
->offset
);
2550 ret
= btrfs_drop_extents(trans
, root
, inode
, start
,
2555 key
.objectid
= btrfs_ino(BTRFS_I(inode
));
2556 key
.type
= BTRFS_EXTENT_DATA_KEY
;
2559 path
->leave_spinning
= 1;
2561 struct btrfs_file_extent_item
*fi
;
2563 struct btrfs_key found_key
;
2565 ret
= btrfs_search_slot(trans
, root
, &key
, path
, 0, 1);
2570 leaf
= path
->nodes
[0];
2571 btrfs_item_key_to_cpu(leaf
, &found_key
, path
->slots
[0]);
2573 fi
= btrfs_item_ptr(leaf
, path
->slots
[0],
2574 struct btrfs_file_extent_item
);
2575 extent_len
= btrfs_file_extent_num_bytes(leaf
, fi
);
2577 if (extent_len
+ found_key
.offset
== start
&&
2578 relink_is_mergable(leaf
, fi
, new)) {
2579 btrfs_set_file_extent_num_bytes(leaf
, fi
,
2581 btrfs_mark_buffer_dirty(leaf
);
2582 inode_add_bytes(inode
, len
);
2588 btrfs_release_path(path
);
2593 ret
= btrfs_insert_empty_item(trans
, root
, path
, &key
,
2596 btrfs_abort_transaction(trans
, ret
);
2600 leaf
= path
->nodes
[0];
2601 item
= btrfs_item_ptr(leaf
, path
->slots
[0],
2602 struct btrfs_file_extent_item
);
2603 btrfs_set_file_extent_disk_bytenr(leaf
, item
, new->bytenr
);
2604 btrfs_set_file_extent_disk_num_bytes(leaf
, item
, new->disk_len
);
2605 btrfs_set_file_extent_offset(leaf
, item
, start
- new->file_pos
);
2606 btrfs_set_file_extent_num_bytes(leaf
, item
, len
);
2607 btrfs_set_file_extent_ram_bytes(leaf
, item
, new->len
);
2608 btrfs_set_file_extent_generation(leaf
, item
, trans
->transid
);
2609 btrfs_set_file_extent_type(leaf
, item
, BTRFS_FILE_EXTENT_REG
);
2610 btrfs_set_file_extent_compression(leaf
, item
, new->compress_type
);
2611 btrfs_set_file_extent_encryption(leaf
, item
, 0);
2612 btrfs_set_file_extent_other_encoding(leaf
, item
, 0);
2614 btrfs_mark_buffer_dirty(leaf
);
2615 inode_add_bytes(inode
, len
);
2616 btrfs_release_path(path
);
2618 ret
= btrfs_inc_extent_ref(trans
, fs_info
, new->bytenr
,
2620 backref
->root_id
, backref
->inum
,
2621 new->file_pos
); /* start - extent_offset */
2623 btrfs_abort_transaction(trans
, ret
);
2629 btrfs_release_path(path
);
2630 path
->leave_spinning
= 0;
2631 btrfs_end_transaction(trans
);
2633 unlock_extent_cached(&BTRFS_I(inode
)->io_tree
, lock_start
, lock_end
,
2639 static void free_sa_defrag_extent(struct new_sa_defrag_extent
*new)
2641 struct old_sa_defrag_extent
*old
, *tmp
;
2646 list_for_each_entry_safe(old
, tmp
, &new->head
, list
) {
2652 static void relink_file_extents(struct new_sa_defrag_extent
*new)
2654 struct btrfs_fs_info
*fs_info
= btrfs_sb(new->inode
->i_sb
);
2655 struct btrfs_path
*path
;
2656 struct sa_defrag_extent_backref
*backref
;
2657 struct sa_defrag_extent_backref
*prev
= NULL
;
2658 struct inode
*inode
;
2659 struct btrfs_root
*root
;
2660 struct rb_node
*node
;
2664 root
= BTRFS_I(inode
)->root
;
2666 path
= btrfs_alloc_path();
2670 if (!record_extent_backrefs(path
, new)) {
2671 btrfs_free_path(path
);
2674 btrfs_release_path(path
);
2677 node
= rb_first(&new->root
);
2680 rb_erase(node
, &new->root
);
2682 backref
= rb_entry(node
, struct sa_defrag_extent_backref
, node
);
2684 ret
= relink_extent_backref(path
, prev
, backref
);
2697 btrfs_free_path(path
);
2699 free_sa_defrag_extent(new);
2701 atomic_dec(&fs_info
->defrag_running
);
2702 wake_up(&fs_info
->transaction_wait
);
2705 static struct new_sa_defrag_extent
*
2706 record_old_file_extents(struct inode
*inode
,
2707 struct btrfs_ordered_extent
*ordered
)
2709 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2710 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
2711 struct btrfs_path
*path
;
2712 struct btrfs_key key
;
2713 struct old_sa_defrag_extent
*old
;
2714 struct new_sa_defrag_extent
*new;
2717 new = kmalloc(sizeof(*new), GFP_NOFS
);
2722 new->file_pos
= ordered
->file_offset
;
2723 new->len
= ordered
->len
;
2724 new->bytenr
= ordered
->start
;
2725 new->disk_len
= ordered
->disk_len
;
2726 new->compress_type
= ordered
->compress_type
;
2727 new->root
= RB_ROOT
;
2728 INIT_LIST_HEAD(&new->head
);
2730 path
= btrfs_alloc_path();
2734 key
.objectid
= btrfs_ino(BTRFS_I(inode
));
2735 key
.type
= BTRFS_EXTENT_DATA_KEY
;
2736 key
.offset
= new->file_pos
;
2738 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
2741 if (ret
> 0 && path
->slots
[0] > 0)
2744 /* find out all the old extents for the file range */
2746 struct btrfs_file_extent_item
*extent
;
2747 struct extent_buffer
*l
;
2756 slot
= path
->slots
[0];
2758 if (slot
>= btrfs_header_nritems(l
)) {
2759 ret
= btrfs_next_leaf(root
, path
);
2767 btrfs_item_key_to_cpu(l
, &key
, slot
);
2769 if (key
.objectid
!= btrfs_ino(BTRFS_I(inode
)))
2771 if (key
.type
!= BTRFS_EXTENT_DATA_KEY
)
2773 if (key
.offset
>= new->file_pos
+ new->len
)
2776 extent
= btrfs_item_ptr(l
, slot
, struct btrfs_file_extent_item
);
2778 num_bytes
= btrfs_file_extent_num_bytes(l
, extent
);
2779 if (key
.offset
+ num_bytes
< new->file_pos
)
2782 disk_bytenr
= btrfs_file_extent_disk_bytenr(l
, extent
);
2786 extent_offset
= btrfs_file_extent_offset(l
, extent
);
2788 old
= kmalloc(sizeof(*old
), GFP_NOFS
);
2792 offset
= max(new->file_pos
, key
.offset
);
2793 end
= min(new->file_pos
+ new->len
, key
.offset
+ num_bytes
);
2795 old
->bytenr
= disk_bytenr
;
2796 old
->extent_offset
= extent_offset
;
2797 old
->offset
= offset
- key
.offset
;
2798 old
->len
= end
- offset
;
2801 list_add_tail(&old
->list
, &new->head
);
2807 btrfs_free_path(path
);
2808 atomic_inc(&fs_info
->defrag_running
);
2813 btrfs_free_path(path
);
2815 free_sa_defrag_extent(new);
2819 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info
*fs_info
,
2822 struct btrfs_block_group_cache
*cache
;
2824 cache
= btrfs_lookup_block_group(fs_info
, start
);
2827 spin_lock(&cache
->lock
);
2828 cache
->delalloc_bytes
-= len
;
2829 spin_unlock(&cache
->lock
);
2831 btrfs_put_block_group(cache
);
2834 /* as ordered data IO finishes, this gets called so we can finish
2835 * an ordered extent if the range of bytes in the file it covers are
2838 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent
*ordered_extent
)
2840 struct inode
*inode
= ordered_extent
->inode
;
2841 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
2842 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
2843 struct btrfs_trans_handle
*trans
= NULL
;
2844 struct extent_io_tree
*io_tree
= &BTRFS_I(inode
)->io_tree
;
2845 struct extent_state
*cached_state
= NULL
;
2846 struct new_sa_defrag_extent
*new = NULL
;
2847 int compress_type
= 0;
2849 u64 logical_len
= ordered_extent
->len
;
2851 bool truncated
= false;
2853 nolock
= btrfs_is_free_space_inode(inode
);
2855 if (test_bit(BTRFS_ORDERED_IOERR
, &ordered_extent
->flags
)) {
2860 btrfs_free_io_failure_record(inode
, ordered_extent
->file_offset
,
2861 ordered_extent
->file_offset
+
2862 ordered_extent
->len
- 1);
2864 if (test_bit(BTRFS_ORDERED_TRUNCATED
, &ordered_extent
->flags
)) {
2866 logical_len
= ordered_extent
->truncated_len
;
2867 /* Truncated the entire extent, don't bother adding */
2872 if (test_bit(BTRFS_ORDERED_NOCOW
, &ordered_extent
->flags
)) {
2873 BUG_ON(!list_empty(&ordered_extent
->list
)); /* Logic error */
2876 * For mwrite(mmap + memset to write) case, we still reserve
2877 * space for NOCOW range.
2878 * As NOCOW won't cause a new delayed ref, just free the space
2880 btrfs_qgroup_free_data(inode
, ordered_extent
->file_offset
,
2881 ordered_extent
->len
);
2882 btrfs_ordered_update_i_size(inode
, 0, ordered_extent
);
2884 trans
= btrfs_join_transaction_nolock(root
);
2886 trans
= btrfs_join_transaction(root
);
2887 if (IS_ERR(trans
)) {
2888 ret
= PTR_ERR(trans
);
2892 trans
->block_rsv
= &fs_info
->delalloc_block_rsv
;
2893 ret
= btrfs_update_inode_fallback(trans
, root
, inode
);
2894 if (ret
) /* -ENOMEM or corruption */
2895 btrfs_abort_transaction(trans
, ret
);
2899 lock_extent_bits(io_tree
, ordered_extent
->file_offset
,
2900 ordered_extent
->file_offset
+ ordered_extent
->len
- 1,
2903 ret
= test_range_bit(io_tree
, ordered_extent
->file_offset
,
2904 ordered_extent
->file_offset
+ ordered_extent
->len
- 1,
2905 EXTENT_DEFRAG
, 1, cached_state
);
2907 u64 last_snapshot
= btrfs_root_last_snapshot(&root
->root_item
);
2908 if (0 && last_snapshot
>= BTRFS_I(inode
)->generation
)
2909 /* the inode is shared */
2910 new = record_old_file_extents(inode
, ordered_extent
);
2912 clear_extent_bit(io_tree
, ordered_extent
->file_offset
,
2913 ordered_extent
->file_offset
+ ordered_extent
->len
- 1,
2914 EXTENT_DEFRAG
, 0, 0, &cached_state
, GFP_NOFS
);
2918 trans
= btrfs_join_transaction_nolock(root
);
2920 trans
= btrfs_join_transaction(root
);
2921 if (IS_ERR(trans
)) {
2922 ret
= PTR_ERR(trans
);
2927 trans
->block_rsv
= &fs_info
->delalloc_block_rsv
;
2929 if (test_bit(BTRFS_ORDERED_COMPRESSED
, &ordered_extent
->flags
))
2930 compress_type
= ordered_extent
->compress_type
;
2931 if (test_bit(BTRFS_ORDERED_PREALLOC
, &ordered_extent
->flags
)) {
2932 BUG_ON(compress_type
);
2933 ret
= btrfs_mark_extent_written(trans
, inode
,
2934 ordered_extent
->file_offset
,
2935 ordered_extent
->file_offset
+
2938 BUG_ON(root
== fs_info
->tree_root
);
2939 ret
= insert_reserved_file_extent(trans
, inode
,
2940 ordered_extent
->file_offset
,
2941 ordered_extent
->start
,
2942 ordered_extent
->disk_len
,
2943 logical_len
, logical_len
,
2944 compress_type
, 0, 0,
2945 BTRFS_FILE_EXTENT_REG
);
2947 btrfs_release_delalloc_bytes(fs_info
,
2948 ordered_extent
->start
,
2949 ordered_extent
->disk_len
);
2951 unpin_extent_cache(&BTRFS_I(inode
)->extent_tree
,
2952 ordered_extent
->file_offset
, ordered_extent
->len
,
2955 btrfs_abort_transaction(trans
, ret
);
2959 add_pending_csums(trans
, inode
, ordered_extent
->file_offset
,
2960 &ordered_extent
->list
);
2962 btrfs_ordered_update_i_size(inode
, 0, ordered_extent
);
2963 ret
= btrfs_update_inode_fallback(trans
, root
, inode
);
2964 if (ret
) { /* -ENOMEM or corruption */
2965 btrfs_abort_transaction(trans
, ret
);
2970 unlock_extent_cached(io_tree
, ordered_extent
->file_offset
,
2971 ordered_extent
->file_offset
+
2972 ordered_extent
->len
- 1, &cached_state
, GFP_NOFS
);
2974 if (root
!= fs_info
->tree_root
)
2975 btrfs_delalloc_release_metadata(inode
, ordered_extent
->len
);
2977 btrfs_end_transaction(trans
);
2979 if (ret
|| truncated
) {
2983 start
= ordered_extent
->file_offset
+ logical_len
;
2985 start
= ordered_extent
->file_offset
;
2986 end
= ordered_extent
->file_offset
+ ordered_extent
->len
- 1;
2987 clear_extent_uptodate(io_tree
, start
, end
, NULL
, GFP_NOFS
);
2989 /* Drop the cache for the part of the extent we didn't write. */
2990 btrfs_drop_extent_cache(inode
, start
, end
, 0);
2993 * If the ordered extent had an IOERR or something else went
2994 * wrong we need to return the space for this ordered extent
2995 * back to the allocator. We only free the extent in the
2996 * truncated case if we didn't write out the extent at all.
2998 if ((ret
|| !logical_len
) &&
2999 !test_bit(BTRFS_ORDERED_NOCOW
, &ordered_extent
->flags
) &&
3000 !test_bit(BTRFS_ORDERED_PREALLOC
, &ordered_extent
->flags
))
3001 btrfs_free_reserved_extent(fs_info
,
3002 ordered_extent
->start
,
3003 ordered_extent
->disk_len
, 1);
3008 * This needs to be done to make sure anybody waiting knows we are done
3009 * updating everything for this ordered extent.
3011 btrfs_remove_ordered_extent(inode
, ordered_extent
);
3013 /* for snapshot-aware defrag */
3016 free_sa_defrag_extent(new);
3017 atomic_dec(&fs_info
->defrag_running
);
3019 relink_file_extents(new);
3024 btrfs_put_ordered_extent(ordered_extent
);
3025 /* once for the tree */
3026 btrfs_put_ordered_extent(ordered_extent
);
3031 static void finish_ordered_fn(struct btrfs_work
*work
)
3033 struct btrfs_ordered_extent
*ordered_extent
;
3034 ordered_extent
= container_of(work
, struct btrfs_ordered_extent
, work
);
3035 btrfs_finish_ordered_io(ordered_extent
);
3038 static int btrfs_writepage_end_io_hook(struct page
*page
, u64 start
, u64 end
,
3039 struct extent_state
*state
, int uptodate
)
3041 struct inode
*inode
= page
->mapping
->host
;
3042 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
3043 struct btrfs_ordered_extent
*ordered_extent
= NULL
;
3044 struct btrfs_workqueue
*wq
;
3045 btrfs_work_func_t func
;
3047 trace_btrfs_writepage_end_io_hook(page
, start
, end
, uptodate
);
3049 ClearPagePrivate2(page
);
3050 if (!btrfs_dec_test_ordered_pending(inode
, &ordered_extent
, start
,
3051 end
- start
+ 1, uptodate
))
3054 if (btrfs_is_free_space_inode(inode
)) {
3055 wq
= fs_info
->endio_freespace_worker
;
3056 func
= btrfs_freespace_write_helper
;
3058 wq
= fs_info
->endio_write_workers
;
3059 func
= btrfs_endio_write_helper
;
3062 btrfs_init_work(&ordered_extent
->work
, func
, finish_ordered_fn
, NULL
,
3064 btrfs_queue_work(wq
, &ordered_extent
->work
);
3069 static int __readpage_endio_check(struct inode
*inode
,
3070 struct btrfs_io_bio
*io_bio
,
3071 int icsum
, struct page
*page
,
3072 int pgoff
, u64 start
, size_t len
)
3078 csum_expected
= *(((u32
*)io_bio
->csum
) + icsum
);
3080 kaddr
= kmap_atomic(page
);
3081 csum
= btrfs_csum_data(kaddr
+ pgoff
, csum
, len
);
3082 btrfs_csum_final(csum
, (u8
*)&csum
);
3083 if (csum
!= csum_expected
)
3086 kunmap_atomic(kaddr
);
3089 btrfs_print_data_csum_error(inode
, start
, csum
, csum_expected
,
3090 io_bio
->mirror_num
);
3091 memset(kaddr
+ pgoff
, 1, len
);
3092 flush_dcache_page(page
);
3093 kunmap_atomic(kaddr
);
3094 if (csum_expected
== 0)
3100 * when reads are done, we need to check csums to verify the data is correct
3101 * if there's a match, we allow the bio to finish. If not, the code in
3102 * extent_io.c will try to find good copies for us.
3104 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio
*io_bio
,
3105 u64 phy_offset
, struct page
*page
,
3106 u64 start
, u64 end
, int mirror
)
3108 size_t offset
= start
- page_offset(page
);
3109 struct inode
*inode
= page
->mapping
->host
;
3110 struct extent_io_tree
*io_tree
= &BTRFS_I(inode
)->io_tree
;
3111 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
3113 if (PageChecked(page
)) {
3114 ClearPageChecked(page
);
3118 if (BTRFS_I(inode
)->flags
& BTRFS_INODE_NODATASUM
)
3121 if (root
->root_key
.objectid
== BTRFS_DATA_RELOC_TREE_OBJECTID
&&
3122 test_range_bit(io_tree
, start
, end
, EXTENT_NODATASUM
, 1, NULL
)) {
3123 clear_extent_bits(io_tree
, start
, end
, EXTENT_NODATASUM
);
3127 phy_offset
>>= inode
->i_sb
->s_blocksize_bits
;
3128 return __readpage_endio_check(inode
, io_bio
, phy_offset
, page
, offset
,
3129 start
, (size_t)(end
- start
+ 1));
3132 void btrfs_add_delayed_iput(struct inode
*inode
)
3134 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
3135 struct btrfs_inode
*binode
= BTRFS_I(inode
);
3137 if (atomic_add_unless(&inode
->i_count
, -1, 1))
3140 spin_lock(&fs_info
->delayed_iput_lock
);
3141 if (binode
->delayed_iput_count
== 0) {
3142 ASSERT(list_empty(&binode
->delayed_iput
));
3143 list_add_tail(&binode
->delayed_iput
, &fs_info
->delayed_iputs
);
3145 binode
->delayed_iput_count
++;
3147 spin_unlock(&fs_info
->delayed_iput_lock
);
3150 void btrfs_run_delayed_iputs(struct btrfs_fs_info
*fs_info
)
3153 spin_lock(&fs_info
->delayed_iput_lock
);
3154 while (!list_empty(&fs_info
->delayed_iputs
)) {
3155 struct btrfs_inode
*inode
;
3157 inode
= list_first_entry(&fs_info
->delayed_iputs
,
3158 struct btrfs_inode
, delayed_iput
);
3159 if (inode
->delayed_iput_count
) {
3160 inode
->delayed_iput_count
--;
3161 list_move_tail(&inode
->delayed_iput
,
3162 &fs_info
->delayed_iputs
);
3164 list_del_init(&inode
->delayed_iput
);
3166 spin_unlock(&fs_info
->delayed_iput_lock
);
3167 iput(&inode
->vfs_inode
);
3168 spin_lock(&fs_info
->delayed_iput_lock
);
3170 spin_unlock(&fs_info
->delayed_iput_lock
);
3174 * This is called in transaction commit time. If there are no orphan
3175 * files in the subvolume, it removes orphan item and frees block_rsv
3178 void btrfs_orphan_commit_root(struct btrfs_trans_handle
*trans
,
3179 struct btrfs_root
*root
)
3181 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
3182 struct btrfs_block_rsv
*block_rsv
;
3185 if (atomic_read(&root
->orphan_inodes
) ||
3186 root
->orphan_cleanup_state
!= ORPHAN_CLEANUP_DONE
)
3189 spin_lock(&root
->orphan_lock
);
3190 if (atomic_read(&root
->orphan_inodes
)) {
3191 spin_unlock(&root
->orphan_lock
);
3195 if (root
->orphan_cleanup_state
!= ORPHAN_CLEANUP_DONE
) {
3196 spin_unlock(&root
->orphan_lock
);
3200 block_rsv
= root
->orphan_block_rsv
;
3201 root
->orphan_block_rsv
= NULL
;
3202 spin_unlock(&root
->orphan_lock
);
3204 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED
, &root
->state
) &&
3205 btrfs_root_refs(&root
->root_item
) > 0) {
3206 ret
= btrfs_del_orphan_item(trans
, fs_info
->tree_root
,
3207 root
->root_key
.objectid
);
3209 btrfs_abort_transaction(trans
, ret
);
3211 clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED
,
3216 WARN_ON(block_rsv
->size
> 0);
3217 btrfs_free_block_rsv(fs_info
, block_rsv
);
3222 * This creates an orphan entry for the given inode in case something goes
3223 * wrong in the middle of an unlink/truncate.
3225 * NOTE: caller of this function should reserve 5 units of metadata for
3228 int btrfs_orphan_add(struct btrfs_trans_handle
*trans
, struct inode
*inode
)
3230 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
3231 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
3232 struct btrfs_block_rsv
*block_rsv
= NULL
;
3237 if (!root
->orphan_block_rsv
) {
3238 block_rsv
= btrfs_alloc_block_rsv(fs_info
,
3239 BTRFS_BLOCK_RSV_TEMP
);
3244 spin_lock(&root
->orphan_lock
);
3245 if (!root
->orphan_block_rsv
) {
3246 root
->orphan_block_rsv
= block_rsv
;
3247 } else if (block_rsv
) {
3248 btrfs_free_block_rsv(fs_info
, block_rsv
);
3252 if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM
,
3253 &BTRFS_I(inode
)->runtime_flags
)) {
3256 * For proper ENOSPC handling, we should do orphan
3257 * cleanup when mounting. But this introduces backward
3258 * compatibility issue.
3260 if (!xchg(&root
->orphan_item_inserted
, 1))
3266 atomic_inc(&root
->orphan_inodes
);
3269 if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED
,
3270 &BTRFS_I(inode
)->runtime_flags
))
3272 spin_unlock(&root
->orphan_lock
);
3274 /* grab metadata reservation from transaction handle */
3276 ret
= btrfs_orphan_reserve_metadata(trans
, inode
);
3279 atomic_dec(&root
->orphan_inodes
);
3280 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED
,
3281 &BTRFS_I(inode
)->runtime_flags
);
3283 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM
,
3284 &BTRFS_I(inode
)->runtime_flags
);
3289 /* insert an orphan item to track this unlinked/truncated file */
3291 ret
= btrfs_insert_orphan_item(trans
, root
,
3292 btrfs_ino(BTRFS_I(inode
)));
3294 atomic_dec(&root
->orphan_inodes
);
3296 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED
,
3297 &BTRFS_I(inode
)->runtime_flags
);
3298 btrfs_orphan_release_metadata(inode
);
3300 if (ret
!= -EEXIST
) {
3301 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM
,
3302 &BTRFS_I(inode
)->runtime_flags
);
3303 btrfs_abort_transaction(trans
, ret
);
3310 /* insert an orphan item to track subvolume contains orphan files */
3312 ret
= btrfs_insert_orphan_item(trans
, fs_info
->tree_root
,
3313 root
->root_key
.objectid
);
3314 if (ret
&& ret
!= -EEXIST
) {
3315 btrfs_abort_transaction(trans
, ret
);
3323 * We have done the truncate/delete so we can go ahead and remove the orphan
3324 * item for this particular inode.
3326 static int btrfs_orphan_del(struct btrfs_trans_handle
*trans
,
3327 struct inode
*inode
)
3329 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
3330 int delete_item
= 0;
3331 int release_rsv
= 0;
3334 spin_lock(&root
->orphan_lock
);
3335 if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM
,
3336 &BTRFS_I(inode
)->runtime_flags
))
3339 if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED
,
3340 &BTRFS_I(inode
)->runtime_flags
))
3342 spin_unlock(&root
->orphan_lock
);
3345 atomic_dec(&root
->orphan_inodes
);
3347 ret
= btrfs_del_orphan_item(trans
, root
,
3348 btrfs_ino(BTRFS_I(inode
)));
3352 btrfs_orphan_release_metadata(inode
);
3358 * this cleans up any orphans that may be left on the list from the last use
3361 int btrfs_orphan_cleanup(struct btrfs_root
*root
)
3363 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
3364 struct btrfs_path
*path
;
3365 struct extent_buffer
*leaf
;
3366 struct btrfs_key key
, found_key
;
3367 struct btrfs_trans_handle
*trans
;
3368 struct inode
*inode
;
3369 u64 last_objectid
= 0;
3370 int ret
= 0, nr_unlink
= 0, nr_truncate
= 0;
3372 if (cmpxchg(&root
->orphan_cleanup_state
, 0, ORPHAN_CLEANUP_STARTED
))
3375 path
= btrfs_alloc_path();
3380 path
->reada
= READA_BACK
;
3382 key
.objectid
= BTRFS_ORPHAN_OBJECTID
;
3383 key
.type
= BTRFS_ORPHAN_ITEM_KEY
;
3384 key
.offset
= (u64
)-1;
3387 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
3392 * if ret == 0 means we found what we were searching for, which
3393 * is weird, but possible, so only screw with path if we didn't
3394 * find the key and see if we have stuff that matches
3398 if (path
->slots
[0] == 0)
3403 /* pull out the item */
3404 leaf
= path
->nodes
[0];
3405 btrfs_item_key_to_cpu(leaf
, &found_key
, path
->slots
[0]);
3407 /* make sure the item matches what we want */
3408 if (found_key
.objectid
!= BTRFS_ORPHAN_OBJECTID
)
3410 if (found_key
.type
!= BTRFS_ORPHAN_ITEM_KEY
)
3413 /* release the path since we're done with it */
3414 btrfs_release_path(path
);
3417 * this is where we are basically btrfs_lookup, without the
3418 * crossing root thing. we store the inode number in the
3419 * offset of the orphan item.
3422 if (found_key
.offset
== last_objectid
) {
3424 "Error removing orphan entry, stopping orphan cleanup");
3429 last_objectid
= found_key
.offset
;
3431 found_key
.objectid
= found_key
.offset
;
3432 found_key
.type
= BTRFS_INODE_ITEM_KEY
;
3433 found_key
.offset
= 0;
3434 inode
= btrfs_iget(fs_info
->sb
, &found_key
, root
, NULL
);
3435 ret
= PTR_ERR_OR_ZERO(inode
);
3436 if (ret
&& ret
!= -ENOENT
)
3439 if (ret
== -ENOENT
&& root
== fs_info
->tree_root
) {
3440 struct btrfs_root
*dead_root
;
3441 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
3442 int is_dead_root
= 0;
3445 * this is an orphan in the tree root. Currently these
3446 * could come from 2 sources:
3447 * a) a snapshot deletion in progress
3448 * b) a free space cache inode
3449 * We need to distinguish those two, as the snapshot
3450 * orphan must not get deleted.
3451 * find_dead_roots already ran before us, so if this
3452 * is a snapshot deletion, we should find the root
3453 * in the dead_roots list
3455 spin_lock(&fs_info
->trans_lock
);
3456 list_for_each_entry(dead_root
, &fs_info
->dead_roots
,
3458 if (dead_root
->root_key
.objectid
==
3459 found_key
.objectid
) {
3464 spin_unlock(&fs_info
->trans_lock
);
3466 /* prevent this orphan from being found again */
3467 key
.offset
= found_key
.objectid
- 1;
3472 * Inode is already gone but the orphan item is still there,
3473 * kill the orphan item.
3475 if (ret
== -ENOENT
) {
3476 trans
= btrfs_start_transaction(root
, 1);
3477 if (IS_ERR(trans
)) {
3478 ret
= PTR_ERR(trans
);
3481 btrfs_debug(fs_info
, "auto deleting %Lu",
3482 found_key
.objectid
);
3483 ret
= btrfs_del_orphan_item(trans
, root
,
3484 found_key
.objectid
);
3485 btrfs_end_transaction(trans
);
3492 * add this inode to the orphan list so btrfs_orphan_del does
3493 * the proper thing when we hit it
3495 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM
,
3496 &BTRFS_I(inode
)->runtime_flags
);
3497 atomic_inc(&root
->orphan_inodes
);
3499 /* if we have links, this was a truncate, lets do that */
3500 if (inode
->i_nlink
) {
3501 if (WARN_ON(!S_ISREG(inode
->i_mode
))) {
3507 /* 1 for the orphan item deletion. */
3508 trans
= btrfs_start_transaction(root
, 1);
3509 if (IS_ERR(trans
)) {
3511 ret
= PTR_ERR(trans
);
3514 ret
= btrfs_orphan_add(trans
, inode
);
3515 btrfs_end_transaction(trans
);
3521 ret
= btrfs_truncate(inode
);
3523 btrfs_orphan_del(NULL
, inode
);
3528 /* this will do delete_inode and everything for us */
3533 /* release the path since we're done with it */
3534 btrfs_release_path(path
);
3536 root
->orphan_cleanup_state
= ORPHAN_CLEANUP_DONE
;
3538 if (root
->orphan_block_rsv
)
3539 btrfs_block_rsv_release(fs_info
, root
->orphan_block_rsv
,
3542 if (root
->orphan_block_rsv
||
3543 test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED
, &root
->state
)) {
3544 trans
= btrfs_join_transaction(root
);
3546 btrfs_end_transaction(trans
);
3550 btrfs_debug(fs_info
, "unlinked %d orphans", nr_unlink
);
3552 btrfs_debug(fs_info
, "truncated %d orphans", nr_truncate
);
3556 btrfs_err(fs_info
, "could not do orphan cleanup %d", ret
);
3557 btrfs_free_path(path
);
3562 * very simple check to peek ahead in the leaf looking for xattrs. If we
3563 * don't find any xattrs, we know there can't be any acls.
3565 * slot is the slot the inode is in, objectid is the objectid of the inode
3567 static noinline
int acls_after_inode_item(struct extent_buffer
*leaf
,
3568 int slot
, u64 objectid
,
3569 int *first_xattr_slot
)
3571 u32 nritems
= btrfs_header_nritems(leaf
);
3572 struct btrfs_key found_key
;
3573 static u64 xattr_access
= 0;
3574 static u64 xattr_default
= 0;
3577 if (!xattr_access
) {
3578 xattr_access
= btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS
,
3579 strlen(XATTR_NAME_POSIX_ACL_ACCESS
));
3580 xattr_default
= btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT
,
3581 strlen(XATTR_NAME_POSIX_ACL_DEFAULT
));
3585 *first_xattr_slot
= -1;
3586 while (slot
< nritems
) {
3587 btrfs_item_key_to_cpu(leaf
, &found_key
, slot
);
3589 /* we found a different objectid, there must not be acls */
3590 if (found_key
.objectid
!= objectid
)
3593 /* we found an xattr, assume we've got an acl */
3594 if (found_key
.type
== BTRFS_XATTR_ITEM_KEY
) {
3595 if (*first_xattr_slot
== -1)
3596 *first_xattr_slot
= slot
;
3597 if (found_key
.offset
== xattr_access
||
3598 found_key
.offset
== xattr_default
)
3603 * we found a key greater than an xattr key, there can't
3604 * be any acls later on
3606 if (found_key
.type
> BTRFS_XATTR_ITEM_KEY
)
3613 * it goes inode, inode backrefs, xattrs, extents,
3614 * so if there are a ton of hard links to an inode there can
3615 * be a lot of backrefs. Don't waste time searching too hard,
3616 * this is just an optimization
3621 /* we hit the end of the leaf before we found an xattr or
3622 * something larger than an xattr. We have to assume the inode
3625 if (*first_xattr_slot
== -1)
3626 *first_xattr_slot
= slot
;
3631 * read an inode from the btree into the in-memory inode
3633 static int btrfs_read_locked_inode(struct inode
*inode
)
3635 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
3636 struct btrfs_path
*path
;
3637 struct extent_buffer
*leaf
;
3638 struct btrfs_inode_item
*inode_item
;
3639 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
3640 struct btrfs_key location
;
3645 bool filled
= false;
3646 int first_xattr_slot
;
3648 ret
= btrfs_fill_inode(inode
, &rdev
);
3652 path
= btrfs_alloc_path();
3658 memcpy(&location
, &BTRFS_I(inode
)->location
, sizeof(location
));
3660 ret
= btrfs_lookup_inode(NULL
, root
, path
, &location
, 0);
3667 leaf
= path
->nodes
[0];
3672 inode_item
= btrfs_item_ptr(leaf
, path
->slots
[0],
3673 struct btrfs_inode_item
);
3674 inode
->i_mode
= btrfs_inode_mode(leaf
, inode_item
);
3675 set_nlink(inode
, btrfs_inode_nlink(leaf
, inode_item
));
3676 i_uid_write(inode
, btrfs_inode_uid(leaf
, inode_item
));
3677 i_gid_write(inode
, btrfs_inode_gid(leaf
, inode_item
));
3678 btrfs_i_size_write(inode
, btrfs_inode_size(leaf
, inode_item
));
3680 inode
->i_atime
.tv_sec
= btrfs_timespec_sec(leaf
, &inode_item
->atime
);
3681 inode
->i_atime
.tv_nsec
= btrfs_timespec_nsec(leaf
, &inode_item
->atime
);
3683 inode
->i_mtime
.tv_sec
= btrfs_timespec_sec(leaf
, &inode_item
->mtime
);
3684 inode
->i_mtime
.tv_nsec
= btrfs_timespec_nsec(leaf
, &inode_item
->mtime
);
3686 inode
->i_ctime
.tv_sec
= btrfs_timespec_sec(leaf
, &inode_item
->ctime
);
3687 inode
->i_ctime
.tv_nsec
= btrfs_timespec_nsec(leaf
, &inode_item
->ctime
);
3689 BTRFS_I(inode
)->i_otime
.tv_sec
=
3690 btrfs_timespec_sec(leaf
, &inode_item
->otime
);
3691 BTRFS_I(inode
)->i_otime
.tv_nsec
=
3692 btrfs_timespec_nsec(leaf
, &inode_item
->otime
);
3694 inode_set_bytes(inode
, btrfs_inode_nbytes(leaf
, inode_item
));
3695 BTRFS_I(inode
)->generation
= btrfs_inode_generation(leaf
, inode_item
);
3696 BTRFS_I(inode
)->last_trans
= btrfs_inode_transid(leaf
, inode_item
);
3698 inode
->i_version
= btrfs_inode_sequence(leaf
, inode_item
);
3699 inode
->i_generation
= BTRFS_I(inode
)->generation
;
3701 rdev
= btrfs_inode_rdev(leaf
, inode_item
);
3703 BTRFS_I(inode
)->index_cnt
= (u64
)-1;
3704 BTRFS_I(inode
)->flags
= btrfs_inode_flags(leaf
, inode_item
);
3708 * If we were modified in the current generation and evicted from memory
3709 * and then re-read we need to do a full sync since we don't have any
3710 * idea about which extents were modified before we were evicted from
3713 * This is required for both inode re-read from disk and delayed inode
3714 * in delayed_nodes_tree.
3716 if (BTRFS_I(inode
)->last_trans
== fs_info
->generation
)
3717 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC
,
3718 &BTRFS_I(inode
)->runtime_flags
);
3721 * We don't persist the id of the transaction where an unlink operation
3722 * against the inode was last made. So here we assume the inode might
3723 * have been evicted, and therefore the exact value of last_unlink_trans
3724 * lost, and set it to last_trans to avoid metadata inconsistencies
3725 * between the inode and its parent if the inode is fsync'ed and the log
3726 * replayed. For example, in the scenario:
3729 * ln mydir/foo mydir/bar
3732 * echo 2 > /proc/sys/vm/drop_caches # evicts inode
3733 * xfs_io -c fsync mydir/foo
3735 * mount fs, triggers fsync log replay
3737 * We must make sure that when we fsync our inode foo we also log its
3738 * parent inode, otherwise after log replay the parent still has the
3739 * dentry with the "bar" name but our inode foo has a link count of 1
3740 * and doesn't have an inode ref with the name "bar" anymore.
3742 * Setting last_unlink_trans to last_trans is a pessimistic approach,
3743 * but it guarantees correctness at the expense of occasional full
3744 * transaction commits on fsync if our inode is a directory, or if our
3745 * inode is not a directory, logging its parent unnecessarily.
3747 BTRFS_I(inode
)->last_unlink_trans
= BTRFS_I(inode
)->last_trans
;
3750 if (inode
->i_nlink
!= 1 ||
3751 path
->slots
[0] >= btrfs_header_nritems(leaf
))
3754 btrfs_item_key_to_cpu(leaf
, &location
, path
->slots
[0]);
3755 if (location
.objectid
!= btrfs_ino(BTRFS_I(inode
)))
3758 ptr
= btrfs_item_ptr_offset(leaf
, path
->slots
[0]);
3759 if (location
.type
== BTRFS_INODE_REF_KEY
) {
3760 struct btrfs_inode_ref
*ref
;
3762 ref
= (struct btrfs_inode_ref
*)ptr
;
3763 BTRFS_I(inode
)->dir_index
= btrfs_inode_ref_index(leaf
, ref
);
3764 } else if (location
.type
== BTRFS_INODE_EXTREF_KEY
) {
3765 struct btrfs_inode_extref
*extref
;
3767 extref
= (struct btrfs_inode_extref
*)ptr
;
3768 BTRFS_I(inode
)->dir_index
= btrfs_inode_extref_index(leaf
,
3773 * try to precache a NULL acl entry for files that don't have
3774 * any xattrs or acls
3776 maybe_acls
= acls_after_inode_item(leaf
, path
->slots
[0],
3777 btrfs_ino(BTRFS_I(inode
)), &first_xattr_slot
);
3778 if (first_xattr_slot
!= -1) {
3779 path
->slots
[0] = first_xattr_slot
;
3780 ret
= btrfs_load_inode_props(inode
, path
);
3783 "error loading props for ino %llu (root %llu): %d",
3784 btrfs_ino(BTRFS_I(inode
)),
3785 root
->root_key
.objectid
, ret
);
3787 btrfs_free_path(path
);
3790 cache_no_acl(inode
);
3792 switch (inode
->i_mode
& S_IFMT
) {
3794 inode
->i_mapping
->a_ops
= &btrfs_aops
;
3795 BTRFS_I(inode
)->io_tree
.ops
= &btrfs_extent_io_ops
;
3796 inode
->i_fop
= &btrfs_file_operations
;
3797 inode
->i_op
= &btrfs_file_inode_operations
;
3800 inode
->i_fop
= &btrfs_dir_file_operations
;
3801 inode
->i_op
= &btrfs_dir_inode_operations
;
3804 inode
->i_op
= &btrfs_symlink_inode_operations
;
3805 inode_nohighmem(inode
);
3806 inode
->i_mapping
->a_ops
= &btrfs_symlink_aops
;
3809 inode
->i_op
= &btrfs_special_inode_operations
;
3810 init_special_inode(inode
, inode
->i_mode
, rdev
);
3814 btrfs_update_iflags(inode
);
3818 btrfs_free_path(path
);
3819 make_bad_inode(inode
);
3824 * given a leaf and an inode, copy the inode fields into the leaf
3826 static void fill_inode_item(struct btrfs_trans_handle
*trans
,
3827 struct extent_buffer
*leaf
,
3828 struct btrfs_inode_item
*item
,
3829 struct inode
*inode
)
3831 struct btrfs_map_token token
;
3833 btrfs_init_map_token(&token
);
3835 btrfs_set_token_inode_uid(leaf
, item
, i_uid_read(inode
), &token
);
3836 btrfs_set_token_inode_gid(leaf
, item
, i_gid_read(inode
), &token
);
3837 btrfs_set_token_inode_size(leaf
, item
, BTRFS_I(inode
)->disk_i_size
,
3839 btrfs_set_token_inode_mode(leaf
, item
, inode
->i_mode
, &token
);
3840 btrfs_set_token_inode_nlink(leaf
, item
, inode
->i_nlink
, &token
);
3842 btrfs_set_token_timespec_sec(leaf
, &item
->atime
,
3843 inode
->i_atime
.tv_sec
, &token
);
3844 btrfs_set_token_timespec_nsec(leaf
, &item
->atime
,
3845 inode
->i_atime
.tv_nsec
, &token
);
3847 btrfs_set_token_timespec_sec(leaf
, &item
->mtime
,
3848 inode
->i_mtime
.tv_sec
, &token
);
3849 btrfs_set_token_timespec_nsec(leaf
, &item
->mtime
,
3850 inode
->i_mtime
.tv_nsec
, &token
);
3852 btrfs_set_token_timespec_sec(leaf
, &item
->ctime
,
3853 inode
->i_ctime
.tv_sec
, &token
);
3854 btrfs_set_token_timespec_nsec(leaf
, &item
->ctime
,
3855 inode
->i_ctime
.tv_nsec
, &token
);
3857 btrfs_set_token_timespec_sec(leaf
, &item
->otime
,
3858 BTRFS_I(inode
)->i_otime
.tv_sec
, &token
);
3859 btrfs_set_token_timespec_nsec(leaf
, &item
->otime
,
3860 BTRFS_I(inode
)->i_otime
.tv_nsec
, &token
);
3862 btrfs_set_token_inode_nbytes(leaf
, item
, inode_get_bytes(inode
),
3864 btrfs_set_token_inode_generation(leaf
, item
, BTRFS_I(inode
)->generation
,
3866 btrfs_set_token_inode_sequence(leaf
, item
, inode
->i_version
, &token
);
3867 btrfs_set_token_inode_transid(leaf
, item
, trans
->transid
, &token
);
3868 btrfs_set_token_inode_rdev(leaf
, item
, inode
->i_rdev
, &token
);
3869 btrfs_set_token_inode_flags(leaf
, item
, BTRFS_I(inode
)->flags
, &token
);
3870 btrfs_set_token_inode_block_group(leaf
, item
, 0, &token
);
3874 * copy everything in the in-memory inode into the btree.
3876 static noinline
int btrfs_update_inode_item(struct btrfs_trans_handle
*trans
,
3877 struct btrfs_root
*root
, struct inode
*inode
)
3879 struct btrfs_inode_item
*inode_item
;
3880 struct btrfs_path
*path
;
3881 struct extent_buffer
*leaf
;
3884 path
= btrfs_alloc_path();
3888 path
->leave_spinning
= 1;
3889 ret
= btrfs_lookup_inode(trans
, root
, path
, &BTRFS_I(inode
)->location
,
3897 leaf
= path
->nodes
[0];
3898 inode_item
= btrfs_item_ptr(leaf
, path
->slots
[0],
3899 struct btrfs_inode_item
);
3901 fill_inode_item(trans
, leaf
, inode_item
, inode
);
3902 btrfs_mark_buffer_dirty(leaf
);
3903 btrfs_set_inode_last_trans(trans
, inode
);
3906 btrfs_free_path(path
);
3911 * copy everything in the in-memory inode into the btree.
3913 noinline
int btrfs_update_inode(struct btrfs_trans_handle
*trans
,
3914 struct btrfs_root
*root
, struct inode
*inode
)
3916 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
3920 * If the inode is a free space inode, we can deadlock during commit
3921 * if we put it into the delayed code.
3923 * The data relocation inode should also be directly updated
3926 if (!btrfs_is_free_space_inode(inode
)
3927 && root
->root_key
.objectid
!= BTRFS_DATA_RELOC_TREE_OBJECTID
3928 && !test_bit(BTRFS_FS_LOG_RECOVERING
, &fs_info
->flags
)) {
3929 btrfs_update_root_times(trans
, root
);
3931 ret
= btrfs_delayed_update_inode(trans
, root
, inode
);
3933 btrfs_set_inode_last_trans(trans
, inode
);
3937 return btrfs_update_inode_item(trans
, root
, inode
);
3940 noinline
int btrfs_update_inode_fallback(struct btrfs_trans_handle
*trans
,
3941 struct btrfs_root
*root
,
3942 struct inode
*inode
)
3946 ret
= btrfs_update_inode(trans
, root
, inode
);
3948 return btrfs_update_inode_item(trans
, root
, inode
);
3953 * unlink helper that gets used here in inode.c and in the tree logging
3954 * recovery code. It remove a link in a directory with a given name, and
3955 * also drops the back refs in the inode to the directory
3957 static int __btrfs_unlink_inode(struct btrfs_trans_handle
*trans
,
3958 struct btrfs_root
*root
,
3959 struct btrfs_inode
*dir
,
3960 struct btrfs_inode
*inode
,
3961 const char *name
, int name_len
)
3963 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
3964 struct btrfs_path
*path
;
3966 struct extent_buffer
*leaf
;
3967 struct btrfs_dir_item
*di
;
3968 struct btrfs_key key
;
3970 u64 ino
= btrfs_ino(inode
);
3971 u64 dir_ino
= btrfs_ino(dir
);
3973 path
= btrfs_alloc_path();
3979 path
->leave_spinning
= 1;
3980 di
= btrfs_lookup_dir_item(trans
, root
, path
, dir_ino
,
3981 name
, name_len
, -1);
3990 leaf
= path
->nodes
[0];
3991 btrfs_dir_item_key_to_cpu(leaf
, di
, &key
);
3992 ret
= btrfs_delete_one_dir_name(trans
, root
, path
, di
);
3995 btrfs_release_path(path
);
3998 * If we don't have dir index, we have to get it by looking up
3999 * the inode ref, since we get the inode ref, remove it directly,
4000 * it is unnecessary to do delayed deletion.
4002 * But if we have dir index, needn't search inode ref to get it.
4003 * Since the inode ref is close to the inode item, it is better
4004 * that we delay to delete it, and just do this deletion when
4005 * we update the inode item.
4007 if (inode
->dir_index
) {
4008 ret
= btrfs_delayed_delete_inode_ref(inode
);
4010 index
= inode
->dir_index
;
4015 ret
= btrfs_del_inode_ref(trans
, root
, name
, name_len
, ino
,
4019 "failed to delete reference to %.*s, inode %llu parent %llu",
4020 name_len
, name
, ino
, dir_ino
);
4021 btrfs_abort_transaction(trans
, ret
);
4025 ret
= btrfs_delete_delayed_dir_index(trans
, fs_info
, dir
, index
);
4027 btrfs_abort_transaction(trans
, ret
);
4031 ret
= btrfs_del_inode_ref_in_log(trans
, root
, name
, name_len
, inode
,
4033 if (ret
!= 0 && ret
!= -ENOENT
) {
4034 btrfs_abort_transaction(trans
, ret
);
4038 ret
= btrfs_del_dir_entries_in_log(trans
, root
, name
, name_len
, dir
,
4043 btrfs_abort_transaction(trans
, ret
);
4045 btrfs_free_path(path
);
4049 btrfs_i_size_write(&dir
->vfs_inode
,
4050 dir
->vfs_inode
.i_size
- name_len
* 2);
4051 inode_inc_iversion(&inode
->vfs_inode
);
4052 inode_inc_iversion(&dir
->vfs_inode
);
4053 inode
->vfs_inode
.i_ctime
= dir
->vfs_inode
.i_mtime
=
4054 dir
->vfs_inode
.i_ctime
= current_time(&inode
->vfs_inode
);
4055 ret
= btrfs_update_inode(trans
, root
, &dir
->vfs_inode
);
4060 int btrfs_unlink_inode(struct btrfs_trans_handle
*trans
,
4061 struct btrfs_root
*root
,
4062 struct btrfs_inode
*dir
, struct btrfs_inode
*inode
,
4063 const char *name
, int name_len
)
4066 ret
= __btrfs_unlink_inode(trans
, root
, dir
, inode
, name
, name_len
);
4068 drop_nlink(&inode
->vfs_inode
);
4069 ret
= btrfs_update_inode(trans
, root
, &inode
->vfs_inode
);
4075 * helper to start transaction for unlink and rmdir.
4077 * unlink and rmdir are special in btrfs, they do not always free space, so
4078 * if we cannot make our reservations the normal way try and see if there is
4079 * plenty of slack room in the global reserve to migrate, otherwise we cannot
4080 * allow the unlink to occur.
4082 static struct btrfs_trans_handle
*__unlink_start_trans(struct inode
*dir
)
4084 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
4087 * 1 for the possible orphan item
4088 * 1 for the dir item
4089 * 1 for the dir index
4090 * 1 for the inode ref
4093 return btrfs_start_transaction_fallback_global_rsv(root
, 5, 5);
4096 static int btrfs_unlink(struct inode
*dir
, struct dentry
*dentry
)
4098 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
4099 struct btrfs_trans_handle
*trans
;
4100 struct inode
*inode
= d_inode(dentry
);
4103 trans
= __unlink_start_trans(dir
);
4105 return PTR_ERR(trans
);
4107 btrfs_record_unlink_dir(trans
, BTRFS_I(dir
), BTRFS_I(d_inode(dentry
)),
4110 ret
= btrfs_unlink_inode(trans
, root
, BTRFS_I(dir
),
4111 BTRFS_I(d_inode(dentry
)), dentry
->d_name
.name
,
4112 dentry
->d_name
.len
);
4116 if (inode
->i_nlink
== 0) {
4117 ret
= btrfs_orphan_add(trans
, inode
);
4123 btrfs_end_transaction(trans
);
4124 btrfs_btree_balance_dirty(root
->fs_info
);
4128 int btrfs_unlink_subvol(struct btrfs_trans_handle
*trans
,
4129 struct btrfs_root
*root
,
4130 struct inode
*dir
, u64 objectid
,
4131 const char *name
, int name_len
)
4133 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
4134 struct btrfs_path
*path
;
4135 struct extent_buffer
*leaf
;
4136 struct btrfs_dir_item
*di
;
4137 struct btrfs_key key
;
4140 u64 dir_ino
= btrfs_ino(BTRFS_I(dir
));
4142 path
= btrfs_alloc_path();
4146 di
= btrfs_lookup_dir_item(trans
, root
, path
, dir_ino
,
4147 name
, name_len
, -1);
4148 if (IS_ERR_OR_NULL(di
)) {
4156 leaf
= path
->nodes
[0];
4157 btrfs_dir_item_key_to_cpu(leaf
, di
, &key
);
4158 WARN_ON(key
.type
!= BTRFS_ROOT_ITEM_KEY
|| key
.objectid
!= objectid
);
4159 ret
= btrfs_delete_one_dir_name(trans
, root
, path
, di
);
4161 btrfs_abort_transaction(trans
, ret
);
4164 btrfs_release_path(path
);
4166 ret
= btrfs_del_root_ref(trans
, fs_info
, objectid
,
4167 root
->root_key
.objectid
, dir_ino
,
4168 &index
, name
, name_len
);
4170 if (ret
!= -ENOENT
) {
4171 btrfs_abort_transaction(trans
, ret
);
4174 di
= btrfs_search_dir_index_item(root
, path
, dir_ino
,
4176 if (IS_ERR_OR_NULL(di
)) {
4181 btrfs_abort_transaction(trans
, ret
);
4185 leaf
= path
->nodes
[0];
4186 btrfs_item_key_to_cpu(leaf
, &key
, path
->slots
[0]);
4187 btrfs_release_path(path
);
4190 btrfs_release_path(path
);
4192 ret
= btrfs_delete_delayed_dir_index(trans
, fs_info
, BTRFS_I(dir
), index
);
4194 btrfs_abort_transaction(trans
, ret
);
4198 btrfs_i_size_write(dir
, dir
->i_size
- name_len
* 2);
4199 inode_inc_iversion(dir
);
4200 dir
->i_mtime
= dir
->i_ctime
= current_time(dir
);
4201 ret
= btrfs_update_inode_fallback(trans
, root
, dir
);
4203 btrfs_abort_transaction(trans
, ret
);
4205 btrfs_free_path(path
);
4209 static int btrfs_rmdir(struct inode
*dir
, struct dentry
*dentry
)
4211 struct inode
*inode
= d_inode(dentry
);
4213 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
4214 struct btrfs_trans_handle
*trans
;
4215 u64 last_unlink_trans
;
4217 if (inode
->i_size
> BTRFS_EMPTY_DIR_SIZE
)
4219 if (btrfs_ino(BTRFS_I(inode
)) == BTRFS_FIRST_FREE_OBJECTID
)
4222 trans
= __unlink_start_trans(dir
);
4224 return PTR_ERR(trans
);
4226 if (unlikely(btrfs_ino(BTRFS_I(inode
)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID
)) {
4227 err
= btrfs_unlink_subvol(trans
, root
, dir
,
4228 BTRFS_I(inode
)->location
.objectid
,
4229 dentry
->d_name
.name
,
4230 dentry
->d_name
.len
);
4234 err
= btrfs_orphan_add(trans
, inode
);
4238 last_unlink_trans
= BTRFS_I(inode
)->last_unlink_trans
;
4240 /* now the directory is empty */
4241 err
= btrfs_unlink_inode(trans
, root
, BTRFS_I(dir
),
4242 BTRFS_I(d_inode(dentry
)), dentry
->d_name
.name
,
4243 dentry
->d_name
.len
);
4245 btrfs_i_size_write(inode
, 0);
4247 * Propagate the last_unlink_trans value of the deleted dir to
4248 * its parent directory. This is to prevent an unrecoverable
4249 * log tree in the case we do something like this:
4251 * 2) create snapshot under dir foo
4252 * 3) delete the snapshot
4255 * 6) fsync foo or some file inside foo
4257 if (last_unlink_trans
>= trans
->transid
)
4258 BTRFS_I(dir
)->last_unlink_trans
= last_unlink_trans
;
4261 btrfs_end_transaction(trans
);
4262 btrfs_btree_balance_dirty(root
->fs_info
);
4267 static int truncate_space_check(struct btrfs_trans_handle
*trans
,
4268 struct btrfs_root
*root
,
4271 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
4275 * This is only used to apply pressure to the enospc system, we don't
4276 * intend to use this reservation at all.
4278 bytes_deleted
= btrfs_csum_bytes_to_leaves(fs_info
, bytes_deleted
);
4279 bytes_deleted
*= fs_info
->nodesize
;
4280 ret
= btrfs_block_rsv_add(root
, &fs_info
->trans_block_rsv
,
4281 bytes_deleted
, BTRFS_RESERVE_NO_FLUSH
);
4283 trace_btrfs_space_reservation(fs_info
, "transaction",
4286 trans
->bytes_reserved
+= bytes_deleted
;
4292 static int truncate_inline_extent(struct inode
*inode
,
4293 struct btrfs_path
*path
,
4294 struct btrfs_key
*found_key
,
4298 struct extent_buffer
*leaf
= path
->nodes
[0];
4299 int slot
= path
->slots
[0];
4300 struct btrfs_file_extent_item
*fi
;
4301 u32 size
= (u32
)(new_size
- found_key
->offset
);
4302 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
4304 fi
= btrfs_item_ptr(leaf
, slot
, struct btrfs_file_extent_item
);
4306 if (btrfs_file_extent_compression(leaf
, fi
) != BTRFS_COMPRESS_NONE
) {
4307 loff_t offset
= new_size
;
4308 loff_t page_end
= ALIGN(offset
, PAGE_SIZE
);
4311 * Zero out the remaining of the last page of our inline extent,
4312 * instead of directly truncating our inline extent here - that
4313 * would be much more complex (decompressing all the data, then
4314 * compressing the truncated data, which might be bigger than
4315 * the size of the inline extent, resize the extent, etc).
4316 * We release the path because to get the page we might need to
4317 * read the extent item from disk (data not in the page cache).
4319 btrfs_release_path(path
);
4320 return btrfs_truncate_block(inode
, offset
, page_end
- offset
,
4324 btrfs_set_file_extent_ram_bytes(leaf
, fi
, size
);
4325 size
= btrfs_file_extent_calc_inline_size(size
);
4326 btrfs_truncate_item(root
->fs_info
, path
, size
, 1);
4328 if (test_bit(BTRFS_ROOT_REF_COWS
, &root
->state
))
4329 inode_sub_bytes(inode
, item_end
+ 1 - new_size
);
4335 * this can truncate away extent items, csum items and directory items.
4336 * It starts at a high offset and removes keys until it can't find
4337 * any higher than new_size
4339 * csum items that cross the new i_size are truncated to the new size
4342 * min_type is the minimum key type to truncate down to. If set to 0, this
4343 * will kill all the items on this inode, including the INODE_ITEM_KEY.
4345 int btrfs_truncate_inode_items(struct btrfs_trans_handle
*trans
,
4346 struct btrfs_root
*root
,
4347 struct inode
*inode
,
4348 u64 new_size
, u32 min_type
)
4350 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
4351 struct btrfs_path
*path
;
4352 struct extent_buffer
*leaf
;
4353 struct btrfs_file_extent_item
*fi
;
4354 struct btrfs_key key
;
4355 struct btrfs_key found_key
;
4356 u64 extent_start
= 0;
4357 u64 extent_num_bytes
= 0;
4358 u64 extent_offset
= 0;
4360 u64 last_size
= new_size
;
4361 u32 found_type
= (u8
)-1;
4364 int pending_del_nr
= 0;
4365 int pending_del_slot
= 0;
4366 int extent_type
= -1;
4369 u64 ino
= btrfs_ino(BTRFS_I(inode
));
4370 u64 bytes_deleted
= 0;
4372 bool should_throttle
= 0;
4373 bool should_end
= 0;
4375 BUG_ON(new_size
> 0 && min_type
!= BTRFS_EXTENT_DATA_KEY
);
4378 * for non-free space inodes and ref cows, we want to back off from
4381 if (!btrfs_is_free_space_inode(inode
) &&
4382 test_bit(BTRFS_ROOT_REF_COWS
, &root
->state
))
4385 path
= btrfs_alloc_path();
4388 path
->reada
= READA_BACK
;
4391 * We want to drop from the next block forward in case this new size is
4392 * not block aligned since we will be keeping the last block of the
4393 * extent just the way it is.
4395 if (test_bit(BTRFS_ROOT_REF_COWS
, &root
->state
) ||
4396 root
== fs_info
->tree_root
)
4397 btrfs_drop_extent_cache(inode
, ALIGN(new_size
,
4398 fs_info
->sectorsize
),
4402 * This function is also used to drop the items in the log tree before
4403 * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4404 * it is used to drop the loged items. So we shouldn't kill the delayed
4407 if (min_type
== 0 && root
== BTRFS_I(inode
)->root
)
4408 btrfs_kill_delayed_inode_items(BTRFS_I(inode
));
4411 key
.offset
= (u64
)-1;
4416 * with a 16K leaf size and 128MB extents, you can actually queue
4417 * up a huge file in a single leaf. Most of the time that
4418 * bytes_deleted is > 0, it will be huge by the time we get here
4420 if (be_nice
&& bytes_deleted
> SZ_32M
) {
4421 if (btrfs_should_end_transaction(trans
)) {
4428 path
->leave_spinning
= 1;
4429 ret
= btrfs_search_slot(trans
, root
, &key
, path
, -1, 1);
4436 /* there are no items in the tree for us to truncate, we're
4439 if (path
->slots
[0] == 0)
4446 leaf
= path
->nodes
[0];
4447 btrfs_item_key_to_cpu(leaf
, &found_key
, path
->slots
[0]);
4448 found_type
= found_key
.type
;
4450 if (found_key
.objectid
!= ino
)
4453 if (found_type
< min_type
)
4456 item_end
= found_key
.offset
;
4457 if (found_type
== BTRFS_EXTENT_DATA_KEY
) {
4458 fi
= btrfs_item_ptr(leaf
, path
->slots
[0],
4459 struct btrfs_file_extent_item
);
4460 extent_type
= btrfs_file_extent_type(leaf
, fi
);
4461 if (extent_type
!= BTRFS_FILE_EXTENT_INLINE
) {
4463 btrfs_file_extent_num_bytes(leaf
, fi
);
4464 } else if (extent_type
== BTRFS_FILE_EXTENT_INLINE
) {
4465 item_end
+= btrfs_file_extent_inline_len(leaf
,
4466 path
->slots
[0], fi
);
4470 if (found_type
> min_type
) {
4473 if (item_end
< new_size
) {
4475 * With NO_HOLES mode, for the following mapping
4477 * [0-4k][hole][8k-12k]
4479 * if truncating isize down to 6k, it ends up
4482 if (btrfs_fs_incompat(root
->fs_info
, NO_HOLES
))
4483 last_size
= new_size
;
4486 if (found_key
.offset
>= new_size
)
4492 /* FIXME, shrink the extent if the ref count is only 1 */
4493 if (found_type
!= BTRFS_EXTENT_DATA_KEY
)
4497 last_size
= found_key
.offset
;
4499 last_size
= new_size
;
4501 if (extent_type
!= BTRFS_FILE_EXTENT_INLINE
) {
4503 extent_start
= btrfs_file_extent_disk_bytenr(leaf
, fi
);
4505 u64 orig_num_bytes
=
4506 btrfs_file_extent_num_bytes(leaf
, fi
);
4507 extent_num_bytes
= ALIGN(new_size
-
4509 fs_info
->sectorsize
);
4510 btrfs_set_file_extent_num_bytes(leaf
, fi
,
4512 num_dec
= (orig_num_bytes
-
4514 if (test_bit(BTRFS_ROOT_REF_COWS
,
4517 inode_sub_bytes(inode
, num_dec
);
4518 btrfs_mark_buffer_dirty(leaf
);
4521 btrfs_file_extent_disk_num_bytes(leaf
,
4523 extent_offset
= found_key
.offset
-
4524 btrfs_file_extent_offset(leaf
, fi
);
4526 /* FIXME blocksize != 4096 */
4527 num_dec
= btrfs_file_extent_num_bytes(leaf
, fi
);
4528 if (extent_start
!= 0) {
4530 if (test_bit(BTRFS_ROOT_REF_COWS
,
4532 inode_sub_bytes(inode
, num_dec
);
4535 } else if (extent_type
== BTRFS_FILE_EXTENT_INLINE
) {
4537 * we can't truncate inline items that have had
4541 btrfs_file_extent_encryption(leaf
, fi
) == 0 &&
4542 btrfs_file_extent_other_encoding(leaf
, fi
) == 0) {
4545 * Need to release path in order to truncate a
4546 * compressed extent. So delete any accumulated
4547 * extent items so far.
4549 if (btrfs_file_extent_compression(leaf
, fi
) !=
4550 BTRFS_COMPRESS_NONE
&& pending_del_nr
) {
4551 err
= btrfs_del_items(trans
, root
, path
,
4555 btrfs_abort_transaction(trans
,
4562 err
= truncate_inline_extent(inode
, path
,
4567 btrfs_abort_transaction(trans
, err
);
4570 } else if (test_bit(BTRFS_ROOT_REF_COWS
,
4572 inode_sub_bytes(inode
, item_end
+ 1 - new_size
);
4577 if (!pending_del_nr
) {
4578 /* no pending yet, add ourselves */
4579 pending_del_slot
= path
->slots
[0];
4581 } else if (pending_del_nr
&&
4582 path
->slots
[0] + 1 == pending_del_slot
) {
4583 /* hop on the pending chunk */
4585 pending_del_slot
= path
->slots
[0];
4592 should_throttle
= 0;
4595 (test_bit(BTRFS_ROOT_REF_COWS
, &root
->state
) ||
4596 root
== fs_info
->tree_root
)) {
4597 btrfs_set_path_blocking(path
);
4598 bytes_deleted
+= extent_num_bytes
;
4599 ret
= btrfs_free_extent(trans
, fs_info
, extent_start
,
4600 extent_num_bytes
, 0,
4601 btrfs_header_owner(leaf
),
4602 ino
, extent_offset
);
4604 if (btrfs_should_throttle_delayed_refs(trans
, fs_info
))
4605 btrfs_async_run_delayed_refs(fs_info
,
4606 trans
->delayed_ref_updates
* 2,
4609 if (truncate_space_check(trans
, root
,
4610 extent_num_bytes
)) {
4613 if (btrfs_should_throttle_delayed_refs(trans
,
4615 should_throttle
= 1;
4619 if (found_type
== BTRFS_INODE_ITEM_KEY
)
4622 if (path
->slots
[0] == 0 ||
4623 path
->slots
[0] != pending_del_slot
||
4624 should_throttle
|| should_end
) {
4625 if (pending_del_nr
) {
4626 ret
= btrfs_del_items(trans
, root
, path
,
4630 btrfs_abort_transaction(trans
, ret
);
4635 btrfs_release_path(path
);
4636 if (should_throttle
) {
4637 unsigned long updates
= trans
->delayed_ref_updates
;
4639 trans
->delayed_ref_updates
= 0;
4640 ret
= btrfs_run_delayed_refs(trans
,
4648 * if we failed to refill our space rsv, bail out
4649 * and let the transaction restart
4661 if (pending_del_nr
) {
4662 ret
= btrfs_del_items(trans
, root
, path
, pending_del_slot
,
4665 btrfs_abort_transaction(trans
, ret
);
4668 if (root
->root_key
.objectid
!= BTRFS_TREE_LOG_OBJECTID
)
4669 btrfs_ordered_update_i_size(inode
, last_size
, NULL
);
4671 btrfs_free_path(path
);
4674 /* only inline file may have last_size != new_size */
4675 if (new_size
>= fs_info
->sectorsize
||
4676 new_size
> fs_info
->max_inline
)
4677 ASSERT(last_size
== new_size
);
4680 if (be_nice
&& bytes_deleted
> SZ_32M
) {
4681 unsigned long updates
= trans
->delayed_ref_updates
;
4683 trans
->delayed_ref_updates
= 0;
4684 ret
= btrfs_run_delayed_refs(trans
, fs_info
,
4694 * btrfs_truncate_block - read, zero a chunk and write a block
4695 * @inode - inode that we're zeroing
4696 * @from - the offset to start zeroing
4697 * @len - the length to zero, 0 to zero the entire range respective to the
4699 * @front - zero up to the offset instead of from the offset on
4701 * This will find the block for the "from" offset and cow the block and zero the
4702 * part we want to zero. This is used with truncate and hole punching.
4704 int btrfs_truncate_block(struct inode
*inode
, loff_t from
, loff_t len
,
4707 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
4708 struct address_space
*mapping
= inode
->i_mapping
;
4709 struct extent_io_tree
*io_tree
= &BTRFS_I(inode
)->io_tree
;
4710 struct btrfs_ordered_extent
*ordered
;
4711 struct extent_state
*cached_state
= NULL
;
4713 u32 blocksize
= fs_info
->sectorsize
;
4714 pgoff_t index
= from
>> PAGE_SHIFT
;
4715 unsigned offset
= from
& (blocksize
- 1);
4717 gfp_t mask
= btrfs_alloc_write_mask(mapping
);
4722 if ((offset
& (blocksize
- 1)) == 0 &&
4723 (!len
|| ((len
& (blocksize
- 1)) == 0)))
4726 ret
= btrfs_delalloc_reserve_space(inode
,
4727 round_down(from
, blocksize
), blocksize
);
4732 page
= find_or_create_page(mapping
, index
, mask
);
4734 btrfs_delalloc_release_space(inode
,
4735 round_down(from
, blocksize
),
4741 block_start
= round_down(from
, blocksize
);
4742 block_end
= block_start
+ blocksize
- 1;
4744 if (!PageUptodate(page
)) {
4745 ret
= btrfs_readpage(NULL
, page
);
4747 if (page
->mapping
!= mapping
) {
4752 if (!PageUptodate(page
)) {
4757 wait_on_page_writeback(page
);
4759 lock_extent_bits(io_tree
, block_start
, block_end
, &cached_state
);
4760 set_page_extent_mapped(page
);
4762 ordered
= btrfs_lookup_ordered_extent(inode
, block_start
);
4764 unlock_extent_cached(io_tree
, block_start
, block_end
,
4765 &cached_state
, GFP_NOFS
);
4768 btrfs_start_ordered_extent(inode
, ordered
, 1);
4769 btrfs_put_ordered_extent(ordered
);
4773 clear_extent_bit(&BTRFS_I(inode
)->io_tree
, block_start
, block_end
,
4774 EXTENT_DIRTY
| EXTENT_DELALLOC
|
4775 EXTENT_DO_ACCOUNTING
| EXTENT_DEFRAG
,
4776 0, 0, &cached_state
, GFP_NOFS
);
4778 ret
= btrfs_set_extent_delalloc(inode
, block_start
, block_end
,
4781 unlock_extent_cached(io_tree
, block_start
, block_end
,
4782 &cached_state
, GFP_NOFS
);
4786 if (offset
!= blocksize
) {
4788 len
= blocksize
- offset
;
4791 memset(kaddr
+ (block_start
- page_offset(page
)),
4794 memset(kaddr
+ (block_start
- page_offset(page
)) + offset
,
4796 flush_dcache_page(page
);
4799 ClearPageChecked(page
);
4800 set_page_dirty(page
);
4801 unlock_extent_cached(io_tree
, block_start
, block_end
, &cached_state
,
4806 btrfs_delalloc_release_space(inode
, block_start
,
4814 static int maybe_insert_hole(struct btrfs_root
*root
, struct inode
*inode
,
4815 u64 offset
, u64 len
)
4817 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
4818 struct btrfs_trans_handle
*trans
;
4822 * Still need to make sure the inode looks like it's been updated so
4823 * that any holes get logged if we fsync.
4825 if (btrfs_fs_incompat(fs_info
, NO_HOLES
)) {
4826 BTRFS_I(inode
)->last_trans
= fs_info
->generation
;
4827 BTRFS_I(inode
)->last_sub_trans
= root
->log_transid
;
4828 BTRFS_I(inode
)->last_log_commit
= root
->last_log_commit
;
4833 * 1 - for the one we're dropping
4834 * 1 - for the one we're adding
4835 * 1 - for updating the inode.
4837 trans
= btrfs_start_transaction(root
, 3);
4839 return PTR_ERR(trans
);
4841 ret
= btrfs_drop_extents(trans
, root
, inode
, offset
, offset
+ len
, 1);
4843 btrfs_abort_transaction(trans
, ret
);
4844 btrfs_end_transaction(trans
);
4848 ret
= btrfs_insert_file_extent(trans
, root
, btrfs_ino(BTRFS_I(inode
)),
4849 offset
, 0, 0, len
, 0, len
, 0, 0, 0);
4851 btrfs_abort_transaction(trans
, ret
);
4853 btrfs_update_inode(trans
, root
, inode
);
4854 btrfs_end_transaction(trans
);
4859 * This function puts in dummy file extents for the area we're creating a hole
4860 * for. So if we are truncating this file to a larger size we need to insert
4861 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4862 * the range between oldsize and size
4864 int btrfs_cont_expand(struct inode
*inode
, loff_t oldsize
, loff_t size
)
4866 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
4867 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
4868 struct extent_io_tree
*io_tree
= &BTRFS_I(inode
)->io_tree
;
4869 struct extent_map
*em
= NULL
;
4870 struct extent_state
*cached_state
= NULL
;
4871 struct extent_map_tree
*em_tree
= &BTRFS_I(inode
)->extent_tree
;
4872 u64 hole_start
= ALIGN(oldsize
, fs_info
->sectorsize
);
4873 u64 block_end
= ALIGN(size
, fs_info
->sectorsize
);
4880 * If our size started in the middle of a block we need to zero out the
4881 * rest of the block before we expand the i_size, otherwise we could
4882 * expose stale data.
4884 err
= btrfs_truncate_block(inode
, oldsize
, 0, 0);
4888 if (size
<= hole_start
)
4892 struct btrfs_ordered_extent
*ordered
;
4894 lock_extent_bits(io_tree
, hole_start
, block_end
- 1,
4896 ordered
= btrfs_lookup_ordered_range(inode
, hole_start
,
4897 block_end
- hole_start
);
4900 unlock_extent_cached(io_tree
, hole_start
, block_end
- 1,
4901 &cached_state
, GFP_NOFS
);
4902 btrfs_start_ordered_extent(inode
, ordered
, 1);
4903 btrfs_put_ordered_extent(ordered
);
4906 cur_offset
= hole_start
;
4908 em
= btrfs_get_extent(inode
, NULL
, 0, cur_offset
,
4909 block_end
- cur_offset
, 0);
4915 last_byte
= min(extent_map_end(em
), block_end
);
4916 last_byte
= ALIGN(last_byte
, fs_info
->sectorsize
);
4917 if (!test_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
)) {
4918 struct extent_map
*hole_em
;
4919 hole_size
= last_byte
- cur_offset
;
4921 err
= maybe_insert_hole(root
, inode
, cur_offset
,
4925 btrfs_drop_extent_cache(inode
, cur_offset
,
4926 cur_offset
+ hole_size
- 1, 0);
4927 hole_em
= alloc_extent_map();
4929 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC
,
4930 &BTRFS_I(inode
)->runtime_flags
);
4933 hole_em
->start
= cur_offset
;
4934 hole_em
->len
= hole_size
;
4935 hole_em
->orig_start
= cur_offset
;
4937 hole_em
->block_start
= EXTENT_MAP_HOLE
;
4938 hole_em
->block_len
= 0;
4939 hole_em
->orig_block_len
= 0;
4940 hole_em
->ram_bytes
= hole_size
;
4941 hole_em
->bdev
= fs_info
->fs_devices
->latest_bdev
;
4942 hole_em
->compress_type
= BTRFS_COMPRESS_NONE
;
4943 hole_em
->generation
= fs_info
->generation
;
4946 write_lock(&em_tree
->lock
);
4947 err
= add_extent_mapping(em_tree
, hole_em
, 1);
4948 write_unlock(&em_tree
->lock
);
4951 btrfs_drop_extent_cache(inode
, cur_offset
,
4955 free_extent_map(hole_em
);
4958 free_extent_map(em
);
4960 cur_offset
= last_byte
;
4961 if (cur_offset
>= block_end
)
4964 free_extent_map(em
);
4965 unlock_extent_cached(io_tree
, hole_start
, block_end
- 1, &cached_state
,
4970 static int btrfs_setsize(struct inode
*inode
, struct iattr
*attr
)
4972 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
4973 struct btrfs_trans_handle
*trans
;
4974 loff_t oldsize
= i_size_read(inode
);
4975 loff_t newsize
= attr
->ia_size
;
4976 int mask
= attr
->ia_valid
;
4980 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
4981 * special case where we need to update the times despite not having
4982 * these flags set. For all other operations the VFS set these flags
4983 * explicitly if it wants a timestamp update.
4985 if (newsize
!= oldsize
) {
4986 inode_inc_iversion(inode
);
4987 if (!(mask
& (ATTR_CTIME
| ATTR_MTIME
)))
4988 inode
->i_ctime
= inode
->i_mtime
=
4989 current_time(inode
);
4992 if (newsize
> oldsize
) {
4994 * Don't do an expanding truncate while snapshoting is ongoing.
4995 * This is to ensure the snapshot captures a fully consistent
4996 * state of this file - if the snapshot captures this expanding
4997 * truncation, it must capture all writes that happened before
5000 btrfs_wait_for_snapshot_creation(root
);
5001 ret
= btrfs_cont_expand(inode
, oldsize
, newsize
);
5003 btrfs_end_write_no_snapshoting(root
);
5007 trans
= btrfs_start_transaction(root
, 1);
5008 if (IS_ERR(trans
)) {
5009 btrfs_end_write_no_snapshoting(root
);
5010 return PTR_ERR(trans
);
5013 i_size_write(inode
, newsize
);
5014 btrfs_ordered_update_i_size(inode
, i_size_read(inode
), NULL
);
5015 pagecache_isize_extended(inode
, oldsize
, newsize
);
5016 ret
= btrfs_update_inode(trans
, root
, inode
);
5017 btrfs_end_write_no_snapshoting(root
);
5018 btrfs_end_transaction(trans
);
5022 * We're truncating a file that used to have good data down to
5023 * zero. Make sure it gets into the ordered flush list so that
5024 * any new writes get down to disk quickly.
5027 set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE
,
5028 &BTRFS_I(inode
)->runtime_flags
);
5031 * 1 for the orphan item we're going to add
5032 * 1 for the orphan item deletion.
5034 trans
= btrfs_start_transaction(root
, 2);
5036 return PTR_ERR(trans
);
5039 * We need to do this in case we fail at _any_ point during the
5040 * actual truncate. Once we do the truncate_setsize we could
5041 * invalidate pages which forces any outstanding ordered io to
5042 * be instantly completed which will give us extents that need
5043 * to be truncated. If we fail to get an orphan inode down we
5044 * could have left over extents that were never meant to live,
5045 * so we need to guarantee from this point on that everything
5046 * will be consistent.
5048 ret
= btrfs_orphan_add(trans
, inode
);
5049 btrfs_end_transaction(trans
);
5053 /* we don't support swapfiles, so vmtruncate shouldn't fail */
5054 truncate_setsize(inode
, newsize
);
5056 /* Disable nonlocked read DIO to avoid the end less truncate */
5057 btrfs_inode_block_unlocked_dio(inode
);
5058 inode_dio_wait(inode
);
5059 btrfs_inode_resume_unlocked_dio(inode
);
5061 ret
= btrfs_truncate(inode
);
5062 if (ret
&& inode
->i_nlink
) {
5065 /* To get a stable disk_i_size */
5066 err
= btrfs_wait_ordered_range(inode
, 0, (u64
)-1);
5068 btrfs_orphan_del(NULL
, inode
);
5073 * failed to truncate, disk_i_size is only adjusted down
5074 * as we remove extents, so it should represent the true
5075 * size of the inode, so reset the in memory size and
5076 * delete our orphan entry.
5078 trans
= btrfs_join_transaction(root
);
5079 if (IS_ERR(trans
)) {
5080 btrfs_orphan_del(NULL
, inode
);
5083 i_size_write(inode
, BTRFS_I(inode
)->disk_i_size
);
5084 err
= btrfs_orphan_del(trans
, inode
);
5086 btrfs_abort_transaction(trans
, err
);
5087 btrfs_end_transaction(trans
);
5094 static int btrfs_setattr(struct dentry
*dentry
, struct iattr
*attr
)
5096 struct inode
*inode
= d_inode(dentry
);
5097 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
5100 if (btrfs_root_readonly(root
))
5103 err
= setattr_prepare(dentry
, attr
);
5107 if (S_ISREG(inode
->i_mode
) && (attr
->ia_valid
& ATTR_SIZE
)) {
5108 err
= btrfs_setsize(inode
, attr
);
5113 if (attr
->ia_valid
) {
5114 setattr_copy(inode
, attr
);
5115 inode_inc_iversion(inode
);
5116 err
= btrfs_dirty_inode(inode
);
5118 if (!err
&& attr
->ia_valid
& ATTR_MODE
)
5119 err
= posix_acl_chmod(inode
, inode
->i_mode
);
5126 * While truncating the inode pages during eviction, we get the VFS calling
5127 * btrfs_invalidatepage() against each page of the inode. This is slow because
5128 * the calls to btrfs_invalidatepage() result in a huge amount of calls to
5129 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
5130 * extent_state structures over and over, wasting lots of time.
5132 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
5133 * those expensive operations on a per page basis and do only the ordered io
5134 * finishing, while we release here the extent_map and extent_state structures,
5135 * without the excessive merging and splitting.
5137 static void evict_inode_truncate_pages(struct inode
*inode
)
5139 struct extent_io_tree
*io_tree
= &BTRFS_I(inode
)->io_tree
;
5140 struct extent_map_tree
*map_tree
= &BTRFS_I(inode
)->extent_tree
;
5141 struct rb_node
*node
;
5143 ASSERT(inode
->i_state
& I_FREEING
);
5144 truncate_inode_pages_final(&inode
->i_data
);
5146 write_lock(&map_tree
->lock
);
5147 while (!RB_EMPTY_ROOT(&map_tree
->map
)) {
5148 struct extent_map
*em
;
5150 node
= rb_first(&map_tree
->map
);
5151 em
= rb_entry(node
, struct extent_map
, rb_node
);
5152 clear_bit(EXTENT_FLAG_PINNED
, &em
->flags
);
5153 clear_bit(EXTENT_FLAG_LOGGING
, &em
->flags
);
5154 remove_extent_mapping(map_tree
, em
);
5155 free_extent_map(em
);
5156 if (need_resched()) {
5157 write_unlock(&map_tree
->lock
);
5159 write_lock(&map_tree
->lock
);
5162 write_unlock(&map_tree
->lock
);
5165 * Keep looping until we have no more ranges in the io tree.
5166 * We can have ongoing bios started by readpages (called from readahead)
5167 * that have their endio callback (extent_io.c:end_bio_extent_readpage)
5168 * still in progress (unlocked the pages in the bio but did not yet
5169 * unlocked the ranges in the io tree). Therefore this means some
5170 * ranges can still be locked and eviction started because before
5171 * submitting those bios, which are executed by a separate task (work
5172 * queue kthread), inode references (inode->i_count) were not taken
5173 * (which would be dropped in the end io callback of each bio).
5174 * Therefore here we effectively end up waiting for those bios and
5175 * anyone else holding locked ranges without having bumped the inode's
5176 * reference count - if we don't do it, when they access the inode's
5177 * io_tree to unlock a range it may be too late, leading to an
5178 * use-after-free issue.
5180 spin_lock(&io_tree
->lock
);
5181 while (!RB_EMPTY_ROOT(&io_tree
->state
)) {
5182 struct extent_state
*state
;
5183 struct extent_state
*cached_state
= NULL
;
5187 node
= rb_first(&io_tree
->state
);
5188 state
= rb_entry(node
, struct extent_state
, rb_node
);
5189 start
= state
->start
;
5191 spin_unlock(&io_tree
->lock
);
5193 lock_extent_bits(io_tree
, start
, end
, &cached_state
);
5196 * If still has DELALLOC flag, the extent didn't reach disk,
5197 * and its reserved space won't be freed by delayed_ref.
5198 * So we need to free its reserved space here.
5199 * (Refer to comment in btrfs_invalidatepage, case 2)
5201 * Note, end is the bytenr of last byte, so we need + 1 here.
5203 if (state
->state
& EXTENT_DELALLOC
)
5204 btrfs_qgroup_free_data(inode
, start
, end
- start
+ 1);
5206 clear_extent_bit(io_tree
, start
, end
,
5207 EXTENT_LOCKED
| EXTENT_DIRTY
|
5208 EXTENT_DELALLOC
| EXTENT_DO_ACCOUNTING
|
5209 EXTENT_DEFRAG
, 1, 1,
5210 &cached_state
, GFP_NOFS
);
5213 spin_lock(&io_tree
->lock
);
5215 spin_unlock(&io_tree
->lock
);
5218 void btrfs_evict_inode(struct inode
*inode
)
5220 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
5221 struct btrfs_trans_handle
*trans
;
5222 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
5223 struct btrfs_block_rsv
*rsv
, *global_rsv
;
5224 int steal_from_global
= 0;
5228 trace_btrfs_inode_evict(inode
);
5231 kmem_cache_free(btrfs_inode_cachep
, BTRFS_I(inode
));
5235 min_size
= btrfs_calc_trunc_metadata_size(fs_info
, 1);
5237 evict_inode_truncate_pages(inode
);
5239 if (inode
->i_nlink
&&
5240 ((btrfs_root_refs(&root
->root_item
) != 0 &&
5241 root
->root_key
.objectid
!= BTRFS_ROOT_TREE_OBJECTID
) ||
5242 btrfs_is_free_space_inode(inode
)))
5245 if (is_bad_inode(inode
)) {
5246 btrfs_orphan_del(NULL
, inode
);
5249 /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
5250 if (!special_file(inode
->i_mode
))
5251 btrfs_wait_ordered_range(inode
, 0, (u64
)-1);
5253 btrfs_free_io_failure_record(inode
, 0, (u64
)-1);
5255 if (test_bit(BTRFS_FS_LOG_RECOVERING
, &fs_info
->flags
)) {
5256 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM
,
5257 &BTRFS_I(inode
)->runtime_flags
));
5261 if (inode
->i_nlink
> 0) {
5262 BUG_ON(btrfs_root_refs(&root
->root_item
) != 0 &&
5263 root
->root_key
.objectid
!= BTRFS_ROOT_TREE_OBJECTID
);
5267 ret
= btrfs_commit_inode_delayed_inode(BTRFS_I(inode
));
5269 btrfs_orphan_del(NULL
, inode
);
5273 rsv
= btrfs_alloc_block_rsv(fs_info
, BTRFS_BLOCK_RSV_TEMP
);
5275 btrfs_orphan_del(NULL
, inode
);
5278 rsv
->size
= min_size
;
5280 global_rsv
= &fs_info
->global_block_rsv
;
5282 btrfs_i_size_write(inode
, 0);
5285 * This is a bit simpler than btrfs_truncate since we've already
5286 * reserved our space for our orphan item in the unlink, so we just
5287 * need to reserve some slack space in case we add bytes and update
5288 * inode item when doing the truncate.
5291 ret
= btrfs_block_rsv_refill(root
, rsv
, min_size
,
5292 BTRFS_RESERVE_FLUSH_LIMIT
);
5295 * Try and steal from the global reserve since we will
5296 * likely not use this space anyway, we want to try as
5297 * hard as possible to get this to work.
5300 steal_from_global
++;
5302 steal_from_global
= 0;
5306 * steal_from_global == 0: we reserved stuff, hooray!
5307 * steal_from_global == 1: we didn't reserve stuff, boo!
5308 * steal_from_global == 2: we've committed, still not a lot of
5309 * room but maybe we'll have room in the global reserve this
5311 * steal_from_global == 3: abandon all hope!
5313 if (steal_from_global
> 2) {
5315 "Could not get space for a delete, will truncate on mount %d",
5317 btrfs_orphan_del(NULL
, inode
);
5318 btrfs_free_block_rsv(fs_info
, rsv
);
5322 trans
= btrfs_join_transaction(root
);
5323 if (IS_ERR(trans
)) {
5324 btrfs_orphan_del(NULL
, inode
);
5325 btrfs_free_block_rsv(fs_info
, rsv
);
5330 * We can't just steal from the global reserve, we need to make
5331 * sure there is room to do it, if not we need to commit and try
5334 if (steal_from_global
) {
5335 if (!btrfs_check_space_for_delayed_refs(trans
, fs_info
))
5336 ret
= btrfs_block_rsv_migrate(global_rsv
, rsv
,
5343 * Couldn't steal from the global reserve, we have too much
5344 * pending stuff built up, commit the transaction and try it
5348 ret
= btrfs_commit_transaction(trans
);
5350 btrfs_orphan_del(NULL
, inode
);
5351 btrfs_free_block_rsv(fs_info
, rsv
);
5356 steal_from_global
= 0;
5359 trans
->block_rsv
= rsv
;
5361 ret
= btrfs_truncate_inode_items(trans
, root
, inode
, 0, 0);
5362 if (ret
!= -ENOSPC
&& ret
!= -EAGAIN
)
5365 trans
->block_rsv
= &fs_info
->trans_block_rsv
;
5366 btrfs_end_transaction(trans
);
5368 btrfs_btree_balance_dirty(fs_info
);
5371 btrfs_free_block_rsv(fs_info
, rsv
);
5374 * Errors here aren't a big deal, it just means we leave orphan items
5375 * in the tree. They will be cleaned up on the next mount.
5378 trans
->block_rsv
= root
->orphan_block_rsv
;
5379 btrfs_orphan_del(trans
, inode
);
5381 btrfs_orphan_del(NULL
, inode
);
5384 trans
->block_rsv
= &fs_info
->trans_block_rsv
;
5385 if (!(root
== fs_info
->tree_root
||
5386 root
->root_key
.objectid
== BTRFS_TREE_RELOC_OBJECTID
))
5387 btrfs_return_ino(root
, btrfs_ino(BTRFS_I(inode
)));
5389 btrfs_end_transaction(trans
);
5390 btrfs_btree_balance_dirty(fs_info
);
5392 btrfs_remove_delayed_node(BTRFS_I(inode
));
5397 * this returns the key found in the dir entry in the location pointer.
5398 * If no dir entries were found, location->objectid is 0.
5400 static int btrfs_inode_by_name(struct inode
*dir
, struct dentry
*dentry
,
5401 struct btrfs_key
*location
)
5403 const char *name
= dentry
->d_name
.name
;
5404 int namelen
= dentry
->d_name
.len
;
5405 struct btrfs_dir_item
*di
;
5406 struct btrfs_path
*path
;
5407 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
5410 path
= btrfs_alloc_path();
5414 di
= btrfs_lookup_dir_item(NULL
, root
, path
, btrfs_ino(BTRFS_I(dir
)),
5419 if (IS_ERR_OR_NULL(di
))
5422 btrfs_dir_item_key_to_cpu(path
->nodes
[0], di
, location
);
5424 btrfs_free_path(path
);
5427 location
->objectid
= 0;
5432 * when we hit a tree root in a directory, the btrfs part of the inode
5433 * needs to be changed to reflect the root directory of the tree root. This
5434 * is kind of like crossing a mount point.
5436 static int fixup_tree_root_location(struct btrfs_fs_info
*fs_info
,
5438 struct dentry
*dentry
,
5439 struct btrfs_key
*location
,
5440 struct btrfs_root
**sub_root
)
5442 struct btrfs_path
*path
;
5443 struct btrfs_root
*new_root
;
5444 struct btrfs_root_ref
*ref
;
5445 struct extent_buffer
*leaf
;
5446 struct btrfs_key key
;
5450 path
= btrfs_alloc_path();
5457 key
.objectid
= BTRFS_I(dir
)->root
->root_key
.objectid
;
5458 key
.type
= BTRFS_ROOT_REF_KEY
;
5459 key
.offset
= location
->objectid
;
5461 ret
= btrfs_search_slot(NULL
, fs_info
->tree_root
, &key
, path
, 0, 0);
5468 leaf
= path
->nodes
[0];
5469 ref
= btrfs_item_ptr(leaf
, path
->slots
[0], struct btrfs_root_ref
);
5470 if (btrfs_root_ref_dirid(leaf
, ref
) != btrfs_ino(BTRFS_I(dir
)) ||
5471 btrfs_root_ref_name_len(leaf
, ref
) != dentry
->d_name
.len
)
5474 ret
= memcmp_extent_buffer(leaf
, dentry
->d_name
.name
,
5475 (unsigned long)(ref
+ 1),
5476 dentry
->d_name
.len
);
5480 btrfs_release_path(path
);
5482 new_root
= btrfs_read_fs_root_no_name(fs_info
, location
);
5483 if (IS_ERR(new_root
)) {
5484 err
= PTR_ERR(new_root
);
5488 *sub_root
= new_root
;
5489 location
->objectid
= btrfs_root_dirid(&new_root
->root_item
);
5490 location
->type
= BTRFS_INODE_ITEM_KEY
;
5491 location
->offset
= 0;
5494 btrfs_free_path(path
);
5498 static void inode_tree_add(struct inode
*inode
)
5500 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
5501 struct btrfs_inode
*entry
;
5503 struct rb_node
*parent
;
5504 struct rb_node
*new = &BTRFS_I(inode
)->rb_node
;
5505 u64 ino
= btrfs_ino(BTRFS_I(inode
));
5507 if (inode_unhashed(inode
))
5510 spin_lock(&root
->inode_lock
);
5511 p
= &root
->inode_tree
.rb_node
;
5514 entry
= rb_entry(parent
, struct btrfs_inode
, rb_node
);
5516 if (ino
< btrfs_ino(BTRFS_I(&entry
->vfs_inode
)))
5517 p
= &parent
->rb_left
;
5518 else if (ino
> btrfs_ino(BTRFS_I(&entry
->vfs_inode
)))
5519 p
= &parent
->rb_right
;
5521 WARN_ON(!(entry
->vfs_inode
.i_state
&
5522 (I_WILL_FREE
| I_FREEING
)));
5523 rb_replace_node(parent
, new, &root
->inode_tree
);
5524 RB_CLEAR_NODE(parent
);
5525 spin_unlock(&root
->inode_lock
);
5529 rb_link_node(new, parent
, p
);
5530 rb_insert_color(new, &root
->inode_tree
);
5531 spin_unlock(&root
->inode_lock
);
5534 static void inode_tree_del(struct inode
*inode
)
5536 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
5537 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
5540 spin_lock(&root
->inode_lock
);
5541 if (!RB_EMPTY_NODE(&BTRFS_I(inode
)->rb_node
)) {
5542 rb_erase(&BTRFS_I(inode
)->rb_node
, &root
->inode_tree
);
5543 RB_CLEAR_NODE(&BTRFS_I(inode
)->rb_node
);
5544 empty
= RB_EMPTY_ROOT(&root
->inode_tree
);
5546 spin_unlock(&root
->inode_lock
);
5548 if (empty
&& btrfs_root_refs(&root
->root_item
) == 0) {
5549 synchronize_srcu(&fs_info
->subvol_srcu
);
5550 spin_lock(&root
->inode_lock
);
5551 empty
= RB_EMPTY_ROOT(&root
->inode_tree
);
5552 spin_unlock(&root
->inode_lock
);
5554 btrfs_add_dead_root(root
);
5558 void btrfs_invalidate_inodes(struct btrfs_root
*root
)
5560 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
5561 struct rb_node
*node
;
5562 struct rb_node
*prev
;
5563 struct btrfs_inode
*entry
;
5564 struct inode
*inode
;
5567 if (!test_bit(BTRFS_FS_STATE_ERROR
, &fs_info
->fs_state
))
5568 WARN_ON(btrfs_root_refs(&root
->root_item
) != 0);
5570 spin_lock(&root
->inode_lock
);
5572 node
= root
->inode_tree
.rb_node
;
5576 entry
= rb_entry(node
, struct btrfs_inode
, rb_node
);
5578 if (objectid
< btrfs_ino(BTRFS_I(&entry
->vfs_inode
)))
5579 node
= node
->rb_left
;
5580 else if (objectid
> btrfs_ino(BTRFS_I(&entry
->vfs_inode
)))
5581 node
= node
->rb_right
;
5587 entry
= rb_entry(prev
, struct btrfs_inode
, rb_node
);
5588 if (objectid
<= btrfs_ino(BTRFS_I(&entry
->vfs_inode
))) {
5592 prev
= rb_next(prev
);
5596 entry
= rb_entry(node
, struct btrfs_inode
, rb_node
);
5597 objectid
= btrfs_ino(BTRFS_I(&entry
->vfs_inode
)) + 1;
5598 inode
= igrab(&entry
->vfs_inode
);
5600 spin_unlock(&root
->inode_lock
);
5601 if (atomic_read(&inode
->i_count
) > 1)
5602 d_prune_aliases(inode
);
5604 * btrfs_drop_inode will have it removed from
5605 * the inode cache when its usage count
5610 spin_lock(&root
->inode_lock
);
5614 if (cond_resched_lock(&root
->inode_lock
))
5617 node
= rb_next(node
);
5619 spin_unlock(&root
->inode_lock
);
5622 static int btrfs_init_locked_inode(struct inode
*inode
, void *p
)
5624 struct btrfs_iget_args
*args
= p
;
5625 inode
->i_ino
= args
->location
->objectid
;
5626 memcpy(&BTRFS_I(inode
)->location
, args
->location
,
5627 sizeof(*args
->location
));
5628 BTRFS_I(inode
)->root
= args
->root
;
5632 static int btrfs_find_actor(struct inode
*inode
, void *opaque
)
5634 struct btrfs_iget_args
*args
= opaque
;
5635 return args
->location
->objectid
== BTRFS_I(inode
)->location
.objectid
&&
5636 args
->root
== BTRFS_I(inode
)->root
;
5639 static struct inode
*btrfs_iget_locked(struct super_block
*s
,
5640 struct btrfs_key
*location
,
5641 struct btrfs_root
*root
)
5643 struct inode
*inode
;
5644 struct btrfs_iget_args args
;
5645 unsigned long hashval
= btrfs_inode_hash(location
->objectid
, root
);
5647 args
.location
= location
;
5650 inode
= iget5_locked(s
, hashval
, btrfs_find_actor
,
5651 btrfs_init_locked_inode
,
5656 /* Get an inode object given its location and corresponding root.
5657 * Returns in *is_new if the inode was read from disk
5659 struct inode
*btrfs_iget(struct super_block
*s
, struct btrfs_key
*location
,
5660 struct btrfs_root
*root
, int *new)
5662 struct inode
*inode
;
5664 inode
= btrfs_iget_locked(s
, location
, root
);
5666 return ERR_PTR(-ENOMEM
);
5668 if (inode
->i_state
& I_NEW
) {
5671 ret
= btrfs_read_locked_inode(inode
);
5672 if (!is_bad_inode(inode
)) {
5673 inode_tree_add(inode
);
5674 unlock_new_inode(inode
);
5678 unlock_new_inode(inode
);
5681 inode
= ERR_PTR(ret
< 0 ? ret
: -ESTALE
);
5688 static struct inode
*new_simple_dir(struct super_block
*s
,
5689 struct btrfs_key
*key
,
5690 struct btrfs_root
*root
)
5692 struct inode
*inode
= new_inode(s
);
5695 return ERR_PTR(-ENOMEM
);
5697 BTRFS_I(inode
)->root
= root
;
5698 memcpy(&BTRFS_I(inode
)->location
, key
, sizeof(*key
));
5699 set_bit(BTRFS_INODE_DUMMY
, &BTRFS_I(inode
)->runtime_flags
);
5701 inode
->i_ino
= BTRFS_EMPTY_SUBVOL_DIR_OBJECTID
;
5702 inode
->i_op
= &btrfs_dir_ro_inode_operations
;
5703 inode
->i_opflags
&= ~IOP_XATTR
;
5704 inode
->i_fop
= &simple_dir_operations
;
5705 inode
->i_mode
= S_IFDIR
| S_IRUGO
| S_IWUSR
| S_IXUGO
;
5706 inode
->i_mtime
= current_time(inode
);
5707 inode
->i_atime
= inode
->i_mtime
;
5708 inode
->i_ctime
= inode
->i_mtime
;
5709 BTRFS_I(inode
)->i_otime
= inode
->i_mtime
;
5714 struct inode
*btrfs_lookup_dentry(struct inode
*dir
, struct dentry
*dentry
)
5716 struct btrfs_fs_info
*fs_info
= btrfs_sb(dir
->i_sb
);
5717 struct inode
*inode
;
5718 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
5719 struct btrfs_root
*sub_root
= root
;
5720 struct btrfs_key location
;
5724 if (dentry
->d_name
.len
> BTRFS_NAME_LEN
)
5725 return ERR_PTR(-ENAMETOOLONG
);
5727 ret
= btrfs_inode_by_name(dir
, dentry
, &location
);
5729 return ERR_PTR(ret
);
5731 if (location
.objectid
== 0)
5732 return ERR_PTR(-ENOENT
);
5734 if (location
.type
== BTRFS_INODE_ITEM_KEY
) {
5735 inode
= btrfs_iget(dir
->i_sb
, &location
, root
, NULL
);
5739 BUG_ON(location
.type
!= BTRFS_ROOT_ITEM_KEY
);
5741 index
= srcu_read_lock(&fs_info
->subvol_srcu
);
5742 ret
= fixup_tree_root_location(fs_info
, dir
, dentry
,
5743 &location
, &sub_root
);
5746 inode
= ERR_PTR(ret
);
5748 inode
= new_simple_dir(dir
->i_sb
, &location
, sub_root
);
5750 inode
= btrfs_iget(dir
->i_sb
, &location
, sub_root
, NULL
);
5752 srcu_read_unlock(&fs_info
->subvol_srcu
, index
);
5754 if (!IS_ERR(inode
) && root
!= sub_root
) {
5755 down_read(&fs_info
->cleanup_work_sem
);
5756 if (!(inode
->i_sb
->s_flags
& MS_RDONLY
))
5757 ret
= btrfs_orphan_cleanup(sub_root
);
5758 up_read(&fs_info
->cleanup_work_sem
);
5761 inode
= ERR_PTR(ret
);
5768 static int btrfs_dentry_delete(const struct dentry
*dentry
)
5770 struct btrfs_root
*root
;
5771 struct inode
*inode
= d_inode(dentry
);
5773 if (!inode
&& !IS_ROOT(dentry
))
5774 inode
= d_inode(dentry
->d_parent
);
5777 root
= BTRFS_I(inode
)->root
;
5778 if (btrfs_root_refs(&root
->root_item
) == 0)
5781 if (btrfs_ino(BTRFS_I(inode
)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID
)
5787 static void btrfs_dentry_release(struct dentry
*dentry
)
5789 kfree(dentry
->d_fsdata
);
5792 static struct dentry
*btrfs_lookup(struct inode
*dir
, struct dentry
*dentry
,
5795 struct inode
*inode
;
5797 inode
= btrfs_lookup_dentry(dir
, dentry
);
5798 if (IS_ERR(inode
)) {
5799 if (PTR_ERR(inode
) == -ENOENT
)
5802 return ERR_CAST(inode
);
5805 return d_splice_alias(inode
, dentry
);
5808 unsigned char btrfs_filetype_table
[] = {
5809 DT_UNKNOWN
, DT_REG
, DT_DIR
, DT_CHR
, DT_BLK
, DT_FIFO
, DT_SOCK
, DT_LNK
5812 static int btrfs_real_readdir(struct file
*file
, struct dir_context
*ctx
)
5814 struct inode
*inode
= file_inode(file
);
5815 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
5816 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
5817 struct btrfs_item
*item
;
5818 struct btrfs_dir_item
*di
;
5819 struct btrfs_key key
;
5820 struct btrfs_key found_key
;
5821 struct btrfs_path
*path
;
5822 struct list_head ins_list
;
5823 struct list_head del_list
;
5825 struct extent_buffer
*leaf
;
5827 unsigned char d_type
;
5833 struct btrfs_key location
;
5835 if (!dir_emit_dots(file
, ctx
))
5838 path
= btrfs_alloc_path();
5842 path
->reada
= READA_FORWARD
;
5844 INIT_LIST_HEAD(&ins_list
);
5845 INIT_LIST_HEAD(&del_list
);
5846 put
= btrfs_readdir_get_delayed_items(inode
, &ins_list
, &del_list
);
5848 key
.type
= BTRFS_DIR_INDEX_KEY
;
5849 key
.offset
= ctx
->pos
;
5850 key
.objectid
= btrfs_ino(BTRFS_I(inode
));
5852 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
5857 leaf
= path
->nodes
[0];
5858 slot
= path
->slots
[0];
5859 if (slot
>= btrfs_header_nritems(leaf
)) {
5860 ret
= btrfs_next_leaf(root
, path
);
5868 item
= btrfs_item_nr(slot
);
5869 btrfs_item_key_to_cpu(leaf
, &found_key
, slot
);
5871 if (found_key
.objectid
!= key
.objectid
)
5873 if (found_key
.type
!= BTRFS_DIR_INDEX_KEY
)
5875 if (found_key
.offset
< ctx
->pos
)
5877 if (btrfs_should_delete_dir_index(&del_list
, found_key
.offset
))
5880 ctx
->pos
= found_key
.offset
;
5882 di
= btrfs_item_ptr(leaf
, slot
, struct btrfs_dir_item
);
5883 if (verify_dir_item(fs_info
, leaf
, di
))
5886 name_len
= btrfs_dir_name_len(leaf
, di
);
5887 if (name_len
<= sizeof(tmp_name
)) {
5888 name_ptr
= tmp_name
;
5890 name_ptr
= kmalloc(name_len
, GFP_KERNEL
);
5896 read_extent_buffer(leaf
, name_ptr
, (unsigned long)(di
+ 1),
5899 d_type
= btrfs_filetype_table
[btrfs_dir_type(leaf
, di
)];
5900 btrfs_dir_item_key_to_cpu(leaf
, di
, &location
);
5902 over
= !dir_emit(ctx
, name_ptr
, name_len
, location
.objectid
,
5905 if (name_ptr
!= tmp_name
)
5915 ret
= btrfs_readdir_delayed_dir_index(ctx
, &ins_list
);
5920 * Stop new entries from being returned after we return the last
5923 * New directory entries are assigned a strictly increasing
5924 * offset. This means that new entries created during readdir
5925 * are *guaranteed* to be seen in the future by that readdir.
5926 * This has broken buggy programs which operate on names as
5927 * they're returned by readdir. Until we re-use freed offsets
5928 * we have this hack to stop new entries from being returned
5929 * under the assumption that they'll never reach this huge
5932 * This is being careful not to overflow 32bit loff_t unless the
5933 * last entry requires it because doing so has broken 32bit apps
5936 if (ctx
->pos
>= INT_MAX
)
5937 ctx
->pos
= LLONG_MAX
;
5944 btrfs_readdir_put_delayed_items(inode
, &ins_list
, &del_list
);
5945 btrfs_free_path(path
);
5949 int btrfs_write_inode(struct inode
*inode
, struct writeback_control
*wbc
)
5951 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
5952 struct btrfs_trans_handle
*trans
;
5954 bool nolock
= false;
5956 if (test_bit(BTRFS_INODE_DUMMY
, &BTRFS_I(inode
)->runtime_flags
))
5959 if (btrfs_fs_closing(root
->fs_info
) && btrfs_is_free_space_inode(inode
))
5962 if (wbc
->sync_mode
== WB_SYNC_ALL
) {
5964 trans
= btrfs_join_transaction_nolock(root
);
5966 trans
= btrfs_join_transaction(root
);
5968 return PTR_ERR(trans
);
5969 ret
= btrfs_commit_transaction(trans
);
5975 * This is somewhat expensive, updating the tree every time the
5976 * inode changes. But, it is most likely to find the inode in cache.
5977 * FIXME, needs more benchmarking...there are no reasons other than performance
5978 * to keep or drop this code.
5980 static int btrfs_dirty_inode(struct inode
*inode
)
5982 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
5983 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
5984 struct btrfs_trans_handle
*trans
;
5987 if (test_bit(BTRFS_INODE_DUMMY
, &BTRFS_I(inode
)->runtime_flags
))
5990 trans
= btrfs_join_transaction(root
);
5992 return PTR_ERR(trans
);
5994 ret
= btrfs_update_inode(trans
, root
, inode
);
5995 if (ret
&& ret
== -ENOSPC
) {
5996 /* whoops, lets try again with the full transaction */
5997 btrfs_end_transaction(trans
);
5998 trans
= btrfs_start_transaction(root
, 1);
6000 return PTR_ERR(trans
);
6002 ret
= btrfs_update_inode(trans
, root
, inode
);
6004 btrfs_end_transaction(trans
);
6005 if (BTRFS_I(inode
)->delayed_node
)
6006 btrfs_balance_delayed_items(fs_info
);
6012 * This is a copy of file_update_time. We need this so we can return error on
6013 * ENOSPC for updating the inode in the case of file write and mmap writes.
6015 static int btrfs_update_time(struct inode
*inode
, struct timespec
*now
,
6018 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
6020 if (btrfs_root_readonly(root
))
6023 if (flags
& S_VERSION
)
6024 inode_inc_iversion(inode
);
6025 if (flags
& S_CTIME
)
6026 inode
->i_ctime
= *now
;
6027 if (flags
& S_MTIME
)
6028 inode
->i_mtime
= *now
;
6029 if (flags
& S_ATIME
)
6030 inode
->i_atime
= *now
;
6031 return btrfs_dirty_inode(inode
);
6035 * find the highest existing sequence number in a directory
6036 * and then set the in-memory index_cnt variable to reflect
6037 * free sequence numbers
6039 static int btrfs_set_inode_index_count(struct inode
*inode
)
6041 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
6042 struct btrfs_key key
, found_key
;
6043 struct btrfs_path
*path
;
6044 struct extent_buffer
*leaf
;
6047 key
.objectid
= btrfs_ino(BTRFS_I(inode
));
6048 key
.type
= BTRFS_DIR_INDEX_KEY
;
6049 key
.offset
= (u64
)-1;
6051 path
= btrfs_alloc_path();
6055 ret
= btrfs_search_slot(NULL
, root
, &key
, path
, 0, 0);
6058 /* FIXME: we should be able to handle this */
6064 * MAGIC NUMBER EXPLANATION:
6065 * since we search a directory based on f_pos we have to start at 2
6066 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
6067 * else has to start at 2
6069 if (path
->slots
[0] == 0) {
6070 BTRFS_I(inode
)->index_cnt
= 2;
6076 leaf
= path
->nodes
[0];
6077 btrfs_item_key_to_cpu(leaf
, &found_key
, path
->slots
[0]);
6079 if (found_key
.objectid
!= btrfs_ino(BTRFS_I(inode
)) ||
6080 found_key
.type
!= BTRFS_DIR_INDEX_KEY
) {
6081 BTRFS_I(inode
)->index_cnt
= 2;
6085 BTRFS_I(inode
)->index_cnt
= found_key
.offset
+ 1;
6087 btrfs_free_path(path
);
6092 * helper to find a free sequence number in a given directory. This current
6093 * code is very simple, later versions will do smarter things in the btree
6095 int btrfs_set_inode_index(struct inode
*dir
, u64
*index
)
6099 if (BTRFS_I(dir
)->index_cnt
== (u64
)-1) {
6100 ret
= btrfs_inode_delayed_dir_index_count(BTRFS_I(dir
));
6102 ret
= btrfs_set_inode_index_count(dir
);
6108 *index
= BTRFS_I(dir
)->index_cnt
;
6109 BTRFS_I(dir
)->index_cnt
++;
6114 static int btrfs_insert_inode_locked(struct inode
*inode
)
6116 struct btrfs_iget_args args
;
6117 args
.location
= &BTRFS_I(inode
)->location
;
6118 args
.root
= BTRFS_I(inode
)->root
;
6120 return insert_inode_locked4(inode
,
6121 btrfs_inode_hash(inode
->i_ino
, BTRFS_I(inode
)->root
),
6122 btrfs_find_actor
, &args
);
6125 static struct inode
*btrfs_new_inode(struct btrfs_trans_handle
*trans
,
6126 struct btrfs_root
*root
,
6128 const char *name
, int name_len
,
6129 u64 ref_objectid
, u64 objectid
,
6130 umode_t mode
, u64
*index
)
6132 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
6133 struct inode
*inode
;
6134 struct btrfs_inode_item
*inode_item
;
6135 struct btrfs_key
*location
;
6136 struct btrfs_path
*path
;
6137 struct btrfs_inode_ref
*ref
;
6138 struct btrfs_key key
[2];
6140 int nitems
= name
? 2 : 1;
6144 path
= btrfs_alloc_path();
6146 return ERR_PTR(-ENOMEM
);
6148 inode
= new_inode(fs_info
->sb
);
6150 btrfs_free_path(path
);
6151 return ERR_PTR(-ENOMEM
);
6155 * O_TMPFILE, set link count to 0, so that after this point,
6156 * we fill in an inode item with the correct link count.
6159 set_nlink(inode
, 0);
6162 * we have to initialize this early, so we can reclaim the inode
6163 * number if we fail afterwards in this function.
6165 inode
->i_ino
= objectid
;
6168 trace_btrfs_inode_request(dir
);
6170 ret
= btrfs_set_inode_index(dir
, index
);
6172 btrfs_free_path(path
);
6174 return ERR_PTR(ret
);
6180 * index_cnt is ignored for everything but a dir,
6181 * btrfs_get_inode_index_count has an explanation for the magic
6184 BTRFS_I(inode
)->index_cnt
= 2;
6185 BTRFS_I(inode
)->dir_index
= *index
;
6186 BTRFS_I(inode
)->root
= root
;
6187 BTRFS_I(inode
)->generation
= trans
->transid
;
6188 inode
->i_generation
= BTRFS_I(inode
)->generation
;
6191 * We could have gotten an inode number from somebody who was fsynced
6192 * and then removed in this same transaction, so let's just set full
6193 * sync since it will be a full sync anyway and this will blow away the
6194 * old info in the log.
6196 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC
, &BTRFS_I(inode
)->runtime_flags
);
6198 key
[0].objectid
= objectid
;
6199 key
[0].type
= BTRFS_INODE_ITEM_KEY
;
6202 sizes
[0] = sizeof(struct btrfs_inode_item
);
6206 * Start new inodes with an inode_ref. This is slightly more
6207 * efficient for small numbers of hard links since they will
6208 * be packed into one item. Extended refs will kick in if we
6209 * add more hard links than can fit in the ref item.
6211 key
[1].objectid
= objectid
;
6212 key
[1].type
= BTRFS_INODE_REF_KEY
;
6213 key
[1].offset
= ref_objectid
;
6215 sizes
[1] = name_len
+ sizeof(*ref
);
6218 location
= &BTRFS_I(inode
)->location
;
6219 location
->objectid
= objectid
;
6220 location
->offset
= 0;
6221 location
->type
= BTRFS_INODE_ITEM_KEY
;
6223 ret
= btrfs_insert_inode_locked(inode
);
6227 path
->leave_spinning
= 1;
6228 ret
= btrfs_insert_empty_items(trans
, root
, path
, key
, sizes
, nitems
);
6232 inode_init_owner(inode
, dir
, mode
);
6233 inode_set_bytes(inode
, 0);
6235 inode
->i_mtime
= current_time(inode
);
6236 inode
->i_atime
= inode
->i_mtime
;
6237 inode
->i_ctime
= inode
->i_mtime
;
6238 BTRFS_I(inode
)->i_otime
= inode
->i_mtime
;
6240 inode_item
= btrfs_item_ptr(path
->nodes
[0], path
->slots
[0],
6241 struct btrfs_inode_item
);
6242 memzero_extent_buffer(path
->nodes
[0], (unsigned long)inode_item
,
6243 sizeof(*inode_item
));
6244 fill_inode_item(trans
, path
->nodes
[0], inode_item
, inode
);
6247 ref
= btrfs_item_ptr(path
->nodes
[0], path
->slots
[0] + 1,
6248 struct btrfs_inode_ref
);
6249 btrfs_set_inode_ref_name_len(path
->nodes
[0], ref
, name_len
);
6250 btrfs_set_inode_ref_index(path
->nodes
[0], ref
, *index
);
6251 ptr
= (unsigned long)(ref
+ 1);
6252 write_extent_buffer(path
->nodes
[0], name
, ptr
, name_len
);
6255 btrfs_mark_buffer_dirty(path
->nodes
[0]);
6256 btrfs_free_path(path
);
6258 btrfs_inherit_iflags(inode
, dir
);
6260 if (S_ISREG(mode
)) {
6261 if (btrfs_test_opt(fs_info
, NODATASUM
))
6262 BTRFS_I(inode
)->flags
|= BTRFS_INODE_NODATASUM
;
6263 if (btrfs_test_opt(fs_info
, NODATACOW
))
6264 BTRFS_I(inode
)->flags
|= BTRFS_INODE_NODATACOW
|
6265 BTRFS_INODE_NODATASUM
;
6268 inode_tree_add(inode
);
6270 trace_btrfs_inode_new(inode
);
6271 btrfs_set_inode_last_trans(trans
, inode
);
6273 btrfs_update_root_times(trans
, root
);
6275 ret
= btrfs_inode_inherit_props(trans
, inode
, dir
);
6278 "error inheriting props for ino %llu (root %llu): %d",
6279 btrfs_ino(BTRFS_I(inode
)), root
->root_key
.objectid
, ret
);
6284 unlock_new_inode(inode
);
6287 BTRFS_I(dir
)->index_cnt
--;
6288 btrfs_free_path(path
);
6290 return ERR_PTR(ret
);
6293 static inline u8
btrfs_inode_type(struct inode
*inode
)
6295 return btrfs_type_by_mode
[(inode
->i_mode
& S_IFMT
) >> S_SHIFT
];
6299 * utility function to add 'inode' into 'parent_inode' with
6300 * a give name and a given sequence number.
6301 * if 'add_backref' is true, also insert a backref from the
6302 * inode to the parent directory.
6304 int btrfs_add_link(struct btrfs_trans_handle
*trans
,
6305 struct inode
*parent_inode
, struct inode
*inode
,
6306 const char *name
, int name_len
, int add_backref
, u64 index
)
6308 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
6310 struct btrfs_key key
;
6311 struct btrfs_root
*root
= BTRFS_I(parent_inode
)->root
;
6312 u64 ino
= btrfs_ino(BTRFS_I(inode
));
6313 u64 parent_ino
= btrfs_ino(BTRFS_I(parent_inode
));
6315 if (unlikely(ino
== BTRFS_FIRST_FREE_OBJECTID
)) {
6316 memcpy(&key
, &BTRFS_I(inode
)->root
->root_key
, sizeof(key
));
6319 key
.type
= BTRFS_INODE_ITEM_KEY
;
6323 if (unlikely(ino
== BTRFS_FIRST_FREE_OBJECTID
)) {
6324 ret
= btrfs_add_root_ref(trans
, fs_info
, key
.objectid
,
6325 root
->root_key
.objectid
, parent_ino
,
6326 index
, name
, name_len
);
6327 } else if (add_backref
) {
6328 ret
= btrfs_insert_inode_ref(trans
, root
, name
, name_len
, ino
,
6332 /* Nothing to clean up yet */
6336 ret
= btrfs_insert_dir_item(trans
, root
, name
, name_len
,
6338 btrfs_inode_type(inode
), index
);
6339 if (ret
== -EEXIST
|| ret
== -EOVERFLOW
)
6342 btrfs_abort_transaction(trans
, ret
);
6346 btrfs_i_size_write(parent_inode
, parent_inode
->i_size
+
6348 inode_inc_iversion(parent_inode
);
6349 parent_inode
->i_mtime
= parent_inode
->i_ctime
=
6350 current_time(parent_inode
);
6351 ret
= btrfs_update_inode(trans
, root
, parent_inode
);
6353 btrfs_abort_transaction(trans
, ret
);
6357 if (unlikely(ino
== BTRFS_FIRST_FREE_OBJECTID
)) {
6360 err
= btrfs_del_root_ref(trans
, fs_info
, key
.objectid
,
6361 root
->root_key
.objectid
, parent_ino
,
6362 &local_index
, name
, name_len
);
6364 } else if (add_backref
) {
6368 err
= btrfs_del_inode_ref(trans
, root
, name
, name_len
,
6369 ino
, parent_ino
, &local_index
);
6374 static int btrfs_add_nondir(struct btrfs_trans_handle
*trans
,
6375 struct inode
*dir
, struct dentry
*dentry
,
6376 struct inode
*inode
, int backref
, u64 index
)
6378 int err
= btrfs_add_link(trans
, dir
, inode
,
6379 dentry
->d_name
.name
, dentry
->d_name
.len
,
6386 static int btrfs_mknod(struct inode
*dir
, struct dentry
*dentry
,
6387 umode_t mode
, dev_t rdev
)
6389 struct btrfs_fs_info
*fs_info
= btrfs_sb(dir
->i_sb
);
6390 struct btrfs_trans_handle
*trans
;
6391 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
6392 struct inode
*inode
= NULL
;
6399 * 2 for inode item and ref
6401 * 1 for xattr if selinux is on
6403 trans
= btrfs_start_transaction(root
, 5);
6405 return PTR_ERR(trans
);
6407 err
= btrfs_find_free_ino(root
, &objectid
);
6411 inode
= btrfs_new_inode(trans
, root
, dir
, dentry
->d_name
.name
,
6412 dentry
->d_name
.len
, btrfs_ino(BTRFS_I(dir
)), objectid
,
6414 if (IS_ERR(inode
)) {
6415 err
= PTR_ERR(inode
);
6420 * If the active LSM wants to access the inode during
6421 * d_instantiate it needs these. Smack checks to see
6422 * if the filesystem supports xattrs by looking at the
6425 inode
->i_op
= &btrfs_special_inode_operations
;
6426 init_special_inode(inode
, inode
->i_mode
, rdev
);
6428 err
= btrfs_init_inode_security(trans
, inode
, dir
, &dentry
->d_name
);
6430 goto out_unlock_inode
;
6432 err
= btrfs_add_nondir(trans
, dir
, dentry
, inode
, 0, index
);
6434 goto out_unlock_inode
;
6436 btrfs_update_inode(trans
, root
, inode
);
6437 unlock_new_inode(inode
);
6438 d_instantiate(dentry
, inode
);
6442 btrfs_end_transaction(trans
);
6443 btrfs_balance_delayed_items(fs_info
);
6444 btrfs_btree_balance_dirty(fs_info
);
6446 inode_dec_link_count(inode
);
6453 unlock_new_inode(inode
);
6458 static int btrfs_create(struct inode
*dir
, struct dentry
*dentry
,
6459 umode_t mode
, bool excl
)
6461 struct btrfs_fs_info
*fs_info
= btrfs_sb(dir
->i_sb
);
6462 struct btrfs_trans_handle
*trans
;
6463 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
6464 struct inode
*inode
= NULL
;
6465 int drop_inode_on_err
= 0;
6471 * 2 for inode item and ref
6473 * 1 for xattr if selinux is on
6475 trans
= btrfs_start_transaction(root
, 5);
6477 return PTR_ERR(trans
);
6479 err
= btrfs_find_free_ino(root
, &objectid
);
6483 inode
= btrfs_new_inode(trans
, root
, dir
, dentry
->d_name
.name
,
6484 dentry
->d_name
.len
, btrfs_ino(BTRFS_I(dir
)), objectid
,
6486 if (IS_ERR(inode
)) {
6487 err
= PTR_ERR(inode
);
6490 drop_inode_on_err
= 1;
6492 * If the active LSM wants to access the inode during
6493 * d_instantiate it needs these. Smack checks to see
6494 * if the filesystem supports xattrs by looking at the
6497 inode
->i_fop
= &btrfs_file_operations
;
6498 inode
->i_op
= &btrfs_file_inode_operations
;
6499 inode
->i_mapping
->a_ops
= &btrfs_aops
;
6501 err
= btrfs_init_inode_security(trans
, inode
, dir
, &dentry
->d_name
);
6503 goto out_unlock_inode
;
6505 err
= btrfs_update_inode(trans
, root
, inode
);
6507 goto out_unlock_inode
;
6509 err
= btrfs_add_nondir(trans
, dir
, dentry
, inode
, 0, index
);
6511 goto out_unlock_inode
;
6513 BTRFS_I(inode
)->io_tree
.ops
= &btrfs_extent_io_ops
;
6514 unlock_new_inode(inode
);
6515 d_instantiate(dentry
, inode
);
6518 btrfs_end_transaction(trans
);
6519 if (err
&& drop_inode_on_err
) {
6520 inode_dec_link_count(inode
);
6523 btrfs_balance_delayed_items(fs_info
);
6524 btrfs_btree_balance_dirty(fs_info
);
6528 unlock_new_inode(inode
);
6533 static int btrfs_link(struct dentry
*old_dentry
, struct inode
*dir
,
6534 struct dentry
*dentry
)
6536 struct btrfs_trans_handle
*trans
= NULL
;
6537 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
6538 struct inode
*inode
= d_inode(old_dentry
);
6539 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
6544 /* do not allow sys_link's with other subvols of the same device */
6545 if (root
->objectid
!= BTRFS_I(inode
)->root
->objectid
)
6548 if (inode
->i_nlink
>= BTRFS_LINK_MAX
)
6551 err
= btrfs_set_inode_index(dir
, &index
);
6556 * 2 items for inode and inode ref
6557 * 2 items for dir items
6558 * 1 item for parent inode
6560 trans
= btrfs_start_transaction(root
, 5);
6561 if (IS_ERR(trans
)) {
6562 err
= PTR_ERR(trans
);
6567 /* There are several dir indexes for this inode, clear the cache. */
6568 BTRFS_I(inode
)->dir_index
= 0ULL;
6570 inode_inc_iversion(inode
);
6571 inode
->i_ctime
= current_time(inode
);
6573 set_bit(BTRFS_INODE_COPY_EVERYTHING
, &BTRFS_I(inode
)->runtime_flags
);
6575 err
= btrfs_add_nondir(trans
, dir
, dentry
, inode
, 1, index
);
6580 struct dentry
*parent
= dentry
->d_parent
;
6581 err
= btrfs_update_inode(trans
, root
, inode
);
6584 if (inode
->i_nlink
== 1) {
6586 * If new hard link count is 1, it's a file created
6587 * with open(2) O_TMPFILE flag.
6589 err
= btrfs_orphan_del(trans
, inode
);
6593 d_instantiate(dentry
, inode
);
6594 btrfs_log_new_name(trans
, BTRFS_I(inode
), NULL
, parent
);
6597 btrfs_balance_delayed_items(fs_info
);
6600 btrfs_end_transaction(trans
);
6602 inode_dec_link_count(inode
);
6605 btrfs_btree_balance_dirty(fs_info
);
6609 static int btrfs_mkdir(struct inode
*dir
, struct dentry
*dentry
, umode_t mode
)
6611 struct btrfs_fs_info
*fs_info
= btrfs_sb(dir
->i_sb
);
6612 struct inode
*inode
= NULL
;
6613 struct btrfs_trans_handle
*trans
;
6614 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
6616 int drop_on_err
= 0;
6621 * 2 items for inode and ref
6622 * 2 items for dir items
6623 * 1 for xattr if selinux is on
6625 trans
= btrfs_start_transaction(root
, 5);
6627 return PTR_ERR(trans
);
6629 err
= btrfs_find_free_ino(root
, &objectid
);
6633 inode
= btrfs_new_inode(trans
, root
, dir
, dentry
->d_name
.name
,
6634 dentry
->d_name
.len
, btrfs_ino(BTRFS_I(dir
)), objectid
,
6635 S_IFDIR
| mode
, &index
);
6636 if (IS_ERR(inode
)) {
6637 err
= PTR_ERR(inode
);
6642 /* these must be set before we unlock the inode */
6643 inode
->i_op
= &btrfs_dir_inode_operations
;
6644 inode
->i_fop
= &btrfs_dir_file_operations
;
6646 err
= btrfs_init_inode_security(trans
, inode
, dir
, &dentry
->d_name
);
6648 goto out_fail_inode
;
6650 btrfs_i_size_write(inode
, 0);
6651 err
= btrfs_update_inode(trans
, root
, inode
);
6653 goto out_fail_inode
;
6655 err
= btrfs_add_link(trans
, dir
, inode
, dentry
->d_name
.name
,
6656 dentry
->d_name
.len
, 0, index
);
6658 goto out_fail_inode
;
6660 d_instantiate(dentry
, inode
);
6662 * mkdir is special. We're unlocking after we call d_instantiate
6663 * to avoid a race with nfsd calling d_instantiate.
6665 unlock_new_inode(inode
);
6669 btrfs_end_transaction(trans
);
6671 inode_dec_link_count(inode
);
6674 btrfs_balance_delayed_items(fs_info
);
6675 btrfs_btree_balance_dirty(fs_info
);
6679 unlock_new_inode(inode
);
6683 /* Find next extent map of a given extent map, caller needs to ensure locks */
6684 static struct extent_map
*next_extent_map(struct extent_map
*em
)
6686 struct rb_node
*next
;
6688 next
= rb_next(&em
->rb_node
);
6691 return container_of(next
, struct extent_map
, rb_node
);
6694 static struct extent_map
*prev_extent_map(struct extent_map
*em
)
6696 struct rb_node
*prev
;
6698 prev
= rb_prev(&em
->rb_node
);
6701 return container_of(prev
, struct extent_map
, rb_node
);
6704 /* helper for btfs_get_extent. Given an existing extent in the tree,
6705 * the existing extent is the nearest extent to map_start,
6706 * and an extent that you want to insert, deal with overlap and insert
6707 * the best fitted new extent into the tree.
6709 static int merge_extent_mapping(struct extent_map_tree
*em_tree
,
6710 struct extent_map
*existing
,
6711 struct extent_map
*em
,
6714 struct extent_map
*prev
;
6715 struct extent_map
*next
;
6720 BUG_ON(map_start
< em
->start
|| map_start
>= extent_map_end(em
));
6722 if (existing
->start
> map_start
) {
6724 prev
= prev_extent_map(next
);
6727 next
= next_extent_map(prev
);
6730 start
= prev
? extent_map_end(prev
) : em
->start
;
6731 start
= max_t(u64
, start
, em
->start
);
6732 end
= next
? next
->start
: extent_map_end(em
);
6733 end
= min_t(u64
, end
, extent_map_end(em
));
6734 start_diff
= start
- em
->start
;
6736 em
->len
= end
- start
;
6737 if (em
->block_start
< EXTENT_MAP_LAST_BYTE
&&
6738 !test_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
)) {
6739 em
->block_start
+= start_diff
;
6740 em
->block_len
-= start_diff
;
6742 return add_extent_mapping(em_tree
, em
, 0);
6745 static noinline
int uncompress_inline(struct btrfs_path
*path
,
6747 size_t pg_offset
, u64 extent_offset
,
6748 struct btrfs_file_extent_item
*item
)
6751 struct extent_buffer
*leaf
= path
->nodes
[0];
6754 unsigned long inline_size
;
6758 WARN_ON(pg_offset
!= 0);
6759 compress_type
= btrfs_file_extent_compression(leaf
, item
);
6760 max_size
= btrfs_file_extent_ram_bytes(leaf
, item
);
6761 inline_size
= btrfs_file_extent_inline_item_len(leaf
,
6762 btrfs_item_nr(path
->slots
[0]));
6763 tmp
= kmalloc(inline_size
, GFP_NOFS
);
6766 ptr
= btrfs_file_extent_inline_start(item
);
6768 read_extent_buffer(leaf
, tmp
, ptr
, inline_size
);
6770 max_size
= min_t(unsigned long, PAGE_SIZE
, max_size
);
6771 ret
= btrfs_decompress(compress_type
, tmp
, page
,
6772 extent_offset
, inline_size
, max_size
);
6778 * a bit scary, this does extent mapping from logical file offset to the disk.
6779 * the ugly parts come from merging extents from the disk with the in-ram
6780 * representation. This gets more complex because of the data=ordered code,
6781 * where the in-ram extents might be locked pending data=ordered completion.
6783 * This also copies inline extents directly into the page.
6786 struct extent_map
*btrfs_get_extent(struct inode
*inode
, struct page
*page
,
6787 size_t pg_offset
, u64 start
, u64 len
,
6790 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
6793 u64 extent_start
= 0;
6795 u64 objectid
= btrfs_ino(BTRFS_I(inode
));
6797 struct btrfs_path
*path
= NULL
;
6798 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
6799 struct btrfs_file_extent_item
*item
;
6800 struct extent_buffer
*leaf
;
6801 struct btrfs_key found_key
;
6802 struct extent_map
*em
= NULL
;
6803 struct extent_map_tree
*em_tree
= &BTRFS_I(inode
)->extent_tree
;
6804 struct extent_io_tree
*io_tree
= &BTRFS_I(inode
)->io_tree
;
6805 struct btrfs_trans_handle
*trans
= NULL
;
6806 const bool new_inline
= !page
|| create
;
6809 read_lock(&em_tree
->lock
);
6810 em
= lookup_extent_mapping(em_tree
, start
, len
);
6812 em
->bdev
= fs_info
->fs_devices
->latest_bdev
;
6813 read_unlock(&em_tree
->lock
);
6816 if (em
->start
> start
|| em
->start
+ em
->len
<= start
)
6817 free_extent_map(em
);
6818 else if (em
->block_start
== EXTENT_MAP_INLINE
&& page
)
6819 free_extent_map(em
);
6823 em
= alloc_extent_map();
6828 em
->bdev
= fs_info
->fs_devices
->latest_bdev
;
6829 em
->start
= EXTENT_MAP_HOLE
;
6830 em
->orig_start
= EXTENT_MAP_HOLE
;
6832 em
->block_len
= (u64
)-1;
6835 path
= btrfs_alloc_path();
6841 * Chances are we'll be called again, so go ahead and do
6844 path
->reada
= READA_FORWARD
;
6847 ret
= btrfs_lookup_file_extent(trans
, root
, path
,
6848 objectid
, start
, trans
!= NULL
);
6855 if (path
->slots
[0] == 0)
6860 leaf
= path
->nodes
[0];
6861 item
= btrfs_item_ptr(leaf
, path
->slots
[0],
6862 struct btrfs_file_extent_item
);
6863 /* are we inside the extent that was found? */
6864 btrfs_item_key_to_cpu(leaf
, &found_key
, path
->slots
[0]);
6865 found_type
= found_key
.type
;
6866 if (found_key
.objectid
!= objectid
||
6867 found_type
!= BTRFS_EXTENT_DATA_KEY
) {
6869 * If we backup past the first extent we want to move forward
6870 * and see if there is an extent in front of us, otherwise we'll
6871 * say there is a hole for our whole search range which can
6878 found_type
= btrfs_file_extent_type(leaf
, item
);
6879 extent_start
= found_key
.offset
;
6880 if (found_type
== BTRFS_FILE_EXTENT_REG
||
6881 found_type
== BTRFS_FILE_EXTENT_PREALLOC
) {
6882 extent_end
= extent_start
+
6883 btrfs_file_extent_num_bytes(leaf
, item
);
6884 } else if (found_type
== BTRFS_FILE_EXTENT_INLINE
) {
6886 size
= btrfs_file_extent_inline_len(leaf
, path
->slots
[0], item
);
6887 extent_end
= ALIGN(extent_start
+ size
,
6888 fs_info
->sectorsize
);
6891 if (start
>= extent_end
) {
6893 if (path
->slots
[0] >= btrfs_header_nritems(leaf
)) {
6894 ret
= btrfs_next_leaf(root
, path
);
6901 leaf
= path
->nodes
[0];
6903 btrfs_item_key_to_cpu(leaf
, &found_key
, path
->slots
[0]);
6904 if (found_key
.objectid
!= objectid
||
6905 found_key
.type
!= BTRFS_EXTENT_DATA_KEY
)
6907 if (start
+ len
<= found_key
.offset
)
6909 if (start
> found_key
.offset
)
6912 em
->orig_start
= start
;
6913 em
->len
= found_key
.offset
- start
;
6917 btrfs_extent_item_to_extent_map(inode
, path
, item
, new_inline
, em
);
6919 if (found_type
== BTRFS_FILE_EXTENT_REG
||
6920 found_type
== BTRFS_FILE_EXTENT_PREALLOC
) {
6922 } else if (found_type
== BTRFS_FILE_EXTENT_INLINE
) {
6926 size_t extent_offset
;
6932 size
= btrfs_file_extent_inline_len(leaf
, path
->slots
[0], item
);
6933 extent_offset
= page_offset(page
) + pg_offset
- extent_start
;
6934 copy_size
= min_t(u64
, PAGE_SIZE
- pg_offset
,
6935 size
- extent_offset
);
6936 em
->start
= extent_start
+ extent_offset
;
6937 em
->len
= ALIGN(copy_size
, fs_info
->sectorsize
);
6938 em
->orig_block_len
= em
->len
;
6939 em
->orig_start
= em
->start
;
6940 ptr
= btrfs_file_extent_inline_start(item
) + extent_offset
;
6941 if (create
== 0 && !PageUptodate(page
)) {
6942 if (btrfs_file_extent_compression(leaf
, item
) !=
6943 BTRFS_COMPRESS_NONE
) {
6944 ret
= uncompress_inline(path
, page
, pg_offset
,
6945 extent_offset
, item
);
6952 read_extent_buffer(leaf
, map
+ pg_offset
, ptr
,
6954 if (pg_offset
+ copy_size
< PAGE_SIZE
) {
6955 memset(map
+ pg_offset
+ copy_size
, 0,
6956 PAGE_SIZE
- pg_offset
-
6961 flush_dcache_page(page
);
6962 } else if (create
&& PageUptodate(page
)) {
6966 free_extent_map(em
);
6969 btrfs_release_path(path
);
6970 trans
= btrfs_join_transaction(root
);
6973 return ERR_CAST(trans
);
6977 write_extent_buffer(leaf
, map
+ pg_offset
, ptr
,
6980 btrfs_mark_buffer_dirty(leaf
);
6982 set_extent_uptodate(io_tree
, em
->start
,
6983 extent_map_end(em
) - 1, NULL
, GFP_NOFS
);
6988 em
->orig_start
= start
;
6991 em
->block_start
= EXTENT_MAP_HOLE
;
6992 set_bit(EXTENT_FLAG_VACANCY
, &em
->flags
);
6994 btrfs_release_path(path
);
6995 if (em
->start
> start
|| extent_map_end(em
) <= start
) {
6997 "bad extent! em: [%llu %llu] passed [%llu %llu]",
6998 em
->start
, em
->len
, start
, len
);
7004 write_lock(&em_tree
->lock
);
7005 ret
= add_extent_mapping(em_tree
, em
, 0);
7006 /* it is possible that someone inserted the extent into the tree
7007 * while we had the lock dropped. It is also possible that
7008 * an overlapping map exists in the tree
7010 if (ret
== -EEXIST
) {
7011 struct extent_map
*existing
;
7015 existing
= search_extent_mapping(em_tree
, start
, len
);
7017 * existing will always be non-NULL, since there must be
7018 * extent causing the -EEXIST.
7020 if (existing
->start
== em
->start
&&
7021 extent_map_end(existing
) >= extent_map_end(em
) &&
7022 em
->block_start
== existing
->block_start
) {
7024 * The existing extent map already encompasses the
7025 * entire extent map we tried to add.
7027 free_extent_map(em
);
7031 } else if (start
>= extent_map_end(existing
) ||
7032 start
<= existing
->start
) {
7034 * The existing extent map is the one nearest to
7035 * the [start, start + len) range which overlaps
7037 err
= merge_extent_mapping(em_tree
, existing
,
7039 free_extent_map(existing
);
7041 free_extent_map(em
);
7045 free_extent_map(em
);
7050 write_unlock(&em_tree
->lock
);
7053 trace_btrfs_get_extent(root
, BTRFS_I(inode
), em
);
7055 btrfs_free_path(path
);
7057 ret
= btrfs_end_transaction(trans
);
7062 free_extent_map(em
);
7063 return ERR_PTR(err
);
7065 BUG_ON(!em
); /* Error is always set */
7069 struct extent_map
*btrfs_get_extent_fiemap(struct inode
*inode
, struct page
*page
,
7070 size_t pg_offset
, u64 start
, u64 len
,
7073 struct extent_map
*em
;
7074 struct extent_map
*hole_em
= NULL
;
7075 u64 range_start
= start
;
7081 em
= btrfs_get_extent(inode
, page
, pg_offset
, start
, len
, create
);
7088 * - a pre-alloc extent,
7089 * there might actually be delalloc bytes behind it.
7091 if (em
->block_start
!= EXTENT_MAP_HOLE
&&
7092 !test_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
))
7098 /* check to see if we've wrapped (len == -1 or similar) */
7107 /* ok, we didn't find anything, lets look for delalloc */
7108 found
= count_range_bits(&BTRFS_I(inode
)->io_tree
, &range_start
,
7109 end
, len
, EXTENT_DELALLOC
, 1);
7110 found_end
= range_start
+ found
;
7111 if (found_end
< range_start
)
7112 found_end
= (u64
)-1;
7115 * we didn't find anything useful, return
7116 * the original results from get_extent()
7118 if (range_start
> end
|| found_end
<= start
) {
7124 /* adjust the range_start to make sure it doesn't
7125 * go backwards from the start they passed in
7127 range_start
= max(start
, range_start
);
7128 found
= found_end
- range_start
;
7131 u64 hole_start
= start
;
7134 em
= alloc_extent_map();
7140 * when btrfs_get_extent can't find anything it
7141 * returns one huge hole
7143 * make sure what it found really fits our range, and
7144 * adjust to make sure it is based on the start from
7148 u64 calc_end
= extent_map_end(hole_em
);
7150 if (calc_end
<= start
|| (hole_em
->start
> end
)) {
7151 free_extent_map(hole_em
);
7154 hole_start
= max(hole_em
->start
, start
);
7155 hole_len
= calc_end
- hole_start
;
7159 if (hole_em
&& range_start
> hole_start
) {
7160 /* our hole starts before our delalloc, so we
7161 * have to return just the parts of the hole
7162 * that go until the delalloc starts
7164 em
->len
= min(hole_len
,
7165 range_start
- hole_start
);
7166 em
->start
= hole_start
;
7167 em
->orig_start
= hole_start
;
7169 * don't adjust block start at all,
7170 * it is fixed at EXTENT_MAP_HOLE
7172 em
->block_start
= hole_em
->block_start
;
7173 em
->block_len
= hole_len
;
7174 if (test_bit(EXTENT_FLAG_PREALLOC
, &hole_em
->flags
))
7175 set_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
);
7177 em
->start
= range_start
;
7179 em
->orig_start
= range_start
;
7180 em
->block_start
= EXTENT_MAP_DELALLOC
;
7181 em
->block_len
= found
;
7183 } else if (hole_em
) {
7188 free_extent_map(hole_em
);
7190 free_extent_map(em
);
7191 return ERR_PTR(err
);
7196 static struct extent_map
*btrfs_create_dio_extent(struct inode
*inode
,
7199 const u64 orig_start
,
7200 const u64 block_start
,
7201 const u64 block_len
,
7202 const u64 orig_block_len
,
7203 const u64 ram_bytes
,
7206 struct extent_map
*em
= NULL
;
7209 if (type
!= BTRFS_ORDERED_NOCOW
) {
7210 em
= create_pinned_em(inode
, start
, len
, orig_start
,
7211 block_start
, block_len
, orig_block_len
,
7216 ret
= btrfs_add_ordered_extent_dio(inode
, start
, block_start
,
7217 len
, block_len
, type
);
7220 free_extent_map(em
);
7221 btrfs_drop_extent_cache(inode
, start
,
7222 start
+ len
- 1, 0);
7231 static struct extent_map
*btrfs_new_extent_direct(struct inode
*inode
,
7234 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
7235 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
7236 struct extent_map
*em
;
7237 struct btrfs_key ins
;
7241 alloc_hint
= get_extent_allocation_hint(inode
, start
, len
);
7242 ret
= btrfs_reserve_extent(root
, len
, len
, fs_info
->sectorsize
,
7243 0, alloc_hint
, &ins
, 1, 1);
7245 return ERR_PTR(ret
);
7247 em
= btrfs_create_dio_extent(inode
, start
, ins
.offset
, start
,
7248 ins
.objectid
, ins
.offset
, ins
.offset
,
7250 btrfs_dec_block_group_reservations(fs_info
, ins
.objectid
);
7252 btrfs_free_reserved_extent(fs_info
, ins
.objectid
,
7259 * returns 1 when the nocow is safe, < 1 on error, 0 if the
7260 * block must be cow'd
7262 noinline
int can_nocow_extent(struct inode
*inode
, u64 offset
, u64
*len
,
7263 u64
*orig_start
, u64
*orig_block_len
,
7266 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
7267 struct btrfs_path
*path
;
7269 struct extent_buffer
*leaf
;
7270 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
7271 struct extent_io_tree
*io_tree
= &BTRFS_I(inode
)->io_tree
;
7272 struct btrfs_file_extent_item
*fi
;
7273 struct btrfs_key key
;
7280 bool nocow
= (BTRFS_I(inode
)->flags
& BTRFS_INODE_NODATACOW
);
7282 path
= btrfs_alloc_path();
7286 ret
= btrfs_lookup_file_extent(NULL
, root
, path
,
7287 btrfs_ino(BTRFS_I(inode
)), offset
, 0);
7291 slot
= path
->slots
[0];
7294 /* can't find the item, must cow */
7301 leaf
= path
->nodes
[0];
7302 btrfs_item_key_to_cpu(leaf
, &key
, slot
);
7303 if (key
.objectid
!= btrfs_ino(BTRFS_I(inode
)) ||
7304 key
.type
!= BTRFS_EXTENT_DATA_KEY
) {
7305 /* not our file or wrong item type, must cow */
7309 if (key
.offset
> offset
) {
7310 /* Wrong offset, must cow */
7314 fi
= btrfs_item_ptr(leaf
, slot
, struct btrfs_file_extent_item
);
7315 found_type
= btrfs_file_extent_type(leaf
, fi
);
7316 if (found_type
!= BTRFS_FILE_EXTENT_REG
&&
7317 found_type
!= BTRFS_FILE_EXTENT_PREALLOC
) {
7318 /* not a regular extent, must cow */
7322 if (!nocow
&& found_type
== BTRFS_FILE_EXTENT_REG
)
7325 extent_end
= key
.offset
+ btrfs_file_extent_num_bytes(leaf
, fi
);
7326 if (extent_end
<= offset
)
7329 disk_bytenr
= btrfs_file_extent_disk_bytenr(leaf
, fi
);
7330 if (disk_bytenr
== 0)
7333 if (btrfs_file_extent_compression(leaf
, fi
) ||
7334 btrfs_file_extent_encryption(leaf
, fi
) ||
7335 btrfs_file_extent_other_encoding(leaf
, fi
))
7338 backref_offset
= btrfs_file_extent_offset(leaf
, fi
);
7341 *orig_start
= key
.offset
- backref_offset
;
7342 *orig_block_len
= btrfs_file_extent_disk_num_bytes(leaf
, fi
);
7343 *ram_bytes
= btrfs_file_extent_ram_bytes(leaf
, fi
);
7346 if (btrfs_extent_readonly(fs_info
, disk_bytenr
))
7349 num_bytes
= min(offset
+ *len
, extent_end
) - offset
;
7350 if (!nocow
&& found_type
== BTRFS_FILE_EXTENT_PREALLOC
) {
7353 range_end
= round_up(offset
+ num_bytes
,
7354 root
->fs_info
->sectorsize
) - 1;
7355 ret
= test_range_bit(io_tree
, offset
, range_end
,
7356 EXTENT_DELALLOC
, 0, NULL
);
7363 btrfs_release_path(path
);
7366 * look for other files referencing this extent, if we
7367 * find any we must cow
7370 ret
= btrfs_cross_ref_exist(root
, btrfs_ino(BTRFS_I(inode
)),
7371 key
.offset
- backref_offset
, disk_bytenr
);
7378 * adjust disk_bytenr and num_bytes to cover just the bytes
7379 * in this extent we are about to write. If there
7380 * are any csums in that range we have to cow in order
7381 * to keep the csums correct
7383 disk_bytenr
+= backref_offset
;
7384 disk_bytenr
+= offset
- key
.offset
;
7385 if (csum_exist_in_range(fs_info
, disk_bytenr
, num_bytes
))
7388 * all of the above have passed, it is safe to overwrite this extent
7394 btrfs_free_path(path
);
7398 bool btrfs_page_exists_in_range(struct inode
*inode
, loff_t start
, loff_t end
)
7400 struct radix_tree_root
*root
= &inode
->i_mapping
->page_tree
;
7402 void **pagep
= NULL
;
7403 struct page
*page
= NULL
;
7407 start_idx
= start
>> PAGE_SHIFT
;
7410 * end is the last byte in the last page. end == start is legal
7412 end_idx
= end
>> PAGE_SHIFT
;
7416 /* Most of the code in this while loop is lifted from
7417 * find_get_page. It's been modified to begin searching from a
7418 * page and return just the first page found in that range. If the
7419 * found idx is less than or equal to the end idx then we know that
7420 * a page exists. If no pages are found or if those pages are
7421 * outside of the range then we're fine (yay!) */
7422 while (page
== NULL
&&
7423 radix_tree_gang_lookup_slot(root
, &pagep
, NULL
, start_idx
, 1)) {
7424 page
= radix_tree_deref_slot(pagep
);
7425 if (unlikely(!page
))
7428 if (radix_tree_exception(page
)) {
7429 if (radix_tree_deref_retry(page
)) {
7434 * Otherwise, shmem/tmpfs must be storing a swap entry
7435 * here as an exceptional entry: so return it without
7436 * attempting to raise page count.
7439 break; /* TODO: Is this relevant for this use case? */
7442 if (!page_cache_get_speculative(page
)) {
7448 * Has the page moved?
7449 * This is part of the lockless pagecache protocol. See
7450 * include/linux/pagemap.h for details.
7452 if (unlikely(page
!= *pagep
)) {
7459 if (page
->index
<= end_idx
)
7468 static int lock_extent_direct(struct inode
*inode
, u64 lockstart
, u64 lockend
,
7469 struct extent_state
**cached_state
, int writing
)
7471 struct btrfs_ordered_extent
*ordered
;
7475 lock_extent_bits(&BTRFS_I(inode
)->io_tree
, lockstart
, lockend
,
7478 * We're concerned with the entire range that we're going to be
7479 * doing DIO to, so we need to make sure there's no ordered
7480 * extents in this range.
7482 ordered
= btrfs_lookup_ordered_range(inode
, lockstart
,
7483 lockend
- lockstart
+ 1);
7486 * We need to make sure there are no buffered pages in this
7487 * range either, we could have raced between the invalidate in
7488 * generic_file_direct_write and locking the extent. The
7489 * invalidate needs to happen so that reads after a write do not
7494 !btrfs_page_exists_in_range(inode
, lockstart
, lockend
)))
7497 unlock_extent_cached(&BTRFS_I(inode
)->io_tree
, lockstart
, lockend
,
7498 cached_state
, GFP_NOFS
);
7502 * If we are doing a DIO read and the ordered extent we
7503 * found is for a buffered write, we can not wait for it
7504 * to complete and retry, because if we do so we can
7505 * deadlock with concurrent buffered writes on page
7506 * locks. This happens only if our DIO read covers more
7507 * than one extent map, if at this point has already
7508 * created an ordered extent for a previous extent map
7509 * and locked its range in the inode's io tree, and a
7510 * concurrent write against that previous extent map's
7511 * range and this range started (we unlock the ranges
7512 * in the io tree only when the bios complete and
7513 * buffered writes always lock pages before attempting
7514 * to lock range in the io tree).
7517 test_bit(BTRFS_ORDERED_DIRECT
, &ordered
->flags
))
7518 btrfs_start_ordered_extent(inode
, ordered
, 1);
7521 btrfs_put_ordered_extent(ordered
);
7524 * We could trigger writeback for this range (and wait
7525 * for it to complete) and then invalidate the pages for
7526 * this range (through invalidate_inode_pages2_range()),
7527 * but that can lead us to a deadlock with a concurrent
7528 * call to readpages() (a buffered read or a defrag call
7529 * triggered a readahead) on a page lock due to an
7530 * ordered dio extent we created before but did not have
7531 * yet a corresponding bio submitted (whence it can not
7532 * complete), which makes readpages() wait for that
7533 * ordered extent to complete while holding a lock on
7548 static struct extent_map
*create_pinned_em(struct inode
*inode
, u64 start
,
7549 u64 len
, u64 orig_start
,
7550 u64 block_start
, u64 block_len
,
7551 u64 orig_block_len
, u64 ram_bytes
,
7554 struct extent_map_tree
*em_tree
;
7555 struct extent_map
*em
;
7556 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
7559 em_tree
= &BTRFS_I(inode
)->extent_tree
;
7560 em
= alloc_extent_map();
7562 return ERR_PTR(-ENOMEM
);
7565 em
->orig_start
= orig_start
;
7566 em
->mod_start
= start
;
7569 em
->block_len
= block_len
;
7570 em
->block_start
= block_start
;
7571 em
->bdev
= root
->fs_info
->fs_devices
->latest_bdev
;
7572 em
->orig_block_len
= orig_block_len
;
7573 em
->ram_bytes
= ram_bytes
;
7574 em
->generation
= -1;
7575 set_bit(EXTENT_FLAG_PINNED
, &em
->flags
);
7576 if (type
== BTRFS_ORDERED_PREALLOC
)
7577 set_bit(EXTENT_FLAG_FILLING
, &em
->flags
);
7580 btrfs_drop_extent_cache(inode
, em
->start
,
7581 em
->start
+ em
->len
- 1, 0);
7582 write_lock(&em_tree
->lock
);
7583 ret
= add_extent_mapping(em_tree
, em
, 1);
7584 write_unlock(&em_tree
->lock
);
7585 } while (ret
== -EEXIST
);
7588 free_extent_map(em
);
7589 return ERR_PTR(ret
);
7595 static void adjust_dio_outstanding_extents(struct inode
*inode
,
7596 struct btrfs_dio_data
*dio_data
,
7599 unsigned num_extents
= count_max_extents(len
);
7602 * If we have an outstanding_extents count still set then we're
7603 * within our reservation, otherwise we need to adjust our inode
7604 * counter appropriately.
7606 if (dio_data
->outstanding_extents
>= num_extents
) {
7607 dio_data
->outstanding_extents
-= num_extents
;
7610 * If dio write length has been split due to no large enough
7611 * contiguous space, we need to compensate our inode counter
7614 u64 num_needed
= num_extents
- dio_data
->outstanding_extents
;
7616 spin_lock(&BTRFS_I(inode
)->lock
);
7617 BTRFS_I(inode
)->outstanding_extents
+= num_needed
;
7618 spin_unlock(&BTRFS_I(inode
)->lock
);
7622 static int btrfs_get_blocks_direct(struct inode
*inode
, sector_t iblock
,
7623 struct buffer_head
*bh_result
, int create
)
7625 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
7626 struct extent_map
*em
;
7627 struct extent_state
*cached_state
= NULL
;
7628 struct btrfs_dio_data
*dio_data
= NULL
;
7629 u64 start
= iblock
<< inode
->i_blkbits
;
7630 u64 lockstart
, lockend
;
7631 u64 len
= bh_result
->b_size
;
7632 int unlock_bits
= EXTENT_LOCKED
;
7636 unlock_bits
|= EXTENT_DIRTY
;
7638 len
= min_t(u64
, len
, fs_info
->sectorsize
);
7641 lockend
= start
+ len
- 1;
7643 if (current
->journal_info
) {
7645 * Need to pull our outstanding extents and set journal_info to NULL so
7646 * that anything that needs to check if there's a transaction doesn't get
7649 dio_data
= current
->journal_info
;
7650 current
->journal_info
= NULL
;
7654 * If this errors out it's because we couldn't invalidate pagecache for
7655 * this range and we need to fallback to buffered.
7657 if (lock_extent_direct(inode
, lockstart
, lockend
, &cached_state
,
7663 em
= btrfs_get_extent(inode
, NULL
, 0, start
, len
, 0);
7670 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7671 * io. INLINE is special, and we could probably kludge it in here, but
7672 * it's still buffered so for safety lets just fall back to the generic
7675 * For COMPRESSED we _have_ to read the entire extent in so we can
7676 * decompress it, so there will be buffering required no matter what we
7677 * do, so go ahead and fallback to buffered.
7679 * We return -ENOTBLK because that's what makes DIO go ahead and go back
7680 * to buffered IO. Don't blame me, this is the price we pay for using
7683 if (test_bit(EXTENT_FLAG_COMPRESSED
, &em
->flags
) ||
7684 em
->block_start
== EXTENT_MAP_INLINE
) {
7685 free_extent_map(em
);
7690 /* Just a good old fashioned hole, return */
7691 if (!create
&& (em
->block_start
== EXTENT_MAP_HOLE
||
7692 test_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
))) {
7693 free_extent_map(em
);
7698 * We don't allocate a new extent in the following cases
7700 * 1) The inode is marked as NODATACOW. In this case we'll just use the
7702 * 2) The extent is marked as PREALLOC. We're good to go here and can
7703 * just use the extent.
7707 len
= min(len
, em
->len
- (start
- em
->start
));
7708 lockstart
= start
+ len
;
7712 if (test_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
) ||
7713 ((BTRFS_I(inode
)->flags
& BTRFS_INODE_NODATACOW
) &&
7714 em
->block_start
!= EXTENT_MAP_HOLE
)) {
7716 u64 block_start
, orig_start
, orig_block_len
, ram_bytes
;
7718 if (test_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
))
7719 type
= BTRFS_ORDERED_PREALLOC
;
7721 type
= BTRFS_ORDERED_NOCOW
;
7722 len
= min(len
, em
->len
- (start
- em
->start
));
7723 block_start
= em
->block_start
+ (start
- em
->start
);
7725 if (can_nocow_extent(inode
, start
, &len
, &orig_start
,
7726 &orig_block_len
, &ram_bytes
) == 1 &&
7727 btrfs_inc_nocow_writers(fs_info
, block_start
)) {
7728 struct extent_map
*em2
;
7730 em2
= btrfs_create_dio_extent(inode
, start
, len
,
7731 orig_start
, block_start
,
7732 len
, orig_block_len
,
7734 btrfs_dec_nocow_writers(fs_info
, block_start
);
7735 if (type
== BTRFS_ORDERED_PREALLOC
) {
7736 free_extent_map(em
);
7739 if (em2
&& IS_ERR(em2
)) {
7744 * For inode marked NODATACOW or extent marked PREALLOC,
7745 * use the existing or preallocated extent, so does not
7746 * need to adjust btrfs_space_info's bytes_may_use.
7748 btrfs_free_reserved_data_space_noquota(inode
,
7755 * this will cow the extent, reset the len in case we changed
7758 len
= bh_result
->b_size
;
7759 free_extent_map(em
);
7760 em
= btrfs_new_extent_direct(inode
, start
, len
);
7765 len
= min(len
, em
->len
- (start
- em
->start
));
7767 bh_result
->b_blocknr
= (em
->block_start
+ (start
- em
->start
)) >>
7769 bh_result
->b_size
= len
;
7770 bh_result
->b_bdev
= em
->bdev
;
7771 set_buffer_mapped(bh_result
);
7773 if (!test_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
))
7774 set_buffer_new(bh_result
);
7777 * Need to update the i_size under the extent lock so buffered
7778 * readers will get the updated i_size when we unlock.
7780 if (!dio_data
->overwrite
&& start
+ len
> i_size_read(inode
))
7781 i_size_write(inode
, start
+ len
);
7783 adjust_dio_outstanding_extents(inode
, dio_data
, len
);
7784 WARN_ON(dio_data
->reserve
< len
);
7785 dio_data
->reserve
-= len
;
7786 dio_data
->unsubmitted_oe_range_end
= start
+ len
;
7787 current
->journal_info
= dio_data
;
7791 * In the case of write we need to clear and unlock the entire range,
7792 * in the case of read we need to unlock only the end area that we
7793 * aren't using if there is any left over space.
7795 if (lockstart
< lockend
) {
7796 clear_extent_bit(&BTRFS_I(inode
)->io_tree
, lockstart
,
7797 lockend
, unlock_bits
, 1, 0,
7798 &cached_state
, GFP_NOFS
);
7800 free_extent_state(cached_state
);
7803 free_extent_map(em
);
7808 clear_extent_bit(&BTRFS_I(inode
)->io_tree
, lockstart
, lockend
,
7809 unlock_bits
, 1, 0, &cached_state
, GFP_NOFS
);
7812 current
->journal_info
= dio_data
;
7814 * Compensate the delalloc release we do in btrfs_direct_IO() when we
7815 * write less data then expected, so that we don't underflow our inode's
7816 * outstanding extents counter.
7818 if (create
&& dio_data
)
7819 adjust_dio_outstanding_extents(inode
, dio_data
, len
);
7824 static inline int submit_dio_repair_bio(struct inode
*inode
, struct bio
*bio
,
7827 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
7830 BUG_ON(bio_op(bio
) == REQ_OP_WRITE
);
7834 ret
= btrfs_bio_wq_end_io(fs_info
, bio
, BTRFS_WQ_ENDIO_DIO_REPAIR
);
7838 ret
= btrfs_map_bio(fs_info
, bio
, mirror_num
, 0);
7844 static int btrfs_check_dio_repairable(struct inode
*inode
,
7845 struct bio
*failed_bio
,
7846 struct io_failure_record
*failrec
,
7849 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
7852 num_copies
= btrfs_num_copies(fs_info
, failrec
->logical
, failrec
->len
);
7853 if (num_copies
== 1) {
7855 * we only have a single copy of the data, so don't bother with
7856 * all the retry and error correction code that follows. no
7857 * matter what the error is, it is very likely to persist.
7859 btrfs_debug(fs_info
,
7860 "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
7861 num_copies
, failrec
->this_mirror
, failed_mirror
);
7865 failrec
->failed_mirror
= failed_mirror
;
7866 failrec
->this_mirror
++;
7867 if (failrec
->this_mirror
== failed_mirror
)
7868 failrec
->this_mirror
++;
7870 if (failrec
->this_mirror
> num_copies
) {
7871 btrfs_debug(fs_info
,
7872 "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
7873 num_copies
, failrec
->this_mirror
, failed_mirror
);
7880 static int dio_read_error(struct inode
*inode
, struct bio
*failed_bio
,
7881 struct page
*page
, unsigned int pgoff
,
7882 u64 start
, u64 end
, int failed_mirror
,
7883 bio_end_io_t
*repair_endio
, void *repair_arg
)
7885 struct io_failure_record
*failrec
;
7891 BUG_ON(bio_op(failed_bio
) == REQ_OP_WRITE
);
7893 ret
= btrfs_get_io_failure_record(inode
, start
, end
, &failrec
);
7897 ret
= btrfs_check_dio_repairable(inode
, failed_bio
, failrec
,
7900 free_io_failure(inode
, failrec
);
7904 if ((failed_bio
->bi_vcnt
> 1)
7905 || (failed_bio
->bi_io_vec
->bv_len
7906 > btrfs_inode_sectorsize(inode
)))
7907 read_mode
|= REQ_FAILFAST_DEV
;
7909 isector
= start
- btrfs_io_bio(failed_bio
)->logical
;
7910 isector
>>= inode
->i_sb
->s_blocksize_bits
;
7911 bio
= btrfs_create_repair_bio(inode
, failed_bio
, failrec
, page
,
7912 pgoff
, isector
, repair_endio
, repair_arg
);
7914 free_io_failure(inode
, failrec
);
7917 bio_set_op_attrs(bio
, REQ_OP_READ
, read_mode
);
7919 btrfs_debug(BTRFS_I(inode
)->root
->fs_info
,
7920 "Repair DIO Read Error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d\n",
7921 read_mode
, failrec
->this_mirror
, failrec
->in_validation
);
7923 ret
= submit_dio_repair_bio(inode
, bio
, failrec
->this_mirror
);
7925 free_io_failure(inode
, failrec
);
7932 struct btrfs_retry_complete
{
7933 struct completion done
;
7934 struct inode
*inode
;
7939 static void btrfs_retry_endio_nocsum(struct bio
*bio
)
7941 struct btrfs_retry_complete
*done
= bio
->bi_private
;
7942 struct inode
*inode
;
7943 struct bio_vec
*bvec
;
7949 ASSERT(bio
->bi_vcnt
== 1);
7950 inode
= bio
->bi_io_vec
->bv_page
->mapping
->host
;
7951 ASSERT(bio
->bi_io_vec
->bv_len
== btrfs_inode_sectorsize(inode
));
7954 bio_for_each_segment_all(bvec
, bio
, i
)
7955 clean_io_failure(done
->inode
, done
->start
, bvec
->bv_page
, 0);
7957 complete(&done
->done
);
7961 static int __btrfs_correct_data_nocsum(struct inode
*inode
,
7962 struct btrfs_io_bio
*io_bio
)
7964 struct btrfs_fs_info
*fs_info
;
7965 struct bio_vec
*bvec
;
7966 struct btrfs_retry_complete done
;
7974 fs_info
= BTRFS_I(inode
)->root
->fs_info
;
7975 sectorsize
= fs_info
->sectorsize
;
7977 start
= io_bio
->logical
;
7980 bio_for_each_segment_all(bvec
, &io_bio
->bio
, i
) {
7981 nr_sectors
= BTRFS_BYTES_TO_BLKS(fs_info
, bvec
->bv_len
);
7982 pgoff
= bvec
->bv_offset
;
7984 next_block_or_try_again
:
7987 init_completion(&done
.done
);
7989 ret
= dio_read_error(inode
, &io_bio
->bio
, bvec
->bv_page
,
7990 pgoff
, start
, start
+ sectorsize
- 1,
7992 btrfs_retry_endio_nocsum
, &done
);
7996 wait_for_completion(&done
.done
);
7998 if (!done
.uptodate
) {
7999 /* We might have another mirror, so try again */
8000 goto next_block_or_try_again
;
8003 start
+= sectorsize
;
8006 pgoff
+= sectorsize
;
8007 goto next_block_or_try_again
;
8014 static void btrfs_retry_endio(struct bio
*bio
)
8016 struct btrfs_retry_complete
*done
= bio
->bi_private
;
8017 struct btrfs_io_bio
*io_bio
= btrfs_io_bio(bio
);
8018 struct inode
*inode
;
8019 struct bio_vec
*bvec
;
8030 start
= done
->start
;
8032 ASSERT(bio
->bi_vcnt
== 1);
8033 inode
= bio
->bi_io_vec
->bv_page
->mapping
->host
;
8034 ASSERT(bio
->bi_io_vec
->bv_len
== btrfs_inode_sectorsize(inode
));
8036 bio_for_each_segment_all(bvec
, bio
, i
) {
8037 ret
= __readpage_endio_check(done
->inode
, io_bio
, i
,
8038 bvec
->bv_page
, bvec
->bv_offset
,
8039 done
->start
, bvec
->bv_len
);
8041 clean_io_failure(done
->inode
, done
->start
,
8042 bvec
->bv_page
, bvec
->bv_offset
);
8047 done
->uptodate
= uptodate
;
8049 complete(&done
->done
);
8053 static int __btrfs_subio_endio_read(struct inode
*inode
,
8054 struct btrfs_io_bio
*io_bio
, int err
)
8056 struct btrfs_fs_info
*fs_info
;
8057 struct bio_vec
*bvec
;
8058 struct btrfs_retry_complete done
;
8068 fs_info
= BTRFS_I(inode
)->root
->fs_info
;
8069 sectorsize
= fs_info
->sectorsize
;
8072 start
= io_bio
->logical
;
8075 bio_for_each_segment_all(bvec
, &io_bio
->bio
, i
) {
8076 nr_sectors
= BTRFS_BYTES_TO_BLKS(fs_info
, bvec
->bv_len
);
8078 pgoff
= bvec
->bv_offset
;
8080 csum_pos
= BTRFS_BYTES_TO_BLKS(fs_info
, offset
);
8081 ret
= __readpage_endio_check(inode
, io_bio
, csum_pos
,
8082 bvec
->bv_page
, pgoff
, start
,
8089 init_completion(&done
.done
);
8091 ret
= dio_read_error(inode
, &io_bio
->bio
, bvec
->bv_page
,
8092 pgoff
, start
, start
+ sectorsize
- 1,
8094 btrfs_retry_endio
, &done
);
8100 wait_for_completion(&done
.done
);
8102 if (!done
.uptodate
) {
8103 /* We might have another mirror, so try again */
8107 offset
+= sectorsize
;
8108 start
+= sectorsize
;
8113 pgoff
+= sectorsize
;
8121 static int btrfs_subio_endio_read(struct inode
*inode
,
8122 struct btrfs_io_bio
*io_bio
, int err
)
8124 bool skip_csum
= BTRFS_I(inode
)->flags
& BTRFS_INODE_NODATASUM
;
8128 return __btrfs_correct_data_nocsum(inode
, io_bio
);
8132 return __btrfs_subio_endio_read(inode
, io_bio
, err
);
8136 static void btrfs_endio_direct_read(struct bio
*bio
)
8138 struct btrfs_dio_private
*dip
= bio
->bi_private
;
8139 struct inode
*inode
= dip
->inode
;
8140 struct bio
*dio_bio
;
8141 struct btrfs_io_bio
*io_bio
= btrfs_io_bio(bio
);
8142 int err
= bio
->bi_error
;
8144 if (dip
->flags
& BTRFS_DIO_ORIG_BIO_SUBMITTED
)
8145 err
= btrfs_subio_endio_read(inode
, io_bio
, err
);
8147 unlock_extent(&BTRFS_I(inode
)->io_tree
, dip
->logical_offset
,
8148 dip
->logical_offset
+ dip
->bytes
- 1);
8149 dio_bio
= dip
->dio_bio
;
8153 dio_bio
->bi_error
= bio
->bi_error
;
8154 dio_end_io(dio_bio
, bio
->bi_error
);
8157 io_bio
->end_io(io_bio
, err
);
8161 static void btrfs_endio_direct_write_update_ordered(struct inode
*inode
,
8166 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
8167 struct btrfs_ordered_extent
*ordered
= NULL
;
8168 u64 ordered_offset
= offset
;
8169 u64 ordered_bytes
= bytes
;
8173 ret
= btrfs_dec_test_first_ordered_pending(inode
, &ordered
,
8180 btrfs_init_work(&ordered
->work
, btrfs_endio_write_helper
,
8181 finish_ordered_fn
, NULL
, NULL
);
8182 btrfs_queue_work(fs_info
->endio_write_workers
, &ordered
->work
);
8185 * our bio might span multiple ordered extents. If we haven't
8186 * completed the accounting for the whole dio, go back and try again
8188 if (ordered_offset
< offset
+ bytes
) {
8189 ordered_bytes
= offset
+ bytes
- ordered_offset
;
8195 static void btrfs_endio_direct_write(struct bio
*bio
)
8197 struct btrfs_dio_private
*dip
= bio
->bi_private
;
8198 struct bio
*dio_bio
= dip
->dio_bio
;
8200 btrfs_endio_direct_write_update_ordered(dip
->inode
,
8201 dip
->logical_offset
,
8207 dio_bio
->bi_error
= bio
->bi_error
;
8208 dio_end_io(dio_bio
, bio
->bi_error
);
8212 static int __btrfs_submit_bio_start_direct_io(struct inode
*inode
,
8213 struct bio
*bio
, int mirror_num
,
8214 unsigned long bio_flags
, u64 offset
)
8217 ret
= btrfs_csum_one_bio(inode
, bio
, offset
, 1);
8218 BUG_ON(ret
); /* -ENOMEM */
8222 static void btrfs_end_dio_bio(struct bio
*bio
)
8224 struct btrfs_dio_private
*dip
= bio
->bi_private
;
8225 int err
= bio
->bi_error
;
8228 btrfs_warn(BTRFS_I(dip
->inode
)->root
->fs_info
,
8229 "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
8230 btrfs_ino(BTRFS_I(dip
->inode
)), bio_op(bio
),
8232 (unsigned long long)bio
->bi_iter
.bi_sector
,
8233 bio
->bi_iter
.bi_size
, err
);
8235 if (dip
->subio_endio
)
8236 err
= dip
->subio_endio(dip
->inode
, btrfs_io_bio(bio
), err
);
8242 * before atomic variable goto zero, we must make sure
8243 * dip->errors is perceived to be set.
8245 smp_mb__before_atomic();
8248 /* if there are more bios still pending for this dio, just exit */
8249 if (!atomic_dec_and_test(&dip
->pending_bios
))
8253 bio_io_error(dip
->orig_bio
);
8255 dip
->dio_bio
->bi_error
= 0;
8256 bio_endio(dip
->orig_bio
);
8262 static struct bio
*btrfs_dio_bio_alloc(struct block_device
*bdev
,
8263 u64 first_sector
, gfp_t gfp_flags
)
8266 bio
= btrfs_bio_alloc(bdev
, first_sector
, BIO_MAX_PAGES
, gfp_flags
);
8268 bio_associate_current(bio
);
8272 static inline int btrfs_lookup_and_bind_dio_csum(struct inode
*inode
,
8273 struct btrfs_dio_private
*dip
,
8277 struct btrfs_io_bio
*io_bio
= btrfs_io_bio(bio
);
8278 struct btrfs_io_bio
*orig_io_bio
= btrfs_io_bio(dip
->orig_bio
);
8282 * We load all the csum data we need when we submit
8283 * the first bio to reduce the csum tree search and
8286 if (dip
->logical_offset
== file_offset
) {
8287 ret
= btrfs_lookup_bio_sums_dio(inode
, dip
->orig_bio
,
8293 if (bio
== dip
->orig_bio
)
8296 file_offset
-= dip
->logical_offset
;
8297 file_offset
>>= inode
->i_sb
->s_blocksize_bits
;
8298 io_bio
->csum
= (u8
*)(((u32
*)orig_io_bio
->csum
) + file_offset
);
8303 static inline int __btrfs_submit_dio_bio(struct bio
*bio
, struct inode
*inode
,
8304 u64 file_offset
, int skip_sum
,
8307 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
8308 struct btrfs_dio_private
*dip
= bio
->bi_private
;
8309 bool write
= bio_op(bio
) == REQ_OP_WRITE
;
8313 async_submit
= !atomic_read(&BTRFS_I(inode
)->sync_writers
);
8318 ret
= btrfs_bio_wq_end_io(fs_info
, bio
, BTRFS_WQ_ENDIO_DATA
);
8326 if (write
&& async_submit
) {
8327 ret
= btrfs_wq_submit_bio(fs_info
, inode
, bio
, 0, 0,
8329 __btrfs_submit_bio_start_direct_io
,
8330 __btrfs_submit_bio_done
);
8334 * If we aren't doing async submit, calculate the csum of the
8337 ret
= btrfs_csum_one_bio(inode
, bio
, file_offset
, 1);
8341 ret
= btrfs_lookup_and_bind_dio_csum(inode
, dip
, bio
,
8347 ret
= btrfs_map_bio(fs_info
, bio
, 0, async_submit
);
8353 static int btrfs_submit_direct_hook(struct btrfs_dio_private
*dip
,
8356 struct inode
*inode
= dip
->inode
;
8357 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
8358 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
8360 struct bio
*orig_bio
= dip
->orig_bio
;
8361 struct bio_vec
*bvec
;
8362 u64 start_sector
= orig_bio
->bi_iter
.bi_sector
;
8363 u64 file_offset
= dip
->logical_offset
;
8366 u32 blocksize
= fs_info
->sectorsize
;
8367 int async_submit
= 0;
8372 map_length
= orig_bio
->bi_iter
.bi_size
;
8373 ret
= btrfs_map_block(fs_info
, btrfs_op(orig_bio
), start_sector
<< 9,
8374 &map_length
, NULL
, 0);
8378 if (map_length
>= orig_bio
->bi_iter
.bi_size
) {
8380 dip
->flags
|= BTRFS_DIO_ORIG_BIO_SUBMITTED
;
8384 /* async crcs make it difficult to collect full stripe writes. */
8385 if (btrfs_get_alloc_profile(root
, 1) & BTRFS_BLOCK_GROUP_RAID56_MASK
)
8390 bio
= btrfs_dio_bio_alloc(orig_bio
->bi_bdev
, start_sector
, GFP_NOFS
);
8394 bio
->bi_opf
= orig_bio
->bi_opf
;
8395 bio
->bi_private
= dip
;
8396 bio
->bi_end_io
= btrfs_end_dio_bio
;
8397 btrfs_io_bio(bio
)->logical
= file_offset
;
8398 atomic_inc(&dip
->pending_bios
);
8400 bio_for_each_segment_all(bvec
, orig_bio
, j
) {
8401 nr_sectors
= BTRFS_BYTES_TO_BLKS(fs_info
, bvec
->bv_len
);
8404 if (unlikely(map_length
< submit_len
+ blocksize
||
8405 bio_add_page(bio
, bvec
->bv_page
, blocksize
,
8406 bvec
->bv_offset
+ (i
* blocksize
)) < blocksize
)) {
8408 * inc the count before we submit the bio so
8409 * we know the end IO handler won't happen before
8410 * we inc the count. Otherwise, the dip might get freed
8411 * before we're done setting it up
8413 atomic_inc(&dip
->pending_bios
);
8414 ret
= __btrfs_submit_dio_bio(bio
, inode
,
8415 file_offset
, skip_sum
,
8419 atomic_dec(&dip
->pending_bios
);
8423 start_sector
+= submit_len
>> 9;
8424 file_offset
+= submit_len
;
8428 bio
= btrfs_dio_bio_alloc(orig_bio
->bi_bdev
,
8429 start_sector
, GFP_NOFS
);
8432 bio
->bi_opf
= orig_bio
->bi_opf
;
8433 bio
->bi_private
= dip
;
8434 bio
->bi_end_io
= btrfs_end_dio_bio
;
8435 btrfs_io_bio(bio
)->logical
= file_offset
;
8437 map_length
= orig_bio
->bi_iter
.bi_size
;
8438 ret
= btrfs_map_block(fs_info
, btrfs_op(orig_bio
),
8440 &map_length
, NULL
, 0);
8448 submit_len
+= blocksize
;
8457 ret
= __btrfs_submit_dio_bio(bio
, inode
, file_offset
, skip_sum
,
8466 * before atomic variable goto zero, we must
8467 * make sure dip->errors is perceived to be set.
8469 smp_mb__before_atomic();
8470 if (atomic_dec_and_test(&dip
->pending_bios
))
8471 bio_io_error(dip
->orig_bio
);
8473 /* bio_end_io() will handle error, so we needn't return it */
8477 static void btrfs_submit_direct(struct bio
*dio_bio
, struct inode
*inode
,
8480 struct btrfs_dio_private
*dip
= NULL
;
8481 struct bio
*io_bio
= NULL
;
8482 struct btrfs_io_bio
*btrfs_bio
;
8484 bool write
= (bio_op(dio_bio
) == REQ_OP_WRITE
);
8487 skip_sum
= BTRFS_I(inode
)->flags
& BTRFS_INODE_NODATASUM
;
8489 io_bio
= btrfs_bio_clone(dio_bio
, GFP_NOFS
);
8495 dip
= kzalloc(sizeof(*dip
), GFP_NOFS
);
8501 dip
->private = dio_bio
->bi_private
;
8503 dip
->logical_offset
= file_offset
;
8504 dip
->bytes
= dio_bio
->bi_iter
.bi_size
;
8505 dip
->disk_bytenr
= (u64
)dio_bio
->bi_iter
.bi_sector
<< 9;
8506 io_bio
->bi_private
= dip
;
8507 dip
->orig_bio
= io_bio
;
8508 dip
->dio_bio
= dio_bio
;
8509 atomic_set(&dip
->pending_bios
, 0);
8510 btrfs_bio
= btrfs_io_bio(io_bio
);
8511 btrfs_bio
->logical
= file_offset
;
8514 io_bio
->bi_end_io
= btrfs_endio_direct_write
;
8516 io_bio
->bi_end_io
= btrfs_endio_direct_read
;
8517 dip
->subio_endio
= btrfs_subio_endio_read
;
8521 * Reset the range for unsubmitted ordered extents (to a 0 length range)
8522 * even if we fail to submit a bio, because in such case we do the
8523 * corresponding error handling below and it must not be done a second
8524 * time by btrfs_direct_IO().
8527 struct btrfs_dio_data
*dio_data
= current
->journal_info
;
8529 dio_data
->unsubmitted_oe_range_end
= dip
->logical_offset
+
8531 dio_data
->unsubmitted_oe_range_start
=
8532 dio_data
->unsubmitted_oe_range_end
;
8535 ret
= btrfs_submit_direct_hook(dip
, skip_sum
);
8539 if (btrfs_bio
->end_io
)
8540 btrfs_bio
->end_io(btrfs_bio
, ret
);
8544 * If we arrived here it means either we failed to submit the dip
8545 * or we either failed to clone the dio_bio or failed to allocate the
8546 * dip. If we cloned the dio_bio and allocated the dip, we can just
8547 * call bio_endio against our io_bio so that we get proper resource
8548 * cleanup if we fail to submit the dip, otherwise, we must do the
8549 * same as btrfs_endio_direct_[write|read] because we can't call these
8550 * callbacks - they require an allocated dip and a clone of dio_bio.
8552 if (io_bio
&& dip
) {
8553 io_bio
->bi_error
= -EIO
;
8556 * The end io callbacks free our dip, do the final put on io_bio
8557 * and all the cleanup and final put for dio_bio (through
8564 btrfs_endio_direct_write_update_ordered(inode
,
8566 dio_bio
->bi_iter
.bi_size
,
8569 unlock_extent(&BTRFS_I(inode
)->io_tree
, file_offset
,
8570 file_offset
+ dio_bio
->bi_iter
.bi_size
- 1);
8572 dio_bio
->bi_error
= -EIO
;
8574 * Releases and cleans up our dio_bio, no need to bio_put()
8575 * nor bio_endio()/bio_io_error() against dio_bio.
8577 dio_end_io(dio_bio
, ret
);
8584 static ssize_t
check_direct_IO(struct btrfs_fs_info
*fs_info
,
8586 const struct iov_iter
*iter
, loff_t offset
)
8590 unsigned int blocksize_mask
= fs_info
->sectorsize
- 1;
8591 ssize_t retval
= -EINVAL
;
8593 if (offset
& blocksize_mask
)
8596 if (iov_iter_alignment(iter
) & blocksize_mask
)
8599 /* If this is a write we don't need to check anymore */
8600 if (iov_iter_rw(iter
) != READ
|| !iter_is_iovec(iter
))
8603 * Check to make sure we don't have duplicate iov_base's in this
8604 * iovec, if so return EINVAL, otherwise we'll get csum errors
8605 * when reading back.
8607 for (seg
= 0; seg
< iter
->nr_segs
; seg
++) {
8608 for (i
= seg
+ 1; i
< iter
->nr_segs
; i
++) {
8609 if (iter
->iov
[seg
].iov_base
== iter
->iov
[i
].iov_base
)
8618 static ssize_t
btrfs_direct_IO(struct kiocb
*iocb
, struct iov_iter
*iter
)
8620 struct file
*file
= iocb
->ki_filp
;
8621 struct inode
*inode
= file
->f_mapping
->host
;
8622 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
8623 struct btrfs_dio_data dio_data
= { 0 };
8624 loff_t offset
= iocb
->ki_pos
;
8628 bool relock
= false;
8631 if (check_direct_IO(fs_info
, iocb
, iter
, offset
))
8634 inode_dio_begin(inode
);
8635 smp_mb__after_atomic();
8638 * The generic stuff only does filemap_write_and_wait_range, which
8639 * isn't enough if we've written compressed pages to this area, so
8640 * we need to flush the dirty pages again to make absolutely sure
8641 * that any outstanding dirty pages are on disk.
8643 count
= iov_iter_count(iter
);
8644 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT
,
8645 &BTRFS_I(inode
)->runtime_flags
))
8646 filemap_fdatawrite_range(inode
->i_mapping
, offset
,
8647 offset
+ count
- 1);
8649 if (iov_iter_rw(iter
) == WRITE
) {
8651 * If the write DIO is beyond the EOF, we need update
8652 * the isize, but it is protected by i_mutex. So we can
8653 * not unlock the i_mutex at this case.
8655 if (offset
+ count
<= inode
->i_size
) {
8656 dio_data
.overwrite
= 1;
8657 inode_unlock(inode
);
8660 ret
= btrfs_delalloc_reserve_space(inode
, offset
, count
);
8663 dio_data
.outstanding_extents
= count_max_extents(count
);
8666 * We need to know how many extents we reserved so that we can
8667 * do the accounting properly if we go over the number we
8668 * originally calculated. Abuse current->journal_info for this.
8670 dio_data
.reserve
= round_up(count
,
8671 fs_info
->sectorsize
);
8672 dio_data
.unsubmitted_oe_range_start
= (u64
)offset
;
8673 dio_data
.unsubmitted_oe_range_end
= (u64
)offset
;
8674 current
->journal_info
= &dio_data
;
8675 down_read(&BTRFS_I(inode
)->dio_sem
);
8676 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK
,
8677 &BTRFS_I(inode
)->runtime_flags
)) {
8678 inode_dio_end(inode
);
8679 flags
= DIO_LOCKING
| DIO_SKIP_HOLES
;
8683 ret
= __blockdev_direct_IO(iocb
, inode
,
8684 fs_info
->fs_devices
->latest_bdev
,
8685 iter
, btrfs_get_blocks_direct
, NULL
,
8686 btrfs_submit_direct
, flags
);
8687 if (iov_iter_rw(iter
) == WRITE
) {
8688 up_read(&BTRFS_I(inode
)->dio_sem
);
8689 current
->journal_info
= NULL
;
8690 if (ret
< 0 && ret
!= -EIOCBQUEUED
) {
8691 if (dio_data
.reserve
)
8692 btrfs_delalloc_release_space(inode
, offset
,
8695 * On error we might have left some ordered extents
8696 * without submitting corresponding bios for them, so
8697 * cleanup them up to avoid other tasks getting them
8698 * and waiting for them to complete forever.
8700 if (dio_data
.unsubmitted_oe_range_start
<
8701 dio_data
.unsubmitted_oe_range_end
)
8702 btrfs_endio_direct_write_update_ordered(inode
,
8703 dio_data
.unsubmitted_oe_range_start
,
8704 dio_data
.unsubmitted_oe_range_end
-
8705 dio_data
.unsubmitted_oe_range_start
,
8707 } else if (ret
>= 0 && (size_t)ret
< count
)
8708 btrfs_delalloc_release_space(inode
, offset
,
8709 count
- (size_t)ret
);
8713 inode_dio_end(inode
);
8720 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC)
8722 static int btrfs_fiemap(struct inode
*inode
, struct fiemap_extent_info
*fieinfo
,
8723 __u64 start
, __u64 len
)
8727 ret
= fiemap_check_flags(fieinfo
, BTRFS_FIEMAP_FLAGS
);
8731 return extent_fiemap(inode
, fieinfo
, start
, len
, btrfs_get_extent_fiemap
);
8734 int btrfs_readpage(struct file
*file
, struct page
*page
)
8736 struct extent_io_tree
*tree
;
8737 tree
= &BTRFS_I(page
->mapping
->host
)->io_tree
;
8738 return extent_read_full_page(tree
, page
, btrfs_get_extent
, 0);
8741 static int btrfs_writepage(struct page
*page
, struct writeback_control
*wbc
)
8743 struct extent_io_tree
*tree
;
8744 struct inode
*inode
= page
->mapping
->host
;
8747 if (current
->flags
& PF_MEMALLOC
) {
8748 redirty_page_for_writepage(wbc
, page
);
8754 * If we are under memory pressure we will call this directly from the
8755 * VM, we need to make sure we have the inode referenced for the ordered
8756 * extent. If not just return like we didn't do anything.
8758 if (!igrab(inode
)) {
8759 redirty_page_for_writepage(wbc
, page
);
8760 return AOP_WRITEPAGE_ACTIVATE
;
8762 tree
= &BTRFS_I(page
->mapping
->host
)->io_tree
;
8763 ret
= extent_write_full_page(tree
, page
, btrfs_get_extent
, wbc
);
8764 btrfs_add_delayed_iput(inode
);
8768 static int btrfs_writepages(struct address_space
*mapping
,
8769 struct writeback_control
*wbc
)
8771 struct extent_io_tree
*tree
;
8773 tree
= &BTRFS_I(mapping
->host
)->io_tree
;
8774 return extent_writepages(tree
, mapping
, btrfs_get_extent
, wbc
);
8778 btrfs_readpages(struct file
*file
, struct address_space
*mapping
,
8779 struct list_head
*pages
, unsigned nr_pages
)
8781 struct extent_io_tree
*tree
;
8782 tree
= &BTRFS_I(mapping
->host
)->io_tree
;
8783 return extent_readpages(tree
, mapping
, pages
, nr_pages
,
8786 static int __btrfs_releasepage(struct page
*page
, gfp_t gfp_flags
)
8788 struct extent_io_tree
*tree
;
8789 struct extent_map_tree
*map
;
8792 tree
= &BTRFS_I(page
->mapping
->host
)->io_tree
;
8793 map
= &BTRFS_I(page
->mapping
->host
)->extent_tree
;
8794 ret
= try_release_extent_mapping(map
, tree
, page
, gfp_flags
);
8796 ClearPagePrivate(page
);
8797 set_page_private(page
, 0);
8803 static int btrfs_releasepage(struct page
*page
, gfp_t gfp_flags
)
8805 if (PageWriteback(page
) || PageDirty(page
))
8807 return __btrfs_releasepage(page
, gfp_flags
);
8810 static void btrfs_invalidatepage(struct page
*page
, unsigned int offset
,
8811 unsigned int length
)
8813 struct inode
*inode
= page
->mapping
->host
;
8814 struct extent_io_tree
*tree
;
8815 struct btrfs_ordered_extent
*ordered
;
8816 struct extent_state
*cached_state
= NULL
;
8817 u64 page_start
= page_offset(page
);
8818 u64 page_end
= page_start
+ PAGE_SIZE
- 1;
8821 int inode_evicting
= inode
->i_state
& I_FREEING
;
8824 * we have the page locked, so new writeback can't start,
8825 * and the dirty bit won't be cleared while we are here.
8827 * Wait for IO on this page so that we can safely clear
8828 * the PagePrivate2 bit and do ordered accounting
8830 wait_on_page_writeback(page
);
8832 tree
= &BTRFS_I(inode
)->io_tree
;
8834 btrfs_releasepage(page
, GFP_NOFS
);
8838 if (!inode_evicting
)
8839 lock_extent_bits(tree
, page_start
, page_end
, &cached_state
);
8842 ordered
= btrfs_lookup_ordered_range(inode
, start
,
8843 page_end
- start
+ 1);
8845 end
= min(page_end
, ordered
->file_offset
+ ordered
->len
- 1);
8847 * IO on this page will never be started, so we need
8848 * to account for any ordered extents now
8850 if (!inode_evicting
)
8851 clear_extent_bit(tree
, start
, end
,
8852 EXTENT_DIRTY
| EXTENT_DELALLOC
|
8853 EXTENT_LOCKED
| EXTENT_DO_ACCOUNTING
|
8854 EXTENT_DEFRAG
, 1, 0, &cached_state
,
8857 * whoever cleared the private bit is responsible
8858 * for the finish_ordered_io
8860 if (TestClearPagePrivate2(page
)) {
8861 struct btrfs_ordered_inode_tree
*tree
;
8864 tree
= &BTRFS_I(inode
)->ordered_tree
;
8866 spin_lock_irq(&tree
->lock
);
8867 set_bit(BTRFS_ORDERED_TRUNCATED
, &ordered
->flags
);
8868 new_len
= start
- ordered
->file_offset
;
8869 if (new_len
< ordered
->truncated_len
)
8870 ordered
->truncated_len
= new_len
;
8871 spin_unlock_irq(&tree
->lock
);
8873 if (btrfs_dec_test_ordered_pending(inode
, &ordered
,
8875 end
- start
+ 1, 1))
8876 btrfs_finish_ordered_io(ordered
);
8878 btrfs_put_ordered_extent(ordered
);
8879 if (!inode_evicting
) {
8880 cached_state
= NULL
;
8881 lock_extent_bits(tree
, start
, end
,
8886 if (start
< page_end
)
8891 * Qgroup reserved space handler
8892 * Page here will be either
8893 * 1) Already written to disk
8894 * In this case, its reserved space is released from data rsv map
8895 * and will be freed by delayed_ref handler finally.
8896 * So even we call qgroup_free_data(), it won't decrease reserved
8898 * 2) Not written to disk
8899 * This means the reserved space should be freed here. However,
8900 * if a truncate invalidates the page (by clearing PageDirty)
8901 * and the page is accounted for while allocating extent
8902 * in btrfs_check_data_free_space() we let delayed_ref to
8903 * free the entire extent.
8905 if (PageDirty(page
))
8906 btrfs_qgroup_free_data(inode
, page_start
, PAGE_SIZE
);
8907 if (!inode_evicting
) {
8908 clear_extent_bit(tree
, page_start
, page_end
,
8909 EXTENT_LOCKED
| EXTENT_DIRTY
|
8910 EXTENT_DELALLOC
| EXTENT_DO_ACCOUNTING
|
8911 EXTENT_DEFRAG
, 1, 1,
8912 &cached_state
, GFP_NOFS
);
8914 __btrfs_releasepage(page
, GFP_NOFS
);
8917 ClearPageChecked(page
);
8918 if (PagePrivate(page
)) {
8919 ClearPagePrivate(page
);
8920 set_page_private(page
, 0);
8926 * btrfs_page_mkwrite() is not allowed to change the file size as it gets
8927 * called from a page fault handler when a page is first dirtied. Hence we must
8928 * be careful to check for EOF conditions here. We set the page up correctly
8929 * for a written page which means we get ENOSPC checking when writing into
8930 * holes and correct delalloc and unwritten extent mapping on filesystems that
8931 * support these features.
8933 * We are not allowed to take the i_mutex here so we have to play games to
8934 * protect against truncate races as the page could now be beyond EOF. Because
8935 * vmtruncate() writes the inode size before removing pages, once we have the
8936 * page lock we can determine safely if the page is beyond EOF. If it is not
8937 * beyond EOF, then the page is guaranteed safe against truncation until we
8940 int btrfs_page_mkwrite(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
8942 struct page
*page
= vmf
->page
;
8943 struct inode
*inode
= file_inode(vma
->vm_file
);
8944 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
8945 struct extent_io_tree
*io_tree
= &BTRFS_I(inode
)->io_tree
;
8946 struct btrfs_ordered_extent
*ordered
;
8947 struct extent_state
*cached_state
= NULL
;
8949 unsigned long zero_start
;
8958 reserved_space
= PAGE_SIZE
;
8960 sb_start_pagefault(inode
->i_sb
);
8961 page_start
= page_offset(page
);
8962 page_end
= page_start
+ PAGE_SIZE
- 1;
8966 * Reserving delalloc space after obtaining the page lock can lead to
8967 * deadlock. For example, if a dirty page is locked by this function
8968 * and the call to btrfs_delalloc_reserve_space() ends up triggering
8969 * dirty page write out, then the btrfs_writepage() function could
8970 * end up waiting indefinitely to get a lock on the page currently
8971 * being processed by btrfs_page_mkwrite() function.
8973 ret
= btrfs_delalloc_reserve_space(inode
, page_start
,
8976 ret
= file_update_time(vma
->vm_file
);
8982 else /* -ENOSPC, -EIO, etc */
8983 ret
= VM_FAULT_SIGBUS
;
8989 ret
= VM_FAULT_NOPAGE
; /* make the VM retry the fault */
8992 size
= i_size_read(inode
);
8994 if ((page
->mapping
!= inode
->i_mapping
) ||
8995 (page_start
>= size
)) {
8996 /* page got truncated out from underneath us */
8999 wait_on_page_writeback(page
);
9001 lock_extent_bits(io_tree
, page_start
, page_end
, &cached_state
);
9002 set_page_extent_mapped(page
);
9005 * we can't set the delalloc bits if there are pending ordered
9006 * extents. Drop our locks and wait for them to finish
9008 ordered
= btrfs_lookup_ordered_range(inode
, page_start
, PAGE_SIZE
);
9010 unlock_extent_cached(io_tree
, page_start
, page_end
,
9011 &cached_state
, GFP_NOFS
);
9013 btrfs_start_ordered_extent(inode
, ordered
, 1);
9014 btrfs_put_ordered_extent(ordered
);
9018 if (page
->index
== ((size
- 1) >> PAGE_SHIFT
)) {
9019 reserved_space
= round_up(size
- page_start
,
9020 fs_info
->sectorsize
);
9021 if (reserved_space
< PAGE_SIZE
) {
9022 end
= page_start
+ reserved_space
- 1;
9023 spin_lock(&BTRFS_I(inode
)->lock
);
9024 BTRFS_I(inode
)->outstanding_extents
++;
9025 spin_unlock(&BTRFS_I(inode
)->lock
);
9026 btrfs_delalloc_release_space(inode
, page_start
,
9027 PAGE_SIZE
- reserved_space
);
9032 * page_mkwrite gets called when the page is firstly dirtied after it's
9033 * faulted in, but write(2) could also dirty a page and set delalloc
9034 * bits, thus in this case for space account reason, we still need to
9035 * clear any delalloc bits within this page range since we have to
9036 * reserve data&meta space before lock_page() (see above comments).
9038 clear_extent_bit(&BTRFS_I(inode
)->io_tree
, page_start
, end
,
9039 EXTENT_DIRTY
| EXTENT_DELALLOC
|
9040 EXTENT_DO_ACCOUNTING
| EXTENT_DEFRAG
,
9041 0, 0, &cached_state
, GFP_NOFS
);
9043 ret
= btrfs_set_extent_delalloc(inode
, page_start
, end
,
9046 unlock_extent_cached(io_tree
, page_start
, page_end
,
9047 &cached_state
, GFP_NOFS
);
9048 ret
= VM_FAULT_SIGBUS
;
9053 /* page is wholly or partially inside EOF */
9054 if (page_start
+ PAGE_SIZE
> size
)
9055 zero_start
= size
& ~PAGE_MASK
;
9057 zero_start
= PAGE_SIZE
;
9059 if (zero_start
!= PAGE_SIZE
) {
9061 memset(kaddr
+ zero_start
, 0, PAGE_SIZE
- zero_start
);
9062 flush_dcache_page(page
);
9065 ClearPageChecked(page
);
9066 set_page_dirty(page
);
9067 SetPageUptodate(page
);
9069 BTRFS_I(inode
)->last_trans
= fs_info
->generation
;
9070 BTRFS_I(inode
)->last_sub_trans
= BTRFS_I(inode
)->root
->log_transid
;
9071 BTRFS_I(inode
)->last_log_commit
= BTRFS_I(inode
)->root
->last_log_commit
;
9073 unlock_extent_cached(io_tree
, page_start
, page_end
, &cached_state
, GFP_NOFS
);
9077 sb_end_pagefault(inode
->i_sb
);
9078 return VM_FAULT_LOCKED
;
9082 btrfs_delalloc_release_space(inode
, page_start
, reserved_space
);
9084 sb_end_pagefault(inode
->i_sb
);
9088 static int btrfs_truncate(struct inode
*inode
)
9090 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
9091 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
9092 struct btrfs_block_rsv
*rsv
;
9095 struct btrfs_trans_handle
*trans
;
9096 u64 mask
= fs_info
->sectorsize
- 1;
9097 u64 min_size
= btrfs_calc_trunc_metadata_size(fs_info
, 1);
9099 ret
= btrfs_wait_ordered_range(inode
, inode
->i_size
& (~mask
),
9105 * Yes ladies and gentlemen, this is indeed ugly. The fact is we have
9106 * 3 things going on here
9108 * 1) We need to reserve space for our orphan item and the space to
9109 * delete our orphan item. Lord knows we don't want to have a dangling
9110 * orphan item because we didn't reserve space to remove it.
9112 * 2) We need to reserve space to update our inode.
9114 * 3) We need to have something to cache all the space that is going to
9115 * be free'd up by the truncate operation, but also have some slack
9116 * space reserved in case it uses space during the truncate (thank you
9117 * very much snapshotting).
9119 * And we need these to all be separate. The fact is we can use a lot of
9120 * space doing the truncate, and we have no earthly idea how much space
9121 * we will use, so we need the truncate reservation to be separate so it
9122 * doesn't end up using space reserved for updating the inode or
9123 * removing the orphan item. We also need to be able to stop the
9124 * transaction and start a new one, which means we need to be able to
9125 * update the inode several times, and we have no idea of knowing how
9126 * many times that will be, so we can't just reserve 1 item for the
9127 * entirety of the operation, so that has to be done separately as well.
9128 * Then there is the orphan item, which does indeed need to be held on
9129 * to for the whole operation, and we need nobody to touch this reserved
9130 * space except the orphan code.
9132 * So that leaves us with
9134 * 1) root->orphan_block_rsv - for the orphan deletion.
9135 * 2) rsv - for the truncate reservation, which we will steal from the
9136 * transaction reservation.
9137 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for
9138 * updating the inode.
9140 rsv
= btrfs_alloc_block_rsv(fs_info
, BTRFS_BLOCK_RSV_TEMP
);
9143 rsv
->size
= min_size
;
9147 * 1 for the truncate slack space
9148 * 1 for updating the inode.
9150 trans
= btrfs_start_transaction(root
, 2);
9151 if (IS_ERR(trans
)) {
9152 err
= PTR_ERR(trans
);
9156 /* Migrate the slack space for the truncate to our reserve */
9157 ret
= btrfs_block_rsv_migrate(&fs_info
->trans_block_rsv
, rsv
,
9162 * So if we truncate and then write and fsync we normally would just
9163 * write the extents that changed, which is a problem if we need to
9164 * first truncate that entire inode. So set this flag so we write out
9165 * all of the extents in the inode to the sync log so we're completely
9168 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC
, &BTRFS_I(inode
)->runtime_flags
);
9169 trans
->block_rsv
= rsv
;
9172 ret
= btrfs_truncate_inode_items(trans
, root
, inode
,
9174 BTRFS_EXTENT_DATA_KEY
);
9175 if (ret
!= -ENOSPC
&& ret
!= -EAGAIN
) {
9180 trans
->block_rsv
= &fs_info
->trans_block_rsv
;
9181 ret
= btrfs_update_inode(trans
, root
, inode
);
9187 btrfs_end_transaction(trans
);
9188 btrfs_btree_balance_dirty(fs_info
);
9190 trans
= btrfs_start_transaction(root
, 2);
9191 if (IS_ERR(trans
)) {
9192 ret
= err
= PTR_ERR(trans
);
9197 btrfs_block_rsv_release(fs_info
, rsv
, -1);
9198 ret
= btrfs_block_rsv_migrate(&fs_info
->trans_block_rsv
,
9200 BUG_ON(ret
); /* shouldn't happen */
9201 trans
->block_rsv
= rsv
;
9204 if (ret
== 0 && inode
->i_nlink
> 0) {
9205 trans
->block_rsv
= root
->orphan_block_rsv
;
9206 ret
= btrfs_orphan_del(trans
, inode
);
9212 trans
->block_rsv
= &fs_info
->trans_block_rsv
;
9213 ret
= btrfs_update_inode(trans
, root
, inode
);
9217 ret
= btrfs_end_transaction(trans
);
9218 btrfs_btree_balance_dirty(fs_info
);
9221 btrfs_free_block_rsv(fs_info
, rsv
);
9230 * create a new subvolume directory/inode (helper for the ioctl).
9232 int btrfs_create_subvol_root(struct btrfs_trans_handle
*trans
,
9233 struct btrfs_root
*new_root
,
9234 struct btrfs_root
*parent_root
,
9237 struct inode
*inode
;
9241 inode
= btrfs_new_inode(trans
, new_root
, NULL
, "..", 2,
9242 new_dirid
, new_dirid
,
9243 S_IFDIR
| (~current_umask() & S_IRWXUGO
),
9246 return PTR_ERR(inode
);
9247 inode
->i_op
= &btrfs_dir_inode_operations
;
9248 inode
->i_fop
= &btrfs_dir_file_operations
;
9250 set_nlink(inode
, 1);
9251 btrfs_i_size_write(inode
, 0);
9252 unlock_new_inode(inode
);
9254 err
= btrfs_subvol_inherit_props(trans
, new_root
, parent_root
);
9256 btrfs_err(new_root
->fs_info
,
9257 "error inheriting subvolume %llu properties: %d",
9258 new_root
->root_key
.objectid
, err
);
9260 err
= btrfs_update_inode(trans
, new_root
, inode
);
9266 struct inode
*btrfs_alloc_inode(struct super_block
*sb
)
9268 struct btrfs_inode
*ei
;
9269 struct inode
*inode
;
9271 ei
= kmem_cache_alloc(btrfs_inode_cachep
, GFP_NOFS
);
9278 ei
->last_sub_trans
= 0;
9279 ei
->logged_trans
= 0;
9280 ei
->delalloc_bytes
= 0;
9281 ei
->defrag_bytes
= 0;
9282 ei
->disk_i_size
= 0;
9285 ei
->index_cnt
= (u64
)-1;
9287 ei
->last_unlink_trans
= 0;
9288 ei
->last_log_commit
= 0;
9289 ei
->delayed_iput_count
= 0;
9291 spin_lock_init(&ei
->lock
);
9292 ei
->outstanding_extents
= 0;
9293 ei
->reserved_extents
= 0;
9295 ei
->runtime_flags
= 0;
9296 ei
->force_compress
= BTRFS_COMPRESS_NONE
;
9298 ei
->delayed_node
= NULL
;
9300 ei
->i_otime
.tv_sec
= 0;
9301 ei
->i_otime
.tv_nsec
= 0;
9303 inode
= &ei
->vfs_inode
;
9304 extent_map_tree_init(&ei
->extent_tree
);
9305 extent_io_tree_init(&ei
->io_tree
, &inode
->i_data
);
9306 extent_io_tree_init(&ei
->io_failure_tree
, &inode
->i_data
);
9307 ei
->io_tree
.track_uptodate
= 1;
9308 ei
->io_failure_tree
.track_uptodate
= 1;
9309 atomic_set(&ei
->sync_writers
, 0);
9310 mutex_init(&ei
->log_mutex
);
9311 mutex_init(&ei
->delalloc_mutex
);
9312 btrfs_ordered_inode_tree_init(&ei
->ordered_tree
);
9313 INIT_LIST_HEAD(&ei
->delalloc_inodes
);
9314 INIT_LIST_HEAD(&ei
->delayed_iput
);
9315 RB_CLEAR_NODE(&ei
->rb_node
);
9316 init_rwsem(&ei
->dio_sem
);
9321 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
9322 void btrfs_test_destroy_inode(struct inode
*inode
)
9324 btrfs_drop_extent_cache(inode
, 0, (u64
)-1, 0);
9325 kmem_cache_free(btrfs_inode_cachep
, BTRFS_I(inode
));
9329 static void btrfs_i_callback(struct rcu_head
*head
)
9331 struct inode
*inode
= container_of(head
, struct inode
, i_rcu
);
9332 kmem_cache_free(btrfs_inode_cachep
, BTRFS_I(inode
));
9335 void btrfs_destroy_inode(struct inode
*inode
)
9337 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
9338 struct btrfs_ordered_extent
*ordered
;
9339 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
9341 WARN_ON(!hlist_empty(&inode
->i_dentry
));
9342 WARN_ON(inode
->i_data
.nrpages
);
9343 WARN_ON(BTRFS_I(inode
)->outstanding_extents
);
9344 WARN_ON(BTRFS_I(inode
)->reserved_extents
);
9345 WARN_ON(BTRFS_I(inode
)->delalloc_bytes
);
9346 WARN_ON(BTRFS_I(inode
)->csum_bytes
);
9347 WARN_ON(BTRFS_I(inode
)->defrag_bytes
);
9350 * This can happen where we create an inode, but somebody else also
9351 * created the same inode and we need to destroy the one we already
9357 if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM
,
9358 &BTRFS_I(inode
)->runtime_flags
)) {
9359 btrfs_info(fs_info
, "inode %llu still on the orphan list",
9360 btrfs_ino(BTRFS_I(inode
)));
9361 atomic_dec(&root
->orphan_inodes
);
9365 ordered
= btrfs_lookup_first_ordered_extent(inode
, (u64
)-1);
9370 "found ordered extent %llu %llu on inode cleanup",
9371 ordered
->file_offset
, ordered
->len
);
9372 btrfs_remove_ordered_extent(inode
, ordered
);
9373 btrfs_put_ordered_extent(ordered
);
9374 btrfs_put_ordered_extent(ordered
);
9377 btrfs_qgroup_check_reserved_leak(inode
);
9378 inode_tree_del(inode
);
9379 btrfs_drop_extent_cache(inode
, 0, (u64
)-1, 0);
9381 call_rcu(&inode
->i_rcu
, btrfs_i_callback
);
9384 int btrfs_drop_inode(struct inode
*inode
)
9386 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
9391 /* the snap/subvol tree is on deleting */
9392 if (btrfs_root_refs(&root
->root_item
) == 0)
9395 return generic_drop_inode(inode
);
9398 static void init_once(void *foo
)
9400 struct btrfs_inode
*ei
= (struct btrfs_inode
*) foo
;
9402 inode_init_once(&ei
->vfs_inode
);
9405 void btrfs_destroy_cachep(void)
9408 * Make sure all delayed rcu free inodes are flushed before we
9412 kmem_cache_destroy(btrfs_inode_cachep
);
9413 kmem_cache_destroy(btrfs_trans_handle_cachep
);
9414 kmem_cache_destroy(btrfs_transaction_cachep
);
9415 kmem_cache_destroy(btrfs_path_cachep
);
9416 kmem_cache_destroy(btrfs_free_space_cachep
);
9419 int btrfs_init_cachep(void)
9421 btrfs_inode_cachep
= kmem_cache_create("btrfs_inode",
9422 sizeof(struct btrfs_inode
), 0,
9423 SLAB_RECLAIM_ACCOUNT
| SLAB_MEM_SPREAD
| SLAB_ACCOUNT
,
9425 if (!btrfs_inode_cachep
)
9428 btrfs_trans_handle_cachep
= kmem_cache_create("btrfs_trans_handle",
9429 sizeof(struct btrfs_trans_handle
), 0,
9430 SLAB_TEMPORARY
| SLAB_MEM_SPREAD
, NULL
);
9431 if (!btrfs_trans_handle_cachep
)
9434 btrfs_transaction_cachep
= kmem_cache_create("btrfs_transaction",
9435 sizeof(struct btrfs_transaction
), 0,
9436 SLAB_TEMPORARY
| SLAB_MEM_SPREAD
, NULL
);
9437 if (!btrfs_transaction_cachep
)
9440 btrfs_path_cachep
= kmem_cache_create("btrfs_path",
9441 sizeof(struct btrfs_path
), 0,
9442 SLAB_MEM_SPREAD
, NULL
);
9443 if (!btrfs_path_cachep
)
9446 btrfs_free_space_cachep
= kmem_cache_create("btrfs_free_space",
9447 sizeof(struct btrfs_free_space
), 0,
9448 SLAB_MEM_SPREAD
, NULL
);
9449 if (!btrfs_free_space_cachep
)
9454 btrfs_destroy_cachep();
9458 static int btrfs_getattr(struct vfsmount
*mnt
,
9459 struct dentry
*dentry
, struct kstat
*stat
)
9462 struct inode
*inode
= d_inode(dentry
);
9463 u32 blocksize
= inode
->i_sb
->s_blocksize
;
9465 generic_fillattr(inode
, stat
);
9466 stat
->dev
= BTRFS_I(inode
)->root
->anon_dev
;
9468 spin_lock(&BTRFS_I(inode
)->lock
);
9469 delalloc_bytes
= BTRFS_I(inode
)->delalloc_bytes
;
9470 spin_unlock(&BTRFS_I(inode
)->lock
);
9471 stat
->blocks
= (ALIGN(inode_get_bytes(inode
), blocksize
) +
9472 ALIGN(delalloc_bytes
, blocksize
)) >> 9;
9476 static int btrfs_rename_exchange(struct inode
*old_dir
,
9477 struct dentry
*old_dentry
,
9478 struct inode
*new_dir
,
9479 struct dentry
*new_dentry
)
9481 struct btrfs_fs_info
*fs_info
= btrfs_sb(old_dir
->i_sb
);
9482 struct btrfs_trans_handle
*trans
;
9483 struct btrfs_root
*root
= BTRFS_I(old_dir
)->root
;
9484 struct btrfs_root
*dest
= BTRFS_I(new_dir
)->root
;
9485 struct inode
*new_inode
= new_dentry
->d_inode
;
9486 struct inode
*old_inode
= old_dentry
->d_inode
;
9487 struct timespec ctime
= current_time(old_inode
);
9488 struct dentry
*parent
;
9489 u64 old_ino
= btrfs_ino(BTRFS_I(old_inode
));
9490 u64 new_ino
= btrfs_ino(BTRFS_I(new_inode
));
9495 bool root_log_pinned
= false;
9496 bool dest_log_pinned
= false;
9498 /* we only allow rename subvolume link between subvolumes */
9499 if (old_ino
!= BTRFS_FIRST_FREE_OBJECTID
&& root
!= dest
)
9502 /* close the race window with snapshot create/destroy ioctl */
9503 if (old_ino
== BTRFS_FIRST_FREE_OBJECTID
)
9504 down_read(&fs_info
->subvol_sem
);
9505 if (new_ino
== BTRFS_FIRST_FREE_OBJECTID
)
9506 down_read(&fs_info
->subvol_sem
);
9509 * We want to reserve the absolute worst case amount of items. So if
9510 * both inodes are subvols and we need to unlink them then that would
9511 * require 4 item modifications, but if they are both normal inodes it
9512 * would require 5 item modifications, so we'll assume their normal
9513 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items
9514 * should cover the worst case number of items we'll modify.
9516 trans
= btrfs_start_transaction(root
, 12);
9517 if (IS_ERR(trans
)) {
9518 ret
= PTR_ERR(trans
);
9523 * We need to find a free sequence number both in the source and
9524 * in the destination directory for the exchange.
9526 ret
= btrfs_set_inode_index(new_dir
, &old_idx
);
9529 ret
= btrfs_set_inode_index(old_dir
, &new_idx
);
9533 BTRFS_I(old_inode
)->dir_index
= 0ULL;
9534 BTRFS_I(new_inode
)->dir_index
= 0ULL;
9536 /* Reference for the source. */
9537 if (old_ino
== BTRFS_FIRST_FREE_OBJECTID
) {
9538 /* force full log commit if subvolume involved. */
9539 btrfs_set_log_full_commit(fs_info
, trans
);
9541 btrfs_pin_log_trans(root
);
9542 root_log_pinned
= true;
9543 ret
= btrfs_insert_inode_ref(trans
, dest
,
9544 new_dentry
->d_name
.name
,
9545 new_dentry
->d_name
.len
,
9547 btrfs_ino(BTRFS_I(new_dir
)),
9553 /* And now for the dest. */
9554 if (new_ino
== BTRFS_FIRST_FREE_OBJECTID
) {
9555 /* force full log commit if subvolume involved. */
9556 btrfs_set_log_full_commit(fs_info
, trans
);
9558 btrfs_pin_log_trans(dest
);
9559 dest_log_pinned
= true;
9560 ret
= btrfs_insert_inode_ref(trans
, root
,
9561 old_dentry
->d_name
.name
,
9562 old_dentry
->d_name
.len
,
9564 btrfs_ino(BTRFS_I(old_dir
)),
9570 /* Update inode version and ctime/mtime. */
9571 inode_inc_iversion(old_dir
);
9572 inode_inc_iversion(new_dir
);
9573 inode_inc_iversion(old_inode
);
9574 inode_inc_iversion(new_inode
);
9575 old_dir
->i_ctime
= old_dir
->i_mtime
= ctime
;
9576 new_dir
->i_ctime
= new_dir
->i_mtime
= ctime
;
9577 old_inode
->i_ctime
= ctime
;
9578 new_inode
->i_ctime
= ctime
;
9580 if (old_dentry
->d_parent
!= new_dentry
->d_parent
) {
9581 btrfs_record_unlink_dir(trans
, BTRFS_I(old_dir
),
9582 BTRFS_I(old_inode
), 1);
9583 btrfs_record_unlink_dir(trans
, BTRFS_I(new_dir
),
9584 BTRFS_I(new_inode
), 1);
9587 /* src is a subvolume */
9588 if (old_ino
== BTRFS_FIRST_FREE_OBJECTID
) {
9589 root_objectid
= BTRFS_I(old_inode
)->root
->root_key
.objectid
;
9590 ret
= btrfs_unlink_subvol(trans
, root
, old_dir
,
9592 old_dentry
->d_name
.name
,
9593 old_dentry
->d_name
.len
);
9594 } else { /* src is an inode */
9595 ret
= __btrfs_unlink_inode(trans
, root
, BTRFS_I(old_dir
),
9596 BTRFS_I(old_dentry
->d_inode
),
9597 old_dentry
->d_name
.name
,
9598 old_dentry
->d_name
.len
);
9600 ret
= btrfs_update_inode(trans
, root
, old_inode
);
9603 btrfs_abort_transaction(trans
, ret
);
9607 /* dest is a subvolume */
9608 if (new_ino
== BTRFS_FIRST_FREE_OBJECTID
) {
9609 root_objectid
= BTRFS_I(new_inode
)->root
->root_key
.objectid
;
9610 ret
= btrfs_unlink_subvol(trans
, dest
, new_dir
,
9612 new_dentry
->d_name
.name
,
9613 new_dentry
->d_name
.len
);
9614 } else { /* dest is an inode */
9615 ret
= __btrfs_unlink_inode(trans
, dest
, BTRFS_I(new_dir
),
9616 BTRFS_I(new_dentry
->d_inode
),
9617 new_dentry
->d_name
.name
,
9618 new_dentry
->d_name
.len
);
9620 ret
= btrfs_update_inode(trans
, dest
, new_inode
);
9623 btrfs_abort_transaction(trans
, ret
);
9627 ret
= btrfs_add_link(trans
, new_dir
, old_inode
,
9628 new_dentry
->d_name
.name
,
9629 new_dentry
->d_name
.len
, 0, old_idx
);
9631 btrfs_abort_transaction(trans
, ret
);
9635 ret
= btrfs_add_link(trans
, old_dir
, new_inode
,
9636 old_dentry
->d_name
.name
,
9637 old_dentry
->d_name
.len
, 0, new_idx
);
9639 btrfs_abort_transaction(trans
, ret
);
9643 if (old_inode
->i_nlink
== 1)
9644 BTRFS_I(old_inode
)->dir_index
= old_idx
;
9645 if (new_inode
->i_nlink
== 1)
9646 BTRFS_I(new_inode
)->dir_index
= new_idx
;
9648 if (root_log_pinned
) {
9649 parent
= new_dentry
->d_parent
;
9650 btrfs_log_new_name(trans
, BTRFS_I(old_inode
), BTRFS_I(old_dir
),
9652 btrfs_end_log_trans(root
);
9653 root_log_pinned
= false;
9655 if (dest_log_pinned
) {
9656 parent
= old_dentry
->d_parent
;
9657 btrfs_log_new_name(trans
, BTRFS_I(new_inode
), BTRFS_I(new_dir
),
9659 btrfs_end_log_trans(dest
);
9660 dest_log_pinned
= false;
9664 * If we have pinned a log and an error happened, we unpin tasks
9665 * trying to sync the log and force them to fallback to a transaction
9666 * commit if the log currently contains any of the inodes involved in
9667 * this rename operation (to ensure we do not persist a log with an
9668 * inconsistent state for any of these inodes or leading to any
9669 * inconsistencies when replayed). If the transaction was aborted, the
9670 * abortion reason is propagated to userspace when attempting to commit
9671 * the transaction. If the log does not contain any of these inodes, we
9672 * allow the tasks to sync it.
9674 if (ret
&& (root_log_pinned
|| dest_log_pinned
)) {
9675 if (btrfs_inode_in_log(BTRFS_I(old_dir
), fs_info
->generation
) ||
9676 btrfs_inode_in_log(BTRFS_I(new_dir
), fs_info
->generation
) ||
9677 btrfs_inode_in_log(BTRFS_I(old_inode
), fs_info
->generation
) ||
9679 btrfs_inode_in_log(BTRFS_I(new_inode
), fs_info
->generation
)))
9680 btrfs_set_log_full_commit(fs_info
, trans
);
9682 if (root_log_pinned
) {
9683 btrfs_end_log_trans(root
);
9684 root_log_pinned
= false;
9686 if (dest_log_pinned
) {
9687 btrfs_end_log_trans(dest
);
9688 dest_log_pinned
= false;
9691 ret
= btrfs_end_transaction(trans
);
9693 if (new_ino
== BTRFS_FIRST_FREE_OBJECTID
)
9694 up_read(&fs_info
->subvol_sem
);
9695 if (old_ino
== BTRFS_FIRST_FREE_OBJECTID
)
9696 up_read(&fs_info
->subvol_sem
);
9701 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle
*trans
,
9702 struct btrfs_root
*root
,
9704 struct dentry
*dentry
)
9707 struct inode
*inode
;
9711 ret
= btrfs_find_free_ino(root
, &objectid
);
9715 inode
= btrfs_new_inode(trans
, root
, dir
,
9716 dentry
->d_name
.name
,
9718 btrfs_ino(BTRFS_I(dir
)),
9720 S_IFCHR
| WHITEOUT_MODE
,
9723 if (IS_ERR(inode
)) {
9724 ret
= PTR_ERR(inode
);
9728 inode
->i_op
= &btrfs_special_inode_operations
;
9729 init_special_inode(inode
, inode
->i_mode
,
9732 ret
= btrfs_init_inode_security(trans
, inode
, dir
,
9737 ret
= btrfs_add_nondir(trans
, dir
, dentry
,
9742 ret
= btrfs_update_inode(trans
, root
, inode
);
9744 unlock_new_inode(inode
);
9746 inode_dec_link_count(inode
);
9752 static int btrfs_rename(struct inode
*old_dir
, struct dentry
*old_dentry
,
9753 struct inode
*new_dir
, struct dentry
*new_dentry
,
9756 struct btrfs_fs_info
*fs_info
= btrfs_sb(old_dir
->i_sb
);
9757 struct btrfs_trans_handle
*trans
;
9758 unsigned int trans_num_items
;
9759 struct btrfs_root
*root
= BTRFS_I(old_dir
)->root
;
9760 struct btrfs_root
*dest
= BTRFS_I(new_dir
)->root
;
9761 struct inode
*new_inode
= d_inode(new_dentry
);
9762 struct inode
*old_inode
= d_inode(old_dentry
);
9766 u64 old_ino
= btrfs_ino(BTRFS_I(old_inode
));
9767 bool log_pinned
= false;
9769 if (btrfs_ino(BTRFS_I(new_dir
)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID
)
9772 /* we only allow rename subvolume link between subvolumes */
9773 if (old_ino
!= BTRFS_FIRST_FREE_OBJECTID
&& root
!= dest
)
9776 if (old_ino
== BTRFS_EMPTY_SUBVOL_DIR_OBJECTID
||
9777 (new_inode
&& btrfs_ino(BTRFS_I(new_inode
)) == BTRFS_FIRST_FREE_OBJECTID
))
9780 if (S_ISDIR(old_inode
->i_mode
) && new_inode
&&
9781 new_inode
->i_size
> BTRFS_EMPTY_DIR_SIZE
)
9785 /* check for collisions, even if the name isn't there */
9786 ret
= btrfs_check_dir_item_collision(dest
, new_dir
->i_ino
,
9787 new_dentry
->d_name
.name
,
9788 new_dentry
->d_name
.len
);
9791 if (ret
== -EEXIST
) {
9793 * eexist without a new_inode */
9794 if (WARN_ON(!new_inode
)) {
9798 /* maybe -EOVERFLOW */
9805 * we're using rename to replace one file with another. Start IO on it
9806 * now so we don't add too much work to the end of the transaction
9808 if (new_inode
&& S_ISREG(old_inode
->i_mode
) && new_inode
->i_size
)
9809 filemap_flush(old_inode
->i_mapping
);
9811 /* close the racy window with snapshot create/destroy ioctl */
9812 if (old_ino
== BTRFS_FIRST_FREE_OBJECTID
)
9813 down_read(&fs_info
->subvol_sem
);
9815 * We want to reserve the absolute worst case amount of items. So if
9816 * both inodes are subvols and we need to unlink them then that would
9817 * require 4 item modifications, but if they are both normal inodes it
9818 * would require 5 item modifications, so we'll assume they are normal
9819 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items
9820 * should cover the worst case number of items we'll modify.
9821 * If our rename has the whiteout flag, we need more 5 units for the
9822 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
9823 * when selinux is enabled).
9825 trans_num_items
= 11;
9826 if (flags
& RENAME_WHITEOUT
)
9827 trans_num_items
+= 5;
9828 trans
= btrfs_start_transaction(root
, trans_num_items
);
9829 if (IS_ERR(trans
)) {
9830 ret
= PTR_ERR(trans
);
9835 btrfs_record_root_in_trans(trans
, dest
);
9837 ret
= btrfs_set_inode_index(new_dir
, &index
);
9841 BTRFS_I(old_inode
)->dir_index
= 0ULL;
9842 if (unlikely(old_ino
== BTRFS_FIRST_FREE_OBJECTID
)) {
9843 /* force full log commit if subvolume involved. */
9844 btrfs_set_log_full_commit(fs_info
, trans
);
9846 btrfs_pin_log_trans(root
);
9848 ret
= btrfs_insert_inode_ref(trans
, dest
,
9849 new_dentry
->d_name
.name
,
9850 new_dentry
->d_name
.len
,
9852 btrfs_ino(BTRFS_I(new_dir
)), index
);
9857 inode_inc_iversion(old_dir
);
9858 inode_inc_iversion(new_dir
);
9859 inode_inc_iversion(old_inode
);
9860 old_dir
->i_ctime
= old_dir
->i_mtime
=
9861 new_dir
->i_ctime
= new_dir
->i_mtime
=
9862 old_inode
->i_ctime
= current_time(old_dir
);
9864 if (old_dentry
->d_parent
!= new_dentry
->d_parent
)
9865 btrfs_record_unlink_dir(trans
, BTRFS_I(old_dir
),
9866 BTRFS_I(old_inode
), 1);
9868 if (unlikely(old_ino
== BTRFS_FIRST_FREE_OBJECTID
)) {
9869 root_objectid
= BTRFS_I(old_inode
)->root
->root_key
.objectid
;
9870 ret
= btrfs_unlink_subvol(trans
, root
, old_dir
, root_objectid
,
9871 old_dentry
->d_name
.name
,
9872 old_dentry
->d_name
.len
);
9874 ret
= __btrfs_unlink_inode(trans
, root
, BTRFS_I(old_dir
),
9875 BTRFS_I(d_inode(old_dentry
)),
9876 old_dentry
->d_name
.name
,
9877 old_dentry
->d_name
.len
);
9879 ret
= btrfs_update_inode(trans
, root
, old_inode
);
9882 btrfs_abort_transaction(trans
, ret
);
9887 inode_inc_iversion(new_inode
);
9888 new_inode
->i_ctime
= current_time(new_inode
);
9889 if (unlikely(btrfs_ino(BTRFS_I(new_inode
)) ==
9890 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID
)) {
9891 root_objectid
= BTRFS_I(new_inode
)->location
.objectid
;
9892 ret
= btrfs_unlink_subvol(trans
, dest
, new_dir
,
9894 new_dentry
->d_name
.name
,
9895 new_dentry
->d_name
.len
);
9896 BUG_ON(new_inode
->i_nlink
== 0);
9898 ret
= btrfs_unlink_inode(trans
, dest
, BTRFS_I(new_dir
),
9899 BTRFS_I(d_inode(new_dentry
)),
9900 new_dentry
->d_name
.name
,
9901 new_dentry
->d_name
.len
);
9903 if (!ret
&& new_inode
->i_nlink
== 0)
9904 ret
= btrfs_orphan_add(trans
, d_inode(new_dentry
));
9906 btrfs_abort_transaction(trans
, ret
);
9911 ret
= btrfs_add_link(trans
, new_dir
, old_inode
,
9912 new_dentry
->d_name
.name
,
9913 new_dentry
->d_name
.len
, 0, index
);
9915 btrfs_abort_transaction(trans
, ret
);
9919 if (old_inode
->i_nlink
== 1)
9920 BTRFS_I(old_inode
)->dir_index
= index
;
9923 struct dentry
*parent
= new_dentry
->d_parent
;
9925 btrfs_log_new_name(trans
, BTRFS_I(old_inode
), BTRFS_I(old_dir
),
9927 btrfs_end_log_trans(root
);
9931 if (flags
& RENAME_WHITEOUT
) {
9932 ret
= btrfs_whiteout_for_rename(trans
, root
, old_dir
,
9936 btrfs_abort_transaction(trans
, ret
);
9942 * If we have pinned the log and an error happened, we unpin tasks
9943 * trying to sync the log and force them to fallback to a transaction
9944 * commit if the log currently contains any of the inodes involved in
9945 * this rename operation (to ensure we do not persist a log with an
9946 * inconsistent state for any of these inodes or leading to any
9947 * inconsistencies when replayed). If the transaction was aborted, the
9948 * abortion reason is propagated to userspace when attempting to commit
9949 * the transaction. If the log does not contain any of these inodes, we
9950 * allow the tasks to sync it.
9952 if (ret
&& log_pinned
) {
9953 if (btrfs_inode_in_log(BTRFS_I(old_dir
), fs_info
->generation
) ||
9954 btrfs_inode_in_log(BTRFS_I(new_dir
), fs_info
->generation
) ||
9955 btrfs_inode_in_log(BTRFS_I(old_inode
), fs_info
->generation
) ||
9957 btrfs_inode_in_log(BTRFS_I(new_inode
), fs_info
->generation
)))
9958 btrfs_set_log_full_commit(fs_info
, trans
);
9960 btrfs_end_log_trans(root
);
9963 btrfs_end_transaction(trans
);
9965 if (old_ino
== BTRFS_FIRST_FREE_OBJECTID
)
9966 up_read(&fs_info
->subvol_sem
);
9971 static int btrfs_rename2(struct inode
*old_dir
, struct dentry
*old_dentry
,
9972 struct inode
*new_dir
, struct dentry
*new_dentry
,
9975 if (flags
& ~(RENAME_NOREPLACE
| RENAME_EXCHANGE
| RENAME_WHITEOUT
))
9978 if (flags
& RENAME_EXCHANGE
)
9979 return btrfs_rename_exchange(old_dir
, old_dentry
, new_dir
,
9982 return btrfs_rename(old_dir
, old_dentry
, new_dir
, new_dentry
, flags
);
9985 static void btrfs_run_delalloc_work(struct btrfs_work
*work
)
9987 struct btrfs_delalloc_work
*delalloc_work
;
9988 struct inode
*inode
;
9990 delalloc_work
= container_of(work
, struct btrfs_delalloc_work
,
9992 inode
= delalloc_work
->inode
;
9993 filemap_flush(inode
->i_mapping
);
9994 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT
,
9995 &BTRFS_I(inode
)->runtime_flags
))
9996 filemap_flush(inode
->i_mapping
);
9998 if (delalloc_work
->delay_iput
)
9999 btrfs_add_delayed_iput(inode
);
10002 complete(&delalloc_work
->completion
);
10005 struct btrfs_delalloc_work
*btrfs_alloc_delalloc_work(struct inode
*inode
,
10008 struct btrfs_delalloc_work
*work
;
10010 work
= kmalloc(sizeof(*work
), GFP_NOFS
);
10014 init_completion(&work
->completion
);
10015 INIT_LIST_HEAD(&work
->list
);
10016 work
->inode
= inode
;
10017 work
->delay_iput
= delay_iput
;
10018 WARN_ON_ONCE(!inode
);
10019 btrfs_init_work(&work
->work
, btrfs_flush_delalloc_helper
,
10020 btrfs_run_delalloc_work
, NULL
, NULL
);
10025 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work
*work
)
10027 wait_for_completion(&work
->completion
);
10032 * some fairly slow code that needs optimization. This walks the list
10033 * of all the inodes with pending delalloc and forces them to disk.
10035 static int __start_delalloc_inodes(struct btrfs_root
*root
, int delay_iput
,
10038 struct btrfs_inode
*binode
;
10039 struct inode
*inode
;
10040 struct btrfs_delalloc_work
*work
, *next
;
10041 struct list_head works
;
10042 struct list_head splice
;
10045 INIT_LIST_HEAD(&works
);
10046 INIT_LIST_HEAD(&splice
);
10048 mutex_lock(&root
->delalloc_mutex
);
10049 spin_lock(&root
->delalloc_lock
);
10050 list_splice_init(&root
->delalloc_inodes
, &splice
);
10051 while (!list_empty(&splice
)) {
10052 binode
= list_entry(splice
.next
, struct btrfs_inode
,
10055 list_move_tail(&binode
->delalloc_inodes
,
10056 &root
->delalloc_inodes
);
10057 inode
= igrab(&binode
->vfs_inode
);
10059 cond_resched_lock(&root
->delalloc_lock
);
10062 spin_unlock(&root
->delalloc_lock
);
10064 work
= btrfs_alloc_delalloc_work(inode
, delay_iput
);
10067 btrfs_add_delayed_iput(inode
);
10073 list_add_tail(&work
->list
, &works
);
10074 btrfs_queue_work(root
->fs_info
->flush_workers
,
10077 if (nr
!= -1 && ret
>= nr
)
10080 spin_lock(&root
->delalloc_lock
);
10082 spin_unlock(&root
->delalloc_lock
);
10085 list_for_each_entry_safe(work
, next
, &works
, list
) {
10086 list_del_init(&work
->list
);
10087 btrfs_wait_and_free_delalloc_work(work
);
10090 if (!list_empty_careful(&splice
)) {
10091 spin_lock(&root
->delalloc_lock
);
10092 list_splice_tail(&splice
, &root
->delalloc_inodes
);
10093 spin_unlock(&root
->delalloc_lock
);
10095 mutex_unlock(&root
->delalloc_mutex
);
10099 int btrfs_start_delalloc_inodes(struct btrfs_root
*root
, int delay_iput
)
10101 struct btrfs_fs_info
*fs_info
= root
->fs_info
;
10104 if (test_bit(BTRFS_FS_STATE_ERROR
, &fs_info
->fs_state
))
10107 ret
= __start_delalloc_inodes(root
, delay_iput
, -1);
10111 * the filemap_flush will queue IO into the worker threads, but
10112 * we have to make sure the IO is actually started and that
10113 * ordered extents get created before we return
10115 atomic_inc(&fs_info
->async_submit_draining
);
10116 while (atomic_read(&fs_info
->nr_async_submits
) ||
10117 atomic_read(&fs_info
->async_delalloc_pages
)) {
10118 wait_event(fs_info
->async_submit_wait
,
10119 (atomic_read(&fs_info
->nr_async_submits
) == 0 &&
10120 atomic_read(&fs_info
->async_delalloc_pages
) == 0));
10122 atomic_dec(&fs_info
->async_submit_draining
);
10126 int btrfs_start_delalloc_roots(struct btrfs_fs_info
*fs_info
, int delay_iput
,
10129 struct btrfs_root
*root
;
10130 struct list_head splice
;
10133 if (test_bit(BTRFS_FS_STATE_ERROR
, &fs_info
->fs_state
))
10136 INIT_LIST_HEAD(&splice
);
10138 mutex_lock(&fs_info
->delalloc_root_mutex
);
10139 spin_lock(&fs_info
->delalloc_root_lock
);
10140 list_splice_init(&fs_info
->delalloc_roots
, &splice
);
10141 while (!list_empty(&splice
) && nr
) {
10142 root
= list_first_entry(&splice
, struct btrfs_root
,
10144 root
= btrfs_grab_fs_root(root
);
10146 list_move_tail(&root
->delalloc_root
,
10147 &fs_info
->delalloc_roots
);
10148 spin_unlock(&fs_info
->delalloc_root_lock
);
10150 ret
= __start_delalloc_inodes(root
, delay_iput
, nr
);
10151 btrfs_put_fs_root(root
);
10159 spin_lock(&fs_info
->delalloc_root_lock
);
10161 spin_unlock(&fs_info
->delalloc_root_lock
);
10164 atomic_inc(&fs_info
->async_submit_draining
);
10165 while (atomic_read(&fs_info
->nr_async_submits
) ||
10166 atomic_read(&fs_info
->async_delalloc_pages
)) {
10167 wait_event(fs_info
->async_submit_wait
,
10168 (atomic_read(&fs_info
->nr_async_submits
) == 0 &&
10169 atomic_read(&fs_info
->async_delalloc_pages
) == 0));
10171 atomic_dec(&fs_info
->async_submit_draining
);
10173 if (!list_empty_careful(&splice
)) {
10174 spin_lock(&fs_info
->delalloc_root_lock
);
10175 list_splice_tail(&splice
, &fs_info
->delalloc_roots
);
10176 spin_unlock(&fs_info
->delalloc_root_lock
);
10178 mutex_unlock(&fs_info
->delalloc_root_mutex
);
10182 static int btrfs_symlink(struct inode
*dir
, struct dentry
*dentry
,
10183 const char *symname
)
10185 struct btrfs_fs_info
*fs_info
= btrfs_sb(dir
->i_sb
);
10186 struct btrfs_trans_handle
*trans
;
10187 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
10188 struct btrfs_path
*path
;
10189 struct btrfs_key key
;
10190 struct inode
*inode
= NULL
;
10192 int drop_inode
= 0;
10198 struct btrfs_file_extent_item
*ei
;
10199 struct extent_buffer
*leaf
;
10201 name_len
= strlen(symname
);
10202 if (name_len
> BTRFS_MAX_INLINE_DATA_SIZE(fs_info
))
10203 return -ENAMETOOLONG
;
10206 * 2 items for inode item and ref
10207 * 2 items for dir items
10208 * 1 item for updating parent inode item
10209 * 1 item for the inline extent item
10210 * 1 item for xattr if selinux is on
10212 trans
= btrfs_start_transaction(root
, 7);
10214 return PTR_ERR(trans
);
10216 err
= btrfs_find_free_ino(root
, &objectid
);
10220 inode
= btrfs_new_inode(trans
, root
, dir
, dentry
->d_name
.name
,
10221 dentry
->d_name
.len
, btrfs_ino(BTRFS_I(dir
)),
10222 objectid
, S_IFLNK
|S_IRWXUGO
, &index
);
10223 if (IS_ERR(inode
)) {
10224 err
= PTR_ERR(inode
);
10229 * If the active LSM wants to access the inode during
10230 * d_instantiate it needs these. Smack checks to see
10231 * if the filesystem supports xattrs by looking at the
10234 inode
->i_fop
= &btrfs_file_operations
;
10235 inode
->i_op
= &btrfs_file_inode_operations
;
10236 inode
->i_mapping
->a_ops
= &btrfs_aops
;
10237 BTRFS_I(inode
)->io_tree
.ops
= &btrfs_extent_io_ops
;
10239 err
= btrfs_init_inode_security(trans
, inode
, dir
, &dentry
->d_name
);
10241 goto out_unlock_inode
;
10243 path
= btrfs_alloc_path();
10246 goto out_unlock_inode
;
10248 key
.objectid
= btrfs_ino(BTRFS_I(inode
));
10250 key
.type
= BTRFS_EXTENT_DATA_KEY
;
10251 datasize
= btrfs_file_extent_calc_inline_size(name_len
);
10252 err
= btrfs_insert_empty_item(trans
, root
, path
, &key
,
10255 btrfs_free_path(path
);
10256 goto out_unlock_inode
;
10258 leaf
= path
->nodes
[0];
10259 ei
= btrfs_item_ptr(leaf
, path
->slots
[0],
10260 struct btrfs_file_extent_item
);
10261 btrfs_set_file_extent_generation(leaf
, ei
, trans
->transid
);
10262 btrfs_set_file_extent_type(leaf
, ei
,
10263 BTRFS_FILE_EXTENT_INLINE
);
10264 btrfs_set_file_extent_encryption(leaf
, ei
, 0);
10265 btrfs_set_file_extent_compression(leaf
, ei
, 0);
10266 btrfs_set_file_extent_other_encoding(leaf
, ei
, 0);
10267 btrfs_set_file_extent_ram_bytes(leaf
, ei
, name_len
);
10269 ptr
= btrfs_file_extent_inline_start(ei
);
10270 write_extent_buffer(leaf
, symname
, ptr
, name_len
);
10271 btrfs_mark_buffer_dirty(leaf
);
10272 btrfs_free_path(path
);
10274 inode
->i_op
= &btrfs_symlink_inode_operations
;
10275 inode_nohighmem(inode
);
10276 inode
->i_mapping
->a_ops
= &btrfs_symlink_aops
;
10277 inode_set_bytes(inode
, name_len
);
10278 btrfs_i_size_write(inode
, name_len
);
10279 err
= btrfs_update_inode(trans
, root
, inode
);
10281 * Last step, add directory indexes for our symlink inode. This is the
10282 * last step to avoid extra cleanup of these indexes if an error happens
10286 err
= btrfs_add_nondir(trans
, dir
, dentry
, inode
, 0, index
);
10289 goto out_unlock_inode
;
10292 unlock_new_inode(inode
);
10293 d_instantiate(dentry
, inode
);
10296 btrfs_end_transaction(trans
);
10298 inode_dec_link_count(inode
);
10301 btrfs_btree_balance_dirty(fs_info
);
10306 unlock_new_inode(inode
);
10310 static int __btrfs_prealloc_file_range(struct inode
*inode
, int mode
,
10311 u64 start
, u64 num_bytes
, u64 min_size
,
10312 loff_t actual_len
, u64
*alloc_hint
,
10313 struct btrfs_trans_handle
*trans
)
10315 struct btrfs_fs_info
*fs_info
= btrfs_sb(inode
->i_sb
);
10316 struct extent_map_tree
*em_tree
= &BTRFS_I(inode
)->extent_tree
;
10317 struct extent_map
*em
;
10318 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
10319 struct btrfs_key ins
;
10320 u64 cur_offset
= start
;
10323 u64 last_alloc
= (u64
)-1;
10325 bool own_trans
= true;
10326 u64 end
= start
+ num_bytes
- 1;
10330 while (num_bytes
> 0) {
10332 trans
= btrfs_start_transaction(root
, 3);
10333 if (IS_ERR(trans
)) {
10334 ret
= PTR_ERR(trans
);
10339 cur_bytes
= min_t(u64
, num_bytes
, SZ_256M
);
10340 cur_bytes
= max(cur_bytes
, min_size
);
10342 * If we are severely fragmented we could end up with really
10343 * small allocations, so if the allocator is returning small
10344 * chunks lets make its job easier by only searching for those
10347 cur_bytes
= min(cur_bytes
, last_alloc
);
10348 ret
= btrfs_reserve_extent(root
, cur_bytes
, cur_bytes
,
10349 min_size
, 0, *alloc_hint
, &ins
, 1, 0);
10352 btrfs_end_transaction(trans
);
10355 btrfs_dec_block_group_reservations(fs_info
, ins
.objectid
);
10357 last_alloc
= ins
.offset
;
10358 ret
= insert_reserved_file_extent(trans
, inode
,
10359 cur_offset
, ins
.objectid
,
10360 ins
.offset
, ins
.offset
,
10361 ins
.offset
, 0, 0, 0,
10362 BTRFS_FILE_EXTENT_PREALLOC
);
10364 btrfs_free_reserved_extent(fs_info
, ins
.objectid
,
10366 btrfs_abort_transaction(trans
, ret
);
10368 btrfs_end_transaction(trans
);
10372 btrfs_drop_extent_cache(inode
, cur_offset
,
10373 cur_offset
+ ins
.offset
-1, 0);
10375 em
= alloc_extent_map();
10377 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC
,
10378 &BTRFS_I(inode
)->runtime_flags
);
10382 em
->start
= cur_offset
;
10383 em
->orig_start
= cur_offset
;
10384 em
->len
= ins
.offset
;
10385 em
->block_start
= ins
.objectid
;
10386 em
->block_len
= ins
.offset
;
10387 em
->orig_block_len
= ins
.offset
;
10388 em
->ram_bytes
= ins
.offset
;
10389 em
->bdev
= fs_info
->fs_devices
->latest_bdev
;
10390 set_bit(EXTENT_FLAG_PREALLOC
, &em
->flags
);
10391 em
->generation
= trans
->transid
;
10394 write_lock(&em_tree
->lock
);
10395 ret
= add_extent_mapping(em_tree
, em
, 1);
10396 write_unlock(&em_tree
->lock
);
10397 if (ret
!= -EEXIST
)
10399 btrfs_drop_extent_cache(inode
, cur_offset
,
10400 cur_offset
+ ins
.offset
- 1,
10403 free_extent_map(em
);
10405 num_bytes
-= ins
.offset
;
10406 cur_offset
+= ins
.offset
;
10407 *alloc_hint
= ins
.objectid
+ ins
.offset
;
10409 inode_inc_iversion(inode
);
10410 inode
->i_ctime
= current_time(inode
);
10411 BTRFS_I(inode
)->flags
|= BTRFS_INODE_PREALLOC
;
10412 if (!(mode
& FALLOC_FL_KEEP_SIZE
) &&
10413 (actual_len
> inode
->i_size
) &&
10414 (cur_offset
> inode
->i_size
)) {
10415 if (cur_offset
> actual_len
)
10416 i_size
= actual_len
;
10418 i_size
= cur_offset
;
10419 i_size_write(inode
, i_size
);
10420 btrfs_ordered_update_i_size(inode
, i_size
, NULL
);
10423 ret
= btrfs_update_inode(trans
, root
, inode
);
10426 btrfs_abort_transaction(trans
, ret
);
10428 btrfs_end_transaction(trans
);
10433 btrfs_end_transaction(trans
);
10435 if (cur_offset
< end
)
10436 btrfs_free_reserved_data_space(inode
, cur_offset
,
10437 end
- cur_offset
+ 1);
10441 int btrfs_prealloc_file_range(struct inode
*inode
, int mode
,
10442 u64 start
, u64 num_bytes
, u64 min_size
,
10443 loff_t actual_len
, u64
*alloc_hint
)
10445 return __btrfs_prealloc_file_range(inode
, mode
, start
, num_bytes
,
10446 min_size
, actual_len
, alloc_hint
,
10450 int btrfs_prealloc_file_range_trans(struct inode
*inode
,
10451 struct btrfs_trans_handle
*trans
, int mode
,
10452 u64 start
, u64 num_bytes
, u64 min_size
,
10453 loff_t actual_len
, u64
*alloc_hint
)
10455 return __btrfs_prealloc_file_range(inode
, mode
, start
, num_bytes
,
10456 min_size
, actual_len
, alloc_hint
, trans
);
10459 static int btrfs_set_page_dirty(struct page
*page
)
10461 return __set_page_dirty_nobuffers(page
);
10464 static int btrfs_permission(struct inode
*inode
, int mask
)
10466 struct btrfs_root
*root
= BTRFS_I(inode
)->root
;
10467 umode_t mode
= inode
->i_mode
;
10469 if (mask
& MAY_WRITE
&&
10470 (S_ISREG(mode
) || S_ISDIR(mode
) || S_ISLNK(mode
))) {
10471 if (btrfs_root_readonly(root
))
10473 if (BTRFS_I(inode
)->flags
& BTRFS_INODE_READONLY
)
10476 return generic_permission(inode
, mask
);
10479 static int btrfs_tmpfile(struct inode
*dir
, struct dentry
*dentry
, umode_t mode
)
10481 struct btrfs_fs_info
*fs_info
= btrfs_sb(dir
->i_sb
);
10482 struct btrfs_trans_handle
*trans
;
10483 struct btrfs_root
*root
= BTRFS_I(dir
)->root
;
10484 struct inode
*inode
= NULL
;
10490 * 5 units required for adding orphan entry
10492 trans
= btrfs_start_transaction(root
, 5);
10494 return PTR_ERR(trans
);
10496 ret
= btrfs_find_free_ino(root
, &objectid
);
10500 inode
= btrfs_new_inode(trans
, root
, dir
, NULL
, 0,
10501 btrfs_ino(BTRFS_I(dir
)), objectid
, mode
, &index
);
10502 if (IS_ERR(inode
)) {
10503 ret
= PTR_ERR(inode
);
10508 inode
->i_fop
= &btrfs_file_operations
;
10509 inode
->i_op
= &btrfs_file_inode_operations
;
10511 inode
->i_mapping
->a_ops
= &btrfs_aops
;
10512 BTRFS_I(inode
)->io_tree
.ops
= &btrfs_extent_io_ops
;
10514 ret
= btrfs_init_inode_security(trans
, inode
, dir
, NULL
);
10518 ret
= btrfs_update_inode(trans
, root
, inode
);
10521 ret
= btrfs_orphan_add(trans
, inode
);
10526 * We set number of links to 0 in btrfs_new_inode(), and here we set
10527 * it to 1 because d_tmpfile() will issue a warning if the count is 0,
10530 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
10532 set_nlink(inode
, 1);
10533 unlock_new_inode(inode
);
10534 d_tmpfile(dentry
, inode
);
10535 mark_inode_dirty(inode
);
10538 btrfs_end_transaction(trans
);
10541 btrfs_balance_delayed_items(fs_info
);
10542 btrfs_btree_balance_dirty(fs_info
);
10546 unlock_new_inode(inode
);
10551 static const struct inode_operations btrfs_dir_inode_operations
= {
10552 .getattr
= btrfs_getattr
,
10553 .lookup
= btrfs_lookup
,
10554 .create
= btrfs_create
,
10555 .unlink
= btrfs_unlink
,
10556 .link
= btrfs_link
,
10557 .mkdir
= btrfs_mkdir
,
10558 .rmdir
= btrfs_rmdir
,
10559 .rename
= btrfs_rename2
,
10560 .symlink
= btrfs_symlink
,
10561 .setattr
= btrfs_setattr
,
10562 .mknod
= btrfs_mknod
,
10563 .listxattr
= btrfs_listxattr
,
10564 .permission
= btrfs_permission
,
10565 .get_acl
= btrfs_get_acl
,
10566 .set_acl
= btrfs_set_acl
,
10567 .update_time
= btrfs_update_time
,
10568 .tmpfile
= btrfs_tmpfile
,
10570 static const struct inode_operations btrfs_dir_ro_inode_operations
= {
10571 .lookup
= btrfs_lookup
,
10572 .permission
= btrfs_permission
,
10573 .update_time
= btrfs_update_time
,
10576 static const struct file_operations btrfs_dir_file_operations
= {
10577 .llseek
= generic_file_llseek
,
10578 .read
= generic_read_dir
,
10579 .iterate_shared
= btrfs_real_readdir
,
10580 .unlocked_ioctl
= btrfs_ioctl
,
10581 #ifdef CONFIG_COMPAT
10582 .compat_ioctl
= btrfs_compat_ioctl
,
10584 .release
= btrfs_release_file
,
10585 .fsync
= btrfs_sync_file
,
10588 static const struct extent_io_ops btrfs_extent_io_ops
= {
10589 .fill_delalloc
= run_delalloc_range
,
10590 .submit_bio_hook
= btrfs_submit_bio_hook
,
10591 .merge_bio_hook
= btrfs_merge_bio_hook
,
10592 .readpage_end_io_hook
= btrfs_readpage_end_io_hook
,
10593 .writepage_end_io_hook
= btrfs_writepage_end_io_hook
,
10594 .writepage_start_hook
= btrfs_writepage_start_hook
,
10595 .set_bit_hook
= btrfs_set_bit_hook
,
10596 .clear_bit_hook
= btrfs_clear_bit_hook
,
10597 .merge_extent_hook
= btrfs_merge_extent_hook
,
10598 .split_extent_hook
= btrfs_split_extent_hook
,
10602 * btrfs doesn't support the bmap operation because swapfiles
10603 * use bmap to make a mapping of extents in the file. They assume
10604 * these extents won't change over the life of the file and they
10605 * use the bmap result to do IO directly to the drive.
10607 * the btrfs bmap call would return logical addresses that aren't
10608 * suitable for IO and they also will change frequently as COW
10609 * operations happen. So, swapfile + btrfs == corruption.
10611 * For now we're avoiding this by dropping bmap.
10613 static const struct address_space_operations btrfs_aops
= {
10614 .readpage
= btrfs_readpage
,
10615 .writepage
= btrfs_writepage
,
10616 .writepages
= btrfs_writepages
,
10617 .readpages
= btrfs_readpages
,
10618 .direct_IO
= btrfs_direct_IO
,
10619 .invalidatepage
= btrfs_invalidatepage
,
10620 .releasepage
= btrfs_releasepage
,
10621 .set_page_dirty
= btrfs_set_page_dirty
,
10622 .error_remove_page
= generic_error_remove_page
,
10625 static const struct address_space_operations btrfs_symlink_aops
= {
10626 .readpage
= btrfs_readpage
,
10627 .writepage
= btrfs_writepage
,
10628 .invalidatepage
= btrfs_invalidatepage
,
10629 .releasepage
= btrfs_releasepage
,
10632 static const struct inode_operations btrfs_file_inode_operations
= {
10633 .getattr
= btrfs_getattr
,
10634 .setattr
= btrfs_setattr
,
10635 .listxattr
= btrfs_listxattr
,
10636 .permission
= btrfs_permission
,
10637 .fiemap
= btrfs_fiemap
,
10638 .get_acl
= btrfs_get_acl
,
10639 .set_acl
= btrfs_set_acl
,
10640 .update_time
= btrfs_update_time
,
10642 static const struct inode_operations btrfs_special_inode_operations
= {
10643 .getattr
= btrfs_getattr
,
10644 .setattr
= btrfs_setattr
,
10645 .permission
= btrfs_permission
,
10646 .listxattr
= btrfs_listxattr
,
10647 .get_acl
= btrfs_get_acl
,
10648 .set_acl
= btrfs_set_acl
,
10649 .update_time
= btrfs_update_time
,
10651 static const struct inode_operations btrfs_symlink_inode_operations
= {
10652 .get_link
= page_get_link
,
10653 .getattr
= btrfs_getattr
,
10654 .setattr
= btrfs_setattr
,
10655 .permission
= btrfs_permission
,
10656 .listxattr
= btrfs_listxattr
,
10657 .update_time
= btrfs_update_time
,
10660 const struct dentry_operations btrfs_dentry_operations
= {
10661 .d_delete
= btrfs_dentry_delete
,
10662 .d_release
= btrfs_dentry_release
,