2 * Copyright (c) 2012 Taobao.
3 * Written by Tao Ma <boyu.mt@taobao.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of version 2.1 of the GNU Lesser General Public License
7 * as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 #include "ext4_jbd2.h"
19 #define EXT4_XATTR_SYSTEM_DATA "data"
20 #define EXT4_MIN_INLINE_DATA_SIZE ((sizeof(__le32) * EXT4_N_BLOCKS))
21 #define EXT4_INLINE_DOTDOT_SIZE 4
23 int ext4_get_inline_size(struct inode
*inode
)
25 if (EXT4_I(inode
)->i_inline_off
)
26 return EXT4_I(inode
)->i_inline_size
;
31 static int get_max_inline_xattr_value_size(struct inode
*inode
,
32 struct ext4_iloc
*iloc
)
34 struct ext4_xattr_ibody_header
*header
;
35 struct ext4_xattr_entry
*entry
;
36 struct ext4_inode
*raw_inode
;
39 min_offs
= EXT4_SB(inode
->i_sb
)->s_inode_size
-
40 EXT4_GOOD_OLD_INODE_SIZE
-
41 EXT4_I(inode
)->i_extra_isize
-
42 sizeof(struct ext4_xattr_ibody_header
);
45 * We need to subtract another sizeof(__u32) since an in-inode xattr
46 * needs an empty 4 bytes to indicate the gap between the xattr entry
47 * and the name/value pair.
49 if (!ext4_test_inode_state(inode
, EXT4_STATE_XATTR
))
50 return EXT4_XATTR_SIZE(min_offs
-
51 EXT4_XATTR_LEN(strlen(EXT4_XATTR_SYSTEM_DATA
)) -
52 EXT4_XATTR_ROUND
- sizeof(__u32
));
54 raw_inode
= ext4_raw_inode(iloc
);
55 header
= IHDR(inode
, raw_inode
);
56 entry
= IFIRST(header
);
58 /* Compute min_offs. */
59 for (; !IS_LAST_ENTRY(entry
); entry
= EXT4_XATTR_NEXT(entry
)) {
60 if (!entry
->e_value_block
&& entry
->e_value_size
) {
61 size_t offs
= le16_to_cpu(entry
->e_value_offs
);
67 ((void *)entry
- (void *)IFIRST(header
)) - sizeof(__u32
);
69 if (EXT4_I(inode
)->i_inline_off
) {
70 entry
= (struct ext4_xattr_entry
*)
71 ((void *)raw_inode
+ EXT4_I(inode
)->i_inline_off
);
73 free
+= le32_to_cpu(entry
->e_value_size
);
77 free
-= EXT4_XATTR_LEN(strlen(EXT4_XATTR_SYSTEM_DATA
));
79 if (free
> EXT4_XATTR_ROUND
)
80 free
= EXT4_XATTR_SIZE(free
- EXT4_XATTR_ROUND
);
89 * Get the maximum size we now can store in an inode.
90 * If we can't find the space for a xattr entry, don't use the space
91 * of the extents since we have no space to indicate the inline data.
93 int ext4_get_max_inline_size(struct inode
*inode
)
95 int error
, max_inline_size
;
96 struct ext4_iloc iloc
;
98 if (EXT4_I(inode
)->i_extra_isize
== 0)
101 error
= ext4_get_inode_loc(inode
, &iloc
);
103 ext4_error_inode(inode
, __func__
, __LINE__
, 0,
104 "can't get inode location %lu",
109 down_read(&EXT4_I(inode
)->xattr_sem
);
110 max_inline_size
= get_max_inline_xattr_value_size(inode
, &iloc
);
111 up_read(&EXT4_I(inode
)->xattr_sem
);
115 if (!max_inline_size
)
118 return max_inline_size
+ EXT4_MIN_INLINE_DATA_SIZE
;
121 int ext4_has_inline_data(struct inode
*inode
)
123 return ext4_test_inode_flag(inode
, EXT4_INODE_INLINE_DATA
) &&
124 EXT4_I(inode
)->i_inline_off
;
128 * this function does not take xattr_sem, which is OK because it is
129 * currently only used in a code path coming form ext4_iget, before
130 * the new inode has been unlocked
132 int ext4_find_inline_data_nolock(struct inode
*inode
)
134 struct ext4_xattr_ibody_find is
= {
135 .s
= { .not_found
= -ENODATA
, },
137 struct ext4_xattr_info i
= {
138 .name_index
= EXT4_XATTR_INDEX_SYSTEM
,
139 .name
= EXT4_XATTR_SYSTEM_DATA
,
143 if (EXT4_I(inode
)->i_extra_isize
== 0)
146 error
= ext4_get_inode_loc(inode
, &is
.iloc
);
150 error
= ext4_xattr_ibody_find(inode
, &i
, &is
);
154 if (!is
.s
.not_found
) {
155 EXT4_I(inode
)->i_inline_off
= (u16
)((void *)is
.s
.here
-
156 (void *)ext4_raw_inode(&is
.iloc
));
157 EXT4_I(inode
)->i_inline_size
= EXT4_MIN_INLINE_DATA_SIZE
+
158 le32_to_cpu(is
.s
.here
->e_value_size
);
159 ext4_set_inode_state(inode
, EXT4_STATE_MAY_INLINE_DATA
);
166 static int ext4_read_inline_data(struct inode
*inode
, void *buffer
,
168 struct ext4_iloc
*iloc
)
170 struct ext4_xattr_entry
*entry
;
171 struct ext4_xattr_ibody_header
*header
;
173 struct ext4_inode
*raw_inode
;
178 BUG_ON(len
> EXT4_I(inode
)->i_inline_size
);
180 cp_len
= len
< EXT4_MIN_INLINE_DATA_SIZE
?
181 len
: EXT4_MIN_INLINE_DATA_SIZE
;
183 raw_inode
= ext4_raw_inode(iloc
);
184 memcpy(buffer
, (void *)(raw_inode
->i_block
), cp_len
);
192 header
= IHDR(inode
, raw_inode
);
193 entry
= (struct ext4_xattr_entry
*)((void *)raw_inode
+
194 EXT4_I(inode
)->i_inline_off
);
195 len
= min_t(unsigned int, len
,
196 (unsigned int)le32_to_cpu(entry
->e_value_size
));
199 (void *)IFIRST(header
) + le16_to_cpu(entry
->e_value_offs
), len
);
207 * write the buffer to the inline inode.
208 * If 'create' is set, we don't need to do the extra copy in the xattr
209 * value since it is already handled by ext4_xattr_ibody_set. That saves
212 void ext4_write_inline_data(struct inode
*inode
, struct ext4_iloc
*iloc
,
213 void *buffer
, loff_t pos
, unsigned int len
)
215 struct ext4_xattr_entry
*entry
;
216 struct ext4_xattr_ibody_header
*header
;
217 struct ext4_inode
*raw_inode
;
220 BUG_ON(!EXT4_I(inode
)->i_inline_off
);
221 BUG_ON(pos
+ len
> EXT4_I(inode
)->i_inline_size
);
223 raw_inode
= ext4_raw_inode(iloc
);
226 if (pos
< EXT4_MIN_INLINE_DATA_SIZE
) {
227 cp_len
= pos
+ len
> EXT4_MIN_INLINE_DATA_SIZE
?
228 EXT4_MIN_INLINE_DATA_SIZE
- pos
: len
;
229 memcpy((void *)raw_inode
->i_block
+ pos
, buffer
, cp_len
);
239 pos
-= EXT4_MIN_INLINE_DATA_SIZE
;
240 header
= IHDR(inode
, raw_inode
);
241 entry
= (struct ext4_xattr_entry
*)((void *)raw_inode
+
242 EXT4_I(inode
)->i_inline_off
);
244 memcpy((void *)IFIRST(header
) + le16_to_cpu(entry
->e_value_offs
) + pos
,
248 static int ext4_create_inline_data(handle_t
*handle
,
249 struct inode
*inode
, unsigned len
)
253 struct ext4_xattr_ibody_find is
= {
254 .s
= { .not_found
= -ENODATA
, },
256 struct ext4_xattr_info i
= {
257 .name_index
= EXT4_XATTR_INDEX_SYSTEM
,
258 .name
= EXT4_XATTR_SYSTEM_DATA
,
261 error
= ext4_get_inode_loc(inode
, &is
.iloc
);
265 error
= ext4_journal_get_write_access(handle
, is
.iloc
.bh
);
269 if (len
> EXT4_MIN_INLINE_DATA_SIZE
) {
270 value
= (void *)empty_zero_page
;
271 len
-= EXT4_MIN_INLINE_DATA_SIZE
;
277 /* Insert the the xttr entry. */
281 error
= ext4_xattr_ibody_find(inode
, &i
, &is
);
285 BUG_ON(!is
.s
.not_found
);
287 error
= ext4_xattr_ibody_set(handle
, inode
, &i
, &is
);
289 if (error
== -ENOSPC
)
290 ext4_clear_inode_state(inode
,
291 EXT4_STATE_MAY_INLINE_DATA
);
295 memset((void *)ext4_raw_inode(&is
.iloc
)->i_block
,
296 0, EXT4_MIN_INLINE_DATA_SIZE
);
298 EXT4_I(inode
)->i_inline_off
= (u16
)((void *)is
.s
.here
-
299 (void *)ext4_raw_inode(&is
.iloc
));
300 EXT4_I(inode
)->i_inline_size
= len
+ EXT4_MIN_INLINE_DATA_SIZE
;
301 ext4_clear_inode_flag(inode
, EXT4_INODE_EXTENTS
);
302 ext4_set_inode_flag(inode
, EXT4_INODE_INLINE_DATA
);
304 error
= ext4_mark_iloc_dirty(handle
, inode
, &is
.iloc
);
311 static int ext4_update_inline_data(handle_t
*handle
, struct inode
*inode
,
316 struct ext4_xattr_ibody_find is
= {
317 .s
= { .not_found
= -ENODATA
, },
319 struct ext4_xattr_info i
= {
320 .name_index
= EXT4_XATTR_INDEX_SYSTEM
,
321 .name
= EXT4_XATTR_SYSTEM_DATA
,
324 /* If the old space is ok, write the data directly. */
325 if (len
<= EXT4_I(inode
)->i_inline_size
)
328 error
= ext4_get_inode_loc(inode
, &is
.iloc
);
332 error
= ext4_xattr_ibody_find(inode
, &i
, &is
);
336 BUG_ON(is
.s
.not_found
);
338 len
-= EXT4_MIN_INLINE_DATA_SIZE
;
339 value
= kzalloc(len
, GFP_NOFS
);
343 error
= ext4_xattr_ibody_get(inode
, i
.name_index
, i
.name
,
345 if (error
== -ENODATA
)
348 error
= ext4_journal_get_write_access(handle
, is
.iloc
.bh
);
352 /* Update the xttr entry. */
356 error
= ext4_xattr_ibody_set(handle
, inode
, &i
, &is
);
360 EXT4_I(inode
)->i_inline_off
= (u16
)((void *)is
.s
.here
-
361 (void *)ext4_raw_inode(&is
.iloc
));
362 EXT4_I(inode
)->i_inline_size
= EXT4_MIN_INLINE_DATA_SIZE
+
363 le32_to_cpu(is
.s
.here
->e_value_size
);
364 ext4_set_inode_state(inode
, EXT4_STATE_MAY_INLINE_DATA
);
366 error
= ext4_mark_iloc_dirty(handle
, inode
, &is
.iloc
);
374 int ext4_prepare_inline_data(handle_t
*handle
, struct inode
*inode
,
378 struct ext4_inode_info
*ei
= EXT4_I(inode
);
380 if (!ext4_test_inode_state(inode
, EXT4_STATE_MAY_INLINE_DATA
))
383 size
= ext4_get_max_inline_size(inode
);
387 down_write(&EXT4_I(inode
)->xattr_sem
);
389 if (ei
->i_inline_off
)
390 ret
= ext4_update_inline_data(handle
, inode
, len
);
392 ret
= ext4_create_inline_data(handle
, inode
, len
);
394 up_write(&EXT4_I(inode
)->xattr_sem
);
399 static int ext4_destroy_inline_data_nolock(handle_t
*handle
,
402 struct ext4_inode_info
*ei
= EXT4_I(inode
);
403 struct ext4_xattr_ibody_find is
= {
404 .s
= { .not_found
= 0, },
406 struct ext4_xattr_info i
= {
407 .name_index
= EXT4_XATTR_INDEX_SYSTEM
,
408 .name
= EXT4_XATTR_SYSTEM_DATA
,
414 if (!ei
->i_inline_off
)
417 error
= ext4_get_inode_loc(inode
, &is
.iloc
);
421 error
= ext4_xattr_ibody_find(inode
, &i
, &is
);
425 error
= ext4_journal_get_write_access(handle
, is
.iloc
.bh
);
429 error
= ext4_xattr_ibody_set(handle
, inode
, &i
, &is
);
433 memset((void *)ext4_raw_inode(&is
.iloc
)->i_block
,
434 0, EXT4_MIN_INLINE_DATA_SIZE
);
436 if (EXT4_HAS_INCOMPAT_FEATURE(inode
->i_sb
,
437 EXT4_FEATURE_INCOMPAT_EXTENTS
)) {
438 if (S_ISDIR(inode
->i_mode
) ||
439 S_ISREG(inode
->i_mode
) || S_ISLNK(inode
->i_mode
)) {
440 ext4_set_inode_flag(inode
, EXT4_INODE_EXTENTS
);
441 ext4_ext_tree_init(handle
, inode
);
444 ext4_clear_inode_flag(inode
, EXT4_INODE_INLINE_DATA
);
447 error
= ext4_mark_iloc_dirty(handle
, inode
, &is
.iloc
);
449 EXT4_I(inode
)->i_inline_off
= 0;
450 EXT4_I(inode
)->i_inline_size
= 0;
451 ext4_clear_inode_state(inode
, EXT4_STATE_MAY_INLINE_DATA
);
454 if (error
== -ENODATA
)
459 static int ext4_read_inline_page(struct inode
*inode
, struct page
*page
)
464 struct ext4_iloc iloc
;
466 BUG_ON(!PageLocked(page
));
467 BUG_ON(!ext4_has_inline_data(inode
));
470 if (!EXT4_I(inode
)->i_inline_off
) {
471 ext4_warning(inode
->i_sb
, "inode %lu doesn't have inline data.",
476 ret
= ext4_get_inode_loc(inode
, &iloc
);
480 len
= min_t(size_t, ext4_get_inline_size(inode
), i_size_read(inode
));
481 kaddr
= kmap_atomic(page
);
482 ret
= ext4_read_inline_data(inode
, kaddr
, len
, &iloc
);
483 flush_dcache_page(page
);
484 kunmap_atomic(kaddr
);
485 zero_user_segment(page
, len
, PAGE_CACHE_SIZE
);
486 SetPageUptodate(page
);
493 int ext4_readpage_inline(struct inode
*inode
, struct page
*page
)
497 down_read(&EXT4_I(inode
)->xattr_sem
);
498 if (!ext4_has_inline_data(inode
)) {
499 up_read(&EXT4_I(inode
)->xattr_sem
);
504 * Current inline data can only exist in the 1st page,
505 * So for all the other pages, just set them uptodate.
508 ret
= ext4_read_inline_page(inode
, page
);
509 else if (!PageUptodate(page
)) {
510 zero_user_segment(page
, 0, PAGE_CACHE_SIZE
);
511 SetPageUptodate(page
);
514 up_read(&EXT4_I(inode
)->xattr_sem
);
517 return ret
>= 0 ? 0 : ret
;
520 static int ext4_convert_inline_data_to_extent(struct address_space
*mapping
,
524 int ret
, needed_blocks
;
525 handle_t
*handle
= NULL
;
526 int retries
= 0, sem_held
= 0;
527 struct page
*page
= NULL
;
529 struct ext4_iloc iloc
;
531 if (!ext4_has_inline_data(inode
)) {
533 * clear the flag so that no new write
534 * will trap here again.
536 ext4_clear_inode_state(inode
, EXT4_STATE_MAY_INLINE_DATA
);
540 needed_blocks
= ext4_writepage_trans_blocks(inode
);
542 ret
= ext4_get_inode_loc(inode
, &iloc
);
547 handle
= ext4_journal_start(inode
, needed_blocks
);
548 if (IS_ERR(handle
)) {
549 ret
= PTR_ERR(handle
);
554 /* We cannot recurse into the filesystem as the transaction is already
556 flags
|= AOP_FLAG_NOFS
;
558 page
= grab_cache_page_write_begin(mapping
, 0, flags
);
564 down_write(&EXT4_I(inode
)->xattr_sem
);
566 /* If some one has already done this for us, just exit. */
567 if (!ext4_has_inline_data(inode
)) {
573 to
= ext4_get_inline_size(inode
);
574 if (!PageUptodate(page
)) {
575 ret
= ext4_read_inline_page(inode
, page
);
580 ret
= ext4_destroy_inline_data_nolock(handle
, inode
);
584 if (ext4_should_dioread_nolock(inode
))
585 ret
= __block_write_begin(page
, from
, to
, ext4_get_block_write
);
587 ret
= __block_write_begin(page
, from
, to
, ext4_get_block
);
589 if (!ret
&& ext4_should_journal_data(inode
)) {
590 ret
= ext4_walk_page_buffers(handle
, page_buffers(page
),
592 do_journal_get_write_access
);
597 page_cache_release(page
);
598 ext4_orphan_add(handle
, inode
);
599 up_write(&EXT4_I(inode
)->xattr_sem
);
601 ext4_journal_stop(handle
);
603 ext4_truncate_failed_write(inode
);
605 * If truncate failed early the inode might
606 * still be on the orphan list; we need to
607 * make sure the inode is removed from the
608 * orphan list in that case.
611 ext4_orphan_del(NULL
, inode
);
614 if (ret
== -ENOSPC
&& ext4_should_retry_alloc(inode
->i_sb
, &retries
))
617 block_commit_write(page
, from
, to
);
621 page_cache_release(page
);
624 up_write(&EXT4_I(inode
)->xattr_sem
);
626 ext4_journal_stop(handle
);
632 * Try to write data in the inode.
633 * If the inode has inline data, check whether the new write can be
634 * in the inode also. If not, create the page the handle, move the data
635 * to the page make it update and let the later codes create extent for it.
637 int ext4_try_to_write_inline_data(struct address_space
*mapping
,
639 loff_t pos
, unsigned len
,
646 struct ext4_iloc iloc
;
648 if (pos
+ len
> ext4_get_max_inline_size(inode
))
651 ret
= ext4_get_inode_loc(inode
, &iloc
);
656 * The possible write could happen in the inode,
657 * so try to reserve the space in inode first.
659 handle
= ext4_journal_start(inode
, 1);
660 if (IS_ERR(handle
)) {
661 ret
= PTR_ERR(handle
);
666 ret
= ext4_prepare_inline_data(handle
, inode
, pos
+ len
);
667 if (ret
&& ret
!= -ENOSPC
)
670 /* We don't have space in inline inode, so convert it to extent. */
671 if (ret
== -ENOSPC
) {
672 ext4_journal_stop(handle
);
677 flags
|= AOP_FLAG_NOFS
;
679 page
= grab_cache_page_write_begin(mapping
, 0, flags
);
686 down_read(&EXT4_I(inode
)->xattr_sem
);
687 if (!ext4_has_inline_data(inode
)) {
690 page_cache_release(page
);
694 if (!PageUptodate(page
)) {
695 ret
= ext4_read_inline_page(inode
, page
);
703 up_read(&EXT4_I(inode
)->xattr_sem
);
706 ext4_journal_stop(handle
);
710 return ext4_convert_inline_data_to_extent(mapping
,
714 int ext4_write_inline_data_end(struct inode
*inode
, loff_t pos
, unsigned len
,
715 unsigned copied
, struct page
*page
)
719 struct ext4_iloc iloc
;
721 if (unlikely(copied
< len
)) {
722 if (!PageUptodate(page
)) {
728 ret
= ext4_get_inode_loc(inode
, &iloc
);
730 ext4_std_error(inode
->i_sb
, ret
);
735 down_write(&EXT4_I(inode
)->xattr_sem
);
736 BUG_ON(!ext4_has_inline_data(inode
));
738 kaddr
= kmap_atomic(page
);
739 ext4_write_inline_data(inode
, &iloc
, kaddr
, pos
, len
);
740 kunmap_atomic(kaddr
);
741 SetPageUptodate(page
);
742 /* clear page dirty so that writepages wouldn't work for us. */
743 ClearPageDirty(page
);
745 up_write(&EXT4_I(inode
)->xattr_sem
);
752 ext4_journalled_write_inline_data(struct inode
*inode
,
758 struct ext4_iloc iloc
;
760 ret
= ext4_get_inode_loc(inode
, &iloc
);
762 ext4_std_error(inode
->i_sb
, ret
);
766 down_write(&EXT4_I(inode
)->xattr_sem
);
767 kaddr
= kmap_atomic(page
);
768 ext4_write_inline_data(inode
, &iloc
, kaddr
, 0, len
);
769 kunmap_atomic(kaddr
);
770 up_write(&EXT4_I(inode
)->xattr_sem
);
776 * Try to make the page cache and handle ready for the inline data case.
777 * We can call this function in 2 cases:
778 * 1. The inode is created and the first write exceeds inline size. We can
779 * clear the inode state safely.
780 * 2. The inode has inline data, then we need to read the data, make it
781 * update and dirty so that ext4_da_writepages can handle it. We don't
782 * need to start the journal since the file's metatdata isn't changed now.
784 static int ext4_da_convert_inline_data_to_extent(struct address_space
*mapping
,
789 int ret
= 0, inline_size
;
792 page
= grab_cache_page_write_begin(mapping
, 0, flags
);
796 down_read(&EXT4_I(inode
)->xattr_sem
);
797 if (!ext4_has_inline_data(inode
)) {
798 ext4_clear_inode_state(inode
, EXT4_STATE_MAY_INLINE_DATA
);
802 inline_size
= ext4_get_inline_size(inode
);
804 if (!PageUptodate(page
)) {
805 ret
= ext4_read_inline_page(inode
, page
);
810 ret
= __block_write_begin(page
, 0, inline_size
,
811 ext4_da_get_block_prep
);
813 ext4_truncate_failed_write(inode
);
818 SetPageUptodate(page
);
819 ext4_clear_inode_state(inode
, EXT4_STATE_MAY_INLINE_DATA
);
820 *fsdata
= (void *)CONVERT_INLINE_DATA
;
823 up_read(&EXT4_I(inode
)->xattr_sem
);
826 page_cache_release(page
);
832 * Prepare the write for the inline data.
833 * If the the data can be written into the inode, we just read
834 * the page and make it uptodate, and start the journal.
835 * Otherwise read the page, makes it dirty so that it can be
836 * handle in writepages(the i_disksize update is left to the
837 * normal ext4_da_write_end).
839 int ext4_da_write_inline_data_begin(struct address_space
*mapping
,
841 loff_t pos
, unsigned len
,
846 int ret
, inline_size
;
849 struct ext4_iloc iloc
;
851 ret
= ext4_get_inode_loc(inode
, &iloc
);
855 handle
= ext4_journal_start(inode
, 1);
856 if (IS_ERR(handle
)) {
857 ret
= PTR_ERR(handle
);
862 inline_size
= ext4_get_max_inline_size(inode
);
865 if (inline_size
>= pos
+ len
) {
866 ret
= ext4_prepare_inline_data(handle
, inode
, pos
+ len
);
867 if (ret
&& ret
!= -ENOSPC
)
871 if (ret
== -ENOSPC
) {
872 ret
= ext4_da_convert_inline_data_to_extent(mapping
,
880 * We cannot recurse into the filesystem as the transaction
881 * is already started.
883 flags
|= AOP_FLAG_NOFS
;
885 page
= grab_cache_page_write_begin(mapping
, 0, flags
);
891 down_read(&EXT4_I(inode
)->xattr_sem
);
892 if (!ext4_has_inline_data(inode
)) {
894 goto out_release_page
;
897 if (!PageUptodate(page
)) {
898 ret
= ext4_read_inline_page(inode
, page
);
900 goto out_release_page
;
903 up_read(&EXT4_I(inode
)->xattr_sem
);
909 up_read(&EXT4_I(inode
)->xattr_sem
);
911 page_cache_release(page
);
914 ext4_journal_stop(handle
);
919 int ext4_da_write_inline_data_end(struct inode
*inode
, loff_t pos
,
920 unsigned len
, unsigned copied
,
923 int i_size_changed
= 0;
925 copied
= ext4_write_inline_data_end(inode
, pos
, len
, copied
, page
);
928 * No need to use i_size_read() here, the i_size
929 * cannot change under us because we hold i_mutex.
931 * But it's important to update i_size while still holding page lock:
932 * page writeout could otherwise come in and zero beyond i_size.
934 if (pos
+copied
> inode
->i_size
) {
935 i_size_write(inode
, pos
+copied
);
939 page_cache_release(page
);
942 * Don't mark the inode dirty under page lock. First, it unnecessarily
943 * makes the holding time of page lock longer. Second, it forces lock
944 * ordering of page lock and transaction start for journaling
948 mark_inode_dirty(inode
);
953 #ifdef INLINE_DIR_DEBUG
954 void ext4_show_inline_dir(struct inode
*dir
, struct buffer_head
*bh
,
955 void *inline_start
, int inline_size
)
958 unsigned short de_len
;
959 struct ext4_dir_entry_2
*de
= inline_start
;
960 void *dlimit
= inline_start
+ inline_size
;
962 trace_printk("inode %lu\n", dir
->i_ino
);
964 while ((void *)de
< dlimit
) {
965 de_len
= ext4_rec_len_from_disk(de
->rec_len
, inline_size
);
966 trace_printk("de: off %u rlen %u name %*.s nlen %u ino %u\n",
967 offset
, de_len
, de
->name_len
, de
->name
,
968 de
->name_len
, le32_to_cpu(de
->inode
));
969 if (ext4_check_dir_entry(dir
, NULL
, de
, bh
,
970 inline_start
, inline_size
, offset
))
974 de
= (struct ext4_dir_entry_2
*) ((char *) de
+ de_len
);
978 #define ext4_show_inline_dir(dir, bh, inline_start, inline_size)
982 * Add a new entry into a inline dir.
983 * It will return -ENOSPC if no space is available, and -EIO
984 * and -EEXIST if directory entry already exists.
986 static int ext4_add_dirent_to_inline(handle_t
*handle
,
987 struct dentry
*dentry
,
989 struct ext4_iloc
*iloc
,
990 void *inline_start
, int inline_size
)
992 struct inode
*dir
= dentry
->d_parent
->d_inode
;
993 const char *name
= dentry
->d_name
.name
;
994 int namelen
= dentry
->d_name
.len
;
995 unsigned short reclen
;
997 struct ext4_dir_entry_2
*de
;
999 reclen
= EXT4_DIR_REC_LEN(namelen
);
1000 err
= ext4_find_dest_de(dir
, inode
, iloc
->bh
,
1001 inline_start
, inline_size
,
1002 name
, namelen
, &de
);
1006 err
= ext4_journal_get_write_access(handle
, iloc
->bh
);
1009 ext4_insert_dentry(inode
, de
, inline_size
, name
, namelen
);
1011 ext4_show_inline_dir(dir
, iloc
->bh
, inline_start
, inline_size
);
1014 * XXX shouldn't update any times until successful
1015 * completion of syscall, but too many callers depend
1018 * XXX similarly, too many callers depend on
1019 * ext4_new_inode() setting the times, but error
1020 * recovery deletes the inode, so the worst that can
1021 * happen is that the times are slightly out of date
1022 * and/or different from the directory change time.
1024 dir
->i_mtime
= dir
->i_ctime
= ext4_current_time(dir
);
1025 ext4_update_dx_flag(dir
);
1027 ext4_mark_inode_dirty(handle
, dir
);
1031 static void *ext4_get_inline_xattr_pos(struct inode
*inode
,
1032 struct ext4_iloc
*iloc
)
1034 struct ext4_xattr_entry
*entry
;
1035 struct ext4_xattr_ibody_header
*header
;
1037 BUG_ON(!EXT4_I(inode
)->i_inline_off
);
1039 header
= IHDR(inode
, ext4_raw_inode(iloc
));
1040 entry
= (struct ext4_xattr_entry
*)((void *)ext4_raw_inode(iloc
) +
1041 EXT4_I(inode
)->i_inline_off
);
1043 return (void *)IFIRST(header
) + le16_to_cpu(entry
->e_value_offs
);
1046 /* Set the final de to cover the whole block. */
1047 static void ext4_update_final_de(void *de_buf
, int old_size
, int new_size
)
1049 struct ext4_dir_entry_2
*de
, *prev_de
;
1053 de
= (struct ext4_dir_entry_2
*)de_buf
;
1055 limit
= de_buf
+ old_size
;
1058 de_len
= ext4_rec_len_from_disk(de
->rec_len
, old_size
);
1060 de
= (struct ext4_dir_entry_2
*)de_buf
;
1061 } while (de_buf
< limit
);
1063 prev_de
->rec_len
= ext4_rec_len_to_disk(de_len
+ new_size
-
1064 old_size
, new_size
);
1066 /* this is just created, so create an empty entry. */
1068 de
->rec_len
= ext4_rec_len_to_disk(new_size
, new_size
);
1072 static int ext4_update_inline_dir(handle_t
*handle
, struct inode
*dir
,
1073 struct ext4_iloc
*iloc
)
1076 int old_size
= EXT4_I(dir
)->i_inline_size
- EXT4_MIN_INLINE_DATA_SIZE
;
1077 int new_size
= get_max_inline_xattr_value_size(dir
, iloc
);
1079 if (new_size
- old_size
<= EXT4_DIR_REC_LEN(1))
1082 ret
= ext4_update_inline_data(handle
, dir
,
1083 new_size
+ EXT4_MIN_INLINE_DATA_SIZE
);
1087 ext4_update_final_de(ext4_get_inline_xattr_pos(dir
, iloc
), old_size
,
1088 EXT4_I(dir
)->i_inline_size
-
1089 EXT4_MIN_INLINE_DATA_SIZE
);
1090 dir
->i_size
= EXT4_I(dir
)->i_disksize
= EXT4_I(dir
)->i_inline_size
;
1094 static void ext4_restore_inline_data(handle_t
*handle
, struct inode
*inode
,
1095 struct ext4_iloc
*iloc
,
1096 void *buf
, int inline_size
)
1098 ext4_create_inline_data(handle
, inode
, inline_size
);
1099 ext4_write_inline_data(inode
, iloc
, buf
, 0, inline_size
);
1100 ext4_set_inode_state(inode
, EXT4_STATE_MAY_INLINE_DATA
);
1103 static int ext4_finish_convert_inline_dir(handle_t
*handle
,
1104 struct inode
*inode
,
1105 struct buffer_head
*dir_block
,
1109 int err
, csum_size
= 0, header_size
= 0;
1110 struct ext4_dir_entry_2
*de
;
1111 struct ext4_dir_entry_tail
*t
;
1112 void *target
= dir_block
->b_data
;
1115 * First create "." and ".." and then copy the dir information
1116 * back to the block.
1118 de
= (struct ext4_dir_entry_2
*)target
;
1119 de
= ext4_init_dot_dotdot(inode
, de
,
1120 inode
->i_sb
->s_blocksize
, csum_size
,
1121 le32_to_cpu(((struct ext4_dir_entry_2
*)buf
)->inode
), 1);
1122 header_size
= (void *)de
- target
;
1124 memcpy((void *)de
, buf
+ EXT4_INLINE_DOTDOT_SIZE
,
1125 inline_size
- EXT4_INLINE_DOTDOT_SIZE
);
1127 if (EXT4_HAS_RO_COMPAT_FEATURE(inode
->i_sb
,
1128 EXT4_FEATURE_RO_COMPAT_METADATA_CSUM
))
1129 csum_size
= sizeof(struct ext4_dir_entry_tail
);
1131 inode
->i_size
= inode
->i_sb
->s_blocksize
;
1132 i_size_write(inode
, inode
->i_sb
->s_blocksize
);
1133 EXT4_I(inode
)->i_disksize
= inode
->i_sb
->s_blocksize
;
1134 ext4_update_final_de(dir_block
->b_data
,
1135 inline_size
- EXT4_INLINE_DOTDOT_SIZE
+ header_size
,
1136 inode
->i_sb
->s_blocksize
- csum_size
);
1139 t
= EXT4_DIRENT_TAIL(dir_block
->b_data
,
1140 inode
->i_sb
->s_blocksize
);
1141 initialize_dirent_tail(t
, inode
->i_sb
->s_blocksize
);
1143 set_buffer_uptodate(dir_block
);
1144 err
= ext4_handle_dirty_dirent_node(handle
, inode
, dir_block
);
1147 set_buffer_verified(dir_block
);
1152 static int ext4_convert_inline_data_nolock(handle_t
*handle
,
1153 struct inode
*inode
,
1154 struct ext4_iloc
*iloc
)
1158 struct buffer_head
*data_bh
= NULL
;
1159 struct ext4_map_blocks map
;
1162 inline_size
= ext4_get_inline_size(inode
);
1163 buf
= kmalloc(inline_size
, GFP_NOFS
);
1169 error
= ext4_read_inline_data(inode
, buf
, inline_size
, iloc
);
1173 error
= ext4_destroy_inline_data_nolock(handle
, inode
);
1180 error
= ext4_map_blocks(handle
, inode
, &map
, EXT4_GET_BLOCKS_CREATE
);
1183 if (!(map
.m_flags
& EXT4_MAP_MAPPED
)) {
1188 data_bh
= sb_getblk(inode
->i_sb
, map
.m_pblk
);
1194 lock_buffer(data_bh
);
1195 error
= ext4_journal_get_create_access(handle
, data_bh
);
1197 unlock_buffer(data_bh
);
1201 memset(data_bh
->b_data
, 0, inode
->i_sb
->s_blocksize
);
1203 if (!S_ISDIR(inode
->i_mode
)) {
1204 memcpy(data_bh
->b_data
, buf
, inline_size
);
1205 set_buffer_uptodate(data_bh
);
1206 error
= ext4_handle_dirty_metadata(handle
,
1209 error
= ext4_finish_convert_inline_dir(handle
, inode
, data_bh
,
1213 unlock_buffer(data_bh
);
1216 ext4_restore_inline_data(handle
, inode
, iloc
, buf
, inline_size
);
1225 * Try to add the new entry to the inline data.
1226 * If succeeds, return 0. If not, extended the inline dir and copied data to
1227 * the new created block.
1229 int ext4_try_add_inline_entry(handle_t
*handle
, struct dentry
*dentry
,
1230 struct inode
*inode
)
1232 int ret
, inline_size
;
1234 struct ext4_iloc iloc
;
1235 struct inode
*dir
= dentry
->d_parent
->d_inode
;
1237 ret
= ext4_get_inode_loc(dir
, &iloc
);
1241 down_write(&EXT4_I(dir
)->xattr_sem
);
1242 if (!ext4_has_inline_data(dir
))
1245 inline_start
= (void *)ext4_raw_inode(&iloc
)->i_block
+
1246 EXT4_INLINE_DOTDOT_SIZE
;
1247 inline_size
= EXT4_MIN_INLINE_DATA_SIZE
- EXT4_INLINE_DOTDOT_SIZE
;
1249 ret
= ext4_add_dirent_to_inline(handle
, dentry
, inode
, &iloc
,
1250 inline_start
, inline_size
);
1254 /* check whether it can be inserted to inline xattr space. */
1255 inline_size
= EXT4_I(dir
)->i_inline_size
-
1256 EXT4_MIN_INLINE_DATA_SIZE
;
1258 /* Try to use the xattr space.*/
1259 ret
= ext4_update_inline_dir(handle
, dir
, &iloc
);
1260 if (ret
&& ret
!= -ENOSPC
)
1263 inline_size
= EXT4_I(dir
)->i_inline_size
-
1264 EXT4_MIN_INLINE_DATA_SIZE
;
1268 inline_start
= ext4_get_inline_xattr_pos(dir
, &iloc
);
1270 ret
= ext4_add_dirent_to_inline(handle
, dentry
, inode
, &iloc
,
1271 inline_start
, inline_size
);
1278 * The inline space is filled up, so create a new block for it.
1279 * As the extent tree will be created, we have to save the inline
1282 ret
= ext4_convert_inline_data_nolock(handle
, dir
, &iloc
);
1285 ext4_mark_inode_dirty(handle
, dir
);
1286 up_write(&EXT4_I(dir
)->xattr_sem
);
1291 int ext4_read_inline_dir(struct file
*filp
,
1292 void *dirent
, filldir_t filldir
,
1293 int *has_inline_data
)
1296 unsigned int offset
, parent_ino
;
1298 struct ext4_dir_entry_2
*de
;
1299 struct super_block
*sb
;
1300 struct inode
*inode
= filp
->f_path
.dentry
->d_inode
;
1301 int ret
, inline_size
= 0;
1302 struct ext4_iloc iloc
;
1303 void *dir_buf
= NULL
;
1305 ret
= ext4_get_inode_loc(inode
, &iloc
);
1309 down_read(&EXT4_I(inode
)->xattr_sem
);
1310 if (!ext4_has_inline_data(inode
)) {
1311 up_read(&EXT4_I(inode
)->xattr_sem
);
1312 *has_inline_data
= 0;
1316 inline_size
= ext4_get_inline_size(inode
);
1317 dir_buf
= kmalloc(inline_size
, GFP_NOFS
);
1320 up_read(&EXT4_I(inode
)->xattr_sem
);
1324 ret
= ext4_read_inline_data(inode
, dir_buf
, inline_size
, &iloc
);
1325 up_read(&EXT4_I(inode
)->xattr_sem
);
1331 parent_ino
= le32_to_cpu(((struct ext4_dir_entry_2
*)dir_buf
)->inode
);
1333 while (!error
&& !stored
&& filp
->f_pos
< inode
->i_size
) {
1336 * If the version has changed since the last call to
1337 * readdir(2), then we might be pointing to an invalid
1338 * dirent right now. Scan from the start of the inline
1341 if (filp
->f_version
!= inode
->i_version
) {
1343 i
< inode
->i_size
&& i
< offset
;) {
1345 /* skip "." and ".." if needed. */
1346 i
+= EXT4_INLINE_DOTDOT_SIZE
;
1349 de
= (struct ext4_dir_entry_2
*)
1351 /* It's too expensive to do a full
1352 * dirent test each time round this
1353 * loop, but we do have to test at
1354 * least that it is non-zero. A
1355 * failure will be detected in the
1356 * dirent test below. */
1357 if (ext4_rec_len_from_disk(de
->rec_len
,
1358 inline_size
) < EXT4_DIR_REC_LEN(1))
1360 i
+= ext4_rec_len_from_disk(de
->rec_len
,
1364 filp
->f_pos
= offset
;
1365 filp
->f_version
= inode
->i_version
;
1368 while (!error
&& filp
->f_pos
< inode
->i_size
) {
1369 if (filp
->f_pos
== 0) {
1370 error
= filldir(dirent
, ".", 1, 0, inode
->i_ino
,
1376 error
= filldir(dirent
, "..", 2, 0, parent_ino
,
1382 filp
->f_pos
= offset
= EXT4_INLINE_DOTDOT_SIZE
;
1386 de
= (struct ext4_dir_entry_2
*)(dir_buf
+ offset
);
1387 if (ext4_check_dir_entry(inode
, filp
, de
,
1389 inline_size
, offset
)) {
1393 offset
+= ext4_rec_len_from_disk(de
->rec_len
,
1395 if (le32_to_cpu(de
->inode
)) {
1396 /* We might block in the next section
1397 * if the data destination is
1398 * currently swapped out. So, use a
1399 * version stamp to detect whether or
1400 * not the directory has been modified
1401 * during the copy operation.
1403 u64 version
= filp
->f_version
;
1405 error
= filldir(dirent
, de
->name
,
1408 le32_to_cpu(de
->inode
),
1409 get_dtype(sb
, de
->file_type
));
1412 if (version
!= filp
->f_version
)
1416 filp
->f_pos
+= ext4_rec_len_from_disk(de
->rec_len
,
1427 struct buffer_head
*ext4_get_first_inline_block(struct inode
*inode
,
1428 struct ext4_dir_entry_2
**parent_de
,
1431 struct ext4_iloc iloc
;
1433 *retval
= ext4_get_inode_loc(inode
, &iloc
);
1437 *parent_de
= (struct ext4_dir_entry_2
*)ext4_raw_inode(&iloc
)->i_block
;
1443 * Try to create the inline data for the new dir.
1444 * If it succeeds, return 0, otherwise return the error.
1445 * In case of ENOSPC, the caller should create the normal disk layout dir.
1447 int ext4_try_create_inline_dir(handle_t
*handle
, struct inode
*parent
,
1448 struct inode
*inode
)
1450 int ret
, inline_size
= EXT4_MIN_INLINE_DATA_SIZE
;
1451 struct ext4_iloc iloc
;
1452 struct ext4_dir_entry_2
*de
;
1454 ret
= ext4_get_inode_loc(inode
, &iloc
);
1458 ret
= ext4_prepare_inline_data(handle
, inode
, inline_size
);
1463 * For inline dir, we only save the inode information for the ".."
1464 * and create a fake dentry to cover the left space.
1466 de
= (struct ext4_dir_entry_2
*)ext4_raw_inode(&iloc
)->i_block
;
1467 de
->inode
= cpu_to_le32(parent
->i_ino
);
1468 de
= (struct ext4_dir_entry_2
*)((void *)de
+ EXT4_INLINE_DOTDOT_SIZE
);
1470 de
->rec_len
= ext4_rec_len_to_disk(
1471 inline_size
- EXT4_INLINE_DOTDOT_SIZE
,
1473 set_nlink(inode
, 2);
1474 inode
->i_size
= EXT4_I(inode
)->i_disksize
= inline_size
;
1480 struct buffer_head
*ext4_find_inline_entry(struct inode
*dir
,
1481 const struct qstr
*d_name
,
1482 struct ext4_dir_entry_2
**res_dir
,
1483 int *has_inline_data
)
1486 struct ext4_iloc iloc
;
1490 if (ext4_get_inode_loc(dir
, &iloc
))
1493 down_read(&EXT4_I(dir
)->xattr_sem
);
1494 if (!ext4_has_inline_data(dir
)) {
1495 *has_inline_data
= 0;
1499 inline_start
= (void *)ext4_raw_inode(&iloc
)->i_block
+
1500 EXT4_INLINE_DOTDOT_SIZE
;
1501 inline_size
= EXT4_MIN_INLINE_DATA_SIZE
- EXT4_INLINE_DOTDOT_SIZE
;
1502 ret
= search_dir(iloc
.bh
, inline_start
, inline_size
,
1503 dir
, d_name
, 0, res_dir
);
1509 if (ext4_get_inline_size(dir
) == EXT4_MIN_INLINE_DATA_SIZE
)
1512 inline_start
= ext4_get_inline_xattr_pos(dir
, &iloc
);
1513 inline_size
= ext4_get_inline_size(dir
) - EXT4_MIN_INLINE_DATA_SIZE
;
1515 ret
= search_dir(iloc
.bh
, inline_start
, inline_size
,
1516 dir
, d_name
, 0, res_dir
);
1524 up_read(&EXT4_I(dir
)->xattr_sem
);
1528 int ext4_delete_inline_entry(handle_t
*handle
,
1530 struct ext4_dir_entry_2
*de_del
,
1531 struct buffer_head
*bh
,
1532 int *has_inline_data
)
1534 int err
, inline_size
;
1535 struct ext4_iloc iloc
;
1538 err
= ext4_get_inode_loc(dir
, &iloc
);
1542 down_write(&EXT4_I(dir
)->xattr_sem
);
1543 if (!ext4_has_inline_data(dir
)) {
1544 *has_inline_data
= 0;
1548 if ((void *)de_del
- ((void *)ext4_raw_inode(&iloc
)->i_block
) <
1549 EXT4_MIN_INLINE_DATA_SIZE
) {
1550 inline_start
= (void *)ext4_raw_inode(&iloc
)->i_block
+
1551 EXT4_INLINE_DOTDOT_SIZE
;
1552 inline_size
= EXT4_MIN_INLINE_DATA_SIZE
-
1553 EXT4_INLINE_DOTDOT_SIZE
;
1555 inline_start
= ext4_get_inline_xattr_pos(dir
, &iloc
);
1556 inline_size
= ext4_get_inline_size(dir
) -
1557 EXT4_MIN_INLINE_DATA_SIZE
;
1560 err
= ext4_journal_get_write_access(handle
, bh
);
1564 err
= ext4_generic_delete_entry(handle
, dir
, de_del
, bh
,
1565 inline_start
, inline_size
, 0);
1569 BUFFER_TRACE(bh
, "call ext4_handle_dirty_metadata");
1570 err
= ext4_mark_inode_dirty(handle
, dir
);
1574 ext4_show_inline_dir(dir
, iloc
.bh
, inline_start
, inline_size
);
1576 up_write(&EXT4_I(dir
)->xattr_sem
);
1579 ext4_std_error(dir
->i_sb
, err
);
1584 * Get the inline dentry at offset.
1586 static inline struct ext4_dir_entry_2
*
1587 ext4_get_inline_entry(struct inode
*inode
,
1588 struct ext4_iloc
*iloc
,
1589 unsigned int offset
,
1590 void **inline_start
,
1595 BUG_ON(offset
> ext4_get_inline_size(inode
));
1597 if (offset
< EXT4_MIN_INLINE_DATA_SIZE
) {
1598 inline_pos
= (void *)ext4_raw_inode(iloc
)->i_block
;
1599 *inline_size
= EXT4_MIN_INLINE_DATA_SIZE
;
1601 inline_pos
= ext4_get_inline_xattr_pos(inode
, iloc
);
1602 offset
-= EXT4_MIN_INLINE_DATA_SIZE
;
1603 *inline_size
= ext4_get_inline_size(inode
) -
1604 EXT4_MIN_INLINE_DATA_SIZE
;
1608 *inline_start
= inline_pos
;
1609 return (struct ext4_dir_entry_2
*)(inline_pos
+ offset
);
1612 int empty_inline_dir(struct inode
*dir
, int *has_inline_data
)
1614 int err
, inline_size
;
1615 struct ext4_iloc iloc
;
1617 unsigned int offset
;
1618 struct ext4_dir_entry_2
*de
;
1621 err
= ext4_get_inode_loc(dir
, &iloc
);
1623 EXT4_ERROR_INODE(dir
, "error %d getting inode %lu block",
1628 down_read(&EXT4_I(dir
)->xattr_sem
);
1629 if (!ext4_has_inline_data(dir
)) {
1630 *has_inline_data
= 0;
1634 de
= (struct ext4_dir_entry_2
*)ext4_raw_inode(&iloc
)->i_block
;
1635 if (!le32_to_cpu(de
->inode
)) {
1636 ext4_warning(dir
->i_sb
,
1637 "bad inline directory (dir #%lu) - no `..'",
1643 offset
= EXT4_INLINE_DOTDOT_SIZE
;
1644 while (offset
< dir
->i_size
) {
1645 de
= ext4_get_inline_entry(dir
, &iloc
, offset
,
1646 &inline_pos
, &inline_size
);
1647 if (ext4_check_dir_entry(dir
, NULL
, de
,
1648 iloc
.bh
, inline_pos
,
1649 inline_size
, offset
)) {
1650 ext4_warning(dir
->i_sb
,
1651 "bad inline directory (dir #%lu) - "
1652 "inode %u, rec_len %u, name_len %d"
1654 dir
->i_ino
, le32_to_cpu(de
->inode
),
1655 le16_to_cpu(de
->rec_len
), de
->name_len
,
1660 if (le32_to_cpu(de
->inode
)) {
1664 offset
+= ext4_rec_len_from_disk(de
->rec_len
, inline_size
);
1668 up_read(&EXT4_I(dir
)->xattr_sem
);
1673 int ext4_destroy_inline_data(handle_t
*handle
, struct inode
*inode
)
1677 down_write(&EXT4_I(inode
)->xattr_sem
);
1678 ret
= ext4_destroy_inline_data_nolock(handle
, inode
);
1679 up_write(&EXT4_I(inode
)->xattr_sem
);