2 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19 #include "xfs_shared.h"
20 #include "xfs_format.h"
21 #include "xfs_log_format.h"
22 #include "xfs_trans_resv.h"
23 #include "xfs_mount.h"
24 #include "xfs_inode.h"
25 #include "xfs_trans.h"
26 #include "xfs_inode_item.h"
27 #include "xfs_alloc.h"
28 #include "xfs_error.h"
29 #include "xfs_iomap.h"
30 #include "xfs_trace.h"
32 #include "xfs_bmap_util.h"
33 #include "xfs_bmap_btree.h"
34 #include <linux/gfp.h>
35 #include <linux/mpage.h>
36 #include <linux/pagevec.h>
37 #include <linux/writeback.h>
39 /* flags for direct write completions */
40 #define XFS_DIO_FLAG_UNWRITTEN (1 << 0)
41 #define XFS_DIO_FLAG_APPEND (1 << 1)
44 * structure owned by writepages passed to individual writepage calls
46 struct xfs_writepage_ctx
{
47 struct xfs_bmbt_irec imap
;
50 struct xfs_ioend
*ioend
;
60 struct buffer_head
*bh
, *head
;
62 *delalloc
= *unwritten
= 0;
64 bh
= head
= page_buffers(page
);
66 if (buffer_unwritten(bh
))
68 else if (buffer_delay(bh
))
70 } while ((bh
= bh
->b_this_page
) != head
);
74 xfs_find_bdev_for_inode(
77 struct xfs_inode
*ip
= XFS_I(inode
);
78 struct xfs_mount
*mp
= ip
->i_mount
;
80 if (XFS_IS_REALTIME_INODE(ip
))
81 return mp
->m_rtdev_targp
->bt_bdev
;
83 return mp
->m_ddev_targp
->bt_bdev
;
87 * We're now finished for good with this ioend structure.
88 * Update the page state via the associated buffer_heads,
89 * release holds on the inode and bio, and finally free
90 * up memory. Do not use the ioend after this.
96 struct buffer_head
*bh
, *next
;
98 for (bh
= ioend
->io_buffer_head
; bh
; bh
= next
) {
100 bh
->b_end_io(bh
, !ioend
->io_error
);
103 mempool_free(ioend
, xfs_ioend_pool
);
107 * Fast and loose check if this write could update the on-disk inode size.
109 static inline bool xfs_ioend_is_append(struct xfs_ioend
*ioend
)
111 return ioend
->io_offset
+ ioend
->io_size
>
112 XFS_I(ioend
->io_inode
)->i_d
.di_size
;
116 xfs_setfilesize_trans_alloc(
117 struct xfs_ioend
*ioend
)
119 struct xfs_mount
*mp
= XFS_I(ioend
->io_inode
)->i_mount
;
120 struct xfs_trans
*tp
;
123 tp
= xfs_trans_alloc(mp
, XFS_TRANS_FSYNC_TS
);
125 error
= xfs_trans_reserve(tp
, &M_RES(mp
)->tr_fsyncts
, 0, 0);
127 xfs_trans_cancel(tp
);
131 ioend
->io_append_trans
= tp
;
134 * We may pass freeze protection with a transaction. So tell lockdep
137 __sb_writers_release(ioend
->io_inode
->i_sb
, SB_FREEZE_FS
);
139 * We hand off the transaction to the completion thread now, so
140 * clear the flag here.
142 current_restore_flags_nested(&tp
->t_pflags
, PF_FSTRANS
);
147 * Update on-disk file size now that data has been written to disk.
151 struct xfs_inode
*ip
,
152 struct xfs_trans
*tp
,
158 xfs_ilock(ip
, XFS_ILOCK_EXCL
);
159 isize
= xfs_new_eof(ip
, offset
+ size
);
161 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
162 xfs_trans_cancel(tp
);
166 trace_xfs_setfilesize(ip
, offset
, size
);
168 ip
->i_d
.di_size
= isize
;
169 xfs_trans_ijoin(tp
, ip
, XFS_ILOCK_EXCL
);
170 xfs_trans_log_inode(tp
, ip
, XFS_ILOG_CORE
);
172 return xfs_trans_commit(tp
);
176 xfs_setfilesize_ioend(
177 struct xfs_ioend
*ioend
)
179 struct xfs_inode
*ip
= XFS_I(ioend
->io_inode
);
180 struct xfs_trans
*tp
= ioend
->io_append_trans
;
183 * The transaction may have been allocated in the I/O submission thread,
184 * thus we need to mark ourselves as being in a transaction manually.
185 * Similarly for freeze protection.
187 current_set_flags_nested(&tp
->t_pflags
, PF_FSTRANS
);
188 __sb_writers_acquired(VFS_I(ip
)->i_sb
, SB_FREEZE_FS
);
190 /* we abort the update if there was an IO error */
191 if (ioend
->io_error
) {
192 xfs_trans_cancel(tp
);
193 return ioend
->io_error
;
196 return xfs_setfilesize(ip
, tp
, ioend
->io_offset
, ioend
->io_size
);
200 * Schedule IO completion handling on the final put of an ioend.
202 * If there is no work to do we might as well call it a day and free the
207 struct xfs_ioend
*ioend
)
209 if (atomic_dec_and_test(&ioend
->io_remaining
)) {
210 struct xfs_mount
*mp
= XFS_I(ioend
->io_inode
)->i_mount
;
212 if (ioend
->io_type
== XFS_IO_UNWRITTEN
)
213 queue_work(mp
->m_unwritten_workqueue
, &ioend
->io_work
);
214 else if (ioend
->io_append_trans
)
215 queue_work(mp
->m_data_workqueue
, &ioend
->io_work
);
217 xfs_destroy_ioend(ioend
);
222 * IO write completion.
226 struct work_struct
*work
)
228 xfs_ioend_t
*ioend
= container_of(work
, xfs_ioend_t
, io_work
);
229 struct xfs_inode
*ip
= XFS_I(ioend
->io_inode
);
233 * Set an error if the mount has shut down and proceed with end I/O
234 * processing so it can perform whatever cleanups are necessary.
236 if (XFS_FORCED_SHUTDOWN(ip
->i_mount
))
237 ioend
->io_error
= -EIO
;
240 * For unwritten extents we need to issue transactions to convert a
241 * range to normal written extens after the data I/O has finished.
242 * Detecting and handling completion IO errors is done individually
243 * for each case as different cleanup operations need to be performed
246 if (ioend
->io_type
== XFS_IO_UNWRITTEN
) {
249 error
= xfs_iomap_write_unwritten(ip
, ioend
->io_offset
,
251 } else if (ioend
->io_append_trans
) {
252 error
= xfs_setfilesize_ioend(ioend
);
254 ASSERT(!xfs_ioend_is_append(ioend
));
259 ioend
->io_error
= error
;
260 xfs_destroy_ioend(ioend
);
264 * Allocate and initialise an IO completion structure.
265 * We need to track unwritten extent write completion here initially.
266 * We'll need to extend this for updating the ondisk inode size later
276 ioend
= mempool_alloc(xfs_ioend_pool
, GFP_NOFS
);
277 memset(ioend
, 0, sizeof(*ioend
));
280 * Set the count to 1 initially, which will prevent an I/O
281 * completion callback from happening before we have started
282 * all the I/O from calling the completion routine too early.
284 atomic_set(&ioend
->io_remaining
, 1);
285 INIT_LIST_HEAD(&ioend
->io_list
);
286 ioend
->io_type
= type
;
287 ioend
->io_inode
= inode
;
288 INIT_WORK(&ioend
->io_work
, xfs_end_io
);
296 struct xfs_bmbt_irec
*imap
,
299 struct xfs_inode
*ip
= XFS_I(inode
);
300 struct xfs_mount
*mp
= ip
->i_mount
;
301 ssize_t count
= 1 << inode
->i_blkbits
;
302 xfs_fileoff_t offset_fsb
, end_fsb
;
304 int bmapi_flags
= XFS_BMAPI_ENTIRE
;
307 if (XFS_FORCED_SHUTDOWN(mp
))
310 if (type
== XFS_IO_UNWRITTEN
)
311 bmapi_flags
|= XFS_BMAPI_IGSTATE
;
313 xfs_ilock(ip
, XFS_ILOCK_SHARED
);
314 ASSERT(ip
->i_d
.di_format
!= XFS_DINODE_FMT_BTREE
||
315 (ip
->i_df
.if_flags
& XFS_IFEXTENTS
));
316 ASSERT(offset
<= mp
->m_super
->s_maxbytes
);
318 if (offset
+ count
> mp
->m_super
->s_maxbytes
)
319 count
= mp
->m_super
->s_maxbytes
- offset
;
320 end_fsb
= XFS_B_TO_FSB(mp
, (xfs_ufsize_t
)offset
+ count
);
321 offset_fsb
= XFS_B_TO_FSBT(mp
, offset
);
322 error
= xfs_bmapi_read(ip
, offset_fsb
, end_fsb
- offset_fsb
,
323 imap
, &nimaps
, bmapi_flags
);
324 xfs_iunlock(ip
, XFS_ILOCK_SHARED
);
329 if (type
== XFS_IO_DELALLOC
&&
330 (!nimaps
|| isnullstartblock(imap
->br_startblock
))) {
331 error
= xfs_iomap_write_allocate(ip
, offset
, imap
);
333 trace_xfs_map_blocks_alloc(ip
, offset
, count
, type
, imap
);
338 if (type
== XFS_IO_UNWRITTEN
) {
340 ASSERT(imap
->br_startblock
!= HOLESTARTBLOCK
);
341 ASSERT(imap
->br_startblock
!= DELAYSTARTBLOCK
);
345 trace_xfs_map_blocks_found(ip
, offset
, count
, type
, imap
);
352 struct xfs_bmbt_irec
*imap
,
355 offset
>>= inode
->i_blkbits
;
357 return offset
>= imap
->br_startoff
&&
358 offset
< imap
->br_startoff
+ imap
->br_blockcount
;
362 * BIO completion handler for buffered IO.
368 xfs_ioend_t
*ioend
= bio
->bi_private
;
370 if (!ioend
->io_error
)
371 ioend
->io_error
= bio
->bi_error
;
373 /* Toss bio and pass work off to an xfsdatad thread */
374 bio
->bi_private
= NULL
;
375 bio
->bi_end_io
= NULL
;
378 xfs_finish_ioend(ioend
);
382 xfs_submit_ioend_bio(
383 struct writeback_control
*wbc
,
387 atomic_inc(&ioend
->io_remaining
);
388 bio
->bi_private
= ioend
;
389 bio
->bi_end_io
= xfs_end_bio
;
390 submit_bio(wbc
->sync_mode
== WB_SYNC_ALL
? WRITE_SYNC
: WRITE
, bio
);
395 struct buffer_head
*bh
)
397 struct bio
*bio
= bio_alloc(GFP_NOIO
, BIO_MAX_PAGES
);
399 ASSERT(bio
->bi_private
== NULL
);
400 bio
->bi_iter
.bi_sector
= bh
->b_blocknr
* (bh
->b_size
>> 9);
401 bio
->bi_bdev
= bh
->b_bdev
;
406 xfs_start_buffer_writeback(
407 struct buffer_head
*bh
)
409 ASSERT(buffer_mapped(bh
));
410 ASSERT(buffer_locked(bh
));
411 ASSERT(!buffer_delay(bh
));
412 ASSERT(!buffer_unwritten(bh
));
414 mark_buffer_async_write(bh
);
415 set_buffer_uptodate(bh
);
416 clear_buffer_dirty(bh
);
420 xfs_start_page_writeback(
424 ASSERT(PageLocked(page
));
425 ASSERT(!PageWriteback(page
));
428 * if the page was not fully cleaned, we need to ensure that the higher
429 * layers come back to it correctly. That means we need to keep the page
430 * dirty, and for WB_SYNC_ALL writeback we need to ensure the
431 * PAGECACHE_TAG_TOWRITE index mark is not removed so another attempt to
432 * write this page in this writeback sweep will be made.
435 clear_page_dirty_for_io(page
);
436 set_page_writeback(page
);
438 set_page_writeback_keepwrite(page
);
443 static inline int xfs_bio_add_buffer(struct bio
*bio
, struct buffer_head
*bh
)
445 return bio_add_page(bio
, bh
->b_page
, bh
->b_size
, bh_offset(bh
));
449 * Submit the bio for an ioend. We are passed an ioend with a bio attached to
450 * it, and we submit that bio. The ioend may be used for multiple bio
451 * submissions, so we only want to allocate an append transaction for the ioend
452 * once. In the case of multiple bio submission, each bio will take an IO
453 * reference to the ioend to ensure that the ioend completion is only done once
454 * all bios have been submitted and the ioend is really done.
456 * If @fail is non-zero, it means that we have a situation where some part of
457 * the submission process has failed after we have marked paged for writeback
458 * and unlocked them. In this situation, we need to fail the bio and ioend
459 * rather than submit it to IO. This typically only happens on a filesystem
464 struct writeback_control
*wbc
,
468 /* Reserve log space if we might write beyond the on-disk inode size. */
470 ioend
->io_bio
&& ioend
->io_type
!= XFS_IO_UNWRITTEN
&&
471 xfs_ioend_is_append(ioend
) &&
472 !ioend
->io_append_trans
)
473 status
= xfs_setfilesize_trans_alloc(ioend
);
476 * If we are failing the IO now, just mark the ioend with an
477 * error and finish it. This will run IO completion immediately
478 * as there is only one reference to the ioend at this point in
483 bio_put(ioend
->io_bio
);
484 ioend
->io_error
= status
;
485 xfs_finish_ioend(ioend
);
489 xfs_submit_ioend_bio(wbc
, ioend
, ioend
->io_bio
);
490 ioend
->io_bio
= NULL
;
491 xfs_finish_ioend(ioend
);
496 * Test to see if we've been building up a completion structure for
497 * earlier buffers -- if so, we try to append to this ioend if we
498 * can, otherwise we finish off any current ioend and start another.
499 * Return the ioend we finished off so that the caller can submit it
500 * once it has finished processing the dirty page.
505 struct buffer_head
*bh
,
507 struct xfs_writepage_ctx
*wpc
,
508 struct writeback_control
*wbc
,
509 struct list_head
*iolist
)
511 if (!wpc
->ioend
|| wpc
->io_type
!= wpc
->ioend
->io_type
||
512 bh
->b_blocknr
!= wpc
->last_block
+ 1 ||
513 offset
!= wpc
->ioend
->io_offset
+ wpc
->ioend
->io_size
) {
514 struct xfs_ioend
*new;
517 list_add(&wpc
->ioend
->io_list
, iolist
);
519 new = xfs_alloc_ioend(inode
, wpc
->io_type
);
520 new->io_offset
= offset
;
521 new->io_buffer_head
= bh
;
522 new->io_buffer_tail
= bh
;
525 wpc
->ioend
->io_buffer_tail
->b_private
= bh
;
526 wpc
->ioend
->io_buffer_tail
= bh
;
528 bh
->b_private
= NULL
;
531 if (!wpc
->ioend
->io_bio
)
532 wpc
->ioend
->io_bio
= xfs_alloc_ioend_bio(bh
);
534 if (xfs_bio_add_buffer(wpc
->ioend
->io_bio
, bh
) != bh
->b_size
) {
535 xfs_submit_ioend_bio(wbc
, wpc
->ioend
, wpc
->ioend
->io_bio
);
536 wpc
->ioend
->io_bio
= NULL
;
540 wpc
->ioend
->io_size
+= bh
->b_size
;
541 wpc
->last_block
= bh
->b_blocknr
;
542 xfs_start_buffer_writeback(bh
);
548 struct buffer_head
*bh
,
549 struct xfs_bmbt_irec
*imap
,
553 struct xfs_mount
*m
= XFS_I(inode
)->i_mount
;
554 xfs_off_t iomap_offset
= XFS_FSB_TO_B(m
, imap
->br_startoff
);
555 xfs_daddr_t iomap_bn
= xfs_fsb_to_db(XFS_I(inode
), imap
->br_startblock
);
557 ASSERT(imap
->br_startblock
!= HOLESTARTBLOCK
);
558 ASSERT(imap
->br_startblock
!= DELAYSTARTBLOCK
);
560 bn
= (iomap_bn
>> (inode
->i_blkbits
- BBSHIFT
)) +
561 ((offset
- iomap_offset
) >> inode
->i_blkbits
);
563 ASSERT(bn
|| XFS_IS_REALTIME_INODE(XFS_I(inode
)));
566 set_buffer_mapped(bh
);
572 struct buffer_head
*bh
,
573 struct xfs_bmbt_irec
*imap
,
576 ASSERT(imap
->br_startblock
!= HOLESTARTBLOCK
);
577 ASSERT(imap
->br_startblock
!= DELAYSTARTBLOCK
);
579 xfs_map_buffer(inode
, bh
, imap
, offset
);
580 set_buffer_mapped(bh
);
581 clear_buffer_delay(bh
);
582 clear_buffer_unwritten(bh
);
586 * Test if a given page contains at least one buffer of a given @type.
587 * If @check_all_buffers is true, then we walk all the buffers in the page to
588 * try to find one of the type passed in. If it is not set, then the caller only
589 * needs to check the first buffer on the page for a match.
595 bool check_all_buffers
)
597 struct buffer_head
*bh
;
598 struct buffer_head
*head
;
600 if (PageWriteback(page
))
604 if (!page_has_buffers(page
))
607 bh
= head
= page_buffers(page
);
609 if (buffer_unwritten(bh
)) {
610 if (type
== XFS_IO_UNWRITTEN
)
612 } else if (buffer_delay(bh
)) {
613 if (type
== XFS_IO_DELALLOC
)
615 } else if (buffer_dirty(bh
) && buffer_mapped(bh
)) {
616 if (type
== XFS_IO_OVERWRITE
)
620 /* If we are only checking the first buffer, we are done now. */
621 if (!check_all_buffers
)
623 } while ((bh
= bh
->b_this_page
) != head
);
629 xfs_vm_invalidatepage(
634 trace_xfs_invalidatepage(page
->mapping
->host
, page
, offset
,
636 block_invalidatepage(page
, offset
, length
);
640 * If the page has delalloc buffers on it, we need to punch them out before we
641 * invalidate the page. If we don't, we leave a stale delalloc mapping on the
642 * inode that can trip a BUG() in xfs_get_blocks() later on if a direct IO read
643 * is done on that same region - the delalloc extent is returned when none is
644 * supposed to be there.
646 * We prevent this by truncating away the delalloc regions on the page before
647 * invalidating it. Because they are delalloc, we can do this without needing a
648 * transaction. Indeed - if we get ENOSPC errors, we have to be able to do this
649 * truncation without a transaction as there is no space left for block
650 * reservation (typically why we see a ENOSPC in writeback).
652 * This is not a performance critical path, so for now just do the punching a
653 * buffer head at a time.
656 xfs_aops_discard_page(
659 struct inode
*inode
= page
->mapping
->host
;
660 struct xfs_inode
*ip
= XFS_I(inode
);
661 struct buffer_head
*bh
, *head
;
662 loff_t offset
= page_offset(page
);
664 if (!xfs_check_page_type(page
, XFS_IO_DELALLOC
, true))
667 if (XFS_FORCED_SHUTDOWN(ip
->i_mount
))
670 xfs_alert(ip
->i_mount
,
671 "page discard on page %p, inode 0x%llx, offset %llu.",
672 page
, ip
->i_ino
, offset
);
674 xfs_ilock(ip
, XFS_ILOCK_EXCL
);
675 bh
= head
= page_buffers(page
);
678 xfs_fileoff_t start_fsb
;
680 if (!buffer_delay(bh
))
683 start_fsb
= XFS_B_TO_FSBT(ip
->i_mount
, offset
);
684 error
= xfs_bmap_punch_delalloc_range(ip
, start_fsb
, 1);
686 /* something screwed, just bail */
687 if (!XFS_FORCED_SHUTDOWN(ip
->i_mount
)) {
688 xfs_alert(ip
->i_mount
,
689 "page discard unable to remove delalloc mapping.");
694 offset
+= 1 << inode
->i_blkbits
;
696 } while ((bh
= bh
->b_this_page
) != head
);
698 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
700 xfs_vm_invalidatepage(page
, 0, PAGE_CACHE_SIZE
);
705 * We implement an immediate ioend submission policy here to avoid needing to
706 * chain multiple ioends and hence nest mempool allocations which can violate
707 * forward progress guarantees we need to provide. The current ioend we are
708 * adding buffers to is cached on the writepage context, and if the new buffer
709 * does not append to the cached ioend it will create a new ioend and cache that
712 * If a new ioend is created and cached, the old ioend is returned and queued
713 * locally for submission once the entire page is processed or an error has been
714 * detected. While ioends are submitted immediately after they are completed,
715 * batching optimisations are provided by higher level block plugging.
717 * At the end of a writeback pass, there will be a cached ioend remaining on the
718 * writepage context that the caller will need to submit.
722 struct xfs_writepage_ctx
*wpc
,
723 struct writeback_control
*wbc
,
727 __uint64_t end_offset
)
729 LIST_HEAD(submit_list
);
730 struct xfs_ioend
*ioend
, *next
;
731 struct buffer_head
*bh
, *head
;
732 ssize_t len
= 1 << inode
->i_blkbits
;
737 bh
= head
= page_buffers(page
);
738 offset
= page_offset(page
);
740 if (offset
>= end_offset
)
742 if (!buffer_uptodate(bh
))
746 * set_page_dirty dirties all buffers in a page, independent
747 * of their state. The dirty state however is entirely
748 * meaningless for holes (!mapped && uptodate), so skip
749 * buffers covering holes here.
751 if (!buffer_mapped(bh
) && buffer_uptodate(bh
)) {
752 wpc
->imap_valid
= false;
756 if (buffer_unwritten(bh
)) {
757 if (wpc
->io_type
!= XFS_IO_UNWRITTEN
) {
758 wpc
->io_type
= XFS_IO_UNWRITTEN
;
759 wpc
->imap_valid
= false;
761 } else if (buffer_delay(bh
)) {
762 if (wpc
->io_type
!= XFS_IO_DELALLOC
) {
763 wpc
->io_type
= XFS_IO_DELALLOC
;
764 wpc
->imap_valid
= false;
766 } else if (buffer_uptodate(bh
)) {
767 if (wpc
->io_type
!= XFS_IO_OVERWRITE
) {
768 wpc
->io_type
= XFS_IO_OVERWRITE
;
769 wpc
->imap_valid
= false;
772 if (PageUptodate(page
))
773 ASSERT(buffer_mapped(bh
));
775 * This buffer is not uptodate and will not be
776 * written to disk. Ensure that we will put any
777 * subsequent writeable buffers into a new
780 wpc
->imap_valid
= false;
785 wpc
->imap_valid
= xfs_imap_valid(inode
, &wpc
->imap
,
787 if (!wpc
->imap_valid
) {
788 error
= xfs_map_blocks(inode
, offset
, &wpc
->imap
,
792 wpc
->imap_valid
= xfs_imap_valid(inode
, &wpc
->imap
,
795 if (wpc
->imap_valid
) {
797 if (wpc
->io_type
!= XFS_IO_OVERWRITE
)
798 xfs_map_at_offset(inode
, bh
, &wpc
->imap
, offset
);
799 xfs_add_to_ioend(inode
, bh
, offset
, wpc
, wbc
, &submit_list
);
803 } while (offset
+= len
, ((bh
= bh
->b_this_page
) != head
));
805 if (uptodate
&& bh
== head
)
806 SetPageUptodate(page
);
808 ASSERT(wpc
->ioend
|| list_empty(&submit_list
));
812 * On error, we have to fail the ioend here because we have locked
813 * buffers in the ioend. If we don't do this, we'll deadlock
814 * invalidating the page as that tries to lock the buffers on the page.
815 * Also, because we may have set pages under writeback, we have to make
816 * sure we run IO completion to mark the error state of the IO
817 * appropriately, so we can't cancel the ioend directly here. That means
818 * we have to mark this page as under writeback if we included any
819 * buffers from it in the ioend chain so that completion treats it
822 * If we didn't include the page in the ioend, the on error we can
823 * simply discard and unlock it as there are no other users of the page
824 * or it's buffers right now. The caller will still need to trigger
825 * submission of outstanding ioends on the writepage context so they are
826 * treated correctly on error.
829 xfs_start_page_writeback(page
, !error
);
832 * Preserve the original error if there was one, otherwise catch
833 * submission errors here and propagate into subsequent ioend
836 list_for_each_entry_safe(ioend
, next
, &submit_list
, io_list
) {
839 list_del_init(&ioend
->io_list
);
840 error2
= xfs_submit_ioend(wbc
, ioend
, error
);
841 if (error2
&& !error
)
845 xfs_aops_discard_page(page
);
846 ClearPageUptodate(page
);
850 * We can end up here with no error and nothing to write if we
851 * race with a partial page truncate on a sub-page block sized
852 * filesystem. In that case we need to mark the page clean.
854 xfs_start_page_writeback(page
, 1);
855 end_page_writeback(page
);
858 mapping_set_error(page
->mapping
, error
);
863 * Write out a dirty page.
865 * For delalloc space on the page we need to allocate space and flush it.
866 * For unwritten space on the page we need to start the conversion to
867 * regular allocated space.
868 * For any other dirty buffer heads on the page we should flush them.
873 struct writeback_control
*wbc
,
876 struct xfs_writepage_ctx
*wpc
= data
;
877 struct inode
*inode
= page
->mapping
->host
;
879 __uint64_t end_offset
;
882 trace_xfs_writepage(inode
, page
, 0, 0);
884 ASSERT(page_has_buffers(page
));
887 * Refuse to write the page out if we are called from reclaim context.
889 * This avoids stack overflows when called from deeply used stacks in
890 * random callers for direct reclaim or memcg reclaim. We explicitly
891 * allow reclaim from kswapd as the stack usage there is relatively low.
893 * This should never happen except in the case of a VM regression so
896 if (WARN_ON_ONCE((current
->flags
& (PF_MEMALLOC
|PF_KSWAPD
)) ==
901 * Given that we do not allow direct reclaim to call us, we should
902 * never be called while in a filesystem transaction.
904 if (WARN_ON_ONCE(current
->flags
& PF_FSTRANS
))
908 * Is this page beyond the end of the file?
910 * The page index is less than the end_index, adjust the end_offset
911 * to the highest offset that this page should represent.
912 * -----------------------------------------------------
913 * | file mapping | <EOF> |
914 * -----------------------------------------------------
915 * | Page ... | Page N-2 | Page N-1 | Page N | |
916 * ^--------------------------------^----------|--------
917 * | desired writeback range | see else |
918 * ---------------------------------^------------------|
920 offset
= i_size_read(inode
);
921 end_index
= offset
>> PAGE_CACHE_SHIFT
;
922 if (page
->index
< end_index
)
923 end_offset
= (xfs_off_t
)(page
->index
+ 1) << PAGE_CACHE_SHIFT
;
926 * Check whether the page to write out is beyond or straddles
928 * -------------------------------------------------------
929 * | file mapping | <EOF> |
930 * -------------------------------------------------------
931 * | Page ... | Page N-2 | Page N-1 | Page N | Beyond |
932 * ^--------------------------------^-----------|---------
934 * ---------------------------------^-----------|--------|
936 unsigned offset_into_page
= offset
& (PAGE_CACHE_SIZE
- 1);
939 * Skip the page if it is fully outside i_size, e.g. due to a
940 * truncate operation that is in progress. We must redirty the
941 * page so that reclaim stops reclaiming it. Otherwise
942 * xfs_vm_releasepage() is called on it and gets confused.
944 * Note that the end_index is unsigned long, it would overflow
945 * if the given offset is greater than 16TB on 32-bit system
946 * and if we do check the page is fully outside i_size or not
947 * via "if (page->index >= end_index + 1)" as "end_index + 1"
948 * will be evaluated to 0. Hence this page will be redirtied
949 * and be written out repeatedly which would result in an
950 * infinite loop, the user program that perform this operation
951 * will hang. Instead, we can verify this situation by checking
952 * if the page to write is totally beyond the i_size or if it's
953 * offset is just equal to the EOF.
955 if (page
->index
> end_index
||
956 (page
->index
== end_index
&& offset_into_page
== 0))
960 * The page straddles i_size. It must be zeroed out on each
961 * and every writepage invocation because it may be mmapped.
962 * "A file is mapped in multiples of the page size. For a file
963 * that is not a multiple of the page size, the remaining
964 * memory is zeroed when mapped, and writes to that region are
965 * not written out to the file."
967 zero_user_segment(page
, offset_into_page
, PAGE_CACHE_SIZE
);
969 /* Adjust the end_offset to the end of file */
973 return xfs_writepage_map(wpc
, wbc
, inode
, page
, offset
, end_offset
);
976 redirty_page_for_writepage(wbc
, page
);
984 struct writeback_control
*wbc
)
986 struct xfs_writepage_ctx wpc
= {
987 .io_type
= XFS_IO_INVALID
,
991 ret
= xfs_do_writepage(page
, wbc
, &wpc
);
993 ret
= xfs_submit_ioend(wbc
, wpc
.ioend
, ret
);
999 struct address_space
*mapping
,
1000 struct writeback_control
*wbc
)
1002 struct xfs_writepage_ctx wpc
= {
1003 .io_type
= XFS_IO_INVALID
,
1007 xfs_iflags_clear(XFS_I(mapping
->host
), XFS_ITRUNCATED
);
1008 if (dax_mapping(mapping
))
1009 return dax_writeback_mapping_range(mapping
,
1010 xfs_find_bdev_for_inode(mapping
->host
), wbc
);
1012 ret
= write_cache_pages(mapping
, wbc
, xfs_do_writepage
, &wpc
);
1014 ret
= xfs_submit_ioend(wbc
, wpc
.ioend
, ret
);
1019 * Called to move a page into cleanable state - and from there
1020 * to be released. The page should already be clean. We always
1021 * have buffer heads in this call.
1023 * Returns 1 if the page is ok to release, 0 otherwise.
1030 int delalloc
, unwritten
;
1032 trace_xfs_releasepage(page
->mapping
->host
, page
, 0, 0);
1034 xfs_count_page_state(page
, &delalloc
, &unwritten
);
1036 if (WARN_ON_ONCE(delalloc
))
1038 if (WARN_ON_ONCE(unwritten
))
1041 return try_to_free_buffers(page
);
1045 * When we map a DIO buffer, we may need to pass flags to
1046 * xfs_end_io_direct_write to tell it what kind of write IO we are doing.
1048 * Note that for DIO, an IO to the highest supported file block offset (i.e.
1049 * 2^63 - 1FSB bytes) will result in the offset + count overflowing a signed 64
1050 * bit variable. Hence if we see this overflow, we have to assume that the IO is
1051 * extending the file size. We won't know for sure until IO completion is run
1052 * and the actual max write offset is communicated to the IO completion
1057 struct inode
*inode
,
1058 struct buffer_head
*bh_result
,
1059 struct xfs_bmbt_irec
*imap
,
1062 uintptr_t *flags
= (uintptr_t *)&bh_result
->b_private
;
1063 xfs_off_t size
= bh_result
->b_size
;
1065 trace_xfs_get_blocks_map_direct(XFS_I(inode
), offset
, size
,
1066 ISUNWRITTEN(imap
) ? XFS_IO_UNWRITTEN
: XFS_IO_OVERWRITE
, imap
);
1068 if (ISUNWRITTEN(imap
)) {
1069 *flags
|= XFS_DIO_FLAG_UNWRITTEN
;
1070 set_buffer_defer_completion(bh_result
);
1071 } else if (offset
+ size
> i_size_read(inode
) || offset
+ size
< 0) {
1072 *flags
|= XFS_DIO_FLAG_APPEND
;
1073 set_buffer_defer_completion(bh_result
);
1078 * If this is O_DIRECT or the mpage code calling tell them how large the mapping
1079 * is, so that we can avoid repeated get_blocks calls.
1081 * If the mapping spans EOF, then we have to break the mapping up as the mapping
1082 * for blocks beyond EOF must be marked new so that sub block regions can be
1083 * correctly zeroed. We can't do this for mappings within EOF unless the mapping
1084 * was just allocated or is unwritten, otherwise the callers would overwrite
1085 * existing data with zeros. Hence we have to split the mapping into a range up
1086 * to and including EOF, and a second mapping for beyond EOF.
1090 struct inode
*inode
,
1092 struct buffer_head
*bh_result
,
1093 struct xfs_bmbt_irec
*imap
,
1097 xfs_off_t mapping_size
;
1099 mapping_size
= imap
->br_startoff
+ imap
->br_blockcount
- iblock
;
1100 mapping_size
<<= inode
->i_blkbits
;
1102 ASSERT(mapping_size
> 0);
1103 if (mapping_size
> size
)
1104 mapping_size
= size
;
1105 if (offset
< i_size_read(inode
) &&
1106 offset
+ mapping_size
>= i_size_read(inode
)) {
1107 /* limit mapping to block that spans EOF */
1108 mapping_size
= roundup_64(i_size_read(inode
) - offset
,
1109 1 << inode
->i_blkbits
);
1111 if (mapping_size
> LONG_MAX
)
1112 mapping_size
= LONG_MAX
;
1114 bh_result
->b_size
= mapping_size
;
1119 struct inode
*inode
,
1121 struct buffer_head
*bh_result
,
1126 struct xfs_inode
*ip
= XFS_I(inode
);
1127 struct xfs_mount
*mp
= ip
->i_mount
;
1128 xfs_fileoff_t offset_fsb
, end_fsb
;
1131 struct xfs_bmbt_irec imap
;
1137 if (XFS_FORCED_SHUTDOWN(mp
))
1140 offset
= (xfs_off_t
)iblock
<< inode
->i_blkbits
;
1141 ASSERT(bh_result
->b_size
>= (1 << inode
->i_blkbits
));
1142 size
= bh_result
->b_size
;
1144 if (!create
&& direct
&& offset
>= i_size_read(inode
))
1148 * Direct I/O is usually done on preallocated files, so try getting
1149 * a block mapping without an exclusive lock first. For buffered
1150 * writes we already have the exclusive iolock anyway, so avoiding
1151 * a lock roundtrip here by taking the ilock exclusive from the
1152 * beginning is a useful micro optimization.
1154 if (create
&& !direct
) {
1155 lockmode
= XFS_ILOCK_EXCL
;
1156 xfs_ilock(ip
, lockmode
);
1158 lockmode
= xfs_ilock_data_map_shared(ip
);
1161 ASSERT(offset
<= mp
->m_super
->s_maxbytes
);
1162 if (offset
+ size
> mp
->m_super
->s_maxbytes
)
1163 size
= mp
->m_super
->s_maxbytes
- offset
;
1164 end_fsb
= XFS_B_TO_FSB(mp
, (xfs_ufsize_t
)offset
+ size
);
1165 offset_fsb
= XFS_B_TO_FSBT(mp
, offset
);
1167 error
= xfs_bmapi_read(ip
, offset_fsb
, end_fsb
- offset_fsb
,
1168 &imap
, &nimaps
, XFS_BMAPI_ENTIRE
);
1172 /* for DAX, we convert unwritten extents directly */
1175 (imap
.br_startblock
== HOLESTARTBLOCK
||
1176 imap
.br_startblock
== DELAYSTARTBLOCK
) ||
1177 (IS_DAX(inode
) && ISUNWRITTEN(&imap
)))) {
1178 if (direct
|| xfs_get_extsz_hint(ip
)) {
1180 * xfs_iomap_write_direct() expects the shared lock. It
1181 * is unlocked on return.
1183 if (lockmode
== XFS_ILOCK_EXCL
)
1184 xfs_ilock_demote(ip
, lockmode
);
1186 error
= xfs_iomap_write_direct(ip
, offset
, size
,
1194 * Delalloc reservations do not require a transaction,
1195 * we can go on without dropping the lock here. If we
1196 * are allocating a new delalloc block, make sure that
1197 * we set the new flag so that we mark the buffer new so
1198 * that we know that it is newly allocated if the write
1201 if (nimaps
&& imap
.br_startblock
== HOLESTARTBLOCK
)
1203 error
= xfs_iomap_write_delay(ip
, offset
, size
, &imap
);
1207 xfs_iunlock(ip
, lockmode
);
1209 trace_xfs_get_blocks_alloc(ip
, offset
, size
,
1210 ISUNWRITTEN(&imap
) ? XFS_IO_UNWRITTEN
1211 : XFS_IO_DELALLOC
, &imap
);
1212 } else if (nimaps
) {
1213 trace_xfs_get_blocks_found(ip
, offset
, size
,
1214 ISUNWRITTEN(&imap
) ? XFS_IO_UNWRITTEN
1215 : XFS_IO_OVERWRITE
, &imap
);
1216 xfs_iunlock(ip
, lockmode
);
1218 trace_xfs_get_blocks_notfound(ip
, offset
, size
);
1222 if (IS_DAX(inode
) && create
) {
1223 ASSERT(!ISUNWRITTEN(&imap
));
1224 /* zeroing is not needed at a higher layer */
1228 /* trim mapping down to size requested */
1229 if (direct
|| size
> (1 << inode
->i_blkbits
))
1230 xfs_map_trim_size(inode
, iblock
, bh_result
,
1231 &imap
, offset
, size
);
1234 * For unwritten extents do not report a disk address in the buffered
1235 * read case (treat as if we're reading into a hole).
1237 if (imap
.br_startblock
!= HOLESTARTBLOCK
&&
1238 imap
.br_startblock
!= DELAYSTARTBLOCK
&&
1239 (create
|| !ISUNWRITTEN(&imap
))) {
1240 xfs_map_buffer(inode
, bh_result
, &imap
, offset
);
1241 if (ISUNWRITTEN(&imap
))
1242 set_buffer_unwritten(bh_result
);
1243 /* direct IO needs special help */
1244 if (create
&& direct
) {
1246 ASSERT(!ISUNWRITTEN(&imap
));
1248 xfs_map_direct(inode
, bh_result
, &imap
, offset
);
1253 * If this is a realtime file, data may be on a different device.
1254 * to that pointed to from the buffer_head b_bdev currently.
1256 bh_result
->b_bdev
= xfs_find_bdev_for_inode(inode
);
1259 * If we previously allocated a block out beyond eof and we are now
1260 * coming back to use it then we will need to flag it as new even if it
1261 * has a disk address.
1263 * With sub-block writes into unwritten extents we also need to mark
1264 * the buffer as new so that the unwritten parts of the buffer gets
1268 ((!buffer_mapped(bh_result
) && !buffer_uptodate(bh_result
)) ||
1269 (offset
>= i_size_read(inode
)) ||
1270 (new || ISUNWRITTEN(&imap
))))
1271 set_buffer_new(bh_result
);
1273 if (imap
.br_startblock
== DELAYSTARTBLOCK
) {
1276 set_buffer_uptodate(bh_result
);
1277 set_buffer_mapped(bh_result
);
1278 set_buffer_delay(bh_result
);
1285 xfs_iunlock(ip
, lockmode
);
1291 struct inode
*inode
,
1293 struct buffer_head
*bh_result
,
1296 return __xfs_get_blocks(inode
, iblock
, bh_result
, create
, false, false);
1300 xfs_get_blocks_direct(
1301 struct inode
*inode
,
1303 struct buffer_head
*bh_result
,
1306 return __xfs_get_blocks(inode
, iblock
, bh_result
, create
, true, false);
1310 xfs_get_blocks_dax_fault(
1311 struct inode
*inode
,
1313 struct buffer_head
*bh_result
,
1316 return __xfs_get_blocks(inode
, iblock
, bh_result
, create
, true, true);
1320 * Complete a direct I/O write request.
1322 * xfs_map_direct passes us some flags in the private data to tell us what to
1323 * do. If no flags are set, then the write IO is an overwrite wholly within
1324 * the existing allocated file size and so there is nothing for us to do.
1326 * Note that in this case the completion can be called in interrupt context,
1327 * whereas if we have flags set we will always be called in task context
1328 * (i.e. from a workqueue).
1331 xfs_end_io_direct_write(
1337 struct inode
*inode
= file_inode(iocb
->ki_filp
);
1338 struct xfs_inode
*ip
= XFS_I(inode
);
1339 struct xfs_mount
*mp
= ip
->i_mount
;
1340 uintptr_t flags
= (uintptr_t)private;
1343 trace_xfs_end_io_direct_write(ip
, offset
, size
);
1345 if (XFS_FORCED_SHUTDOWN(mp
))
1352 * The flags tell us whether we are doing unwritten extent conversions
1353 * or an append transaction that updates the on-disk file size. These
1354 * cases are the only cases where we should *potentially* be needing
1355 * to update the VFS inode size.
1358 ASSERT(offset
+ size
<= i_size_read(inode
));
1363 * We need to update the in-core inode size here so that we don't end up
1364 * with the on-disk inode size being outside the in-core inode size. We
1365 * have no other method of updating EOF for AIO, so always do it here
1368 * We need to lock the test/set EOF update as we can be racing with
1369 * other IO completions here to update the EOF. Failing to serialise
1370 * here can result in EOF moving backwards and Bad Things Happen when
1373 spin_lock(&ip
->i_flags_lock
);
1374 if (offset
+ size
> i_size_read(inode
))
1375 i_size_write(inode
, offset
+ size
);
1376 spin_unlock(&ip
->i_flags_lock
);
1378 if (flags
& XFS_DIO_FLAG_UNWRITTEN
) {
1379 trace_xfs_end_io_direct_write_unwritten(ip
, offset
, size
);
1381 error
= xfs_iomap_write_unwritten(ip
, offset
, size
);
1382 } else if (flags
& XFS_DIO_FLAG_APPEND
) {
1383 struct xfs_trans
*tp
;
1385 trace_xfs_end_io_direct_write_append(ip
, offset
, size
);
1387 tp
= xfs_trans_alloc(mp
, XFS_TRANS_FSYNC_TS
);
1388 error
= xfs_trans_reserve(tp
, &M_RES(mp
)->tr_fsyncts
, 0, 0);
1390 xfs_trans_cancel(tp
);
1393 error
= xfs_setfilesize(ip
, tp
, offset
, size
);
1402 struct iov_iter
*iter
,
1405 struct inode
*inode
= iocb
->ki_filp
->f_mapping
->host
;
1406 dio_iodone_t
*endio
= NULL
;
1408 struct block_device
*bdev
;
1410 if (iov_iter_rw(iter
) == WRITE
) {
1411 endio
= xfs_end_io_direct_write
;
1412 flags
= DIO_ASYNC_EXTEND
;
1415 if (IS_DAX(inode
)) {
1416 return dax_do_io(iocb
, inode
, iter
, offset
,
1417 xfs_get_blocks_direct
, endio
, 0);
1420 bdev
= xfs_find_bdev_for_inode(inode
);
1421 return __blockdev_direct_IO(iocb
, inode
, bdev
, iter
, offset
,
1422 xfs_get_blocks_direct
, endio
, NULL
, flags
);
1426 * Punch out the delalloc blocks we have already allocated.
1428 * Don't bother with xfs_setattr given that nothing can have made it to disk yet
1429 * as the page is still locked at this point.
1432 xfs_vm_kill_delalloc_range(
1433 struct inode
*inode
,
1437 struct xfs_inode
*ip
= XFS_I(inode
);
1438 xfs_fileoff_t start_fsb
;
1439 xfs_fileoff_t end_fsb
;
1442 start_fsb
= XFS_B_TO_FSB(ip
->i_mount
, start
);
1443 end_fsb
= XFS_B_TO_FSB(ip
->i_mount
, end
);
1444 if (end_fsb
<= start_fsb
)
1447 xfs_ilock(ip
, XFS_ILOCK_EXCL
);
1448 error
= xfs_bmap_punch_delalloc_range(ip
, start_fsb
,
1449 end_fsb
- start_fsb
);
1451 /* something screwed, just bail */
1452 if (!XFS_FORCED_SHUTDOWN(ip
->i_mount
)) {
1453 xfs_alert(ip
->i_mount
,
1454 "xfs_vm_write_failed: unable to clean up ino %lld",
1458 xfs_iunlock(ip
, XFS_ILOCK_EXCL
);
1462 xfs_vm_write_failed(
1463 struct inode
*inode
,
1468 loff_t block_offset
;
1471 loff_t from
= pos
& (PAGE_CACHE_SIZE
- 1);
1472 loff_t to
= from
+ len
;
1473 struct buffer_head
*bh
, *head
;
1474 struct xfs_mount
*mp
= XFS_I(inode
)->i_mount
;
1477 * The request pos offset might be 32 or 64 bit, this is all fine
1478 * on 64-bit platform. However, for 64-bit pos request on 32-bit
1479 * platform, the high 32-bit will be masked off if we evaluate the
1480 * block_offset via (pos & PAGE_MASK) because the PAGE_MASK is
1481 * 0xfffff000 as an unsigned long, hence the result is incorrect
1482 * which could cause the following ASSERT failed in most cases.
1483 * In order to avoid this, we can evaluate the block_offset of the
1484 * start of the page by using shifts rather than masks the mismatch
1487 block_offset
= (pos
>> PAGE_CACHE_SHIFT
) << PAGE_CACHE_SHIFT
;
1489 ASSERT(block_offset
+ from
== pos
);
1491 head
= page_buffers(page
);
1493 for (bh
= head
; bh
!= head
|| !block_start
;
1494 bh
= bh
->b_this_page
, block_start
= block_end
,
1495 block_offset
+= bh
->b_size
) {
1496 block_end
= block_start
+ bh
->b_size
;
1498 /* skip buffers before the write */
1499 if (block_end
<= from
)
1502 /* if the buffer is after the write, we're done */
1503 if (block_start
>= to
)
1507 * Process delalloc and unwritten buffers beyond EOF. We can
1508 * encounter unwritten buffers in the event that a file has
1509 * post-EOF unwritten extents and an extending write happens to
1510 * fail (e.g., an unaligned write that also involves a delalloc
1511 * to the same page).
1513 if (!buffer_delay(bh
) && !buffer_unwritten(bh
))
1516 if (!xfs_mp_fail_writes(mp
) && !buffer_new(bh
) &&
1517 block_offset
< i_size_read(inode
))
1520 if (buffer_delay(bh
))
1521 xfs_vm_kill_delalloc_range(inode
, block_offset
,
1522 block_offset
+ bh
->b_size
);
1525 * This buffer does not contain data anymore. make sure anyone
1526 * who finds it knows that for certain.
1528 clear_buffer_delay(bh
);
1529 clear_buffer_uptodate(bh
);
1530 clear_buffer_mapped(bh
);
1531 clear_buffer_new(bh
);
1532 clear_buffer_dirty(bh
);
1533 clear_buffer_unwritten(bh
);
1539 * This used to call block_write_begin(), but it unlocks and releases the page
1540 * on error, and we need that page to be able to punch stale delalloc blocks out
1541 * on failure. hence we copy-n-waste it here and call xfs_vm_write_failed() at
1542 * the appropriate point.
1547 struct address_space
*mapping
,
1551 struct page
**pagep
,
1554 pgoff_t index
= pos
>> PAGE_CACHE_SHIFT
;
1557 struct xfs_mount
*mp
= XFS_I(mapping
->host
)->i_mount
;
1559 ASSERT(len
<= PAGE_CACHE_SIZE
);
1561 page
= grab_cache_page_write_begin(mapping
, index
, flags
);
1565 status
= __block_write_begin(page
, pos
, len
, xfs_get_blocks
);
1566 if (xfs_mp_fail_writes(mp
))
1568 if (unlikely(status
)) {
1569 struct inode
*inode
= mapping
->host
;
1570 size_t isize
= i_size_read(inode
);
1572 xfs_vm_write_failed(inode
, page
, pos
, len
);
1576 * If the write is beyond EOF, we only want to kill blocks
1577 * allocated in this write, not blocks that were previously
1578 * written successfully.
1580 if (xfs_mp_fail_writes(mp
))
1582 if (pos
+ len
> isize
) {
1583 ssize_t start
= max_t(ssize_t
, pos
, isize
);
1585 truncate_pagecache_range(inode
, start
, pos
+ len
);
1588 page_cache_release(page
);
1597 * On failure, we only need to kill delalloc blocks beyond EOF in the range of
1598 * this specific write because they will never be written. Previous writes
1599 * beyond EOF where block allocation succeeded do not need to be trashed, so
1600 * only new blocks from this write should be trashed. For blocks within
1601 * EOF, generic_write_end() zeros them so they are safe to leave alone and be
1602 * written with all the other valid data.
1607 struct address_space
*mapping
,
1616 ASSERT(len
<= PAGE_CACHE_SIZE
);
1618 ret
= generic_write_end(file
, mapping
, pos
, len
, copied
, page
, fsdata
);
1619 if (unlikely(ret
< len
)) {
1620 struct inode
*inode
= mapping
->host
;
1621 size_t isize
= i_size_read(inode
);
1622 loff_t to
= pos
+ len
;
1625 /* only kill blocks in this write beyond EOF */
1628 xfs_vm_kill_delalloc_range(inode
, isize
, to
);
1629 truncate_pagecache_range(inode
, isize
, to
);
1637 struct address_space
*mapping
,
1640 struct inode
*inode
= (struct inode
*)mapping
->host
;
1641 struct xfs_inode
*ip
= XFS_I(inode
);
1643 trace_xfs_vm_bmap(XFS_I(inode
));
1644 xfs_ilock(ip
, XFS_IOLOCK_SHARED
);
1645 filemap_write_and_wait(mapping
);
1646 xfs_iunlock(ip
, XFS_IOLOCK_SHARED
);
1647 return generic_block_bmap(mapping
, block
, xfs_get_blocks
);
1652 struct file
*unused
,
1655 trace_xfs_vm_readpage(page
->mapping
->host
, 1);
1656 return mpage_readpage(page
, xfs_get_blocks
);
1661 struct file
*unused
,
1662 struct address_space
*mapping
,
1663 struct list_head
*pages
,
1666 trace_xfs_vm_readpages(mapping
->host
, nr_pages
);
1667 return mpage_readpages(mapping
, pages
, nr_pages
, xfs_get_blocks
);
1671 * This is basically a copy of __set_page_dirty_buffers() with one
1672 * small tweak: buffers beyond EOF do not get marked dirty. If we mark them
1673 * dirty, we'll never be able to clean them because we don't write buffers
1674 * beyond EOF, and that means we can't invalidate pages that span EOF
1675 * that have been marked dirty. Further, the dirty state can leak into
1676 * the file interior if the file is extended, resulting in all sorts of
1677 * bad things happening as the state does not match the underlying data.
1679 * XXX: this really indicates that bufferheads in XFS need to die. Warts like
1680 * this only exist because of bufferheads and how the generic code manages them.
1683 xfs_vm_set_page_dirty(
1686 struct address_space
*mapping
= page
->mapping
;
1687 struct inode
*inode
= mapping
->host
;
1692 if (unlikely(!mapping
))
1693 return !TestSetPageDirty(page
);
1695 end_offset
= i_size_read(inode
);
1696 offset
= page_offset(page
);
1698 spin_lock(&mapping
->private_lock
);
1699 if (page_has_buffers(page
)) {
1700 struct buffer_head
*head
= page_buffers(page
);
1701 struct buffer_head
*bh
= head
;
1704 if (offset
< end_offset
)
1705 set_buffer_dirty(bh
);
1706 bh
= bh
->b_this_page
;
1707 offset
+= 1 << inode
->i_blkbits
;
1708 } while (bh
!= head
);
1711 * Lock out page->mem_cgroup migration to keep PageDirty
1712 * synchronized with per-memcg dirty page counters.
1714 lock_page_memcg(page
);
1715 newly_dirty
= !TestSetPageDirty(page
);
1716 spin_unlock(&mapping
->private_lock
);
1719 /* sigh - __set_page_dirty() is static, so copy it here, too */
1720 unsigned long flags
;
1722 spin_lock_irqsave(&mapping
->tree_lock
, flags
);
1723 if (page
->mapping
) { /* Race with truncate? */
1724 WARN_ON_ONCE(!PageUptodate(page
));
1725 account_page_dirtied(page
, mapping
);
1726 radix_tree_tag_set(&mapping
->page_tree
,
1727 page_index(page
), PAGECACHE_TAG_DIRTY
);
1729 spin_unlock_irqrestore(&mapping
->tree_lock
, flags
);
1731 unlock_page_memcg(page
);
1733 __mark_inode_dirty(mapping
->host
, I_DIRTY_PAGES
);
1737 const struct address_space_operations xfs_address_space_operations
= {
1738 .readpage
= xfs_vm_readpage
,
1739 .readpages
= xfs_vm_readpages
,
1740 .writepage
= xfs_vm_writepage
,
1741 .writepages
= xfs_vm_writepages
,
1742 .set_page_dirty
= xfs_vm_set_page_dirty
,
1743 .releasepage
= xfs_vm_releasepage
,
1744 .invalidatepage
= xfs_vm_invalidatepage
,
1745 .write_begin
= xfs_vm_write_begin
,
1746 .write_end
= xfs_vm_write_end
,
1747 .bmap
= xfs_vm_bmap
,
1748 .direct_IO
= xfs_vm_direct_IO
,
1749 .migratepage
= buffer_migrate_page
,
1750 .is_partially_uptodate
= block_is_partially_uptodate
,
1751 .error_remove_page
= generic_error_remove_page
,