2 * Copyright (C) 2010 Red Hat, Inc.
3 * Copyright (c) 2016 Christoph Hellwig.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #include <linux/module.h>
15 #include <linux/compiler.h>
17 #include <linux/iomap.h>
18 #include <linux/uaccess.h>
19 #include <linux/gfp.h>
21 #include <linux/swap.h>
22 #include <linux/pagemap.h>
23 #include <linux/file.h>
24 #include <linux/uio.h>
25 #include <linux/backing-dev.h>
26 #include <linux/buffer_head.h>
27 #include <linux/task_io_accounting_ops.h>
28 #include <linux/dax.h>
32 * Execute a iomap write on a segment of the mapping that spans a
33 * contiguous range of pages that have identical block mapping state.
35 * This avoids the need to map pages individually, do individual allocations
36 * for each page and most importantly avoid the need for filesystem specific
37 * locking per page. Instead, all the operations are amortised over the entire
38 * range of pages. It is assumed that the filesystems will lock whatever
39 * resources they require in the iomap_begin call, and release them in the
43 iomap_apply(struct inode
*inode
, loff_t pos
, loff_t length
, unsigned flags
,
44 const struct iomap_ops
*ops
, void *data
, iomap_actor_t actor
)
46 struct iomap iomap
= { 0 };
47 loff_t written
= 0, ret
;
50 * Need to map a range from start position for length bytes. This can
51 * span multiple pages - it is only guaranteed to return a range of a
52 * single type of pages (e.g. all into a hole, all mapped or all
53 * unwritten). Failure at this point has nothing to undo.
55 * If allocation is required for this range, reserve the space now so
56 * that the allocation is guaranteed to succeed later on. Once we copy
57 * the data into the page cache pages, then we cannot fail otherwise we
58 * expose transient stale data. If the reserve fails, we can safely
59 * back out at this point as there is nothing to undo.
61 ret
= ops
->iomap_begin(inode
, pos
, length
, flags
, &iomap
);
64 if (WARN_ON(iomap
.offset
> pos
))
68 * Cut down the length to the one actually provided by the filesystem,
69 * as it might not be able to give us the whole size that we requested.
71 if (iomap
.offset
+ iomap
.length
< pos
+ length
)
72 length
= iomap
.offset
+ iomap
.length
- pos
;
75 * Now that we have guaranteed that the space allocation will succeed.
76 * we can do the copy-in page by page without having to worry about
77 * failures exposing transient data.
79 written
= actor(inode
, pos
, length
, data
, &iomap
);
82 * Now the data has been copied, commit the range we've copied. This
83 * should not fail unless the filesystem has had a fatal error.
86 ret
= ops
->iomap_end(inode
, pos
, length
,
87 written
> 0 ? written
: 0,
91 return written
? written
: ret
;
95 iomap_write_failed(struct inode
*inode
, loff_t pos
, unsigned len
)
97 loff_t i_size
= i_size_read(inode
);
100 * Only truncate newly allocated pages beyoned EOF, even if the
101 * write started inside the existing inode size.
103 if (pos
+ len
> i_size
)
104 truncate_pagecache_range(inode
, max(pos
, i_size
), pos
+ len
);
108 iomap_write_begin(struct inode
*inode
, loff_t pos
, unsigned len
, unsigned flags
,
109 struct page
**pagep
, struct iomap
*iomap
)
111 pgoff_t index
= pos
>> PAGE_SHIFT
;
115 BUG_ON(pos
+ len
> iomap
->offset
+ iomap
->length
);
117 if (fatal_signal_pending(current
))
120 page
= grab_cache_page_write_begin(inode
->i_mapping
, index
, flags
);
124 status
= __block_write_begin_int(page
, pos
, len
, NULL
, iomap
);
125 if (unlikely(status
)) {
130 iomap_write_failed(inode
, pos
, len
);
138 iomap_write_end(struct inode
*inode
, loff_t pos
, unsigned len
,
139 unsigned copied
, struct page
*page
)
143 ret
= generic_write_end(NULL
, inode
->i_mapping
, pos
, len
,
146 iomap_write_failed(inode
, pos
, len
);
151 iomap_write_actor(struct inode
*inode
, loff_t pos
, loff_t length
, void *data
,
154 struct iov_iter
*i
= data
;
157 unsigned int flags
= AOP_FLAG_NOFS
;
160 * Copies from kernel address space cannot fail (NFSD is a big user).
162 if (!iter_is_iovec(i
))
163 flags
|= AOP_FLAG_UNINTERRUPTIBLE
;
167 unsigned long offset
; /* Offset into pagecache page */
168 unsigned long bytes
; /* Bytes to write to page */
169 size_t copied
; /* Bytes copied from user */
171 offset
= (pos
& (PAGE_SIZE
- 1));
172 bytes
= min_t(unsigned long, PAGE_SIZE
- offset
,
179 * Bring in the user page that we will copy from _first_.
180 * Otherwise there's a nasty deadlock on copying from the
181 * same page as we're writing to, without it being marked
184 * Not only is this an optimisation, but it is also required
185 * to check that the address is actually valid, when atomic
186 * usercopies are used, below.
188 if (unlikely(iov_iter_fault_in_readable(i
, bytes
))) {
193 status
= iomap_write_begin(inode
, pos
, bytes
, flags
, &page
,
195 if (unlikely(status
))
198 if (mapping_writably_mapped(inode
->i_mapping
))
199 flush_dcache_page(page
);
201 copied
= iov_iter_copy_from_user_atomic(page
, i
, offset
, bytes
);
203 flush_dcache_page(page
);
205 status
= iomap_write_end(inode
, pos
, bytes
, copied
, page
);
206 if (unlikely(status
< 0))
212 iov_iter_advance(i
, copied
);
213 if (unlikely(copied
== 0)) {
215 * If we were unable to copy any data at all, we must
216 * fall back to a single segment length write.
218 * If we didn't fallback here, we could livelock
219 * because not all segments in the iov can be copied at
220 * once without a pagefault.
222 bytes
= min_t(unsigned long, PAGE_SIZE
- offset
,
223 iov_iter_single_seg_count(i
));
230 balance_dirty_pages_ratelimited(inode
->i_mapping
);
231 } while (iov_iter_count(i
) && length
);
233 return written
? written
: status
;
237 iomap_file_buffered_write(struct kiocb
*iocb
, struct iov_iter
*iter
,
238 const struct iomap_ops
*ops
)
240 struct inode
*inode
= iocb
->ki_filp
->f_mapping
->host
;
241 loff_t pos
= iocb
->ki_pos
, ret
= 0, written
= 0;
243 while (iov_iter_count(iter
)) {
244 ret
= iomap_apply(inode
, pos
, iov_iter_count(iter
),
245 IOMAP_WRITE
, ops
, iter
, iomap_write_actor
);
252 return written
? written
: ret
;
254 EXPORT_SYMBOL_GPL(iomap_file_buffered_write
);
257 __iomap_read_page(struct inode
*inode
, loff_t offset
)
259 struct address_space
*mapping
= inode
->i_mapping
;
262 page
= read_mapping_page(mapping
, offset
>> PAGE_SHIFT
, NULL
);
265 if (!PageUptodate(page
)) {
267 return ERR_PTR(-EIO
);
273 iomap_dirty_actor(struct inode
*inode
, loff_t pos
, loff_t length
, void *data
,
280 struct page
*page
, *rpage
;
281 unsigned long offset
; /* Offset into pagecache page */
282 unsigned long bytes
; /* Bytes to write to page */
284 offset
= (pos
& (PAGE_SIZE
- 1));
285 bytes
= min_t(unsigned long, PAGE_SIZE
- offset
, length
);
287 rpage
= __iomap_read_page(inode
, pos
);
289 return PTR_ERR(rpage
);
291 status
= iomap_write_begin(inode
, pos
, bytes
,
292 AOP_FLAG_NOFS
| AOP_FLAG_UNINTERRUPTIBLE
,
295 if (unlikely(status
))
298 WARN_ON_ONCE(!PageUptodate(page
));
300 status
= iomap_write_end(inode
, pos
, bytes
, bytes
, page
);
301 if (unlikely(status
<= 0)) {
302 if (WARN_ON_ONCE(status
== 0))
313 balance_dirty_pages_ratelimited(inode
->i_mapping
);
320 iomap_file_dirty(struct inode
*inode
, loff_t pos
, loff_t len
,
321 const struct iomap_ops
*ops
)
326 ret
= iomap_apply(inode
, pos
, len
, IOMAP_WRITE
, ops
, NULL
,
336 EXPORT_SYMBOL_GPL(iomap_file_dirty
);
338 static int iomap_zero(struct inode
*inode
, loff_t pos
, unsigned offset
,
339 unsigned bytes
, struct iomap
*iomap
)
344 status
= iomap_write_begin(inode
, pos
, bytes
,
345 AOP_FLAG_UNINTERRUPTIBLE
| AOP_FLAG_NOFS
, &page
, iomap
);
349 zero_user(page
, offset
, bytes
);
350 mark_page_accessed(page
);
352 return iomap_write_end(inode
, pos
, bytes
, bytes
, page
);
355 static int iomap_dax_zero(loff_t pos
, unsigned offset
, unsigned bytes
,
358 sector_t sector
= iomap
->blkno
+
359 (((pos
& ~(PAGE_SIZE
- 1)) - iomap
->offset
) >> 9);
361 return __dax_zero_page_range(iomap
->bdev
, sector
, offset
, bytes
);
365 iomap_zero_range_actor(struct inode
*inode
, loff_t pos
, loff_t count
,
366 void *data
, struct iomap
*iomap
)
368 bool *did_zero
= data
;
372 /* already zeroed? we're done. */
373 if (iomap
->type
== IOMAP_HOLE
|| iomap
->type
== IOMAP_UNWRITTEN
)
377 unsigned offset
, bytes
;
379 offset
= pos
& (PAGE_SIZE
- 1); /* Within page */
380 bytes
= min_t(unsigned, PAGE_SIZE
- offset
, count
);
383 status
= iomap_dax_zero(pos
, offset
, bytes
, iomap
);
385 status
= iomap_zero(inode
, pos
, offset
, bytes
, iomap
);
400 iomap_zero_range(struct inode
*inode
, loff_t pos
, loff_t len
, bool *did_zero
,
401 const struct iomap_ops
*ops
)
406 ret
= iomap_apply(inode
, pos
, len
, IOMAP_ZERO
,
407 ops
, did_zero
, iomap_zero_range_actor
);
417 EXPORT_SYMBOL_GPL(iomap_zero_range
);
420 iomap_truncate_page(struct inode
*inode
, loff_t pos
, bool *did_zero
,
421 const struct iomap_ops
*ops
)
423 unsigned blocksize
= (1 << inode
->i_blkbits
);
424 unsigned off
= pos
& (blocksize
- 1);
426 /* Block boundary? Nothing to do */
429 return iomap_zero_range(inode
, pos
, blocksize
- off
, did_zero
, ops
);
431 EXPORT_SYMBOL_GPL(iomap_truncate_page
);
434 iomap_page_mkwrite_actor(struct inode
*inode
, loff_t pos
, loff_t length
,
435 void *data
, struct iomap
*iomap
)
437 struct page
*page
= data
;
440 ret
= __block_write_begin_int(page
, pos
, length
, NULL
, iomap
);
444 block_commit_write(page
, 0, length
);
448 int iomap_page_mkwrite(struct vm_area_struct
*vma
, struct vm_fault
*vmf
,
449 const struct iomap_ops
*ops
)
451 struct page
*page
= vmf
->page
;
452 struct inode
*inode
= file_inode(vma
->vm_file
);
453 unsigned long length
;
458 size
= i_size_read(inode
);
459 if ((page
->mapping
!= inode
->i_mapping
) ||
460 (page_offset(page
) > size
)) {
461 /* We overload EFAULT to mean page got truncated */
466 /* page is wholly or partially inside EOF */
467 if (((page
->index
+ 1) << PAGE_SHIFT
) > size
)
468 length
= size
& ~PAGE_MASK
;
472 offset
= page_offset(page
);
474 ret
= iomap_apply(inode
, offset
, length
,
475 IOMAP_WRITE
| IOMAP_FAULT
, ops
, page
,
476 iomap_page_mkwrite_actor
);
477 if (unlikely(ret
<= 0))
483 set_page_dirty(page
);
484 wait_for_stable_page(page
);
490 EXPORT_SYMBOL_GPL(iomap_page_mkwrite
);
493 struct fiemap_extent_info
*fi
;
497 static int iomap_to_fiemap(struct fiemap_extent_info
*fi
,
498 struct iomap
*iomap
, u32 flags
)
500 switch (iomap
->type
) {
505 flags
|= FIEMAP_EXTENT_DELALLOC
| FIEMAP_EXTENT_UNKNOWN
;
507 case IOMAP_UNWRITTEN
:
508 flags
|= FIEMAP_EXTENT_UNWRITTEN
;
514 if (iomap
->flags
& IOMAP_F_MERGED
)
515 flags
|= FIEMAP_EXTENT_MERGED
;
516 if (iomap
->flags
& IOMAP_F_SHARED
)
517 flags
|= FIEMAP_EXTENT_SHARED
;
519 return fiemap_fill_next_extent(fi
, iomap
->offset
,
520 iomap
->blkno
!= IOMAP_NULL_BLOCK
? iomap
->blkno
<< 9: 0,
521 iomap
->length
, flags
);
526 iomap_fiemap_actor(struct inode
*inode
, loff_t pos
, loff_t length
, void *data
,
529 struct fiemap_ctx
*ctx
= data
;
532 if (iomap
->type
== IOMAP_HOLE
)
535 ret
= iomap_to_fiemap(ctx
->fi
, &ctx
->prev
, 0);
538 case 0: /* success */
540 case 1: /* extent array full */
547 int iomap_fiemap(struct inode
*inode
, struct fiemap_extent_info
*fi
,
548 loff_t start
, loff_t len
, const struct iomap_ops
*ops
)
550 struct fiemap_ctx ctx
;
553 memset(&ctx
, 0, sizeof(ctx
));
555 ctx
.prev
.type
= IOMAP_HOLE
;
557 ret
= fiemap_check_flags(fi
, FIEMAP_FLAG_SYNC
);
561 if (fi
->fi_flags
& FIEMAP_FLAG_SYNC
) {
562 ret
= filemap_write_and_wait(inode
->i_mapping
);
568 ret
= iomap_apply(inode
, start
, len
, IOMAP_REPORT
, ops
, &ctx
,
570 /* inode with no (attribute) mapping will give ENOENT */
582 if (ctx
.prev
.type
!= IOMAP_HOLE
) {
583 ret
= iomap_to_fiemap(fi
, &ctx
.prev
, FIEMAP_EXTENT_LAST
);
590 EXPORT_SYMBOL_GPL(iomap_fiemap
);
593 * Private flags for iomap_dio, must not overlap with the public ones in
596 #define IOMAP_DIO_WRITE (1 << 30)
597 #define IOMAP_DIO_DIRTY (1 << 31)
601 iomap_dio_end_io_t
*end_io
;
609 /* used during submission and for synchronous completion: */
611 struct iov_iter
*iter
;
612 struct task_struct
*waiter
;
613 struct request_queue
*last_queue
;
617 /* used for aio completion: */
619 struct work_struct work
;
624 static ssize_t
iomap_dio_complete(struct iomap_dio
*dio
)
626 struct kiocb
*iocb
= dio
->iocb
;
630 ret
= dio
->end_io(iocb
,
631 dio
->error
? dio
->error
: dio
->size
,
639 /* check for short read */
640 if (iocb
->ki_pos
+ ret
> dio
->i_size
&&
641 !(dio
->flags
& IOMAP_DIO_WRITE
))
642 ret
= dio
->i_size
- iocb
->ki_pos
;
646 inode_dio_end(file_inode(iocb
->ki_filp
));
652 static void iomap_dio_complete_work(struct work_struct
*work
)
654 struct iomap_dio
*dio
= container_of(work
, struct iomap_dio
, aio
.work
);
655 struct kiocb
*iocb
= dio
->iocb
;
656 bool is_write
= (dio
->flags
& IOMAP_DIO_WRITE
);
659 ret
= iomap_dio_complete(dio
);
660 if (is_write
&& ret
> 0)
661 ret
= generic_write_sync(iocb
, ret
);
662 iocb
->ki_complete(iocb
, ret
, 0);
666 * Set an error in the dio if none is set yet. We have to use cmpxchg
667 * as the submission context and the completion context(s) can race to
670 static inline void iomap_dio_set_error(struct iomap_dio
*dio
, int ret
)
672 cmpxchg(&dio
->error
, 0, ret
);
675 static void iomap_dio_bio_end_io(struct bio
*bio
)
677 struct iomap_dio
*dio
= bio
->bi_private
;
678 bool should_dirty
= (dio
->flags
& IOMAP_DIO_DIRTY
);
681 iomap_dio_set_error(dio
, bio
->bi_error
);
683 if (atomic_dec_and_test(&dio
->ref
)) {
684 if (is_sync_kiocb(dio
->iocb
)) {
685 struct task_struct
*waiter
= dio
->submit
.waiter
;
687 WRITE_ONCE(dio
->submit
.waiter
, NULL
);
688 wake_up_process(waiter
);
689 } else if (dio
->flags
& IOMAP_DIO_WRITE
) {
690 struct inode
*inode
= file_inode(dio
->iocb
->ki_filp
);
692 INIT_WORK(&dio
->aio
.work
, iomap_dio_complete_work
);
693 queue_work(inode
->i_sb
->s_dio_done_wq
, &dio
->aio
.work
);
695 iomap_dio_complete_work(&dio
->aio
.work
);
700 bio_check_pages_dirty(bio
);
702 struct bio_vec
*bvec
;
705 bio_for_each_segment_all(bvec
, bio
, i
)
706 put_page(bvec
->bv_page
);
712 iomap_dio_zero(struct iomap_dio
*dio
, struct iomap
*iomap
, loff_t pos
,
715 struct page
*page
= ZERO_PAGE(0);
718 bio
= bio_alloc(GFP_KERNEL
, 1);
719 bio
->bi_bdev
= iomap
->bdev
;
720 bio
->bi_iter
.bi_sector
=
721 iomap
->blkno
+ ((pos
- iomap
->offset
) >> 9);
722 bio
->bi_private
= dio
;
723 bio
->bi_end_io
= iomap_dio_bio_end_io
;
726 if (bio_add_page(bio
, page
, len
, 0) != len
)
728 bio_set_op_attrs(bio
, REQ_OP_WRITE
, REQ_SYNC
| REQ_IDLE
);
730 atomic_inc(&dio
->ref
);
731 return submit_bio(bio
);
735 iomap_dio_actor(struct inode
*inode
, loff_t pos
, loff_t length
,
736 void *data
, struct iomap
*iomap
)
738 struct iomap_dio
*dio
= data
;
739 unsigned blkbits
= blksize_bits(bdev_logical_block_size(iomap
->bdev
));
740 unsigned fs_block_size
= (1 << inode
->i_blkbits
), pad
;
741 unsigned align
= iov_iter_alignment(dio
->submit
.iter
);
742 struct iov_iter iter
;
744 bool need_zeroout
= false;
747 if ((pos
| length
| align
) & ((1 << blkbits
) - 1))
750 switch (iomap
->type
) {
752 if (WARN_ON_ONCE(dio
->flags
& IOMAP_DIO_WRITE
))
755 case IOMAP_UNWRITTEN
:
756 if (!(dio
->flags
& IOMAP_DIO_WRITE
)) {
757 iov_iter_zero(length
, dio
->submit
.iter
);
761 dio
->flags
|= IOMAP_DIO_UNWRITTEN
;
765 if (iomap
->flags
& IOMAP_F_SHARED
)
766 dio
->flags
|= IOMAP_DIO_COW
;
767 if (iomap
->flags
& IOMAP_F_NEW
)
776 * Operate on a partial iter trimmed to the extent we were called for.
777 * We'll update the iter in the dio once we're done with this extent.
779 iter
= *dio
->submit
.iter
;
780 iov_iter_truncate(&iter
, length
);
782 nr_pages
= iov_iter_npages(&iter
, BIO_MAX_PAGES
);
787 /* zero out from the start of the block to the write offset */
788 pad
= pos
& (fs_block_size
- 1);
790 iomap_dio_zero(dio
, iomap
, pos
- pad
, pad
);
797 bio
= bio_alloc(GFP_KERNEL
, nr_pages
);
798 bio
->bi_bdev
= iomap
->bdev
;
799 bio
->bi_iter
.bi_sector
=
800 iomap
->blkno
+ ((pos
- iomap
->offset
) >> 9);
801 bio
->bi_private
= dio
;
802 bio
->bi_end_io
= iomap_dio_bio_end_io
;
804 ret
= bio_iov_iter_get_pages(bio
, &iter
);
810 if (dio
->flags
& IOMAP_DIO_WRITE
) {
811 bio_set_op_attrs(bio
, REQ_OP_WRITE
, REQ_SYNC
| REQ_IDLE
);
812 task_io_account_write(bio
->bi_iter
.bi_size
);
814 bio_set_op_attrs(bio
, REQ_OP_READ
, 0);
815 if (dio
->flags
& IOMAP_DIO_DIRTY
)
816 bio_set_pages_dirty(bio
);
819 dio
->size
+= bio
->bi_iter
.bi_size
;
820 pos
+= bio
->bi_iter
.bi_size
;
822 nr_pages
= iov_iter_npages(&iter
, BIO_MAX_PAGES
);
824 atomic_inc(&dio
->ref
);
826 dio
->submit
.last_queue
= bdev_get_queue(iomap
->bdev
);
827 dio
->submit
.cookie
= submit_bio(bio
);
831 /* zero out from the end of the write to the end of the block */
832 pad
= pos
& (fs_block_size
- 1);
834 iomap_dio_zero(dio
, iomap
, pos
, fs_block_size
- pad
);
837 iov_iter_advance(dio
->submit
.iter
, length
);
842 iomap_dio_rw(struct kiocb
*iocb
, struct iov_iter
*iter
,
843 const struct iomap_ops
*ops
, iomap_dio_end_io_t end_io
)
845 struct address_space
*mapping
= iocb
->ki_filp
->f_mapping
;
846 struct inode
*inode
= file_inode(iocb
->ki_filp
);
847 size_t count
= iov_iter_count(iter
);
848 loff_t pos
= iocb
->ki_pos
, end
= iocb
->ki_pos
+ count
- 1, ret
= 0;
849 unsigned int flags
= IOMAP_DIRECT
;
850 struct blk_plug plug
;
851 struct iomap_dio
*dio
;
853 lockdep_assert_held(&inode
->i_rwsem
);
858 dio
= kmalloc(sizeof(*dio
), GFP_KERNEL
);
863 atomic_set(&dio
->ref
, 1);
865 dio
->i_size
= i_size_read(inode
);
866 dio
->end_io
= end_io
;
870 dio
->submit
.iter
= iter
;
871 if (is_sync_kiocb(iocb
)) {
872 dio
->submit
.waiter
= current
;
873 dio
->submit
.cookie
= BLK_QC_T_NONE
;
874 dio
->submit
.last_queue
= NULL
;
877 if (iov_iter_rw(iter
) == READ
) {
878 if (pos
>= dio
->i_size
)
881 if (iter
->type
== ITER_IOVEC
)
882 dio
->flags
|= IOMAP_DIO_DIRTY
;
884 dio
->flags
|= IOMAP_DIO_WRITE
;
885 flags
|= IOMAP_WRITE
;
888 if (mapping
->nrpages
) {
889 ret
= filemap_write_and_wait_range(mapping
, iocb
->ki_pos
, end
);
893 ret
= invalidate_inode_pages2_range(mapping
,
894 iocb
->ki_pos
>> PAGE_SHIFT
, end
>> PAGE_SHIFT
);
899 inode_dio_begin(inode
);
901 blk_start_plug(&plug
);
903 ret
= iomap_apply(inode
, pos
, count
, flags
, ops
, dio
,
906 /* magic error code to fall back to buffered I/O */
912 } while ((count
= iov_iter_count(iter
)) > 0);
913 blk_finish_plug(&plug
);
916 iomap_dio_set_error(dio
, ret
);
918 if (ret
>= 0 && iov_iter_rw(iter
) == WRITE
&& !is_sync_kiocb(iocb
) &&
919 !inode
->i_sb
->s_dio_done_wq
) {
920 ret
= sb_init_dio_done_wq(inode
->i_sb
);
922 iomap_dio_set_error(dio
, ret
);
925 if (!atomic_dec_and_test(&dio
->ref
)) {
926 if (!is_sync_kiocb(iocb
))
930 set_current_state(TASK_UNINTERRUPTIBLE
);
931 if (!READ_ONCE(dio
->submit
.waiter
))
934 if (!(iocb
->ki_flags
& IOCB_HIPRI
) ||
935 !dio
->submit
.last_queue
||
936 !blk_mq_poll(dio
->submit
.last_queue
,
940 __set_current_state(TASK_RUNNING
);
944 * Try again to invalidate clean pages which might have been cached by
945 * non-direct readahead, or faulted in by get_user_pages() if the source
946 * of the write was an mmap'ed region of the file we're writing. Either
947 * one is a pretty crazy thing to do, so we don't support it 100%. If
948 * this invalidation fails, tough, the write still worked...
950 if (iov_iter_rw(iter
) == WRITE
&& mapping
->nrpages
) {
951 ret
= invalidate_inode_pages2_range(mapping
,
952 iocb
->ki_pos
>> PAGE_SHIFT
, end
>> PAGE_SHIFT
);
956 return iomap_dio_complete(dio
);
962 EXPORT_SYMBOL_GPL(iomap_dio_rw
);