2 * Copyright (C) 2010 Red Hat, Inc.
3 * Copyright (c) 2016 Christoph Hellwig.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 #include <linux/module.h>
15 #include <linux/compiler.h>
17 #include <linux/iomap.h>
18 #include <linux/uaccess.h>
19 #include <linux/gfp.h>
21 #include <linux/swap.h>
22 #include <linux/pagemap.h>
23 #include <linux/file.h>
24 #include <linux/uio.h>
25 #include <linux/backing-dev.h>
26 #include <linux/buffer_head.h>
27 #include <linux/dax.h>
30 typedef loff_t (*iomap_actor_t
)(struct inode
*inode
, loff_t pos
, loff_t len
,
31 void *data
, struct iomap
*iomap
);
34 * Execute a iomap write on a segment of the mapping that spans a
35 * contiguous range of pages that have identical block mapping state.
37 * This avoids the need to map pages individually, do individual allocations
38 * for each page and most importantly avoid the need for filesystem specific
39 * locking per page. Instead, all the operations are amortised over the entire
40 * range of pages. It is assumed that the filesystems will lock whatever
41 * resources they require in the iomap_begin call, and release them in the
45 iomap_apply(struct inode
*inode
, loff_t pos
, loff_t length
, unsigned flags
,
46 struct iomap_ops
*ops
, void *data
, iomap_actor_t actor
)
48 struct iomap iomap
= { 0 };
49 loff_t written
= 0, ret
;
52 * Need to map a range from start position for length bytes. This can
53 * span multiple pages - it is only guaranteed to return a range of a
54 * single type of pages (e.g. all into a hole, all mapped or all
55 * unwritten). Failure at this point has nothing to undo.
57 * If allocation is required for this range, reserve the space now so
58 * that the allocation is guaranteed to succeed later on. Once we copy
59 * the data into the page cache pages, then we cannot fail otherwise we
60 * expose transient stale data. If the reserve fails, we can safely
61 * back out at this point as there is nothing to undo.
63 ret
= ops
->iomap_begin(inode
, pos
, length
, flags
, &iomap
);
66 if (WARN_ON(iomap
.offset
> pos
))
70 * Cut down the length to the one actually provided by the filesystem,
71 * as it might not be able to give us the whole size that we requested.
73 if (iomap
.offset
+ iomap
.length
< pos
+ length
)
74 length
= iomap
.offset
+ iomap
.length
- pos
;
77 * Now that we have guaranteed that the space allocation will succeed.
78 * we can do the copy-in page by page without having to worry about
79 * failures exposing transient data.
81 written
= actor(inode
, pos
, length
, data
, &iomap
);
84 * Now the data has been copied, commit the range we've copied. This
85 * should not fail unless the filesystem has had a fatal error.
87 ret
= ops
->iomap_end(inode
, pos
, length
, written
> 0 ? written
: 0,
90 return written
? written
: ret
;
94 iomap_write_failed(struct inode
*inode
, loff_t pos
, unsigned len
)
96 loff_t i_size
= i_size_read(inode
);
99 * Only truncate newly allocated pages beyoned EOF, even if the
100 * write started inside the existing inode size.
102 if (pos
+ len
> i_size
)
103 truncate_pagecache_range(inode
, max(pos
, i_size
), pos
+ len
);
107 iomap_write_begin(struct inode
*inode
, loff_t pos
, unsigned len
, unsigned flags
,
108 struct page
**pagep
, struct iomap
*iomap
)
110 pgoff_t index
= pos
>> PAGE_SHIFT
;
114 BUG_ON(pos
+ len
> iomap
->offset
+ iomap
->length
);
116 page
= grab_cache_page_write_begin(inode
->i_mapping
, index
, flags
);
120 status
= __block_write_begin_int(page
, pos
, len
, NULL
, iomap
);
121 if (unlikely(status
)) {
126 iomap_write_failed(inode
, pos
, len
);
134 iomap_write_end(struct inode
*inode
, loff_t pos
, unsigned len
,
135 unsigned copied
, struct page
*page
)
139 ret
= generic_write_end(NULL
, inode
->i_mapping
, pos
, len
,
142 iomap_write_failed(inode
, pos
, len
);
147 iomap_write_actor(struct inode
*inode
, loff_t pos
, loff_t length
, void *data
,
150 struct iov_iter
*i
= data
;
153 unsigned int flags
= AOP_FLAG_NOFS
;
156 * Copies from kernel address space cannot fail (NFSD is a big user).
158 if (!iter_is_iovec(i
))
159 flags
|= AOP_FLAG_UNINTERRUPTIBLE
;
163 unsigned long offset
; /* Offset into pagecache page */
164 unsigned long bytes
; /* Bytes to write to page */
165 size_t copied
; /* Bytes copied from user */
167 offset
= (pos
& (PAGE_SIZE
- 1));
168 bytes
= min_t(unsigned long, PAGE_SIZE
- offset
,
175 * Bring in the user page that we will copy from _first_.
176 * Otherwise there's a nasty deadlock on copying from the
177 * same page as we're writing to, without it being marked
180 * Not only is this an optimisation, but it is also required
181 * to check that the address is actually valid, when atomic
182 * usercopies are used, below.
184 if (unlikely(iov_iter_fault_in_readable(i
, bytes
))) {
189 status
= iomap_write_begin(inode
, pos
, bytes
, flags
, &page
,
191 if (unlikely(status
))
194 if (mapping_writably_mapped(inode
->i_mapping
))
195 flush_dcache_page(page
);
198 copied
= iov_iter_copy_from_user_atomic(page
, i
, offset
, bytes
);
201 flush_dcache_page(page
);
203 status
= iomap_write_end(inode
, pos
, bytes
, copied
, page
);
204 if (unlikely(status
< 0))
210 iov_iter_advance(i
, copied
);
211 if (unlikely(copied
== 0)) {
213 * If we were unable to copy any data at all, we must
214 * fall back to a single segment length write.
216 * If we didn't fallback here, we could livelock
217 * because not all segments in the iov can be copied at
218 * once without a pagefault.
220 bytes
= min_t(unsigned long, PAGE_SIZE
- offset
,
221 iov_iter_single_seg_count(i
));
228 balance_dirty_pages_ratelimited(inode
->i_mapping
);
229 } while (iov_iter_count(i
) && length
);
231 return written
? written
: status
;
235 iomap_file_buffered_write(struct kiocb
*iocb
, struct iov_iter
*iter
,
236 struct iomap_ops
*ops
)
238 struct inode
*inode
= iocb
->ki_filp
->f_mapping
->host
;
239 loff_t pos
= iocb
->ki_pos
, ret
= 0, written
= 0;
241 while (iov_iter_count(iter
)) {
242 ret
= iomap_apply(inode
, pos
, iov_iter_count(iter
),
243 IOMAP_WRITE
, ops
, iter
, iomap_write_actor
);
250 return written
? written
: ret
;
252 EXPORT_SYMBOL_GPL(iomap_file_buffered_write
);
254 static int iomap_zero(struct inode
*inode
, loff_t pos
, unsigned offset
,
255 unsigned bytes
, struct iomap
*iomap
)
260 status
= iomap_write_begin(inode
, pos
, bytes
,
261 AOP_FLAG_UNINTERRUPTIBLE
| AOP_FLAG_NOFS
, &page
, iomap
);
265 zero_user(page
, offset
, bytes
);
266 mark_page_accessed(page
);
268 return iomap_write_end(inode
, pos
, bytes
, bytes
, page
);
271 static int iomap_dax_zero(loff_t pos
, unsigned offset
, unsigned bytes
,
274 sector_t sector
= iomap
->blkno
+
275 (((pos
& ~(PAGE_SIZE
- 1)) - iomap
->offset
) >> 9);
277 return __dax_zero_page_range(iomap
->bdev
, sector
, offset
, bytes
);
281 iomap_zero_range_actor(struct inode
*inode
, loff_t pos
, loff_t count
,
282 void *data
, struct iomap
*iomap
)
284 bool *did_zero
= data
;
288 /* already zeroed? we're done. */
289 if (iomap
->type
== IOMAP_HOLE
|| iomap
->type
== IOMAP_UNWRITTEN
)
293 unsigned offset
, bytes
;
295 offset
= pos
& (PAGE_SIZE
- 1); /* Within page */
296 bytes
= min_t(unsigned, PAGE_SIZE
- offset
, count
);
299 status
= iomap_dax_zero(pos
, offset
, bytes
, iomap
);
301 status
= iomap_zero(inode
, pos
, offset
, bytes
, iomap
);
316 iomap_zero_range(struct inode
*inode
, loff_t pos
, loff_t len
, bool *did_zero
,
317 struct iomap_ops
*ops
)
322 ret
= iomap_apply(inode
, pos
, len
, IOMAP_ZERO
,
323 ops
, did_zero
, iomap_zero_range_actor
);
333 EXPORT_SYMBOL_GPL(iomap_zero_range
);
336 iomap_truncate_page(struct inode
*inode
, loff_t pos
, bool *did_zero
,
337 struct iomap_ops
*ops
)
339 unsigned blocksize
= (1 << inode
->i_blkbits
);
340 unsigned off
= pos
& (blocksize
- 1);
342 /* Block boundary? Nothing to do */
345 return iomap_zero_range(inode
, pos
, blocksize
- off
, did_zero
, ops
);
347 EXPORT_SYMBOL_GPL(iomap_truncate_page
);
350 iomap_page_mkwrite_actor(struct inode
*inode
, loff_t pos
, loff_t length
,
351 void *data
, struct iomap
*iomap
)
353 struct page
*page
= data
;
356 ret
= __block_write_begin_int(page
, pos
& ~PAGE_MASK
, length
,
361 block_commit_write(page
, 0, length
);
365 int iomap_page_mkwrite(struct vm_area_struct
*vma
, struct vm_fault
*vmf
,
366 struct iomap_ops
*ops
)
368 struct page
*page
= vmf
->page
;
369 struct inode
*inode
= file_inode(vma
->vm_file
);
370 unsigned long length
;
375 size
= i_size_read(inode
);
376 if ((page
->mapping
!= inode
->i_mapping
) ||
377 (page_offset(page
) > size
)) {
378 /* We overload EFAULT to mean page got truncated */
383 /* page is wholly or partially inside EOF */
384 if (((page
->index
+ 1) << PAGE_SHIFT
) > size
)
385 length
= size
& ~PAGE_MASK
;
389 offset
= page_offset(page
);
391 ret
= iomap_apply(inode
, offset
, length
, IOMAP_WRITE
,
392 ops
, page
, iomap_page_mkwrite_actor
);
393 if (unlikely(ret
<= 0))
399 set_page_dirty(page
);
400 wait_for_stable_page(page
);
406 EXPORT_SYMBOL_GPL(iomap_page_mkwrite
);
409 struct fiemap_extent_info
*fi
;
413 static int iomap_to_fiemap(struct fiemap_extent_info
*fi
,
414 struct iomap
*iomap
, u32 flags
)
416 switch (iomap
->type
) {
421 flags
|= FIEMAP_EXTENT_DELALLOC
| FIEMAP_EXTENT_UNKNOWN
;
423 case IOMAP_UNWRITTEN
:
424 flags
|= FIEMAP_EXTENT_UNWRITTEN
;
430 return fiemap_fill_next_extent(fi
, iomap
->offset
,
431 iomap
->blkno
!= IOMAP_NULL_BLOCK
? iomap
->blkno
<< 9: 0,
432 iomap
->length
, flags
| FIEMAP_EXTENT_MERGED
);
437 iomap_fiemap_actor(struct inode
*inode
, loff_t pos
, loff_t length
, void *data
,
440 struct fiemap_ctx
*ctx
= data
;
443 if (iomap
->type
== IOMAP_HOLE
)
446 ret
= iomap_to_fiemap(ctx
->fi
, &ctx
->prev
, 0);
449 case 0: /* success */
451 case 1: /* extent array full */
458 int iomap_fiemap(struct inode
*inode
, struct fiemap_extent_info
*fi
,
459 loff_t start
, loff_t len
, struct iomap_ops
*ops
)
461 struct fiemap_ctx ctx
;
464 memset(&ctx
, 0, sizeof(ctx
));
466 ctx
.prev
.type
= IOMAP_HOLE
;
468 ret
= fiemap_check_flags(fi
, FIEMAP_FLAG_SYNC
);
472 ret
= filemap_write_and_wait(inode
->i_mapping
);
477 ret
= iomap_apply(inode
, start
, len
, 0, ops
, &ctx
,
488 if (ctx
.prev
.type
!= IOMAP_HOLE
) {
489 ret
= iomap_to_fiemap(fi
, &ctx
.prev
, FIEMAP_EXTENT_LAST
);
496 EXPORT_SYMBOL_GPL(iomap_fiemap
);