1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1994-1999 Linus Torvalds
9 * This file handles the generic file mmap semantics used by
10 * most "normal" filesystems (but you don't /have/ to use this:
11 * the NFS filesystem used to do this differently, for example)
13 #include <linux/export.h>
14 #include <linux/compiler.h>
15 #include <linux/dax.h>
17 #include <linux/sched/signal.h>
18 #include <linux/uaccess.h>
19 #include <linux/capability.h>
20 #include <linux/kernel_stat.h>
21 #include <linux/gfp.h>
23 #include <linux/swap.h>
24 #include <linux/mman.h>
25 #include <linux/pagemap.h>
26 #include <linux/file.h>
27 #include <linux/uio.h>
28 #include <linux/error-injection.h>
29 #include <linux/hash.h>
30 #include <linux/writeback.h>
31 #include <linux/backing-dev.h>
32 #include <linux/pagevec.h>
33 #include <linux/blkdev.h>
34 #include <linux/security.h>
35 #include <linux/cpuset.h>
36 #include <linux/hugetlb.h>
37 #include <linux/memcontrol.h>
38 #include <linux/cleancache.h>
39 #include <linux/shmem_fs.h>
40 #include <linux/rmap.h>
41 #include <linux/delayacct.h>
42 #include <linux/psi.h>
43 #include <linux/ramfs.h>
44 #include <linux/page_idle.h>
45 #include <asm/pgalloc.h>
46 #include <asm/tlbflush.h>
49 #define CREATE_TRACE_POINTS
50 #include <trace/events/filemap.h>
53 * FIXME: remove all knowledge of the buffer layer from the core VM
55 #include <linux/buffer_head.h> /* for try_to_free_buffers */
60 * Shared mappings implemented 30.11.1994. It's not fully working yet,
63 * Shared mappings now work. 15.8.1995 Bruno.
65 * finished 'unifying' the page and buffer cache and SMP-threaded the
66 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
68 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
74 * ->i_mmap_rwsem (truncate_pagecache)
75 * ->private_lock (__free_pte->__set_page_dirty_buffers)
76 * ->swap_lock (exclusive_swap_page, others)
80 * ->invalidate_lock (acquired by fs in truncate path)
81 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
85 * ->page_table_lock or pte_lock (various, mainly in memory.c)
86 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
89 * ->invalidate_lock (filemap_fault)
90 * ->lock_page (filemap_fault, access_process_vm)
92 * ->i_rwsem (generic_perform_write)
93 * ->mmap_lock (fault_in_pages_readable->do_page_fault)
96 * sb_lock (fs/fs-writeback.c)
97 * ->i_pages lock (__sync_single_inode)
100 * ->anon_vma.lock (vma_adjust)
103 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
105 * ->page_table_lock or pte_lock
106 * ->swap_lock (try_to_unmap_one)
107 * ->private_lock (try_to_unmap_one)
108 * ->i_pages lock (try_to_unmap_one)
109 * ->lruvec->lru_lock (follow_page->mark_page_accessed)
110 * ->lruvec->lru_lock (check_pte_range->isolate_lru_page)
111 * ->private_lock (page_remove_rmap->set_page_dirty)
112 * ->i_pages lock (page_remove_rmap->set_page_dirty)
113 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
114 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
115 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
116 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
117 * ->inode->i_lock (zap_pte_range->set_page_dirty)
118 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
121 * ->tasklist_lock (memory_failure, collect_procs_ao)
124 static void page_cache_delete(struct address_space
*mapping
,
125 struct page
*page
, void *shadow
)
127 XA_STATE(xas
, &mapping
->i_pages
, page
->index
);
130 mapping_set_update(&xas
, mapping
);
132 /* hugetlb pages are represented by a single entry in the xarray */
133 if (!PageHuge(page
)) {
134 xas_set_order(&xas
, page
->index
, compound_order(page
));
135 nr
= compound_nr(page
);
138 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
139 VM_BUG_ON_PAGE(PageTail(page
), page
);
140 VM_BUG_ON_PAGE(nr
!= 1 && shadow
, page
);
142 xas_store(&xas
, shadow
);
143 xas_init_marks(&xas
);
145 page
->mapping
= NULL
;
146 /* Leave page->index set: truncation lookup relies upon it */
147 mapping
->nrpages
-= nr
;
150 static void unaccount_page_cache_page(struct address_space
*mapping
,
156 * if we're uptodate, flush out into the cleancache, otherwise
157 * invalidate any existing cleancache entries. We can't leave
158 * stale data around in the cleancache once our page is gone
160 if (PageUptodate(page
) && PageMappedToDisk(page
))
161 cleancache_put_page(page
);
163 cleancache_invalidate_page(mapping
, page
);
165 VM_BUG_ON_PAGE(PageTail(page
), page
);
166 VM_BUG_ON_PAGE(page_mapped(page
), page
);
167 if (!IS_ENABLED(CONFIG_DEBUG_VM
) && unlikely(page_mapped(page
))) {
170 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
171 current
->comm
, page_to_pfn(page
));
172 dump_page(page
, "still mapped when deleted");
174 add_taint(TAINT_BAD_PAGE
, LOCKDEP_NOW_UNRELIABLE
);
176 mapcount
= page_mapcount(page
);
177 if (mapping_exiting(mapping
) &&
178 page_count(page
) >= mapcount
+ 2) {
180 * All vmas have already been torn down, so it's
181 * a good bet that actually the page is unmapped,
182 * and we'd prefer not to leak it: if we're wrong,
183 * some other bad page check should catch it later.
185 page_mapcount_reset(page
);
186 page_ref_sub(page
, mapcount
);
190 /* hugetlb pages do not participate in page cache accounting. */
194 nr
= thp_nr_pages(page
);
196 __mod_lruvec_page_state(page
, NR_FILE_PAGES
, -nr
);
197 if (PageSwapBacked(page
)) {
198 __mod_lruvec_page_state(page
, NR_SHMEM
, -nr
);
199 if (PageTransHuge(page
))
200 __mod_lruvec_page_state(page
, NR_SHMEM_THPS
, -nr
);
201 } else if (PageTransHuge(page
)) {
202 __mod_lruvec_page_state(page
, NR_FILE_THPS
, -nr
);
203 filemap_nr_thps_dec(mapping
);
207 * At this point page must be either written or cleaned by
208 * truncate. Dirty page here signals a bug and loss of
211 * This fixes dirty accounting after removing the page entirely
212 * but leaves PageDirty set: it has no effect for truncated
213 * page and anyway will be cleared before returning page into
216 if (WARN_ON_ONCE(PageDirty(page
)))
217 account_page_cleaned(page
, mapping
, inode_to_wb(mapping
->host
));
221 * Delete a page from the page cache and free it. Caller has to make
222 * sure the page is locked and that nobody else uses it - or that usage
223 * is safe. The caller must hold the i_pages lock.
225 void __delete_from_page_cache(struct page
*page
, void *shadow
)
227 struct address_space
*mapping
= page
->mapping
;
229 trace_mm_filemap_delete_from_page_cache(page
);
231 unaccount_page_cache_page(mapping
, page
);
232 page_cache_delete(mapping
, page
, shadow
);
235 static void page_cache_free_page(struct address_space
*mapping
,
238 void (*freepage
)(struct page
*);
240 freepage
= mapping
->a_ops
->freepage
;
244 if (PageTransHuge(page
) && !PageHuge(page
)) {
245 page_ref_sub(page
, thp_nr_pages(page
));
246 VM_BUG_ON_PAGE(page_count(page
) <= 0, page
);
253 * delete_from_page_cache - delete page from page cache
254 * @page: the page which the kernel is trying to remove from page cache
256 * This must be called only on pages that have been verified to be in the page
257 * cache and locked. It will never put the page into the free list, the caller
258 * has a reference on the page.
260 void delete_from_page_cache(struct page
*page
)
262 struct address_space
*mapping
= page_mapping(page
);
264 BUG_ON(!PageLocked(page
));
265 xa_lock_irq(&mapping
->i_pages
);
266 __delete_from_page_cache(page
, NULL
);
267 xa_unlock_irq(&mapping
->i_pages
);
269 page_cache_free_page(mapping
, page
);
271 EXPORT_SYMBOL(delete_from_page_cache
);
274 * page_cache_delete_batch - delete several pages from page cache
275 * @mapping: the mapping to which pages belong
276 * @pvec: pagevec with pages to delete
278 * The function walks over mapping->i_pages and removes pages passed in @pvec
279 * from the mapping. The function expects @pvec to be sorted by page index
280 * and is optimised for it to be dense.
281 * It tolerates holes in @pvec (mapping entries at those indices are not
282 * modified). The function expects only THP head pages to be present in the
285 * The function expects the i_pages lock to be held.
287 static void page_cache_delete_batch(struct address_space
*mapping
,
288 struct pagevec
*pvec
)
290 XA_STATE(xas
, &mapping
->i_pages
, pvec
->pages
[0]->index
);
295 mapping_set_update(&xas
, mapping
);
296 xas_for_each(&xas
, page
, ULONG_MAX
) {
297 if (i
>= pagevec_count(pvec
))
300 /* A swap/dax/shadow entry got inserted? Skip it. */
301 if (xa_is_value(page
))
304 * A page got inserted in our range? Skip it. We have our
305 * pages locked so they are protected from being removed.
306 * If we see a page whose index is higher than ours, it
307 * means our page has been removed, which shouldn't be
308 * possible because we're holding the PageLock.
310 if (page
!= pvec
->pages
[i
]) {
311 VM_BUG_ON_PAGE(page
->index
> pvec
->pages
[i
]->index
,
316 WARN_ON_ONCE(!PageLocked(page
));
318 if (page
->index
== xas
.xa_index
)
319 page
->mapping
= NULL
;
320 /* Leave page->index set: truncation lookup relies on it */
323 * Move to the next page in the vector if this is a regular
324 * page or the index is of the last sub-page of this compound
327 if (page
->index
+ compound_nr(page
) - 1 == xas
.xa_index
)
329 xas_store(&xas
, NULL
);
332 mapping
->nrpages
-= total_pages
;
335 void delete_from_page_cache_batch(struct address_space
*mapping
,
336 struct pagevec
*pvec
)
340 if (!pagevec_count(pvec
))
343 xa_lock_irq(&mapping
->i_pages
);
344 for (i
= 0; i
< pagevec_count(pvec
); i
++) {
345 trace_mm_filemap_delete_from_page_cache(pvec
->pages
[i
]);
347 unaccount_page_cache_page(mapping
, pvec
->pages
[i
]);
349 page_cache_delete_batch(mapping
, pvec
);
350 xa_unlock_irq(&mapping
->i_pages
);
352 for (i
= 0; i
< pagevec_count(pvec
); i
++)
353 page_cache_free_page(mapping
, pvec
->pages
[i
]);
356 int filemap_check_errors(struct address_space
*mapping
)
359 /* Check for outstanding write errors */
360 if (test_bit(AS_ENOSPC
, &mapping
->flags
) &&
361 test_and_clear_bit(AS_ENOSPC
, &mapping
->flags
))
363 if (test_bit(AS_EIO
, &mapping
->flags
) &&
364 test_and_clear_bit(AS_EIO
, &mapping
->flags
))
368 EXPORT_SYMBOL(filemap_check_errors
);
370 static int filemap_check_and_keep_errors(struct address_space
*mapping
)
372 /* Check for outstanding write errors */
373 if (test_bit(AS_EIO
, &mapping
->flags
))
375 if (test_bit(AS_ENOSPC
, &mapping
->flags
))
381 * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range
382 * @mapping: address space structure to write
383 * @wbc: the writeback_control controlling the writeout
385 * Call writepages on the mapping using the provided wbc to control the
388 * Return: %0 on success, negative error code otherwise.
390 int filemap_fdatawrite_wbc(struct address_space
*mapping
,
391 struct writeback_control
*wbc
)
395 if (!mapping_can_writeback(mapping
) ||
396 !mapping_tagged(mapping
, PAGECACHE_TAG_DIRTY
))
399 wbc_attach_fdatawrite_inode(wbc
, mapping
->host
);
400 ret
= do_writepages(mapping
, wbc
);
401 wbc_detach_inode(wbc
);
404 EXPORT_SYMBOL(filemap_fdatawrite_wbc
);
407 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
408 * @mapping: address space structure to write
409 * @start: offset in bytes where the range starts
410 * @end: offset in bytes where the range ends (inclusive)
411 * @sync_mode: enable synchronous operation
413 * Start writeback against all of a mapping's dirty pages that lie
414 * within the byte offsets <start, end> inclusive.
416 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
417 * opposed to a regular memory cleansing writeback. The difference between
418 * these two operations is that if a dirty page/buffer is encountered, it must
419 * be waited upon, and not just skipped over.
421 * Return: %0 on success, negative error code otherwise.
423 int __filemap_fdatawrite_range(struct address_space
*mapping
, loff_t start
,
424 loff_t end
, int sync_mode
)
426 struct writeback_control wbc
= {
427 .sync_mode
= sync_mode
,
428 .nr_to_write
= LONG_MAX
,
429 .range_start
= start
,
433 return filemap_fdatawrite_wbc(mapping
, &wbc
);
436 static inline int __filemap_fdatawrite(struct address_space
*mapping
,
439 return __filemap_fdatawrite_range(mapping
, 0, LLONG_MAX
, sync_mode
);
442 int filemap_fdatawrite(struct address_space
*mapping
)
444 return __filemap_fdatawrite(mapping
, WB_SYNC_ALL
);
446 EXPORT_SYMBOL(filemap_fdatawrite
);
448 int filemap_fdatawrite_range(struct address_space
*mapping
, loff_t start
,
451 return __filemap_fdatawrite_range(mapping
, start
, end
, WB_SYNC_ALL
);
453 EXPORT_SYMBOL(filemap_fdatawrite_range
);
456 * filemap_flush - mostly a non-blocking flush
457 * @mapping: target address_space
459 * This is a mostly non-blocking flush. Not suitable for data-integrity
460 * purposes - I/O may not be started against all dirty pages.
462 * Return: %0 on success, negative error code otherwise.
464 int filemap_flush(struct address_space
*mapping
)
466 return __filemap_fdatawrite(mapping
, WB_SYNC_NONE
);
468 EXPORT_SYMBOL(filemap_flush
);
471 * filemap_range_has_page - check if a page exists in range.
472 * @mapping: address space within which to check
473 * @start_byte: offset in bytes where the range starts
474 * @end_byte: offset in bytes where the range ends (inclusive)
476 * Find at least one page in the range supplied, usually used to check if
477 * direct writing in this range will trigger a writeback.
479 * Return: %true if at least one page exists in the specified range,
482 bool filemap_range_has_page(struct address_space
*mapping
,
483 loff_t start_byte
, loff_t end_byte
)
486 XA_STATE(xas
, &mapping
->i_pages
, start_byte
>> PAGE_SHIFT
);
487 pgoff_t max
= end_byte
>> PAGE_SHIFT
;
489 if (end_byte
< start_byte
)
494 page
= xas_find(&xas
, max
);
495 if (xas_retry(&xas
, page
))
497 /* Shadow entries don't count */
498 if (xa_is_value(page
))
501 * We don't need to try to pin this page; we're about to
502 * release the RCU lock anyway. It is enough to know that
503 * there was a page here recently.
511 EXPORT_SYMBOL(filemap_range_has_page
);
513 static void __filemap_fdatawait_range(struct address_space
*mapping
,
514 loff_t start_byte
, loff_t end_byte
)
516 pgoff_t index
= start_byte
>> PAGE_SHIFT
;
517 pgoff_t end
= end_byte
>> PAGE_SHIFT
;
521 if (end_byte
< start_byte
)
525 while (index
<= end
) {
528 nr_pages
= pagevec_lookup_range_tag(&pvec
, mapping
, &index
,
529 end
, PAGECACHE_TAG_WRITEBACK
);
533 for (i
= 0; i
< nr_pages
; i
++) {
534 struct page
*page
= pvec
.pages
[i
];
536 wait_on_page_writeback(page
);
537 ClearPageError(page
);
539 pagevec_release(&pvec
);
545 * filemap_fdatawait_range - wait for writeback to complete
546 * @mapping: address space structure to wait for
547 * @start_byte: offset in bytes where the range starts
548 * @end_byte: offset in bytes where the range ends (inclusive)
550 * Walk the list of under-writeback pages of the given address space
551 * in the given range and wait for all of them. Check error status of
552 * the address space and return it.
554 * Since the error status of the address space is cleared by this function,
555 * callers are responsible for checking the return value and handling and/or
556 * reporting the error.
558 * Return: error status of the address space.
560 int filemap_fdatawait_range(struct address_space
*mapping
, loff_t start_byte
,
563 __filemap_fdatawait_range(mapping
, start_byte
, end_byte
);
564 return filemap_check_errors(mapping
);
566 EXPORT_SYMBOL(filemap_fdatawait_range
);
569 * filemap_fdatawait_range_keep_errors - wait for writeback to complete
570 * @mapping: address space structure to wait for
571 * @start_byte: offset in bytes where the range starts
572 * @end_byte: offset in bytes where the range ends (inclusive)
574 * Walk the list of under-writeback pages of the given address space in the
575 * given range and wait for all of them. Unlike filemap_fdatawait_range(),
576 * this function does not clear error status of the address space.
578 * Use this function if callers don't handle errors themselves. Expected
579 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
582 int filemap_fdatawait_range_keep_errors(struct address_space
*mapping
,
583 loff_t start_byte
, loff_t end_byte
)
585 __filemap_fdatawait_range(mapping
, start_byte
, end_byte
);
586 return filemap_check_and_keep_errors(mapping
);
588 EXPORT_SYMBOL(filemap_fdatawait_range_keep_errors
);
591 * file_fdatawait_range - wait for writeback to complete
592 * @file: file pointing to address space structure to wait for
593 * @start_byte: offset in bytes where the range starts
594 * @end_byte: offset in bytes where the range ends (inclusive)
596 * Walk the list of under-writeback pages of the address space that file
597 * refers to, in the given range and wait for all of them. Check error
598 * status of the address space vs. the file->f_wb_err cursor and return it.
600 * Since the error status of the file is advanced by this function,
601 * callers are responsible for checking the return value and handling and/or
602 * reporting the error.
604 * Return: error status of the address space vs. the file->f_wb_err cursor.
606 int file_fdatawait_range(struct file
*file
, loff_t start_byte
, loff_t end_byte
)
608 struct address_space
*mapping
= file
->f_mapping
;
610 __filemap_fdatawait_range(mapping
, start_byte
, end_byte
);
611 return file_check_and_advance_wb_err(file
);
613 EXPORT_SYMBOL(file_fdatawait_range
);
616 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
617 * @mapping: address space structure to wait for
619 * Walk the list of under-writeback pages of the given address space
620 * and wait for all of them. Unlike filemap_fdatawait(), this function
621 * does not clear error status of the address space.
623 * Use this function if callers don't handle errors themselves. Expected
624 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
627 * Return: error status of the address space.
629 int filemap_fdatawait_keep_errors(struct address_space
*mapping
)
631 __filemap_fdatawait_range(mapping
, 0, LLONG_MAX
);
632 return filemap_check_and_keep_errors(mapping
);
634 EXPORT_SYMBOL(filemap_fdatawait_keep_errors
);
636 /* Returns true if writeback might be needed or already in progress. */
637 static bool mapping_needs_writeback(struct address_space
*mapping
)
639 return mapping
->nrpages
;
643 * filemap_range_needs_writeback - check if range potentially needs writeback
644 * @mapping: address space within which to check
645 * @start_byte: offset in bytes where the range starts
646 * @end_byte: offset in bytes where the range ends (inclusive)
648 * Find at least one page in the range supplied, usually used to check if
649 * direct writing in this range will trigger a writeback. Used by O_DIRECT
650 * read/write with IOCB_NOWAIT, to see if the caller needs to do
651 * filemap_write_and_wait_range() before proceeding.
653 * Return: %true if the caller should do filemap_write_and_wait_range() before
654 * doing O_DIRECT to a page in this range, %false otherwise.
656 bool filemap_range_needs_writeback(struct address_space
*mapping
,
657 loff_t start_byte
, loff_t end_byte
)
659 XA_STATE(xas
, &mapping
->i_pages
, start_byte
>> PAGE_SHIFT
);
660 pgoff_t max
= end_byte
>> PAGE_SHIFT
;
663 if (!mapping_needs_writeback(mapping
))
665 if (!mapping_tagged(mapping
, PAGECACHE_TAG_DIRTY
) &&
666 !mapping_tagged(mapping
, PAGECACHE_TAG_WRITEBACK
))
668 if (end_byte
< start_byte
)
672 xas_for_each(&xas
, page
, max
) {
673 if (xas_retry(&xas
, page
))
675 if (xa_is_value(page
))
677 if (PageDirty(page
) || PageLocked(page
) || PageWriteback(page
))
683 EXPORT_SYMBOL_GPL(filemap_range_needs_writeback
);
686 * filemap_write_and_wait_range - write out & wait on a file range
687 * @mapping: the address_space for the pages
688 * @lstart: offset in bytes where the range starts
689 * @lend: offset in bytes where the range ends (inclusive)
691 * Write out and wait upon file offsets lstart->lend, inclusive.
693 * Note that @lend is inclusive (describes the last byte to be written) so
694 * that this function can be used to write to the very end-of-file (end = -1).
696 * Return: error status of the address space.
698 int filemap_write_and_wait_range(struct address_space
*mapping
,
699 loff_t lstart
, loff_t lend
)
703 if (mapping_needs_writeback(mapping
)) {
704 err
= __filemap_fdatawrite_range(mapping
, lstart
, lend
,
707 * Even if the above returned error, the pages may be
708 * written partially (e.g. -ENOSPC), so we wait for it.
709 * But the -EIO is special case, it may indicate the worst
710 * thing (e.g. bug) happened, so we avoid waiting for it.
713 int err2
= filemap_fdatawait_range(mapping
,
718 /* Clear any previously stored errors */
719 filemap_check_errors(mapping
);
722 err
= filemap_check_errors(mapping
);
726 EXPORT_SYMBOL(filemap_write_and_wait_range
);
728 void __filemap_set_wb_err(struct address_space
*mapping
, int err
)
730 errseq_t eseq
= errseq_set(&mapping
->wb_err
, err
);
732 trace_filemap_set_wb_err(mapping
, eseq
);
734 EXPORT_SYMBOL(__filemap_set_wb_err
);
737 * file_check_and_advance_wb_err - report wb error (if any) that was previously
738 * and advance wb_err to current one
739 * @file: struct file on which the error is being reported
741 * When userland calls fsync (or something like nfsd does the equivalent), we
742 * want to report any writeback errors that occurred since the last fsync (or
743 * since the file was opened if there haven't been any).
745 * Grab the wb_err from the mapping. If it matches what we have in the file,
746 * then just quickly return 0. The file is all caught up.
748 * If it doesn't match, then take the mapping value, set the "seen" flag in
749 * it and try to swap it into place. If it works, or another task beat us
750 * to it with the new value, then update the f_wb_err and return the error
751 * portion. The error at this point must be reported via proper channels
752 * (a'la fsync, or NFS COMMIT operation, etc.).
754 * While we handle mapping->wb_err with atomic operations, the f_wb_err
755 * value is protected by the f_lock since we must ensure that it reflects
756 * the latest value swapped in for this file descriptor.
758 * Return: %0 on success, negative error code otherwise.
760 int file_check_and_advance_wb_err(struct file
*file
)
763 errseq_t old
= READ_ONCE(file
->f_wb_err
);
764 struct address_space
*mapping
= file
->f_mapping
;
766 /* Locklessly handle the common case where nothing has changed */
767 if (errseq_check(&mapping
->wb_err
, old
)) {
768 /* Something changed, must use slow path */
769 spin_lock(&file
->f_lock
);
770 old
= file
->f_wb_err
;
771 err
= errseq_check_and_advance(&mapping
->wb_err
,
773 trace_file_check_and_advance_wb_err(file
, old
);
774 spin_unlock(&file
->f_lock
);
778 * We're mostly using this function as a drop in replacement for
779 * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect
780 * that the legacy code would have had on these flags.
782 clear_bit(AS_EIO
, &mapping
->flags
);
783 clear_bit(AS_ENOSPC
, &mapping
->flags
);
786 EXPORT_SYMBOL(file_check_and_advance_wb_err
);
789 * file_write_and_wait_range - write out & wait on a file range
790 * @file: file pointing to address_space with pages
791 * @lstart: offset in bytes where the range starts
792 * @lend: offset in bytes where the range ends (inclusive)
794 * Write out and wait upon file offsets lstart->lend, inclusive.
796 * Note that @lend is inclusive (describes the last byte to be written) so
797 * that this function can be used to write to the very end-of-file (end = -1).
799 * After writing out and waiting on the data, we check and advance the
800 * f_wb_err cursor to the latest value, and return any errors detected there.
802 * Return: %0 on success, negative error code otherwise.
804 int file_write_and_wait_range(struct file
*file
, loff_t lstart
, loff_t lend
)
807 struct address_space
*mapping
= file
->f_mapping
;
809 if (mapping_needs_writeback(mapping
)) {
810 err
= __filemap_fdatawrite_range(mapping
, lstart
, lend
,
812 /* See comment of filemap_write_and_wait() */
814 __filemap_fdatawait_range(mapping
, lstart
, lend
);
816 err2
= file_check_and_advance_wb_err(file
);
821 EXPORT_SYMBOL(file_write_and_wait_range
);
824 * replace_page_cache_page - replace a pagecache page with a new one
825 * @old: page to be replaced
826 * @new: page to replace with
828 * This function replaces a page in the pagecache with a new one. On
829 * success it acquires the pagecache reference for the new page and
830 * drops it for the old page. Both the old and new pages must be
831 * locked. This function does not add the new page to the LRU, the
832 * caller must do that.
834 * The remove + add is atomic. This function cannot fail.
836 void replace_page_cache_page(struct page
*old
, struct page
*new)
838 struct address_space
*mapping
= old
->mapping
;
839 void (*freepage
)(struct page
*) = mapping
->a_ops
->freepage
;
840 pgoff_t offset
= old
->index
;
841 XA_STATE(xas
, &mapping
->i_pages
, offset
);
843 VM_BUG_ON_PAGE(!PageLocked(old
), old
);
844 VM_BUG_ON_PAGE(!PageLocked(new), new);
845 VM_BUG_ON_PAGE(new->mapping
, new);
848 new->mapping
= mapping
;
851 mem_cgroup_migrate(old
, new);
854 xas_store(&xas
, new);
857 /* hugetlb pages do not participate in page cache accounting. */
859 __dec_lruvec_page_state(old
, NR_FILE_PAGES
);
861 __inc_lruvec_page_state(new, NR_FILE_PAGES
);
862 if (PageSwapBacked(old
))
863 __dec_lruvec_page_state(old
, NR_SHMEM
);
864 if (PageSwapBacked(new))
865 __inc_lruvec_page_state(new, NR_SHMEM
);
866 xas_unlock_irq(&xas
);
871 EXPORT_SYMBOL_GPL(replace_page_cache_page
);
873 noinline
int __add_to_page_cache_locked(struct page
*page
,
874 struct address_space
*mapping
,
875 pgoff_t offset
, gfp_t gfp
,
878 XA_STATE(xas
, &mapping
->i_pages
, offset
);
879 int huge
= PageHuge(page
);
881 bool charged
= false;
883 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
884 VM_BUG_ON_PAGE(PageSwapBacked(page
), page
);
885 mapping_set_update(&xas
, mapping
);
888 page
->mapping
= mapping
;
889 page
->index
= offset
;
892 error
= mem_cgroup_charge(page
, NULL
, gfp
);
898 gfp
&= GFP_RECLAIM_MASK
;
901 unsigned int order
= xa_get_order(xas
.xa
, xas
.xa_index
);
902 void *entry
, *old
= NULL
;
904 if (order
> thp_order(page
))
905 xas_split_alloc(&xas
, xa_load(xas
.xa
, xas
.xa_index
),
908 xas_for_each_conflict(&xas
, entry
) {
910 if (!xa_is_value(entry
)) {
911 xas_set_err(&xas
, -EEXIST
);
919 /* entry may have been split before we acquired lock */
920 order
= xa_get_order(xas
.xa
, xas
.xa_index
);
921 if (order
> thp_order(page
)) {
922 xas_split(&xas
, old
, order
);
927 xas_store(&xas
, page
);
933 /* hugetlb pages do not participate in page cache accounting */
935 __inc_lruvec_page_state(page
, NR_FILE_PAGES
);
937 xas_unlock_irq(&xas
);
938 } while (xas_nomem(&xas
, gfp
));
940 if (xas_error(&xas
)) {
941 error
= xas_error(&xas
);
943 mem_cgroup_uncharge(page
);
947 trace_mm_filemap_add_to_page_cache(page
);
950 page
->mapping
= NULL
;
951 /* Leave page->index set: truncation relies upon it */
955 ALLOW_ERROR_INJECTION(__add_to_page_cache_locked
, ERRNO
);
958 * add_to_page_cache_locked - add a locked page to the pagecache
960 * @mapping: the page's address_space
961 * @offset: page index
962 * @gfp_mask: page allocation mode
964 * This function is used to add a page to the pagecache. It must be locked.
965 * This function does not add the page to the LRU. The caller must do that.
967 * Return: %0 on success, negative error code otherwise.
969 int add_to_page_cache_locked(struct page
*page
, struct address_space
*mapping
,
970 pgoff_t offset
, gfp_t gfp_mask
)
972 return __add_to_page_cache_locked(page
, mapping
, offset
,
975 EXPORT_SYMBOL(add_to_page_cache_locked
);
977 int add_to_page_cache_lru(struct page
*page
, struct address_space
*mapping
,
978 pgoff_t offset
, gfp_t gfp_mask
)
983 __SetPageLocked(page
);
984 ret
= __add_to_page_cache_locked(page
, mapping
, offset
,
987 __ClearPageLocked(page
);
990 * The page might have been evicted from cache only
991 * recently, in which case it should be activated like
992 * any other repeatedly accessed page.
993 * The exception is pages getting rewritten; evicting other
994 * data from the working set, only to cache data that will
995 * get overwritten with something else, is a waste of memory.
997 WARN_ON_ONCE(PageActive(page
));
998 if (!(gfp_mask
& __GFP_WRITE
) && shadow
)
999 workingset_refault(page
, shadow
);
1000 lru_cache_add(page
);
1004 EXPORT_SYMBOL_GPL(add_to_page_cache_lru
);
1007 struct page
*__page_cache_alloc(gfp_t gfp
)
1012 if (cpuset_do_page_mem_spread()) {
1013 unsigned int cpuset_mems_cookie
;
1015 cpuset_mems_cookie
= read_mems_allowed_begin();
1016 n
= cpuset_mem_spread_node();
1017 page
= __alloc_pages_node(n
, gfp
, 0);
1018 } while (!page
&& read_mems_allowed_retry(cpuset_mems_cookie
));
1022 return alloc_pages(gfp
, 0);
1024 EXPORT_SYMBOL(__page_cache_alloc
);
1028 * filemap_invalidate_lock_two - lock invalidate_lock for two mappings
1030 * Lock exclusively invalidate_lock of any passed mapping that is not NULL.
1032 * @mapping1: the first mapping to lock
1033 * @mapping2: the second mapping to lock
1035 void filemap_invalidate_lock_two(struct address_space
*mapping1
,
1036 struct address_space
*mapping2
)
1038 if (mapping1
> mapping2
)
1039 swap(mapping1
, mapping2
);
1041 down_write(&mapping1
->invalidate_lock
);
1042 if (mapping2
&& mapping1
!= mapping2
)
1043 down_write_nested(&mapping2
->invalidate_lock
, 1);
1045 EXPORT_SYMBOL(filemap_invalidate_lock_two
);
1048 * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings
1050 * Unlock exclusive invalidate_lock of any passed mapping that is not NULL.
1052 * @mapping1: the first mapping to unlock
1053 * @mapping2: the second mapping to unlock
1055 void filemap_invalidate_unlock_two(struct address_space
*mapping1
,
1056 struct address_space
*mapping2
)
1059 up_write(&mapping1
->invalidate_lock
);
1060 if (mapping2
&& mapping1
!= mapping2
)
1061 up_write(&mapping2
->invalidate_lock
);
1063 EXPORT_SYMBOL(filemap_invalidate_unlock_two
);
1066 * In order to wait for pages to become available there must be
1067 * waitqueues associated with pages. By using a hash table of
1068 * waitqueues where the bucket discipline is to maintain all
1069 * waiters on the same queue and wake all when any of the pages
1070 * become available, and for the woken contexts to check to be
1071 * sure the appropriate page became available, this saves space
1072 * at a cost of "thundering herd" phenomena during rare hash
1075 #define PAGE_WAIT_TABLE_BITS 8
1076 #define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
1077 static wait_queue_head_t page_wait_table
[PAGE_WAIT_TABLE_SIZE
] __cacheline_aligned
;
1079 static wait_queue_head_t
*page_waitqueue(struct page
*page
)
1081 return &page_wait_table
[hash_ptr(page
, PAGE_WAIT_TABLE_BITS
)];
1084 void __init
pagecache_init(void)
1088 for (i
= 0; i
< PAGE_WAIT_TABLE_SIZE
; i
++)
1089 init_waitqueue_head(&page_wait_table
[i
]);
1091 page_writeback_init();
1095 * The page wait code treats the "wait->flags" somewhat unusually, because
1096 * we have multiple different kinds of waits, not just the usual "exclusive"
1101 * (a) no special bits set:
1103 * We're just waiting for the bit to be released, and when a waker
1104 * calls the wakeup function, we set WQ_FLAG_WOKEN and wake it up,
1105 * and remove it from the wait queue.
1107 * Simple and straightforward.
1109 * (b) WQ_FLAG_EXCLUSIVE:
1111 * The waiter is waiting to get the lock, and only one waiter should
1112 * be woken up to avoid any thundering herd behavior. We'll set the
1113 * WQ_FLAG_WOKEN bit, wake it up, and remove it from the wait queue.
1115 * This is the traditional exclusive wait.
1117 * (c) WQ_FLAG_EXCLUSIVE | WQ_FLAG_CUSTOM:
1119 * The waiter is waiting to get the bit, and additionally wants the
1120 * lock to be transferred to it for fair lock behavior. If the lock
1121 * cannot be taken, we stop walking the wait queue without waking
1124 * This is the "fair lock handoff" case, and in addition to setting
1125 * WQ_FLAG_WOKEN, we set WQ_FLAG_DONE to let the waiter easily see
1126 * that it now has the lock.
1128 static int wake_page_function(wait_queue_entry_t
*wait
, unsigned mode
, int sync
, void *arg
)
1131 struct wait_page_key
*key
= arg
;
1132 struct wait_page_queue
*wait_page
1133 = container_of(wait
, struct wait_page_queue
, wait
);
1135 if (!wake_page_match(wait_page
, key
))
1139 * If it's a lock handoff wait, we get the bit for it, and
1140 * stop walking (and do not wake it up) if we can't.
1142 flags
= wait
->flags
;
1143 if (flags
& WQ_FLAG_EXCLUSIVE
) {
1144 if (test_bit(key
->bit_nr
, &key
->page
->flags
))
1146 if (flags
& WQ_FLAG_CUSTOM
) {
1147 if (test_and_set_bit(key
->bit_nr
, &key
->page
->flags
))
1149 flags
|= WQ_FLAG_DONE
;
1154 * We are holding the wait-queue lock, but the waiter that
1155 * is waiting for this will be checking the flags without
1158 * So update the flags atomically, and wake up the waiter
1159 * afterwards to avoid any races. This store-release pairs
1160 * with the load-acquire in wait_on_page_bit_common().
1162 smp_store_release(&wait
->flags
, flags
| WQ_FLAG_WOKEN
);
1163 wake_up_state(wait
->private, mode
);
1166 * Ok, we have successfully done what we're waiting for,
1167 * and we can unconditionally remove the wait entry.
1169 * Note that this pairs with the "finish_wait()" in the
1170 * waiter, and has to be the absolute last thing we do.
1171 * After this list_del_init(&wait->entry) the wait entry
1172 * might be de-allocated and the process might even have
1175 list_del_init_careful(&wait
->entry
);
1176 return (flags
& WQ_FLAG_EXCLUSIVE
) != 0;
1179 static void wake_up_page_bit(struct page
*page
, int bit_nr
)
1181 wait_queue_head_t
*q
= page_waitqueue(page
);
1182 struct wait_page_key key
;
1183 unsigned long flags
;
1184 wait_queue_entry_t bookmark
;
1187 key
.bit_nr
= bit_nr
;
1191 bookmark
.private = NULL
;
1192 bookmark
.func
= NULL
;
1193 INIT_LIST_HEAD(&bookmark
.entry
);
1195 spin_lock_irqsave(&q
->lock
, flags
);
1196 __wake_up_locked_key_bookmark(q
, TASK_NORMAL
, &key
, &bookmark
);
1198 while (bookmark
.flags
& WQ_FLAG_BOOKMARK
) {
1200 * Take a breather from holding the lock,
1201 * allow pages that finish wake up asynchronously
1202 * to acquire the lock and remove themselves
1205 spin_unlock_irqrestore(&q
->lock
, flags
);
1207 spin_lock_irqsave(&q
->lock
, flags
);
1208 __wake_up_locked_key_bookmark(q
, TASK_NORMAL
, &key
, &bookmark
);
1212 * It is possible for other pages to have collided on the waitqueue
1213 * hash, so in that case check for a page match. That prevents a long-
1216 * It is still possible to miss a case here, when we woke page waiters
1217 * and removed them from the waitqueue, but there are still other
1220 if (!waitqueue_active(q
) || !key
.page_match
) {
1221 ClearPageWaiters(page
);
1223 * It's possible to miss clearing Waiters here, when we woke
1224 * our page waiters, but the hashed waitqueue has waiters for
1225 * other pages on it.
1227 * That's okay, it's a rare case. The next waker will clear it.
1230 spin_unlock_irqrestore(&q
->lock
, flags
);
1233 static void wake_up_page(struct page
*page
, int bit
)
1235 if (!PageWaiters(page
))
1237 wake_up_page_bit(page
, bit
);
1241 * A choice of three behaviors for wait_on_page_bit_common():
1244 EXCLUSIVE
, /* Hold ref to page and take the bit when woken, like
1245 * __lock_page() waiting on then setting PG_locked.
1247 SHARED
, /* Hold ref to page and check the bit when woken, like
1248 * wait_on_page_writeback() waiting on PG_writeback.
1250 DROP
, /* Drop ref to page before wait, no check when woken,
1251 * like put_and_wait_on_page_locked() on PG_locked.
1256 * Attempt to check (or get) the page bit, and mark us done
1259 static inline bool trylock_page_bit_common(struct page
*page
, int bit_nr
,
1260 struct wait_queue_entry
*wait
)
1262 if (wait
->flags
& WQ_FLAG_EXCLUSIVE
) {
1263 if (test_and_set_bit(bit_nr
, &page
->flags
))
1265 } else if (test_bit(bit_nr
, &page
->flags
))
1268 wait
->flags
|= WQ_FLAG_WOKEN
| WQ_FLAG_DONE
;
1272 /* How many times do we accept lock stealing from under a waiter? */
1273 int sysctl_page_lock_unfairness
= 5;
1275 static inline int wait_on_page_bit_common(wait_queue_head_t
*q
,
1276 struct page
*page
, int bit_nr
, int state
, enum behavior behavior
)
1278 int unfairness
= sysctl_page_lock_unfairness
;
1279 struct wait_page_queue wait_page
;
1280 wait_queue_entry_t
*wait
= &wait_page
.wait
;
1281 bool thrashing
= false;
1282 bool delayacct
= false;
1283 unsigned long pflags
;
1285 if (bit_nr
== PG_locked
&&
1286 !PageUptodate(page
) && PageWorkingset(page
)) {
1287 if (!PageSwapBacked(page
)) {
1288 delayacct_thrashing_start();
1291 psi_memstall_enter(&pflags
);
1296 wait
->func
= wake_page_function
;
1297 wait_page
.page
= page
;
1298 wait_page
.bit_nr
= bit_nr
;
1302 if (behavior
== EXCLUSIVE
) {
1303 wait
->flags
= WQ_FLAG_EXCLUSIVE
;
1304 if (--unfairness
< 0)
1305 wait
->flags
|= WQ_FLAG_CUSTOM
;
1309 * Do one last check whether we can get the
1310 * page bit synchronously.
1312 * Do the SetPageWaiters() marking before that
1313 * to let any waker we _just_ missed know they
1314 * need to wake us up (otherwise they'll never
1315 * even go to the slow case that looks at the
1316 * page queue), and add ourselves to the wait
1317 * queue if we need to sleep.
1319 * This part needs to be done under the queue
1320 * lock to avoid races.
1322 spin_lock_irq(&q
->lock
);
1323 SetPageWaiters(page
);
1324 if (!trylock_page_bit_common(page
, bit_nr
, wait
))
1325 __add_wait_queue_entry_tail(q
, wait
);
1326 spin_unlock_irq(&q
->lock
);
1329 * From now on, all the logic will be based on
1330 * the WQ_FLAG_WOKEN and WQ_FLAG_DONE flag, to
1331 * see whether the page bit testing has already
1332 * been done by the wake function.
1334 * We can drop our reference to the page.
1336 if (behavior
== DROP
)
1340 * Note that until the "finish_wait()", or until
1341 * we see the WQ_FLAG_WOKEN flag, we need to
1342 * be very careful with the 'wait->flags', because
1343 * we may race with a waker that sets them.
1348 set_current_state(state
);
1350 /* Loop until we've been woken or interrupted */
1351 flags
= smp_load_acquire(&wait
->flags
);
1352 if (!(flags
& WQ_FLAG_WOKEN
)) {
1353 if (signal_pending_state(state
, current
))
1360 /* If we were non-exclusive, we're done */
1361 if (behavior
!= EXCLUSIVE
)
1364 /* If the waker got the lock for us, we're done */
1365 if (flags
& WQ_FLAG_DONE
)
1369 * Otherwise, if we're getting the lock, we need to
1370 * try to get it ourselves.
1372 * And if that fails, we'll have to retry this all.
1374 if (unlikely(test_and_set_bit(bit_nr
, &page
->flags
)))
1377 wait
->flags
|= WQ_FLAG_DONE
;
1382 * If a signal happened, this 'finish_wait()' may remove the last
1383 * waiter from the wait-queues, but the PageWaiters bit will remain
1384 * set. That's ok. The next wakeup will take care of it, and trying
1385 * to do it here would be difficult and prone to races.
1387 finish_wait(q
, wait
);
1391 delayacct_thrashing_end();
1392 psi_memstall_leave(&pflags
);
1396 * NOTE! The wait->flags weren't stable until we've done the
1397 * 'finish_wait()', and we could have exited the loop above due
1398 * to a signal, and had a wakeup event happen after the signal
1399 * test but before the 'finish_wait()'.
1401 * So only after the finish_wait() can we reliably determine
1402 * if we got woken up or not, so we can now figure out the final
1403 * return value based on that state without races.
1405 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive
1406 * waiter, but an exclusive one requires WQ_FLAG_DONE.
1408 if (behavior
== EXCLUSIVE
)
1409 return wait
->flags
& WQ_FLAG_DONE
? 0 : -EINTR
;
1411 return wait
->flags
& WQ_FLAG_WOKEN
? 0 : -EINTR
;
1414 void wait_on_page_bit(struct page
*page
, int bit_nr
)
1416 wait_queue_head_t
*q
= page_waitqueue(page
);
1417 wait_on_page_bit_common(q
, page
, bit_nr
, TASK_UNINTERRUPTIBLE
, SHARED
);
1419 EXPORT_SYMBOL(wait_on_page_bit
);
1421 int wait_on_page_bit_killable(struct page
*page
, int bit_nr
)
1423 wait_queue_head_t
*q
= page_waitqueue(page
);
1424 return wait_on_page_bit_common(q
, page
, bit_nr
, TASK_KILLABLE
, SHARED
);
1426 EXPORT_SYMBOL(wait_on_page_bit_killable
);
1429 * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
1430 * @page: The page to wait for.
1431 * @state: The sleep state (TASK_KILLABLE, TASK_UNINTERRUPTIBLE, etc).
1433 * The caller should hold a reference on @page. They expect the page to
1434 * become unlocked relatively soon, but do not wish to hold up migration
1435 * (for example) by holding the reference while waiting for the page to
1436 * come unlocked. After this function returns, the caller should not
1437 * dereference @page.
1439 * Return: 0 if the page was unlocked or -EINTR if interrupted by a signal.
1441 int put_and_wait_on_page_locked(struct page
*page
, int state
)
1443 wait_queue_head_t
*q
;
1445 page
= compound_head(page
);
1446 q
= page_waitqueue(page
);
1447 return wait_on_page_bit_common(q
, page
, PG_locked
, state
, DROP
);
1451 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
1452 * @page: Page defining the wait queue of interest
1453 * @waiter: Waiter to add to the queue
1455 * Add an arbitrary @waiter to the wait queue for the nominated @page.
1457 void add_page_wait_queue(struct page
*page
, wait_queue_entry_t
*waiter
)
1459 wait_queue_head_t
*q
= page_waitqueue(page
);
1460 unsigned long flags
;
1462 spin_lock_irqsave(&q
->lock
, flags
);
1463 __add_wait_queue_entry_tail(q
, waiter
);
1464 SetPageWaiters(page
);
1465 spin_unlock_irqrestore(&q
->lock
, flags
);
1467 EXPORT_SYMBOL_GPL(add_page_wait_queue
);
1469 #ifndef clear_bit_unlock_is_negative_byte
1472 * PG_waiters is the high bit in the same byte as PG_lock.
1474 * On x86 (and on many other architectures), we can clear PG_lock and
1475 * test the sign bit at the same time. But if the architecture does
1476 * not support that special operation, we just do this all by hand
1479 * The read of PG_waiters has to be after (or concurrently with) PG_locked
1480 * being cleared, but a memory barrier should be unnecessary since it is
1481 * in the same byte as PG_locked.
1483 static inline bool clear_bit_unlock_is_negative_byte(long nr
, volatile void *mem
)
1485 clear_bit_unlock(nr
, mem
);
1486 /* smp_mb__after_atomic(); */
1487 return test_bit(PG_waiters
, mem
);
1493 * unlock_page - unlock a locked page
1496 * Unlocks the page and wakes up sleepers in wait_on_page_locked().
1497 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
1498 * mechanism between PageLocked pages and PageWriteback pages is shared.
1499 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
1501 * Note that this depends on PG_waiters being the sign bit in the byte
1502 * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to
1503 * clear the PG_locked bit and test PG_waiters at the same time fairly
1504 * portably (architectures that do LL/SC can test any bit, while x86 can
1505 * test the sign bit).
1507 void unlock_page(struct page
*page
)
1509 BUILD_BUG_ON(PG_waiters
!= 7);
1510 page
= compound_head(page
);
1511 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
1512 if (clear_bit_unlock_is_negative_byte(PG_locked
, &page
->flags
))
1513 wake_up_page_bit(page
, PG_locked
);
1515 EXPORT_SYMBOL(unlock_page
);
1518 * end_page_private_2 - Clear PG_private_2 and release any waiters
1521 * Clear the PG_private_2 bit on a page and wake up any sleepers waiting for
1522 * this. The page ref held for PG_private_2 being set is released.
1524 * This is, for example, used when a netfs page is being written to a local
1525 * disk cache, thereby allowing writes to the cache for the same page to be
1528 void end_page_private_2(struct page
*page
)
1530 page
= compound_head(page
);
1531 VM_BUG_ON_PAGE(!PagePrivate2(page
), page
);
1532 clear_bit_unlock(PG_private_2
, &page
->flags
);
1533 wake_up_page_bit(page
, PG_private_2
);
1536 EXPORT_SYMBOL(end_page_private_2
);
1539 * wait_on_page_private_2 - Wait for PG_private_2 to be cleared on a page
1540 * @page: The page to wait on
1542 * Wait for PG_private_2 (aka PG_fscache) to be cleared on a page.
1544 void wait_on_page_private_2(struct page
*page
)
1546 page
= compound_head(page
);
1547 while (PagePrivate2(page
))
1548 wait_on_page_bit(page
, PG_private_2
);
1550 EXPORT_SYMBOL(wait_on_page_private_2
);
1553 * wait_on_page_private_2_killable - Wait for PG_private_2 to be cleared on a page
1554 * @page: The page to wait on
1556 * Wait for PG_private_2 (aka PG_fscache) to be cleared on a page or until a
1557 * fatal signal is received by the calling task.
1560 * - 0 if successful.
1561 * - -EINTR if a fatal signal was encountered.
1563 int wait_on_page_private_2_killable(struct page
*page
)
1567 page
= compound_head(page
);
1568 while (PagePrivate2(page
)) {
1569 ret
= wait_on_page_bit_killable(page
, PG_private_2
);
1576 EXPORT_SYMBOL(wait_on_page_private_2_killable
);
1579 * end_page_writeback - end writeback against a page
1582 void end_page_writeback(struct page
*page
)
1585 * TestClearPageReclaim could be used here but it is an atomic
1586 * operation and overkill in this particular case. Failing to
1587 * shuffle a page marked for immediate reclaim is too mild to
1588 * justify taking an atomic operation penalty at the end of
1589 * ever page writeback.
1591 if (PageReclaim(page
)) {
1592 ClearPageReclaim(page
);
1593 rotate_reclaimable_page(page
);
1597 * Writeback does not hold a page reference of its own, relying
1598 * on truncation to wait for the clearing of PG_writeback.
1599 * But here we must make sure that the page is not freed and
1600 * reused before the wake_up_page().
1603 if (!test_clear_page_writeback(page
))
1606 smp_mb__after_atomic();
1607 wake_up_page(page
, PG_writeback
);
1610 EXPORT_SYMBOL(end_page_writeback
);
1613 * After completing I/O on a page, call this routine to update the page
1614 * flags appropriately
1616 void page_endio(struct page
*page
, bool is_write
, int err
)
1620 SetPageUptodate(page
);
1622 ClearPageUptodate(page
);
1628 struct address_space
*mapping
;
1631 mapping
= page_mapping(page
);
1633 mapping_set_error(mapping
, err
);
1635 end_page_writeback(page
);
1638 EXPORT_SYMBOL_GPL(page_endio
);
1641 * __lock_page - get a lock on the page, assuming we need to sleep to get it
1642 * @__page: the page to lock
1644 void __lock_page(struct page
*__page
)
1646 struct page
*page
= compound_head(__page
);
1647 wait_queue_head_t
*q
= page_waitqueue(page
);
1648 wait_on_page_bit_common(q
, page
, PG_locked
, TASK_UNINTERRUPTIBLE
,
1651 EXPORT_SYMBOL(__lock_page
);
1653 int __lock_page_killable(struct page
*__page
)
1655 struct page
*page
= compound_head(__page
);
1656 wait_queue_head_t
*q
= page_waitqueue(page
);
1657 return wait_on_page_bit_common(q
, page
, PG_locked
, TASK_KILLABLE
,
1660 EXPORT_SYMBOL_GPL(__lock_page_killable
);
1662 int __lock_page_async(struct page
*page
, struct wait_page_queue
*wait
)
1664 struct wait_queue_head
*q
= page_waitqueue(page
);
1668 wait
->bit_nr
= PG_locked
;
1670 spin_lock_irq(&q
->lock
);
1671 __add_wait_queue_entry_tail(q
, &wait
->wait
);
1672 SetPageWaiters(page
);
1673 ret
= !trylock_page(page
);
1675 * If we were successful now, we know we're still on the
1676 * waitqueue as we're still under the lock. This means it's
1677 * safe to remove and return success, we know the callback
1678 * isn't going to trigger.
1681 __remove_wait_queue(q
, &wait
->wait
);
1684 spin_unlock_irq(&q
->lock
);
1690 * 1 - page is locked; mmap_lock is still held.
1691 * 0 - page is not locked.
1692 * mmap_lock has been released (mmap_read_unlock(), unless flags had both
1693 * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
1694 * which case mmap_lock is still held.
1696 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
1697 * with the page locked and the mmap_lock unperturbed.
1699 int __lock_page_or_retry(struct page
*page
, struct mm_struct
*mm
,
1702 if (fault_flag_allow_retry_first(flags
)) {
1704 * CAUTION! In this case, mmap_lock is not released
1705 * even though return 0.
1707 if (flags
& FAULT_FLAG_RETRY_NOWAIT
)
1710 mmap_read_unlock(mm
);
1711 if (flags
& FAULT_FLAG_KILLABLE
)
1712 wait_on_page_locked_killable(page
);
1714 wait_on_page_locked(page
);
1717 if (flags
& FAULT_FLAG_KILLABLE
) {
1720 ret
= __lock_page_killable(page
);
1722 mmap_read_unlock(mm
);
1733 * page_cache_next_miss() - Find the next gap in the page cache.
1734 * @mapping: Mapping.
1736 * @max_scan: Maximum range to search.
1738 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1739 * gap with the lowest index.
1741 * This function may be called under the rcu_read_lock. However, this will
1742 * not atomically search a snapshot of the cache at a single point in time.
1743 * For example, if a gap is created at index 5, then subsequently a gap is
1744 * created at index 10, page_cache_next_miss covering both indices may
1745 * return 10 if called under the rcu_read_lock.
1747 * Return: The index of the gap if found, otherwise an index outside the
1748 * range specified (in which case 'return - index >= max_scan' will be true).
1749 * In the rare case of index wrap-around, 0 will be returned.
1751 pgoff_t
page_cache_next_miss(struct address_space
*mapping
,
1752 pgoff_t index
, unsigned long max_scan
)
1754 XA_STATE(xas
, &mapping
->i_pages
, index
);
1756 while (max_scan
--) {
1757 void *entry
= xas_next(&xas
);
1758 if (!entry
|| xa_is_value(entry
))
1760 if (xas
.xa_index
== 0)
1764 return xas
.xa_index
;
1766 EXPORT_SYMBOL(page_cache_next_miss
);
1769 * page_cache_prev_miss() - Find the previous gap in the page cache.
1770 * @mapping: Mapping.
1772 * @max_scan: Maximum range to search.
1774 * Search the range [max(index - max_scan + 1, 0), index] for the
1775 * gap with the highest index.
1777 * This function may be called under the rcu_read_lock. However, this will
1778 * not atomically search a snapshot of the cache at a single point in time.
1779 * For example, if a gap is created at index 10, then subsequently a gap is
1780 * created at index 5, page_cache_prev_miss() covering both indices may
1781 * return 5 if called under the rcu_read_lock.
1783 * Return: The index of the gap if found, otherwise an index outside the
1784 * range specified (in which case 'index - return >= max_scan' will be true).
1785 * In the rare case of wrap-around, ULONG_MAX will be returned.
1787 pgoff_t
page_cache_prev_miss(struct address_space
*mapping
,
1788 pgoff_t index
, unsigned long max_scan
)
1790 XA_STATE(xas
, &mapping
->i_pages
, index
);
1792 while (max_scan
--) {
1793 void *entry
= xas_prev(&xas
);
1794 if (!entry
|| xa_is_value(entry
))
1796 if (xas
.xa_index
== ULONG_MAX
)
1800 return xas
.xa_index
;
1802 EXPORT_SYMBOL(page_cache_prev_miss
);
1805 * mapping_get_entry - Get a page cache entry.
1806 * @mapping: the address_space to search
1807 * @index: The page cache index.
1809 * Looks up the page cache slot at @mapping & @index. If there is a
1810 * page cache page, the head page is returned with an increased refcount.
1812 * If the slot holds a shadow entry of a previously evicted page, or a
1813 * swap entry from shmem/tmpfs, it is returned.
1815 * Return: The head page or shadow entry, %NULL if nothing is found.
1817 static struct page
*mapping_get_entry(struct address_space
*mapping
,
1820 XA_STATE(xas
, &mapping
->i_pages
, index
);
1826 page
= xas_load(&xas
);
1827 if (xas_retry(&xas
, page
))
1830 * A shadow entry of a recently evicted page, or a swap entry from
1831 * shmem/tmpfs. Return it without attempting to raise page count.
1833 if (!page
|| xa_is_value(page
))
1836 if (!page_cache_get_speculative(page
))
1840 * Has the page moved or been split?
1841 * This is part of the lockless pagecache protocol. See
1842 * include/linux/pagemap.h for details.
1844 if (unlikely(page
!= xas_reload(&xas
))) {
1855 * pagecache_get_page - Find and get a reference to a page.
1856 * @mapping: The address_space to search.
1857 * @index: The page index.
1858 * @fgp_flags: %FGP flags modify how the page is returned.
1859 * @gfp_mask: Memory allocation flags to use if %FGP_CREAT is specified.
1861 * Looks up the page cache entry at @mapping & @index.
1863 * @fgp_flags can be zero or more of these flags:
1865 * * %FGP_ACCESSED - The page will be marked accessed.
1866 * * %FGP_LOCK - The page is returned locked.
1867 * * %FGP_HEAD - If the page is present and a THP, return the head page
1868 * rather than the exact page specified by the index.
1869 * * %FGP_ENTRY - If there is a shadow / swap / DAX entry, return it
1870 * instead of allocating a new page to replace it.
1871 * * %FGP_CREAT - If no page is present then a new page is allocated using
1872 * @gfp_mask and added to the page cache and the VM's LRU list.
1873 * The page is returned locked and with an increased refcount.
1874 * * %FGP_FOR_MMAP - The caller wants to do its own locking dance if the
1875 * page is already in cache. If the page was allocated, unlock it before
1876 * returning so the caller can do the same dance.
1877 * * %FGP_WRITE - The page will be written
1878 * * %FGP_NOFS - __GFP_FS will get cleared in gfp mask
1879 * * %FGP_NOWAIT - Don't get blocked by page lock
1881 * If %FGP_LOCK or %FGP_CREAT are specified then the function may sleep even
1882 * if the %GFP flags specified for %FGP_CREAT are atomic.
1884 * If there is a page cache page, it is returned with an increased refcount.
1886 * Return: The found page or %NULL otherwise.
1888 struct page
*pagecache_get_page(struct address_space
*mapping
, pgoff_t index
,
1889 int fgp_flags
, gfp_t gfp_mask
)
1894 page
= mapping_get_entry(mapping
, index
);
1895 if (xa_is_value(page
)) {
1896 if (fgp_flags
& FGP_ENTRY
)
1903 if (fgp_flags
& FGP_LOCK
) {
1904 if (fgp_flags
& FGP_NOWAIT
) {
1905 if (!trylock_page(page
)) {
1913 /* Has the page been truncated? */
1914 if (unlikely(page
->mapping
!= mapping
)) {
1919 VM_BUG_ON_PAGE(!thp_contains(page
, index
), page
);
1922 if (fgp_flags
& FGP_ACCESSED
)
1923 mark_page_accessed(page
);
1924 else if (fgp_flags
& FGP_WRITE
) {
1925 /* Clear idle flag for buffer write */
1926 if (page_is_idle(page
))
1927 clear_page_idle(page
);
1929 if (!(fgp_flags
& FGP_HEAD
))
1930 page
= find_subpage(page
, index
);
1933 if (!page
&& (fgp_flags
& FGP_CREAT
)) {
1935 if ((fgp_flags
& FGP_WRITE
) && mapping_can_writeback(mapping
))
1936 gfp_mask
|= __GFP_WRITE
;
1937 if (fgp_flags
& FGP_NOFS
)
1938 gfp_mask
&= ~__GFP_FS
;
1940 page
= __page_cache_alloc(gfp_mask
);
1944 if (WARN_ON_ONCE(!(fgp_flags
& (FGP_LOCK
| FGP_FOR_MMAP
))))
1945 fgp_flags
|= FGP_LOCK
;
1947 /* Init accessed so avoid atomic mark_page_accessed later */
1948 if (fgp_flags
& FGP_ACCESSED
)
1949 __SetPageReferenced(page
);
1951 err
= add_to_page_cache_lru(page
, mapping
, index
, gfp_mask
);
1952 if (unlikely(err
)) {
1960 * add_to_page_cache_lru locks the page, and for mmap we expect
1963 if (page
&& (fgp_flags
& FGP_FOR_MMAP
))
1969 EXPORT_SYMBOL(pagecache_get_page
);
1971 static inline struct page
*find_get_entry(struct xa_state
*xas
, pgoff_t max
,
1977 if (mark
== XA_PRESENT
)
1978 page
= xas_find(xas
, max
);
1980 page
= xas_find_marked(xas
, max
, mark
);
1982 if (xas_retry(xas
, page
))
1985 * A shadow entry of a recently evicted page, a swap
1986 * entry from shmem/tmpfs or a DAX entry. Return it
1987 * without attempting to raise page count.
1989 if (!page
|| xa_is_value(page
))
1992 if (!page_cache_get_speculative(page
))
1995 /* Has the page moved or been split? */
1996 if (unlikely(page
!= xas_reload(xas
))) {
2008 * find_get_entries - gang pagecache lookup
2009 * @mapping: The address_space to search
2010 * @start: The starting page cache index
2011 * @end: The final page index (inclusive).
2012 * @pvec: Where the resulting entries are placed.
2013 * @indices: The cache indices corresponding to the entries in @entries
2015 * find_get_entries() will search for and return a batch of entries in
2016 * the mapping. The entries are placed in @pvec. find_get_entries()
2017 * takes a reference on any actual pages it returns.
2019 * The search returns a group of mapping-contiguous page cache entries
2020 * with ascending indexes. There may be holes in the indices due to
2021 * not-present pages.
2023 * Any shadow entries of evicted pages, or swap entries from
2024 * shmem/tmpfs, are included in the returned array.
2026 * If it finds a Transparent Huge Page, head or tail, find_get_entries()
2027 * stops at that page: the caller is likely to have a better way to handle
2028 * the compound page as a whole, and then skip its extent, than repeatedly
2029 * calling find_get_entries() to return all its tails.
2031 * Return: the number of pages and shadow entries which were found.
2033 unsigned find_get_entries(struct address_space
*mapping
, pgoff_t start
,
2034 pgoff_t end
, struct pagevec
*pvec
, pgoff_t
*indices
)
2036 XA_STATE(xas
, &mapping
->i_pages
, start
);
2038 unsigned int ret
= 0;
2039 unsigned nr_entries
= PAGEVEC_SIZE
;
2042 while ((page
= find_get_entry(&xas
, end
, XA_PRESENT
))) {
2044 * Terminate early on finding a THP, to allow the caller to
2045 * handle it all at once; but continue if this is hugetlbfs.
2047 if (!xa_is_value(page
) && PageTransHuge(page
) &&
2049 page
= find_subpage(page
, xas
.xa_index
);
2050 nr_entries
= ret
+ 1;
2053 indices
[ret
] = xas
.xa_index
;
2054 pvec
->pages
[ret
] = page
;
2055 if (++ret
== nr_entries
)
2065 * find_lock_entries - Find a batch of pagecache entries.
2066 * @mapping: The address_space to search.
2067 * @start: The starting page cache index.
2068 * @end: The final page index (inclusive).
2069 * @pvec: Where the resulting entries are placed.
2070 * @indices: The cache indices of the entries in @pvec.
2072 * find_lock_entries() will return a batch of entries from @mapping.
2073 * Swap, shadow and DAX entries are included. Pages are returned
2074 * locked and with an incremented refcount. Pages which are locked by
2075 * somebody else or under writeback are skipped. Only the head page of
2076 * a THP is returned. Pages which are partially outside the range are
2079 * The entries have ascending indexes. The indices may not be consecutive
2080 * due to not-present entries, THP pages, pages which could not be locked
2081 * or pages under writeback.
2083 * Return: The number of entries which were found.
2085 unsigned find_lock_entries(struct address_space
*mapping
, pgoff_t start
,
2086 pgoff_t end
, struct pagevec
*pvec
, pgoff_t
*indices
)
2088 XA_STATE(xas
, &mapping
->i_pages
, start
);
2092 while ((page
= find_get_entry(&xas
, end
, XA_PRESENT
))) {
2093 if (!xa_is_value(page
)) {
2094 if (page
->index
< start
)
2096 if (page
->index
+ thp_nr_pages(page
) - 1 > end
)
2098 if (!trylock_page(page
))
2100 if (page
->mapping
!= mapping
|| PageWriteback(page
))
2102 VM_BUG_ON_PAGE(!thp_contains(page
, xas
.xa_index
),
2105 indices
[pvec
->nr
] = xas
.xa_index
;
2106 if (!pagevec_add(pvec
, page
))
2114 if (!xa_is_value(page
) && PageTransHuge(page
)) {
2115 unsigned int nr_pages
= thp_nr_pages(page
);
2117 /* Final THP may cross MAX_LFS_FILESIZE on 32-bit */
2118 xas_set(&xas
, page
->index
+ nr_pages
);
2119 if (xas
.xa_index
< nr_pages
)
2125 return pagevec_count(pvec
);
2129 * find_get_pages_range - gang pagecache lookup
2130 * @mapping: The address_space to search
2131 * @start: The starting page index
2132 * @end: The final page index (inclusive)
2133 * @nr_pages: The maximum number of pages
2134 * @pages: Where the resulting pages are placed
2136 * find_get_pages_range() will search for and return a group of up to @nr_pages
2137 * pages in the mapping starting at index @start and up to index @end
2138 * (inclusive). The pages are placed at @pages. find_get_pages_range() takes
2139 * a reference against the returned pages.
2141 * The search returns a group of mapping-contiguous pages with ascending
2142 * indexes. There may be holes in the indices due to not-present pages.
2143 * We also update @start to index the next page for the traversal.
2145 * Return: the number of pages which were found. If this number is
2146 * smaller than @nr_pages, the end of specified range has been
2149 unsigned find_get_pages_range(struct address_space
*mapping
, pgoff_t
*start
,
2150 pgoff_t end
, unsigned int nr_pages
,
2151 struct page
**pages
)
2153 XA_STATE(xas
, &mapping
->i_pages
, *start
);
2157 if (unlikely(!nr_pages
))
2161 while ((page
= find_get_entry(&xas
, end
, XA_PRESENT
))) {
2162 /* Skip over shadow, swap and DAX entries */
2163 if (xa_is_value(page
))
2166 pages
[ret
] = find_subpage(page
, xas
.xa_index
);
2167 if (++ret
== nr_pages
) {
2168 *start
= xas
.xa_index
+ 1;
2174 * We come here when there is no page beyond @end. We take care to not
2175 * overflow the index @start as it confuses some of the callers. This
2176 * breaks the iteration when there is a page at index -1 but that is
2177 * already broken anyway.
2179 if (end
== (pgoff_t
)-1)
2180 *start
= (pgoff_t
)-1;
2190 * find_get_pages_contig - gang contiguous pagecache lookup
2191 * @mapping: The address_space to search
2192 * @index: The starting page index
2193 * @nr_pages: The maximum number of pages
2194 * @pages: Where the resulting pages are placed
2196 * find_get_pages_contig() works exactly like find_get_pages(), except
2197 * that the returned number of pages are guaranteed to be contiguous.
2199 * Return: the number of pages which were found.
2201 unsigned find_get_pages_contig(struct address_space
*mapping
, pgoff_t index
,
2202 unsigned int nr_pages
, struct page
**pages
)
2204 XA_STATE(xas
, &mapping
->i_pages
, index
);
2206 unsigned int ret
= 0;
2208 if (unlikely(!nr_pages
))
2212 for (page
= xas_load(&xas
); page
; page
= xas_next(&xas
)) {
2213 if (xas_retry(&xas
, page
))
2216 * If the entry has been swapped out, we can stop looking.
2217 * No current caller is looking for DAX entries.
2219 if (xa_is_value(page
))
2222 if (!page_cache_get_speculative(page
))
2225 /* Has the page moved or been split? */
2226 if (unlikely(page
!= xas_reload(&xas
)))
2229 pages
[ret
] = find_subpage(page
, xas
.xa_index
);
2230 if (++ret
== nr_pages
)
2241 EXPORT_SYMBOL(find_get_pages_contig
);
2244 * find_get_pages_range_tag - Find and return head pages matching @tag.
2245 * @mapping: the address_space to search
2246 * @index: the starting page index
2247 * @end: The final page index (inclusive)
2248 * @tag: the tag index
2249 * @nr_pages: the maximum number of pages
2250 * @pages: where the resulting pages are placed
2252 * Like find_get_pages(), except we only return head pages which are tagged
2253 * with @tag. @index is updated to the index immediately after the last
2254 * page we return, ready for the next iteration.
2256 * Return: the number of pages which were found.
2258 unsigned find_get_pages_range_tag(struct address_space
*mapping
, pgoff_t
*index
,
2259 pgoff_t end
, xa_mark_t tag
, unsigned int nr_pages
,
2260 struct page
**pages
)
2262 XA_STATE(xas
, &mapping
->i_pages
, *index
);
2266 if (unlikely(!nr_pages
))
2270 while ((page
= find_get_entry(&xas
, end
, tag
))) {
2272 * Shadow entries should never be tagged, but this iteration
2273 * is lockless so there is a window for page reclaim to evict
2274 * a page we saw tagged. Skip over it.
2276 if (xa_is_value(page
))
2280 if (++ret
== nr_pages
) {
2281 *index
= page
->index
+ thp_nr_pages(page
);
2287 * We come here when we got to @end. We take care to not overflow the
2288 * index @index as it confuses some of the callers. This breaks the
2289 * iteration when there is a page at index -1 but that is already
2292 if (end
== (pgoff_t
)-1)
2293 *index
= (pgoff_t
)-1;
2301 EXPORT_SYMBOL(find_get_pages_range_tag
);
2304 * CD/DVDs are error prone. When a medium error occurs, the driver may fail
2305 * a _large_ part of the i/o request. Imagine the worst scenario:
2307 * ---R__________________________________________B__________
2308 * ^ reading here ^ bad block(assume 4k)
2310 * read(R) => miss => readahead(R...B) => media error => frustrating retries
2311 * => failing the whole request => read(R) => read(R+1) =>
2312 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
2313 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
2314 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
2316 * It is going insane. Fix it by quickly scaling down the readahead size.
2318 static void shrink_readahead_size_eio(struct file_ra_state
*ra
)
2324 * filemap_get_read_batch - Get a batch of pages for read
2326 * Get a batch of pages which represent a contiguous range of bytes
2327 * in the file. No tail pages will be returned. If @index is in the
2328 * middle of a THP, the entire THP will be returned. The last page in
2329 * the batch may have Readahead set or be not Uptodate so that the
2330 * caller can take the appropriate action.
2332 static void filemap_get_read_batch(struct address_space
*mapping
,
2333 pgoff_t index
, pgoff_t max
, struct pagevec
*pvec
)
2335 XA_STATE(xas
, &mapping
->i_pages
, index
);
2339 for (head
= xas_load(&xas
); head
; head
= xas_next(&xas
)) {
2340 if (xas_retry(&xas
, head
))
2342 if (xas
.xa_index
> max
|| xa_is_value(head
))
2344 if (!page_cache_get_speculative(head
))
2347 /* Has the page moved or been split? */
2348 if (unlikely(head
!= xas_reload(&xas
)))
2351 if (!pagevec_add(pvec
, head
))
2353 if (!PageUptodate(head
))
2355 if (PageReadahead(head
))
2357 if (PageHead(head
)) {
2358 xas_set(&xas
, head
->index
+ thp_nr_pages(head
));
2359 /* Handle wrap correctly */
2360 if (xas
.xa_index
- 1 >= max
)
2372 static int filemap_read_page(struct file
*file
, struct address_space
*mapping
,
2378 * A previous I/O error may have been due to temporary failures,
2379 * eg. multipath errors. PG_error will be set again if readpage
2382 ClearPageError(page
);
2383 /* Start the actual read. The read will unlock the page. */
2384 error
= mapping
->a_ops
->readpage(file
, page
);
2388 error
= wait_on_page_locked_killable(page
);
2391 if (PageUptodate(page
))
2393 shrink_readahead_size_eio(&file
->f_ra
);
2397 static bool filemap_range_uptodate(struct address_space
*mapping
,
2398 loff_t pos
, struct iov_iter
*iter
, struct page
*page
)
2402 if (PageUptodate(page
))
2404 /* pipes can't handle partially uptodate pages */
2405 if (iov_iter_is_pipe(iter
))
2407 if (!mapping
->a_ops
->is_partially_uptodate
)
2409 if (mapping
->host
->i_blkbits
>= (PAGE_SHIFT
+ thp_order(page
)))
2412 count
= iter
->count
;
2413 if (page_offset(page
) > pos
) {
2414 count
-= page_offset(page
) - pos
;
2417 pos
-= page_offset(page
);
2420 return mapping
->a_ops
->is_partially_uptodate(page
, pos
, count
);
2423 static int filemap_update_page(struct kiocb
*iocb
,
2424 struct address_space
*mapping
, struct iov_iter
*iter
,
2429 if (iocb
->ki_flags
& IOCB_NOWAIT
) {
2430 if (!filemap_invalidate_trylock_shared(mapping
))
2433 filemap_invalidate_lock_shared(mapping
);
2436 if (!trylock_page(page
)) {
2438 if (iocb
->ki_flags
& (IOCB_NOWAIT
| IOCB_NOIO
))
2439 goto unlock_mapping
;
2440 if (!(iocb
->ki_flags
& IOCB_WAITQ
)) {
2441 filemap_invalidate_unlock_shared(mapping
);
2442 put_and_wait_on_page_locked(page
, TASK_KILLABLE
);
2443 return AOP_TRUNCATED_PAGE
;
2445 error
= __lock_page_async(page
, iocb
->ki_waitq
);
2447 goto unlock_mapping
;
2450 error
= AOP_TRUNCATED_PAGE
;
2455 if (filemap_range_uptodate(mapping
, iocb
->ki_pos
, iter
, page
))
2459 if (iocb
->ki_flags
& (IOCB_NOIO
| IOCB_NOWAIT
| IOCB_WAITQ
))
2462 error
= filemap_read_page(iocb
->ki_filp
, mapping
, page
);
2463 goto unlock_mapping
;
2467 filemap_invalidate_unlock_shared(mapping
);
2468 if (error
== AOP_TRUNCATED_PAGE
)
2473 static int filemap_create_page(struct file
*file
,
2474 struct address_space
*mapping
, pgoff_t index
,
2475 struct pagevec
*pvec
)
2480 page
= page_cache_alloc(mapping
);
2485 * Protect against truncate / hole punch. Grabbing invalidate_lock here
2486 * assures we cannot instantiate and bring uptodate new pagecache pages
2487 * after evicting page cache during truncate and before actually
2488 * freeing blocks. Note that we could release invalidate_lock after
2489 * inserting the page into page cache as the locked page would then be
2490 * enough to synchronize with hole punching. But there are code paths
2491 * such as filemap_update_page() filling in partially uptodate pages or
2492 * ->readpages() that need to hold invalidate_lock while mapping blocks
2493 * for IO so let's hold the lock here as well to keep locking rules
2496 filemap_invalidate_lock_shared(mapping
);
2497 error
= add_to_page_cache_lru(page
, mapping
, index
,
2498 mapping_gfp_constraint(mapping
, GFP_KERNEL
));
2499 if (error
== -EEXIST
)
2500 error
= AOP_TRUNCATED_PAGE
;
2504 error
= filemap_read_page(file
, mapping
, page
);
2508 filemap_invalidate_unlock_shared(mapping
);
2509 pagevec_add(pvec
, page
);
2512 filemap_invalidate_unlock_shared(mapping
);
2517 static int filemap_readahead(struct kiocb
*iocb
, struct file
*file
,
2518 struct address_space
*mapping
, struct page
*page
,
2521 if (iocb
->ki_flags
& IOCB_NOIO
)
2523 page_cache_async_readahead(mapping
, &file
->f_ra
, file
, page
,
2524 page
->index
, last_index
- page
->index
);
2528 static int filemap_get_pages(struct kiocb
*iocb
, struct iov_iter
*iter
,
2529 struct pagevec
*pvec
)
2531 struct file
*filp
= iocb
->ki_filp
;
2532 struct address_space
*mapping
= filp
->f_mapping
;
2533 struct file_ra_state
*ra
= &filp
->f_ra
;
2534 pgoff_t index
= iocb
->ki_pos
>> PAGE_SHIFT
;
2539 last_index
= DIV_ROUND_UP(iocb
->ki_pos
+ iter
->count
, PAGE_SIZE
);
2541 if (fatal_signal_pending(current
))
2544 filemap_get_read_batch(mapping
, index
, last_index
, pvec
);
2545 if (!pagevec_count(pvec
)) {
2546 if (iocb
->ki_flags
& IOCB_NOIO
)
2548 page_cache_sync_readahead(mapping
, ra
, filp
, index
,
2549 last_index
- index
);
2550 filemap_get_read_batch(mapping
, index
, last_index
, pvec
);
2552 if (!pagevec_count(pvec
)) {
2553 if (iocb
->ki_flags
& (IOCB_NOWAIT
| IOCB_WAITQ
))
2555 err
= filemap_create_page(filp
, mapping
,
2556 iocb
->ki_pos
>> PAGE_SHIFT
, pvec
);
2557 if (err
== AOP_TRUNCATED_PAGE
)
2562 page
= pvec
->pages
[pagevec_count(pvec
) - 1];
2563 if (PageReadahead(page
)) {
2564 err
= filemap_readahead(iocb
, filp
, mapping
, page
, last_index
);
2568 if (!PageUptodate(page
)) {
2569 if ((iocb
->ki_flags
& IOCB_WAITQ
) && pagevec_count(pvec
) > 1)
2570 iocb
->ki_flags
|= IOCB_NOWAIT
;
2571 err
= filemap_update_page(iocb
, mapping
, iter
, page
);
2580 if (likely(--pvec
->nr
))
2582 if (err
== AOP_TRUNCATED_PAGE
)
2588 * filemap_read - Read data from the page cache.
2589 * @iocb: The iocb to read.
2590 * @iter: Destination for the data.
2591 * @already_read: Number of bytes already read by the caller.
2593 * Copies data from the page cache. If the data is not currently present,
2594 * uses the readahead and readpage address_space operations to fetch it.
2596 * Return: Total number of bytes copied, including those already read by
2597 * the caller. If an error happens before any bytes are copied, returns
2598 * a negative error number.
2600 ssize_t
filemap_read(struct kiocb
*iocb
, struct iov_iter
*iter
,
2601 ssize_t already_read
)
2603 struct file
*filp
= iocb
->ki_filp
;
2604 struct file_ra_state
*ra
= &filp
->f_ra
;
2605 struct address_space
*mapping
= filp
->f_mapping
;
2606 struct inode
*inode
= mapping
->host
;
2607 struct pagevec pvec
;
2609 bool writably_mapped
;
2610 loff_t isize
, end_offset
;
2612 if (unlikely(iocb
->ki_pos
>= inode
->i_sb
->s_maxbytes
))
2614 if (unlikely(!iov_iter_count(iter
)))
2617 iov_iter_truncate(iter
, inode
->i_sb
->s_maxbytes
);
2618 pagevec_init(&pvec
);
2624 * If we've already successfully copied some data, then we
2625 * can no longer safely return -EIOCBQUEUED. Hence mark
2626 * an async read NOWAIT at that point.
2628 if ((iocb
->ki_flags
& IOCB_WAITQ
) && already_read
)
2629 iocb
->ki_flags
|= IOCB_NOWAIT
;
2631 error
= filemap_get_pages(iocb
, iter
, &pvec
);
2636 * i_size must be checked after we know the pages are Uptodate.
2638 * Checking i_size after the check allows us to calculate
2639 * the correct value for "nr", which means the zero-filled
2640 * part of the page is not copied back to userspace (unless
2641 * another truncate extends the file - this is desired though).
2643 isize
= i_size_read(inode
);
2644 if (unlikely(iocb
->ki_pos
>= isize
))
2646 end_offset
= min_t(loff_t
, isize
, iocb
->ki_pos
+ iter
->count
);
2649 * Once we start copying data, we don't want to be touching any
2650 * cachelines that might be contended:
2652 writably_mapped
= mapping_writably_mapped(mapping
);
2655 * When a sequential read accesses a page several times, only
2656 * mark it as accessed the first time.
2658 if (iocb
->ki_pos
>> PAGE_SHIFT
!=
2659 ra
->prev_pos
>> PAGE_SHIFT
)
2660 mark_page_accessed(pvec
.pages
[0]);
2662 for (i
= 0; i
< pagevec_count(&pvec
); i
++) {
2663 struct page
*page
= pvec
.pages
[i
];
2664 size_t page_size
= thp_size(page
);
2665 size_t offset
= iocb
->ki_pos
& (page_size
- 1);
2666 size_t bytes
= min_t(loff_t
, end_offset
- iocb
->ki_pos
,
2667 page_size
- offset
);
2670 if (end_offset
< page_offset(page
))
2673 mark_page_accessed(page
);
2675 * If users can be writing to this page using arbitrary
2676 * virtual addresses, take care about potential aliasing
2677 * before reading the page on the kernel side.
2679 if (writably_mapped
) {
2682 for (j
= 0; j
< thp_nr_pages(page
); j
++)
2683 flush_dcache_page(page
+ j
);
2686 copied
= copy_page_to_iter(page
, offset
, bytes
, iter
);
2688 already_read
+= copied
;
2689 iocb
->ki_pos
+= copied
;
2690 ra
->prev_pos
= iocb
->ki_pos
;
2692 if (copied
< bytes
) {
2698 for (i
= 0; i
< pagevec_count(&pvec
); i
++)
2699 put_page(pvec
.pages
[i
]);
2700 pagevec_reinit(&pvec
);
2701 } while (iov_iter_count(iter
) && iocb
->ki_pos
< isize
&& !error
);
2703 file_accessed(filp
);
2705 return already_read
? already_read
: error
;
2707 EXPORT_SYMBOL_GPL(filemap_read
);
2710 * generic_file_read_iter - generic filesystem read routine
2711 * @iocb: kernel I/O control block
2712 * @iter: destination for the data read
2714 * This is the "read_iter()" routine for all filesystems
2715 * that can use the page cache directly.
2717 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
2718 * be returned when no data can be read without waiting for I/O requests
2719 * to complete; it doesn't prevent readahead.
2721 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
2722 * requests shall be made for the read or for readahead. When no data
2723 * can be read, -EAGAIN shall be returned. When readahead would be
2724 * triggered, a partial, possibly empty read shall be returned.
2727 * * number of bytes copied, even for partial reads
2728 * * negative error code (or 0 if IOCB_NOIO) if nothing was read
2731 generic_file_read_iter(struct kiocb
*iocb
, struct iov_iter
*iter
)
2733 size_t count
= iov_iter_count(iter
);
2737 return 0; /* skip atime */
2739 if (iocb
->ki_flags
& IOCB_DIRECT
) {
2740 struct file
*file
= iocb
->ki_filp
;
2741 struct address_space
*mapping
= file
->f_mapping
;
2742 struct inode
*inode
= mapping
->host
;
2745 size
= i_size_read(inode
);
2746 if (iocb
->ki_flags
& IOCB_NOWAIT
) {
2747 if (filemap_range_needs_writeback(mapping
, iocb
->ki_pos
,
2748 iocb
->ki_pos
+ count
- 1))
2751 retval
= filemap_write_and_wait_range(mapping
,
2753 iocb
->ki_pos
+ count
- 1);
2758 file_accessed(file
);
2760 retval
= mapping
->a_ops
->direct_IO(iocb
, iter
);
2762 iocb
->ki_pos
+= retval
;
2765 if (retval
!= -EIOCBQUEUED
)
2766 iov_iter_revert(iter
, count
- iov_iter_count(iter
));
2769 * Btrfs can have a short DIO read if we encounter
2770 * compressed extents, so if there was an error, or if
2771 * we've already read everything we wanted to, or if
2772 * there was a short read because we hit EOF, go ahead
2773 * and return. Otherwise fallthrough to buffered io for
2774 * the rest of the read. Buffered reads will not work for
2775 * DAX files, so don't bother trying.
2777 if (retval
< 0 || !count
|| iocb
->ki_pos
>= size
||
2782 return filemap_read(iocb
, iter
, retval
);
2784 EXPORT_SYMBOL(generic_file_read_iter
);
2786 static inline loff_t
page_seek_hole_data(struct xa_state
*xas
,
2787 struct address_space
*mapping
, struct page
*page
,
2788 loff_t start
, loff_t end
, bool seek_data
)
2790 const struct address_space_operations
*ops
= mapping
->a_ops
;
2791 size_t offset
, bsz
= i_blocksize(mapping
->host
);
2793 if (xa_is_value(page
) || PageUptodate(page
))
2794 return seek_data
? start
: end
;
2795 if (!ops
->is_partially_uptodate
)
2796 return seek_data
? end
: start
;
2801 if (unlikely(page
->mapping
!= mapping
))
2804 offset
= offset_in_thp(page
, start
) & ~(bsz
- 1);
2807 if (ops
->is_partially_uptodate(page
, offset
, bsz
) == seek_data
)
2809 start
= (start
+ bsz
) & ~(bsz
- 1);
2811 } while (offset
< thp_size(page
));
2819 unsigned int seek_page_size(struct xa_state
*xas
, struct page
*page
)
2821 if (xa_is_value(page
))
2822 return PAGE_SIZE
<< xa_get_order(xas
->xa
, xas
->xa_index
);
2823 return thp_size(page
);
2827 * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache.
2828 * @mapping: Address space to search.
2829 * @start: First byte to consider.
2830 * @end: Limit of search (exclusive).
2831 * @whence: Either SEEK_HOLE or SEEK_DATA.
2833 * If the page cache knows which blocks contain holes and which blocks
2834 * contain data, your filesystem can use this function to implement
2835 * SEEK_HOLE and SEEK_DATA. This is useful for filesystems which are
2836 * entirely memory-based such as tmpfs, and filesystems which support
2837 * unwritten extents.
2839 * Return: The requested offset on success, or -ENXIO if @whence specifies
2840 * SEEK_DATA and there is no data after @start. There is an implicit hole
2841 * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start
2842 * and @end contain data.
2844 loff_t
mapping_seek_hole_data(struct address_space
*mapping
, loff_t start
,
2845 loff_t end
, int whence
)
2847 XA_STATE(xas
, &mapping
->i_pages
, start
>> PAGE_SHIFT
);
2848 pgoff_t max
= (end
- 1) >> PAGE_SHIFT
;
2849 bool seek_data
= (whence
== SEEK_DATA
);
2856 while ((page
= find_get_entry(&xas
, max
, XA_PRESENT
))) {
2857 loff_t pos
= (u64
)xas
.xa_index
<< PAGE_SHIFT
;
2858 unsigned int seek_size
;
2866 seek_size
= seek_page_size(&xas
, page
);
2867 pos
= round_up(pos
+ 1, seek_size
);
2868 start
= page_seek_hole_data(&xas
, mapping
, page
, start
, pos
,
2874 if (seek_size
> PAGE_SIZE
)
2875 xas_set(&xas
, pos
>> PAGE_SHIFT
);
2876 if (!xa_is_value(page
))
2883 if (page
&& !xa_is_value(page
))
2891 #define MMAP_LOTSAMISS (100)
2893 * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
2894 * @vmf - the vm_fault for this fault.
2895 * @page - the page to lock.
2896 * @fpin - the pointer to the file we may pin (or is already pinned).
2898 * This works similar to lock_page_or_retry in that it can drop the mmap_lock.
2899 * It differs in that it actually returns the page locked if it returns 1 and 0
2900 * if it couldn't lock the page. If we did have to drop the mmap_lock then fpin
2901 * will point to the pinned file and needs to be fput()'ed at a later point.
2903 static int lock_page_maybe_drop_mmap(struct vm_fault
*vmf
, struct page
*page
,
2906 if (trylock_page(page
))
2910 * NOTE! This will make us return with VM_FAULT_RETRY, but with
2911 * the mmap_lock still held. That's how FAULT_FLAG_RETRY_NOWAIT
2912 * is supposed to work. We have way too many special cases..
2914 if (vmf
->flags
& FAULT_FLAG_RETRY_NOWAIT
)
2917 *fpin
= maybe_unlock_mmap_for_io(vmf
, *fpin
);
2918 if (vmf
->flags
& FAULT_FLAG_KILLABLE
) {
2919 if (__lock_page_killable(page
)) {
2921 * We didn't have the right flags to drop the mmap_lock,
2922 * but all fault_handlers only check for fatal signals
2923 * if we return VM_FAULT_RETRY, so we need to drop the
2924 * mmap_lock here and return 0 if we don't have a fpin.
2927 mmap_read_unlock(vmf
->vma
->vm_mm
);
2937 * Synchronous readahead happens when we don't even find a page in the page
2938 * cache at all. We don't want to perform IO under the mmap sem, so if we have
2939 * to drop the mmap sem we return the file that was pinned in order for us to do
2940 * that. If we didn't pin a file then we return NULL. The file that is
2941 * returned needs to be fput()'ed when we're done with it.
2943 static struct file
*do_sync_mmap_readahead(struct vm_fault
*vmf
)
2945 struct file
*file
= vmf
->vma
->vm_file
;
2946 struct file_ra_state
*ra
= &file
->f_ra
;
2947 struct address_space
*mapping
= file
->f_mapping
;
2948 DEFINE_READAHEAD(ractl
, file
, ra
, mapping
, vmf
->pgoff
);
2949 struct file
*fpin
= NULL
;
2950 unsigned int mmap_miss
;
2952 /* If we don't want any read-ahead, don't bother */
2953 if (vmf
->vma
->vm_flags
& VM_RAND_READ
)
2958 if (vmf
->vma
->vm_flags
& VM_SEQ_READ
) {
2959 fpin
= maybe_unlock_mmap_for_io(vmf
, fpin
);
2960 page_cache_sync_ra(&ractl
, ra
->ra_pages
);
2964 /* Avoid banging the cache line if not needed */
2965 mmap_miss
= READ_ONCE(ra
->mmap_miss
);
2966 if (mmap_miss
< MMAP_LOTSAMISS
* 10)
2967 WRITE_ONCE(ra
->mmap_miss
, ++mmap_miss
);
2970 * Do we miss much more than hit in this file? If so,
2971 * stop bothering with read-ahead. It will only hurt.
2973 if (mmap_miss
> MMAP_LOTSAMISS
)
2979 fpin
= maybe_unlock_mmap_for_io(vmf
, fpin
);
2980 ra
->start
= max_t(long, 0, vmf
->pgoff
- ra
->ra_pages
/ 2);
2981 ra
->size
= ra
->ra_pages
;
2982 ra
->async_size
= ra
->ra_pages
/ 4;
2983 ractl
._index
= ra
->start
;
2984 do_page_cache_ra(&ractl
, ra
->size
, ra
->async_size
);
2989 * Asynchronous readahead happens when we find the page and PG_readahead,
2990 * so we want to possibly extend the readahead further. We return the file that
2991 * was pinned if we have to drop the mmap_lock in order to do IO.
2993 static struct file
*do_async_mmap_readahead(struct vm_fault
*vmf
,
2996 struct file
*file
= vmf
->vma
->vm_file
;
2997 struct file_ra_state
*ra
= &file
->f_ra
;
2998 struct address_space
*mapping
= file
->f_mapping
;
2999 struct file
*fpin
= NULL
;
3000 unsigned int mmap_miss
;
3001 pgoff_t offset
= vmf
->pgoff
;
3003 /* If we don't want any read-ahead, don't bother */
3004 if (vmf
->vma
->vm_flags
& VM_RAND_READ
|| !ra
->ra_pages
)
3006 mmap_miss
= READ_ONCE(ra
->mmap_miss
);
3008 WRITE_ONCE(ra
->mmap_miss
, --mmap_miss
);
3009 if (PageReadahead(page
)) {
3010 fpin
= maybe_unlock_mmap_for_io(vmf
, fpin
);
3011 page_cache_async_readahead(mapping
, ra
, file
,
3012 page
, offset
, ra
->ra_pages
);
3018 * filemap_fault - read in file data for page fault handling
3019 * @vmf: struct vm_fault containing details of the fault
3021 * filemap_fault() is invoked via the vma operations vector for a
3022 * mapped memory region to read in file data during a page fault.
3024 * The goto's are kind of ugly, but this streamlines the normal case of having
3025 * it in the page cache, and handles the special cases reasonably without
3026 * having a lot of duplicated code.
3028 * vma->vm_mm->mmap_lock must be held on entry.
3030 * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock
3031 * may be dropped before doing I/O or by lock_page_maybe_drop_mmap().
3033 * If our return value does not have VM_FAULT_RETRY set, the mmap_lock
3034 * has not been released.
3036 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
3038 * Return: bitwise-OR of %VM_FAULT_ codes.
3040 vm_fault_t
filemap_fault(struct vm_fault
*vmf
)
3043 struct file
*file
= vmf
->vma
->vm_file
;
3044 struct file
*fpin
= NULL
;
3045 struct address_space
*mapping
= file
->f_mapping
;
3046 struct inode
*inode
= mapping
->host
;
3047 pgoff_t offset
= vmf
->pgoff
;
3051 bool mapping_locked
= false;
3053 max_off
= DIV_ROUND_UP(i_size_read(inode
), PAGE_SIZE
);
3054 if (unlikely(offset
>= max_off
))
3055 return VM_FAULT_SIGBUS
;
3058 * Do we have something in the page cache already?
3060 page
= find_get_page(mapping
, offset
);
3063 * We found the page, so try async readahead before waiting for
3066 if (!(vmf
->flags
& FAULT_FLAG_TRIED
))
3067 fpin
= do_async_mmap_readahead(vmf
, page
);
3068 if (unlikely(!PageUptodate(page
))) {
3069 filemap_invalidate_lock_shared(mapping
);
3070 mapping_locked
= true;
3073 /* No page in the page cache at all */
3074 count_vm_event(PGMAJFAULT
);
3075 count_memcg_event_mm(vmf
->vma
->vm_mm
, PGMAJFAULT
);
3076 ret
= VM_FAULT_MAJOR
;
3077 fpin
= do_sync_mmap_readahead(vmf
);
3080 * See comment in filemap_create_page() why we need
3083 if (!mapping_locked
) {
3084 filemap_invalidate_lock_shared(mapping
);
3085 mapping_locked
= true;
3087 page
= pagecache_get_page(mapping
, offset
,
3088 FGP_CREAT
|FGP_FOR_MMAP
,
3093 filemap_invalidate_unlock_shared(mapping
);
3094 return VM_FAULT_OOM
;
3098 if (!lock_page_maybe_drop_mmap(vmf
, page
, &fpin
))
3101 /* Did it get truncated? */
3102 if (unlikely(compound_head(page
)->mapping
!= mapping
)) {
3107 VM_BUG_ON_PAGE(page_to_pgoff(page
) != offset
, page
);
3110 * We have a locked page in the page cache, now we need to check
3111 * that it's up-to-date. If not, it is going to be due to an error.
3113 if (unlikely(!PageUptodate(page
))) {
3115 * The page was in cache and uptodate and now it is not.
3116 * Strange but possible since we didn't hold the page lock all
3117 * the time. Let's drop everything get the invalidate lock and
3120 if (!mapping_locked
) {
3125 goto page_not_uptodate
;
3129 * We've made it this far and we had to drop our mmap_lock, now is the
3130 * time to return to the upper layer and have it re-find the vma and
3138 filemap_invalidate_unlock_shared(mapping
);
3141 * Found the page and have a reference on it.
3142 * We must recheck i_size under page lock.
3144 max_off
= DIV_ROUND_UP(i_size_read(inode
), PAGE_SIZE
);
3145 if (unlikely(offset
>= max_off
)) {
3148 return VM_FAULT_SIGBUS
;
3152 return ret
| VM_FAULT_LOCKED
;
3156 * Umm, take care of errors if the page isn't up-to-date.
3157 * Try to re-read it _once_. We do this synchronously,
3158 * because there really aren't any performance issues here
3159 * and we need to check for errors.
3161 fpin
= maybe_unlock_mmap_for_io(vmf
, fpin
);
3162 error
= filemap_read_page(file
, mapping
, page
);
3167 if (!error
|| error
== AOP_TRUNCATED_PAGE
)
3169 filemap_invalidate_unlock_shared(mapping
);
3171 return VM_FAULT_SIGBUS
;
3175 * We dropped the mmap_lock, we need to return to the fault handler to
3176 * re-find the vma and come back and find our hopefully still populated
3182 filemap_invalidate_unlock_shared(mapping
);
3185 return ret
| VM_FAULT_RETRY
;
3187 EXPORT_SYMBOL(filemap_fault
);
3189 static bool filemap_map_pmd(struct vm_fault
*vmf
, struct page
*page
)
3191 struct mm_struct
*mm
= vmf
->vma
->vm_mm
;
3193 /* Huge page is mapped? No need to proceed. */
3194 if (pmd_trans_huge(*vmf
->pmd
)) {
3200 if (pmd_none(*vmf
->pmd
) && PageTransHuge(page
)) {
3201 vm_fault_t ret
= do_set_pmd(vmf
, page
);
3203 /* The page is mapped successfully, reference consumed. */
3209 if (pmd_none(*vmf
->pmd
)) {
3210 vmf
->ptl
= pmd_lock(mm
, vmf
->pmd
);
3211 if (likely(pmd_none(*vmf
->pmd
))) {
3213 pmd_populate(mm
, vmf
->pmd
, vmf
->prealloc_pte
);
3214 vmf
->prealloc_pte
= NULL
;
3216 spin_unlock(vmf
->ptl
);
3219 /* See comment in handle_pte_fault() */
3220 if (pmd_devmap_trans_unstable(vmf
->pmd
)) {
3229 static struct page
*next_uptodate_page(struct page
*page
,
3230 struct address_space
*mapping
,
3231 struct xa_state
*xas
, pgoff_t end_pgoff
)
3233 unsigned long max_idx
;
3238 if (xas_retry(xas
, page
))
3240 if (xa_is_value(page
))
3242 if (PageLocked(page
))
3244 if (!page_cache_get_speculative(page
))
3246 /* Has the page moved or been split? */
3247 if (unlikely(page
!= xas_reload(xas
)))
3249 if (!PageUptodate(page
) || PageReadahead(page
))
3251 if (PageHWPoison(page
))
3253 if (!trylock_page(page
))
3255 if (page
->mapping
!= mapping
)
3257 if (!PageUptodate(page
))
3259 max_idx
= DIV_ROUND_UP(i_size_read(mapping
->host
), PAGE_SIZE
);
3260 if (xas
->xa_index
>= max_idx
)
3267 } while ((page
= xas_next_entry(xas
, end_pgoff
)) != NULL
);
3272 static inline struct page
*first_map_page(struct address_space
*mapping
,
3273 struct xa_state
*xas
,
3276 return next_uptodate_page(xas_find(xas
, end_pgoff
),
3277 mapping
, xas
, end_pgoff
);
3280 static inline struct page
*next_map_page(struct address_space
*mapping
,
3281 struct xa_state
*xas
,
3284 return next_uptodate_page(xas_next_entry(xas
, end_pgoff
),
3285 mapping
, xas
, end_pgoff
);
3288 vm_fault_t
filemap_map_pages(struct vm_fault
*vmf
,
3289 pgoff_t start_pgoff
, pgoff_t end_pgoff
)
3291 struct vm_area_struct
*vma
= vmf
->vma
;
3292 struct file
*file
= vma
->vm_file
;
3293 struct address_space
*mapping
= file
->f_mapping
;
3294 pgoff_t last_pgoff
= start_pgoff
;
3296 XA_STATE(xas
, &mapping
->i_pages
, start_pgoff
);
3297 struct page
*head
, *page
;
3298 unsigned int mmap_miss
= READ_ONCE(file
->f_ra
.mmap_miss
);
3302 head
= first_map_page(mapping
, &xas
, end_pgoff
);
3306 if (filemap_map_pmd(vmf
, head
)) {
3307 ret
= VM_FAULT_NOPAGE
;
3311 addr
= vma
->vm_start
+ ((start_pgoff
- vma
->vm_pgoff
) << PAGE_SHIFT
);
3312 vmf
->pte
= pte_offset_map_lock(vma
->vm_mm
, vmf
->pmd
, addr
, &vmf
->ptl
);
3314 page
= find_subpage(head
, xas
.xa_index
);
3315 if (PageHWPoison(page
))
3321 addr
+= (xas
.xa_index
- last_pgoff
) << PAGE_SHIFT
;
3322 vmf
->pte
+= xas
.xa_index
- last_pgoff
;
3323 last_pgoff
= xas
.xa_index
;
3325 if (!pte_none(*vmf
->pte
))
3328 /* We're about to handle the fault */
3329 if (vmf
->address
== addr
)
3330 ret
= VM_FAULT_NOPAGE
;
3332 do_set_pte(vmf
, page
, addr
);
3333 /* no need to invalidate: a not-present page won't be cached */
3334 update_mmu_cache(vma
, addr
, vmf
->pte
);
3340 } while ((head
= next_map_page(mapping
, &xas
, end_pgoff
)) != NULL
);
3341 pte_unmap_unlock(vmf
->pte
, vmf
->ptl
);
3344 WRITE_ONCE(file
->f_ra
.mmap_miss
, mmap_miss
);
3347 EXPORT_SYMBOL(filemap_map_pages
);
3349 vm_fault_t
filemap_page_mkwrite(struct vm_fault
*vmf
)
3351 struct address_space
*mapping
= vmf
->vma
->vm_file
->f_mapping
;
3352 struct page
*page
= vmf
->page
;
3353 vm_fault_t ret
= VM_FAULT_LOCKED
;
3355 sb_start_pagefault(mapping
->host
->i_sb
);
3356 vma_file_update_time(vmf
->vma
);
3358 if (page
->mapping
!= mapping
) {
3360 ret
= VM_FAULT_NOPAGE
;
3364 * We mark the page dirty already here so that when freeze is in
3365 * progress, we are guaranteed that writeback during freezing will
3366 * see the dirty page and writeprotect it again.
3368 set_page_dirty(page
);
3369 wait_for_stable_page(page
);
3371 sb_end_pagefault(mapping
->host
->i_sb
);
3375 const struct vm_operations_struct generic_file_vm_ops
= {
3376 .fault
= filemap_fault
,
3377 .map_pages
= filemap_map_pages
,
3378 .page_mkwrite
= filemap_page_mkwrite
,
3381 /* This is used for a general mmap of a disk file */
3383 int generic_file_mmap(struct file
*file
, struct vm_area_struct
*vma
)
3385 struct address_space
*mapping
= file
->f_mapping
;
3387 if (!mapping
->a_ops
->readpage
)
3389 file_accessed(file
);
3390 vma
->vm_ops
= &generic_file_vm_ops
;
3395 * This is for filesystems which do not implement ->writepage.
3397 int generic_file_readonly_mmap(struct file
*file
, struct vm_area_struct
*vma
)
3399 if ((vma
->vm_flags
& VM_SHARED
) && (vma
->vm_flags
& VM_MAYWRITE
))
3401 return generic_file_mmap(file
, vma
);
3404 vm_fault_t
filemap_page_mkwrite(struct vm_fault
*vmf
)
3406 return VM_FAULT_SIGBUS
;
3408 int generic_file_mmap(struct file
*file
, struct vm_area_struct
*vma
)
3412 int generic_file_readonly_mmap(struct file
*file
, struct vm_area_struct
*vma
)
3416 #endif /* CONFIG_MMU */
3418 EXPORT_SYMBOL(filemap_page_mkwrite
);
3419 EXPORT_SYMBOL(generic_file_mmap
);
3420 EXPORT_SYMBOL(generic_file_readonly_mmap
);
3422 static struct page
*wait_on_page_read(struct page
*page
)
3424 if (!IS_ERR(page
)) {
3425 wait_on_page_locked(page
);
3426 if (!PageUptodate(page
)) {
3428 page
= ERR_PTR(-EIO
);
3434 static struct page
*do_read_cache_page(struct address_space
*mapping
,
3436 int (*filler
)(void *, struct page
*),
3443 page
= find_get_page(mapping
, index
);
3445 page
= __page_cache_alloc(gfp
);
3447 return ERR_PTR(-ENOMEM
);
3448 err
= add_to_page_cache_lru(page
, mapping
, index
, gfp
);
3449 if (unlikely(err
)) {
3453 /* Presumably ENOMEM for xarray node */
3454 return ERR_PTR(err
);
3459 err
= filler(data
, page
);
3461 err
= mapping
->a_ops
->readpage(data
, page
);
3465 return ERR_PTR(err
);
3468 page
= wait_on_page_read(page
);
3473 if (PageUptodate(page
))
3477 * Page is not up to date and may be locked due to one of the following
3478 * case a: Page is being filled and the page lock is held
3479 * case b: Read/write error clearing the page uptodate status
3480 * case c: Truncation in progress (page locked)
3481 * case d: Reclaim in progress
3483 * Case a, the page will be up to date when the page is unlocked.
3484 * There is no need to serialise on the page lock here as the page
3485 * is pinned so the lock gives no additional protection. Even if the
3486 * page is truncated, the data is still valid if PageUptodate as
3487 * it's a race vs truncate race.
3488 * Case b, the page will not be up to date
3489 * Case c, the page may be truncated but in itself, the data may still
3490 * be valid after IO completes as it's a read vs truncate race. The
3491 * operation must restart if the page is not uptodate on unlock but
3492 * otherwise serialising on page lock to stabilise the mapping gives
3493 * no additional guarantees to the caller as the page lock is
3494 * released before return.
3495 * Case d, similar to truncation. If reclaim holds the page lock, it
3496 * will be a race with remove_mapping that determines if the mapping
3497 * is valid on unlock but otherwise the data is valid and there is
3498 * no need to serialise with page lock.
3500 * As the page lock gives no additional guarantee, we optimistically
3501 * wait on the page to be unlocked and check if it's up to date and
3502 * use the page if it is. Otherwise, the page lock is required to
3503 * distinguish between the different cases. The motivation is that we
3504 * avoid spurious serialisations and wakeups when multiple processes
3505 * wait on the same page for IO to complete.
3507 wait_on_page_locked(page
);
3508 if (PageUptodate(page
))
3511 /* Distinguish between all the cases under the safety of the lock */
3514 /* Case c or d, restart the operation */
3515 if (!page
->mapping
) {
3521 /* Someone else locked and filled the page in a very small window */
3522 if (PageUptodate(page
)) {
3528 * A previous I/O error may have been due to temporary
3530 * Clear page error before actual read, PG_error will be
3531 * set again if read page fails.
3533 ClearPageError(page
);
3537 mark_page_accessed(page
);
3542 * read_cache_page - read into page cache, fill it if needed
3543 * @mapping: the page's address_space
3544 * @index: the page index
3545 * @filler: function to perform the read
3546 * @data: first arg to filler(data, page) function, often left as NULL
3548 * Read into the page cache. If a page already exists, and PageUptodate() is
3549 * not set, try to fill the page and wait for it to become unlocked.
3551 * If the page does not get brought uptodate, return -EIO.
3553 * The function expects mapping->invalidate_lock to be already held.
3555 * Return: up to date page on success, ERR_PTR() on failure.
3557 struct page
*read_cache_page(struct address_space
*mapping
,
3559 int (*filler
)(void *, struct page
*),
3562 return do_read_cache_page(mapping
, index
, filler
, data
,
3563 mapping_gfp_mask(mapping
));
3565 EXPORT_SYMBOL(read_cache_page
);
3568 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
3569 * @mapping: the page's address_space
3570 * @index: the page index
3571 * @gfp: the page allocator flags to use if allocating
3573 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
3574 * any new page allocations done using the specified allocation flags.
3576 * If the page does not get brought uptodate, return -EIO.
3578 * The function expects mapping->invalidate_lock to be already held.
3580 * Return: up to date page on success, ERR_PTR() on failure.
3582 struct page
*read_cache_page_gfp(struct address_space
*mapping
,
3586 return do_read_cache_page(mapping
, index
, NULL
, NULL
, gfp
);
3588 EXPORT_SYMBOL(read_cache_page_gfp
);
3590 int pagecache_write_begin(struct file
*file
, struct address_space
*mapping
,
3591 loff_t pos
, unsigned len
, unsigned flags
,
3592 struct page
**pagep
, void **fsdata
)
3594 const struct address_space_operations
*aops
= mapping
->a_ops
;
3596 return aops
->write_begin(file
, mapping
, pos
, len
, flags
,
3599 EXPORT_SYMBOL(pagecache_write_begin
);
3601 int pagecache_write_end(struct file
*file
, struct address_space
*mapping
,
3602 loff_t pos
, unsigned len
, unsigned copied
,
3603 struct page
*page
, void *fsdata
)
3605 const struct address_space_operations
*aops
= mapping
->a_ops
;
3607 return aops
->write_end(file
, mapping
, pos
, len
, copied
, page
, fsdata
);
3609 EXPORT_SYMBOL(pagecache_write_end
);
3612 * Warn about a page cache invalidation failure during a direct I/O write.
3614 void dio_warn_stale_pagecache(struct file
*filp
)
3616 static DEFINE_RATELIMIT_STATE(_rs
, 86400 * HZ
, DEFAULT_RATELIMIT_BURST
);
3620 errseq_set(&filp
->f_mapping
->wb_err
, -EIO
);
3621 if (__ratelimit(&_rs
)) {
3622 path
= file_path(filp
, pathname
, sizeof(pathname
));
3625 pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n");
3626 pr_crit("File: %s PID: %d Comm: %.20s\n", path
, current
->pid
,
3632 generic_file_direct_write(struct kiocb
*iocb
, struct iov_iter
*from
)
3634 struct file
*file
= iocb
->ki_filp
;
3635 struct address_space
*mapping
= file
->f_mapping
;
3636 struct inode
*inode
= mapping
->host
;
3637 loff_t pos
= iocb
->ki_pos
;
3642 write_len
= iov_iter_count(from
);
3643 end
= (pos
+ write_len
- 1) >> PAGE_SHIFT
;
3645 if (iocb
->ki_flags
& IOCB_NOWAIT
) {
3646 /* If there are pages to writeback, return */
3647 if (filemap_range_has_page(file
->f_mapping
, pos
,
3648 pos
+ write_len
- 1))
3651 written
= filemap_write_and_wait_range(mapping
, pos
,
3652 pos
+ write_len
- 1);
3658 * After a write we want buffered reads to be sure to go to disk to get
3659 * the new data. We invalidate clean cached page from the region we're
3660 * about to write. We do this *before* the write so that we can return
3661 * without clobbering -EIOCBQUEUED from ->direct_IO().
3663 written
= invalidate_inode_pages2_range(mapping
,
3664 pos
>> PAGE_SHIFT
, end
);
3666 * If a page can not be invalidated, return 0 to fall back
3667 * to buffered write.
3670 if (written
== -EBUSY
)
3675 written
= mapping
->a_ops
->direct_IO(iocb
, from
);
3678 * Finally, try again to invalidate clean pages which might have been
3679 * cached by non-direct readahead, or faulted in by get_user_pages()
3680 * if the source of the write was an mmap'ed region of the file
3681 * we're writing. Either one is a pretty crazy thing to do,
3682 * so we don't support it 100%. If this invalidation
3683 * fails, tough, the write still worked...
3685 * Most of the time we do not need this since dio_complete() will do
3686 * the invalidation for us. However there are some file systems that
3687 * do not end up with dio_complete() being called, so let's not break
3688 * them by removing it completely.
3690 * Noticeable example is a blkdev_direct_IO().
3692 * Skip invalidation for async writes or if mapping has no pages.
3694 if (written
> 0 && mapping
->nrpages
&&
3695 invalidate_inode_pages2_range(mapping
, pos
>> PAGE_SHIFT
, end
))
3696 dio_warn_stale_pagecache(file
);
3700 write_len
-= written
;
3701 if (pos
> i_size_read(inode
) && !S_ISBLK(inode
->i_mode
)) {
3702 i_size_write(inode
, pos
);
3703 mark_inode_dirty(inode
);
3707 if (written
!= -EIOCBQUEUED
)
3708 iov_iter_revert(from
, write_len
- iov_iter_count(from
));
3712 EXPORT_SYMBOL(generic_file_direct_write
);
3715 * Find or create a page at the given pagecache position. Return the locked
3716 * page. This function is specifically for buffered writes.
3718 struct page
*grab_cache_page_write_begin(struct address_space
*mapping
,
3719 pgoff_t index
, unsigned flags
)
3722 int fgp_flags
= FGP_LOCK
|FGP_WRITE
|FGP_CREAT
;
3724 if (flags
& AOP_FLAG_NOFS
)
3725 fgp_flags
|= FGP_NOFS
;
3727 page
= pagecache_get_page(mapping
, index
, fgp_flags
,
3728 mapping_gfp_mask(mapping
));
3730 wait_for_stable_page(page
);
3734 EXPORT_SYMBOL(grab_cache_page_write_begin
);
3736 ssize_t
generic_perform_write(struct file
*file
,
3737 struct iov_iter
*i
, loff_t pos
)
3739 struct address_space
*mapping
= file
->f_mapping
;
3740 const struct address_space_operations
*a_ops
= mapping
->a_ops
;
3742 ssize_t written
= 0;
3743 unsigned int flags
= 0;
3747 unsigned long offset
; /* Offset into pagecache page */
3748 unsigned long bytes
; /* Bytes to write to page */
3749 size_t copied
; /* Bytes copied from user */
3752 offset
= (pos
& (PAGE_SIZE
- 1));
3753 bytes
= min_t(unsigned long, PAGE_SIZE
- offset
,
3758 * Bring in the user page that we will copy from _first_.
3759 * Otherwise there's a nasty deadlock on copying from the
3760 * same page as we're writing to, without it being marked
3763 if (unlikely(iov_iter_fault_in_readable(i
, bytes
))) {
3768 if (fatal_signal_pending(current
)) {
3773 status
= a_ops
->write_begin(file
, mapping
, pos
, bytes
, flags
,
3775 if (unlikely(status
< 0))
3778 if (mapping_writably_mapped(mapping
))
3779 flush_dcache_page(page
);
3781 copied
= copy_page_from_iter_atomic(page
, offset
, bytes
, i
);
3782 flush_dcache_page(page
);
3784 status
= a_ops
->write_end(file
, mapping
, pos
, bytes
, copied
,
3786 if (unlikely(status
!= copied
)) {
3787 iov_iter_revert(i
, copied
- max(status
, 0L));
3788 if (unlikely(status
< 0))
3793 if (unlikely(status
== 0)) {
3795 * A short copy made ->write_end() reject the
3796 * thing entirely. Might be memory poisoning
3797 * halfway through, might be a race with munmap,
3798 * might be severe memory pressure.
3807 balance_dirty_pages_ratelimited(mapping
);
3808 } while (iov_iter_count(i
));
3810 return written
? written
: status
;
3812 EXPORT_SYMBOL(generic_perform_write
);
3815 * __generic_file_write_iter - write data to a file
3816 * @iocb: IO state structure (file, offset, etc.)
3817 * @from: iov_iter with data to write
3819 * This function does all the work needed for actually writing data to a
3820 * file. It does all basic checks, removes SUID from the file, updates
3821 * modification times and calls proper subroutines depending on whether we
3822 * do direct IO or a standard buffered write.
3824 * It expects i_rwsem to be grabbed unless we work on a block device or similar
3825 * object which does not need locking at all.
3827 * This function does *not* take care of syncing data in case of O_SYNC write.
3828 * A caller has to handle it. This is mainly due to the fact that we want to
3829 * avoid syncing under i_rwsem.
3832 * * number of bytes written, even for truncated writes
3833 * * negative error code if no data has been written at all
3835 ssize_t
__generic_file_write_iter(struct kiocb
*iocb
, struct iov_iter
*from
)
3837 struct file
*file
= iocb
->ki_filp
;
3838 struct address_space
*mapping
= file
->f_mapping
;
3839 struct inode
*inode
= mapping
->host
;
3840 ssize_t written
= 0;
3844 /* We can write back this queue in page reclaim */
3845 current
->backing_dev_info
= inode_to_bdi(inode
);
3846 err
= file_remove_privs(file
);
3850 err
= file_update_time(file
);
3854 if (iocb
->ki_flags
& IOCB_DIRECT
) {
3855 loff_t pos
, endbyte
;
3857 written
= generic_file_direct_write(iocb
, from
);
3859 * If the write stopped short of completing, fall back to
3860 * buffered writes. Some filesystems do this for writes to
3861 * holes, for example. For DAX files, a buffered write will
3862 * not succeed (even if it did, DAX does not handle dirty
3863 * page-cache pages correctly).
3865 if (written
< 0 || !iov_iter_count(from
) || IS_DAX(inode
))
3868 status
= generic_perform_write(file
, from
, pos
= iocb
->ki_pos
);
3870 * If generic_perform_write() returned a synchronous error
3871 * then we want to return the number of bytes which were
3872 * direct-written, or the error code if that was zero. Note
3873 * that this differs from normal direct-io semantics, which
3874 * will return -EFOO even if some bytes were written.
3876 if (unlikely(status
< 0)) {
3881 * We need to ensure that the page cache pages are written to
3882 * disk and invalidated to preserve the expected O_DIRECT
3885 endbyte
= pos
+ status
- 1;
3886 err
= filemap_write_and_wait_range(mapping
, pos
, endbyte
);
3888 iocb
->ki_pos
= endbyte
+ 1;
3890 invalidate_mapping_pages(mapping
,
3892 endbyte
>> PAGE_SHIFT
);
3895 * We don't know how much we wrote, so just return
3896 * the number of bytes which were direct-written
3900 written
= generic_perform_write(file
, from
, iocb
->ki_pos
);
3901 if (likely(written
> 0))
3902 iocb
->ki_pos
+= written
;
3905 current
->backing_dev_info
= NULL
;
3906 return written
? written
: err
;
3908 EXPORT_SYMBOL(__generic_file_write_iter
);
3911 * generic_file_write_iter - write data to a file
3912 * @iocb: IO state structure
3913 * @from: iov_iter with data to write
3915 * This is a wrapper around __generic_file_write_iter() to be used by most
3916 * filesystems. It takes care of syncing the file in case of O_SYNC file
3917 * and acquires i_rwsem as needed.
3919 * * negative error code if no data has been written at all of
3920 * vfs_fsync_range() failed for a synchronous write
3921 * * number of bytes written, even for truncated writes
3923 ssize_t
generic_file_write_iter(struct kiocb
*iocb
, struct iov_iter
*from
)
3925 struct file
*file
= iocb
->ki_filp
;
3926 struct inode
*inode
= file
->f_mapping
->host
;
3930 ret
= generic_write_checks(iocb
, from
);
3932 ret
= __generic_file_write_iter(iocb
, from
);
3933 inode_unlock(inode
);
3936 ret
= generic_write_sync(iocb
, ret
);
3939 EXPORT_SYMBOL(generic_file_write_iter
);
3942 * try_to_release_page() - release old fs-specific metadata on a page
3944 * @page: the page which the kernel is trying to free
3945 * @gfp_mask: memory allocation flags (and I/O mode)
3947 * The address_space is to try to release any data against the page
3948 * (presumably at page->private).
3950 * This may also be called if PG_fscache is set on a page, indicating that the
3951 * page is known to the local caching routines.
3953 * The @gfp_mask argument specifies whether I/O may be performed to release
3954 * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS).
3956 * Return: %1 if the release was successful, otherwise return zero.
3958 int try_to_release_page(struct page
*page
, gfp_t gfp_mask
)
3960 struct address_space
* const mapping
= page
->mapping
;
3962 BUG_ON(!PageLocked(page
));
3963 if (PageWriteback(page
))
3966 if (mapping
&& mapping
->a_ops
->releasepage
)
3967 return mapping
->a_ops
->releasepage(page
, gfp_mask
);
3968 return try_to_free_buffers(page
);
3971 EXPORT_SYMBOL(try_to_release_page
);