]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - mm/filemap.c
Merge tag 'mac80211-for-davem-2016-06-09' of git://git.kernel.org/pub/scm/linux/kerne...
[mirror_ubuntu-bionic-kernel.git] / mm / filemap.c
CommitLineData
1da177e4
LT
1/*
2 * linux/mm/filemap.c
3 *
4 * Copyright (C) 1994-1999 Linus Torvalds
5 */
6
7/*
8 * This file handles the generic file mmap semantics used by
9 * most "normal" filesystems (but you don't /have/ to use this:
10 * the NFS filesystem used to do this differently, for example)
11 */
b95f1b31 12#include <linux/export.h>
1da177e4 13#include <linux/compiler.h>
f9fe48be 14#include <linux/dax.h>
1da177e4 15#include <linux/fs.h>
c22ce143 16#include <linux/uaccess.h>
c59ede7b 17#include <linux/capability.h>
1da177e4 18#include <linux/kernel_stat.h>
5a0e3ad6 19#include <linux/gfp.h>
1da177e4
LT
20#include <linux/mm.h>
21#include <linux/swap.h>
22#include <linux/mman.h>
23#include <linux/pagemap.h>
24#include <linux/file.h>
25#include <linux/uio.h>
26#include <linux/hash.h>
27#include <linux/writeback.h>
53253383 28#include <linux/backing-dev.h>
1da177e4
LT
29#include <linux/pagevec.h>
30#include <linux/blkdev.h>
31#include <linux/security.h>
44110fe3 32#include <linux/cpuset.h>
2f718ffc 33#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
00501b53 34#include <linux/hugetlb.h>
8a9f3ccd 35#include <linux/memcontrol.h>
c515e1fd 36#include <linux/cleancache.h>
f1820361 37#include <linux/rmap.h>
0f8053a5
NP
38#include "internal.h"
39
fe0bfaaf
RJ
40#define CREATE_TRACE_POINTS
41#include <trace/events/filemap.h>
42
1da177e4 43/*
1da177e4
LT
44 * FIXME: remove all knowledge of the buffer layer from the core VM
45 */
148f948b 46#include <linux/buffer_head.h> /* for try_to_free_buffers */
1da177e4 47
1da177e4
LT
48#include <asm/mman.h>
49
50/*
51 * Shared mappings implemented 30.11.1994. It's not fully working yet,
52 * though.
53 *
54 * Shared mappings now work. 15.8.1995 Bruno.
55 *
56 * finished 'unifying' the page and buffer cache and SMP-threaded the
57 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
58 *
59 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
60 */
61
62/*
63 * Lock ordering:
64 *
c8c06efa 65 * ->i_mmap_rwsem (truncate_pagecache)
1da177e4 66 * ->private_lock (__free_pte->__set_page_dirty_buffers)
5d337b91
HD
67 * ->swap_lock (exclusive_swap_page, others)
68 * ->mapping->tree_lock
1da177e4 69 *
1b1dcc1b 70 * ->i_mutex
c8c06efa 71 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
1da177e4
LT
72 *
73 * ->mmap_sem
c8c06efa 74 * ->i_mmap_rwsem
b8072f09 75 * ->page_table_lock or pte_lock (various, mainly in memory.c)
1da177e4
LT
76 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
77 *
78 * ->mmap_sem
79 * ->lock_page (access_process_vm)
80 *
ccad2365 81 * ->i_mutex (generic_perform_write)
82591e6e 82 * ->mmap_sem (fault_in_pages_readable->do_page_fault)
1da177e4 83 *
f758eeab 84 * bdi->wb.list_lock
a66979ab 85 * sb_lock (fs/fs-writeback.c)
1da177e4
LT
86 * ->mapping->tree_lock (__sync_single_inode)
87 *
c8c06efa 88 * ->i_mmap_rwsem
1da177e4
LT
89 * ->anon_vma.lock (vma_adjust)
90 *
91 * ->anon_vma.lock
b8072f09 92 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
1da177e4 93 *
b8072f09 94 * ->page_table_lock or pte_lock
5d337b91 95 * ->swap_lock (try_to_unmap_one)
1da177e4
LT
96 * ->private_lock (try_to_unmap_one)
97 * ->tree_lock (try_to_unmap_one)
98 * ->zone.lru_lock (follow_page->mark_page_accessed)
053837fc 99 * ->zone.lru_lock (check_pte_range->isolate_lru_page)
1da177e4
LT
100 * ->private_lock (page_remove_rmap->set_page_dirty)
101 * ->tree_lock (page_remove_rmap->set_page_dirty)
f758eeab 102 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
250df6ed 103 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
81f8c3a4 104 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
f758eeab 105 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
250df6ed 106 * ->inode->i_lock (zap_pte_range->set_page_dirty)
1da177e4
LT
107 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
108 *
c8c06efa 109 * ->i_mmap_rwsem
9a3c531d 110 * ->tasklist_lock (memory_failure, collect_procs_ao)
1da177e4
LT
111 */
112
91b0abe3
JW
113static void page_cache_tree_delete(struct address_space *mapping,
114 struct page *page, void *shadow)
115{
449dd698 116 struct radix_tree_node *node;
91b0abe3 117
449dd698
JW
118 VM_BUG_ON(!PageLocked(page));
119
d604c324
MW
120 node = radix_tree_replace_clear_tags(&mapping->page_tree, page->index,
121 shadow);
449dd698
JW
122
123 if (shadow) {
f9fe48be 124 mapping->nrexceptional++;
91b0abe3 125 /*
f9fe48be 126 * Make sure the nrexceptional update is committed before
91b0abe3
JW
127 * the nrpages update so that final truncate racing
128 * with reclaim does not see both counters 0 at the
129 * same time and miss a shadow entry.
130 */
131 smp_wmb();
449dd698 132 }
91b0abe3 133 mapping->nrpages--;
449dd698 134
d604c324 135 if (!node)
449dd698 136 return;
449dd698 137
449dd698
JW
138 workingset_node_pages_dec(node);
139 if (shadow)
140 workingset_node_shadows_inc(node);
141 else
142 if (__radix_tree_delete_node(&mapping->page_tree, node))
143 return;
144
145 /*
ac401cc7
JK
146 * Track node that only contains shadow entries. DAX mappings contain
147 * no shadow entries and may contain other exceptional entries so skip
148 * those.
449dd698
JW
149 *
150 * Avoid acquiring the list_lru lock if already tracked. The
151 * list_empty() test is safe as node->private_list is
152 * protected by mapping->tree_lock.
153 */
ac401cc7 154 if (!dax_mapping(mapping) && !workingset_node_pages(node) &&
449dd698
JW
155 list_empty(&node->private_list)) {
156 node->private_data = mapping;
157 list_lru_add(&workingset_shadow_nodes, &node->private_list);
158 }
91b0abe3
JW
159}
160
1da177e4 161/*
e64a782f 162 * Delete a page from the page cache and free it. Caller has to make
1da177e4 163 * sure the page is locked and that nobody else uses it - or that usage
fdf1cdb9 164 * is safe. The caller must hold the mapping's tree_lock.
1da177e4 165 */
62cccb8c 166void __delete_from_page_cache(struct page *page, void *shadow)
1da177e4
LT
167{
168 struct address_space *mapping = page->mapping;
169
fe0bfaaf 170 trace_mm_filemap_delete_from_page_cache(page);
c515e1fd
DM
171 /*
172 * if we're uptodate, flush out into the cleancache, otherwise
173 * invalidate any existing cleancache entries. We can't leave
174 * stale data around in the cleancache once our page is gone
175 */
176 if (PageUptodate(page) && PageMappedToDisk(page))
177 cleancache_put_page(page);
178 else
3167760f 179 cleancache_invalidate_page(mapping, page);
c515e1fd 180
06b241f3
HD
181 VM_BUG_ON_PAGE(page_mapped(page), page);
182 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
183 int mapcount;
184
185 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
186 current->comm, page_to_pfn(page));
187 dump_page(page, "still mapped when deleted");
188 dump_stack();
189 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
190
191 mapcount = page_mapcount(page);
192 if (mapping_exiting(mapping) &&
193 page_count(page) >= mapcount + 2) {
194 /*
195 * All vmas have already been torn down, so it's
196 * a good bet that actually the page is unmapped,
197 * and we'd prefer not to leak it: if we're wrong,
198 * some other bad page check should catch it later.
199 */
200 page_mapcount_reset(page);
6d061f9f 201 page_ref_sub(page, mapcount);
06b241f3
HD
202 }
203 }
204
91b0abe3
JW
205 page_cache_tree_delete(mapping, page, shadow);
206
1da177e4 207 page->mapping = NULL;
b85e0eff 208 /* Leave page->index set: truncation lookup relies upon it */
91b0abe3 209
4165b9b4
MH
210 /* hugetlb pages do not participate in page cache accounting. */
211 if (!PageHuge(page))
212 __dec_zone_page_state(page, NR_FILE_PAGES);
4b02108a
KM
213 if (PageSwapBacked(page))
214 __dec_zone_page_state(page, NR_SHMEM);
3a692790
LT
215
216 /*
b9ea2515
KK
217 * At this point page must be either written or cleaned by truncate.
218 * Dirty page here signals a bug and loss of unwritten data.
3a692790 219 *
b9ea2515
KK
220 * This fixes dirty accounting after removing the page entirely but
221 * leaves PageDirty set: it has no effect for truncated page and
222 * anyway will be cleared before returning page into buddy allocator.
3a692790 223 */
b9ea2515 224 if (WARN_ON_ONCE(PageDirty(page)))
62cccb8c 225 account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
1da177e4
LT
226}
227
702cfbf9
MK
228/**
229 * delete_from_page_cache - delete page from page cache
230 * @page: the page which the kernel is trying to remove from page cache
231 *
232 * This must be called only on pages that have been verified to be in the page
233 * cache and locked. It will never put the page into the free list, the caller
234 * has a reference on the page.
235 */
236void delete_from_page_cache(struct page *page)
1da177e4
LT
237{
238 struct address_space *mapping = page->mapping;
c4843a75
GT
239 unsigned long flags;
240
6072d13c 241 void (*freepage)(struct page *);
1da177e4 242
cd7619d6 243 BUG_ON(!PageLocked(page));
1da177e4 244
6072d13c 245 freepage = mapping->a_ops->freepage;
c4843a75 246
c4843a75 247 spin_lock_irqsave(&mapping->tree_lock, flags);
62cccb8c 248 __delete_from_page_cache(page, NULL);
c4843a75 249 spin_unlock_irqrestore(&mapping->tree_lock, flags);
6072d13c
LT
250
251 if (freepage)
252 freepage(page);
09cbfeaf 253 put_page(page);
97cecb5a
MK
254}
255EXPORT_SYMBOL(delete_from_page_cache);
256
865ffef3
DM
257static int filemap_check_errors(struct address_space *mapping)
258{
259 int ret = 0;
260 /* Check for outstanding write errors */
7fcbbaf1
JA
261 if (test_bit(AS_ENOSPC, &mapping->flags) &&
262 test_and_clear_bit(AS_ENOSPC, &mapping->flags))
865ffef3 263 ret = -ENOSPC;
7fcbbaf1
JA
264 if (test_bit(AS_EIO, &mapping->flags) &&
265 test_and_clear_bit(AS_EIO, &mapping->flags))
865ffef3
DM
266 ret = -EIO;
267 return ret;
268}
269
1da177e4 270/**
485bb99b 271 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
67be2dd1
MW
272 * @mapping: address space structure to write
273 * @start: offset in bytes where the range starts
469eb4d0 274 * @end: offset in bytes where the range ends (inclusive)
67be2dd1 275 * @sync_mode: enable synchronous operation
1da177e4 276 *
485bb99b
RD
277 * Start writeback against all of a mapping's dirty pages that lie
278 * within the byte offsets <start, end> inclusive.
279 *
1da177e4 280 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
485bb99b 281 * opposed to a regular memory cleansing writeback. The difference between
1da177e4
LT
282 * these two operations is that if a dirty page/buffer is encountered, it must
283 * be waited upon, and not just skipped over.
284 */
ebcf28e1
AM
285int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
286 loff_t end, int sync_mode)
1da177e4
LT
287{
288 int ret;
289 struct writeback_control wbc = {
290 .sync_mode = sync_mode,
05fe478d 291 .nr_to_write = LONG_MAX,
111ebb6e
OH
292 .range_start = start,
293 .range_end = end,
1da177e4
LT
294 };
295
296 if (!mapping_cap_writeback_dirty(mapping))
297 return 0;
298
b16b1deb 299 wbc_attach_fdatawrite_inode(&wbc, mapping->host);
1da177e4 300 ret = do_writepages(mapping, &wbc);
b16b1deb 301 wbc_detach_inode(&wbc);
1da177e4
LT
302 return ret;
303}
304
305static inline int __filemap_fdatawrite(struct address_space *mapping,
306 int sync_mode)
307{
111ebb6e 308 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
1da177e4
LT
309}
310
311int filemap_fdatawrite(struct address_space *mapping)
312{
313 return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
314}
315EXPORT_SYMBOL(filemap_fdatawrite);
316
f4c0a0fd 317int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
ebcf28e1 318 loff_t end)
1da177e4
LT
319{
320 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
321}
f4c0a0fd 322EXPORT_SYMBOL(filemap_fdatawrite_range);
1da177e4 323
485bb99b
RD
324/**
325 * filemap_flush - mostly a non-blocking flush
326 * @mapping: target address_space
327 *
1da177e4
LT
328 * This is a mostly non-blocking flush. Not suitable for data-integrity
329 * purposes - I/O may not be started against all dirty pages.
330 */
331int filemap_flush(struct address_space *mapping)
332{
333 return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
334}
335EXPORT_SYMBOL(filemap_flush);
336
aa750fd7
JN
337static int __filemap_fdatawait_range(struct address_space *mapping,
338 loff_t start_byte, loff_t end_byte)
1da177e4 339{
09cbfeaf
KS
340 pgoff_t index = start_byte >> PAGE_SHIFT;
341 pgoff_t end = end_byte >> PAGE_SHIFT;
1da177e4
LT
342 struct pagevec pvec;
343 int nr_pages;
aa750fd7 344 int ret = 0;
1da177e4 345
94004ed7 346 if (end_byte < start_byte)
865ffef3 347 goto out;
1da177e4
LT
348
349 pagevec_init(&pvec, 0);
1da177e4
LT
350 while ((index <= end) &&
351 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
352 PAGECACHE_TAG_WRITEBACK,
353 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
354 unsigned i;
355
356 for (i = 0; i < nr_pages; i++) {
357 struct page *page = pvec.pages[i];
358
359 /* until radix tree lookup accepts end_index */
360 if (page->index > end)
361 continue;
362
363 wait_on_page_writeback(page);
212260aa 364 if (TestClearPageError(page))
1da177e4
LT
365 ret = -EIO;
366 }
367 pagevec_release(&pvec);
368 cond_resched();
369 }
865ffef3 370out:
aa750fd7
JN
371 return ret;
372}
373
374/**
375 * filemap_fdatawait_range - wait for writeback to complete
376 * @mapping: address space structure to wait for
377 * @start_byte: offset in bytes where the range starts
378 * @end_byte: offset in bytes where the range ends (inclusive)
379 *
380 * Walk the list of under-writeback pages of the given address space
381 * in the given range and wait for all of them. Check error status of
382 * the address space and return it.
383 *
384 * Since the error status of the address space is cleared by this function,
385 * callers are responsible for checking the return value and handling and/or
386 * reporting the error.
387 */
388int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
389 loff_t end_byte)
390{
391 int ret, ret2;
392
393 ret = __filemap_fdatawait_range(mapping, start_byte, end_byte);
865ffef3
DM
394 ret2 = filemap_check_errors(mapping);
395 if (!ret)
396 ret = ret2;
1da177e4
LT
397
398 return ret;
399}
d3bccb6f
JK
400EXPORT_SYMBOL(filemap_fdatawait_range);
401
aa750fd7
JN
402/**
403 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
404 * @mapping: address space structure to wait for
405 *
406 * Walk the list of under-writeback pages of the given address space
407 * and wait for all of them. Unlike filemap_fdatawait(), this function
408 * does not clear error status of the address space.
409 *
410 * Use this function if callers don't handle errors themselves. Expected
411 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
412 * fsfreeze(8)
413 */
414void filemap_fdatawait_keep_errors(struct address_space *mapping)
415{
416 loff_t i_size = i_size_read(mapping->host);
417
418 if (i_size == 0)
419 return;
420
421 __filemap_fdatawait_range(mapping, 0, i_size - 1);
422}
423
1da177e4 424/**
485bb99b 425 * filemap_fdatawait - wait for all under-writeback pages to complete
1da177e4 426 * @mapping: address space structure to wait for
485bb99b
RD
427 *
428 * Walk the list of under-writeback pages of the given address space
aa750fd7
JN
429 * and wait for all of them. Check error status of the address space
430 * and return it.
431 *
432 * Since the error status of the address space is cleared by this function,
433 * callers are responsible for checking the return value and handling and/or
434 * reporting the error.
1da177e4
LT
435 */
436int filemap_fdatawait(struct address_space *mapping)
437{
438 loff_t i_size = i_size_read(mapping->host);
439
440 if (i_size == 0)
441 return 0;
442
94004ed7 443 return filemap_fdatawait_range(mapping, 0, i_size - 1);
1da177e4
LT
444}
445EXPORT_SYMBOL(filemap_fdatawait);
446
447int filemap_write_and_wait(struct address_space *mapping)
448{
28fd1298 449 int err = 0;
1da177e4 450
7f6d5b52
RZ
451 if ((!dax_mapping(mapping) && mapping->nrpages) ||
452 (dax_mapping(mapping) && mapping->nrexceptional)) {
28fd1298
OH
453 err = filemap_fdatawrite(mapping);
454 /*
455 * Even if the above returned error, the pages may be
456 * written partially (e.g. -ENOSPC), so we wait for it.
457 * But the -EIO is special case, it may indicate the worst
458 * thing (e.g. bug) happened, so we avoid waiting for it.
459 */
460 if (err != -EIO) {
461 int err2 = filemap_fdatawait(mapping);
462 if (!err)
463 err = err2;
464 }
865ffef3
DM
465 } else {
466 err = filemap_check_errors(mapping);
1da177e4 467 }
28fd1298 468 return err;
1da177e4 469}
28fd1298 470EXPORT_SYMBOL(filemap_write_and_wait);
1da177e4 471
485bb99b
RD
472/**
473 * filemap_write_and_wait_range - write out & wait on a file range
474 * @mapping: the address_space for the pages
475 * @lstart: offset in bytes where the range starts
476 * @lend: offset in bytes where the range ends (inclusive)
477 *
469eb4d0
AM
478 * Write out and wait upon file offsets lstart->lend, inclusive.
479 *
480 * Note that `lend' is inclusive (describes the last byte to be written) so
481 * that this function can be used to write to the very end-of-file (end = -1).
482 */
1da177e4
LT
483int filemap_write_and_wait_range(struct address_space *mapping,
484 loff_t lstart, loff_t lend)
485{
28fd1298 486 int err = 0;
1da177e4 487
7f6d5b52
RZ
488 if ((!dax_mapping(mapping) && mapping->nrpages) ||
489 (dax_mapping(mapping) && mapping->nrexceptional)) {
28fd1298
OH
490 err = __filemap_fdatawrite_range(mapping, lstart, lend,
491 WB_SYNC_ALL);
492 /* See comment of filemap_write_and_wait() */
493 if (err != -EIO) {
94004ed7
CH
494 int err2 = filemap_fdatawait_range(mapping,
495 lstart, lend);
28fd1298
OH
496 if (!err)
497 err = err2;
498 }
865ffef3
DM
499 } else {
500 err = filemap_check_errors(mapping);
1da177e4 501 }
28fd1298 502 return err;
1da177e4 503}
f6995585 504EXPORT_SYMBOL(filemap_write_and_wait_range);
1da177e4 505
ef6a3c63
MS
506/**
507 * replace_page_cache_page - replace a pagecache page with a new one
508 * @old: page to be replaced
509 * @new: page to replace with
510 * @gfp_mask: allocation mode
511 *
512 * This function replaces a page in the pagecache with a new one. On
513 * success it acquires the pagecache reference for the new page and
514 * drops it for the old page. Both the old and new pages must be
515 * locked. This function does not add the new page to the LRU, the
516 * caller must do that.
517 *
518 * The remove + add is atomic. The only way this function can fail is
519 * memory allocation failure.
520 */
521int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
522{
523 int error;
ef6a3c63 524
309381fe
SL
525 VM_BUG_ON_PAGE(!PageLocked(old), old);
526 VM_BUG_ON_PAGE(!PageLocked(new), new);
527 VM_BUG_ON_PAGE(new->mapping, new);
ef6a3c63 528
ef6a3c63
MS
529 error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
530 if (!error) {
531 struct address_space *mapping = old->mapping;
532 void (*freepage)(struct page *);
c4843a75 533 unsigned long flags;
ef6a3c63
MS
534
535 pgoff_t offset = old->index;
536 freepage = mapping->a_ops->freepage;
537
09cbfeaf 538 get_page(new);
ef6a3c63
MS
539 new->mapping = mapping;
540 new->index = offset;
541
c4843a75 542 spin_lock_irqsave(&mapping->tree_lock, flags);
62cccb8c 543 __delete_from_page_cache(old, NULL);
ef6a3c63
MS
544 error = radix_tree_insert(&mapping->page_tree, offset, new);
545 BUG_ON(error);
546 mapping->nrpages++;
4165b9b4
MH
547
548 /*
549 * hugetlb pages do not participate in page cache accounting.
550 */
551 if (!PageHuge(new))
552 __inc_zone_page_state(new, NR_FILE_PAGES);
ef6a3c63
MS
553 if (PageSwapBacked(new))
554 __inc_zone_page_state(new, NR_SHMEM);
c4843a75 555 spin_unlock_irqrestore(&mapping->tree_lock, flags);
6a93ca8f 556 mem_cgroup_migrate(old, new);
ef6a3c63
MS
557 radix_tree_preload_end();
558 if (freepage)
559 freepage(old);
09cbfeaf 560 put_page(old);
ef6a3c63
MS
561 }
562
563 return error;
564}
565EXPORT_SYMBOL_GPL(replace_page_cache_page);
566
0cd6144a 567static int page_cache_tree_insert(struct address_space *mapping,
a528910e 568 struct page *page, void **shadowp)
0cd6144a 569{
449dd698 570 struct radix_tree_node *node;
0cd6144a
JW
571 void **slot;
572 int error;
573
e6145236 574 error = __radix_tree_create(&mapping->page_tree, page->index, 0,
449dd698
JW
575 &node, &slot);
576 if (error)
577 return error;
578 if (*slot) {
0cd6144a
JW
579 void *p;
580
581 p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
582 if (!radix_tree_exceptional_entry(p))
583 return -EEXIST;
f9fe48be 584
f9fe48be 585 mapping->nrexceptional--;
4f622938
JK
586 if (!dax_mapping(mapping)) {
587 if (shadowp)
588 *shadowp = p;
589 if (node)
590 workingset_node_shadows_dec(node);
591 } else {
592 /* DAX can replace empty locked entry with a hole */
593 WARN_ON_ONCE(p !=
594 (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
595 RADIX_DAX_ENTRY_LOCK));
596 /* DAX accounts exceptional entries as normal pages */
597 if (node)
598 workingset_node_pages_dec(node);
ac401cc7
JK
599 /* Wakeup waiters for exceptional entry lock */
600 dax_wake_mapping_entry_waiter(mapping, page->index,
601 false);
4f622938 602 }
0cd6144a 603 }
449dd698
JW
604 radix_tree_replace_slot(slot, page);
605 mapping->nrpages++;
606 if (node) {
607 workingset_node_pages_inc(node);
608 /*
609 * Don't track node that contains actual pages.
610 *
611 * Avoid acquiring the list_lru lock if already
612 * untracked. The list_empty() test is safe as
613 * node->private_list is protected by
614 * mapping->tree_lock.
615 */
616 if (!list_empty(&node->private_list))
617 list_lru_del(&workingset_shadow_nodes,
618 &node->private_list);
619 }
620 return 0;
0cd6144a
JW
621}
622
a528910e
JW
623static int __add_to_page_cache_locked(struct page *page,
624 struct address_space *mapping,
625 pgoff_t offset, gfp_t gfp_mask,
626 void **shadowp)
1da177e4 627{
00501b53
JW
628 int huge = PageHuge(page);
629 struct mem_cgroup *memcg;
e286781d
NP
630 int error;
631
309381fe
SL
632 VM_BUG_ON_PAGE(!PageLocked(page), page);
633 VM_BUG_ON_PAGE(PageSwapBacked(page), page);
e286781d 634
00501b53
JW
635 if (!huge) {
636 error = mem_cgroup_try_charge(page, current->mm,
f627c2f5 637 gfp_mask, &memcg, false);
00501b53
JW
638 if (error)
639 return error;
640 }
1da177e4 641
5e4c0d97 642 error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM);
66a0c8ee 643 if (error) {
00501b53 644 if (!huge)
f627c2f5 645 mem_cgroup_cancel_charge(page, memcg, false);
66a0c8ee
KS
646 return error;
647 }
648
09cbfeaf 649 get_page(page);
66a0c8ee
KS
650 page->mapping = mapping;
651 page->index = offset;
652
653 spin_lock_irq(&mapping->tree_lock);
a528910e 654 error = page_cache_tree_insert(mapping, page, shadowp);
66a0c8ee
KS
655 radix_tree_preload_end();
656 if (unlikely(error))
657 goto err_insert;
4165b9b4
MH
658
659 /* hugetlb pages do not participate in page cache accounting. */
660 if (!huge)
661 __inc_zone_page_state(page, NR_FILE_PAGES);
66a0c8ee 662 spin_unlock_irq(&mapping->tree_lock);
00501b53 663 if (!huge)
f627c2f5 664 mem_cgroup_commit_charge(page, memcg, false, false);
66a0c8ee
KS
665 trace_mm_filemap_add_to_page_cache(page);
666 return 0;
667err_insert:
668 page->mapping = NULL;
669 /* Leave page->index set: truncation relies upon it */
670 spin_unlock_irq(&mapping->tree_lock);
00501b53 671 if (!huge)
f627c2f5 672 mem_cgroup_cancel_charge(page, memcg, false);
09cbfeaf 673 put_page(page);
1da177e4
LT
674 return error;
675}
a528910e
JW
676
677/**
678 * add_to_page_cache_locked - add a locked page to the pagecache
679 * @page: page to add
680 * @mapping: the page's address_space
681 * @offset: page index
682 * @gfp_mask: page allocation mode
683 *
684 * This function is used to add a page to the pagecache. It must be locked.
685 * This function does not add the page to the LRU. The caller must do that.
686 */
687int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
688 pgoff_t offset, gfp_t gfp_mask)
689{
690 return __add_to_page_cache_locked(page, mapping, offset,
691 gfp_mask, NULL);
692}
e286781d 693EXPORT_SYMBOL(add_to_page_cache_locked);
1da177e4
LT
694
695int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
6daa0e28 696 pgoff_t offset, gfp_t gfp_mask)
1da177e4 697{
a528910e 698 void *shadow = NULL;
4f98a2fe
RR
699 int ret;
700
48c935ad 701 __SetPageLocked(page);
a528910e
JW
702 ret = __add_to_page_cache_locked(page, mapping, offset,
703 gfp_mask, &shadow);
704 if (unlikely(ret))
48c935ad 705 __ClearPageLocked(page);
a528910e
JW
706 else {
707 /*
708 * The page might have been evicted from cache only
709 * recently, in which case it should be activated like
710 * any other repeatedly accessed page.
f0281a00
RR
711 * The exception is pages getting rewritten; evicting other
712 * data from the working set, only to cache data that will
713 * get overwritten with something else, is a waste of memory.
a528910e 714 */
f0281a00
RR
715 if (!(gfp_mask & __GFP_WRITE) &&
716 shadow && workingset_refault(shadow)) {
a528910e
JW
717 SetPageActive(page);
718 workingset_activation(page);
719 } else
720 ClearPageActive(page);
721 lru_cache_add(page);
722 }
1da177e4
LT
723 return ret;
724}
18bc0bbd 725EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
1da177e4 726
44110fe3 727#ifdef CONFIG_NUMA
2ae88149 728struct page *__page_cache_alloc(gfp_t gfp)
44110fe3 729{
c0ff7453
MX
730 int n;
731 struct page *page;
732
44110fe3 733 if (cpuset_do_page_mem_spread()) {
cc9a6c87
MG
734 unsigned int cpuset_mems_cookie;
735 do {
d26914d1 736 cpuset_mems_cookie = read_mems_allowed_begin();
cc9a6c87 737 n = cpuset_mem_spread_node();
96db800f 738 page = __alloc_pages_node(n, gfp, 0);
d26914d1 739 } while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
cc9a6c87 740
c0ff7453 741 return page;
44110fe3 742 }
2ae88149 743 return alloc_pages(gfp, 0);
44110fe3 744}
2ae88149 745EXPORT_SYMBOL(__page_cache_alloc);
44110fe3
PJ
746#endif
747
1da177e4
LT
748/*
749 * In order to wait for pages to become available there must be
750 * waitqueues associated with pages. By using a hash table of
751 * waitqueues where the bucket discipline is to maintain all
752 * waiters on the same queue and wake all when any of the pages
753 * become available, and for the woken contexts to check to be
754 * sure the appropriate page became available, this saves space
755 * at a cost of "thundering herd" phenomena during rare hash
756 * collisions.
757 */
a4796e37 758wait_queue_head_t *page_waitqueue(struct page *page)
1da177e4
LT
759{
760 const struct zone *zone = page_zone(page);
761
762 return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
763}
a4796e37 764EXPORT_SYMBOL(page_waitqueue);
1da177e4 765
920c7a5d 766void wait_on_page_bit(struct page *page, int bit_nr)
1da177e4
LT
767{
768 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
769
770 if (test_bit(bit_nr, &page->flags))
74316201 771 __wait_on_bit(page_waitqueue(page), &wait, bit_wait_io,
1da177e4
LT
772 TASK_UNINTERRUPTIBLE);
773}
774EXPORT_SYMBOL(wait_on_page_bit);
775
f62e00cc
KM
776int wait_on_page_bit_killable(struct page *page, int bit_nr)
777{
778 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
779
780 if (!test_bit(bit_nr, &page->flags))
781 return 0;
782
783 return __wait_on_bit(page_waitqueue(page), &wait,
74316201 784 bit_wait_io, TASK_KILLABLE);
f62e00cc
KM
785}
786
cbbce822
N
787int wait_on_page_bit_killable_timeout(struct page *page,
788 int bit_nr, unsigned long timeout)
789{
790 DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
791
792 wait.key.timeout = jiffies + timeout;
793 if (!test_bit(bit_nr, &page->flags))
794 return 0;
795 return __wait_on_bit(page_waitqueue(page), &wait,
796 bit_wait_io_timeout, TASK_KILLABLE);
797}
798EXPORT_SYMBOL_GPL(wait_on_page_bit_killable_timeout);
799
385e1ca5
DH
800/**
801 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
697f619f
RD
802 * @page: Page defining the wait queue of interest
803 * @waiter: Waiter to add to the queue
385e1ca5
DH
804 *
805 * Add an arbitrary @waiter to the wait queue for the nominated @page.
806 */
807void add_page_wait_queue(struct page *page, wait_queue_t *waiter)
808{
809 wait_queue_head_t *q = page_waitqueue(page);
810 unsigned long flags;
811
812 spin_lock_irqsave(&q->lock, flags);
813 __add_wait_queue(q, waiter);
814 spin_unlock_irqrestore(&q->lock, flags);
815}
816EXPORT_SYMBOL_GPL(add_page_wait_queue);
817
1da177e4 818/**
485bb99b 819 * unlock_page - unlock a locked page
1da177e4
LT
820 * @page: the page
821 *
822 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
823 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
da3dae54 824 * mechanism between PageLocked pages and PageWriteback pages is shared.
1da177e4
LT
825 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
826 *
8413ac9d
NP
827 * The mb is necessary to enforce ordering between the clear_bit and the read
828 * of the waitqueue (to avoid SMP races with a parallel wait_on_page_locked()).
1da177e4 829 */
920c7a5d 830void unlock_page(struct page *page)
1da177e4 831{
48c935ad 832 page = compound_head(page);
309381fe 833 VM_BUG_ON_PAGE(!PageLocked(page), page);
8413ac9d 834 clear_bit_unlock(PG_locked, &page->flags);
4e857c58 835 smp_mb__after_atomic();
1da177e4
LT
836 wake_up_page(page, PG_locked);
837}
838EXPORT_SYMBOL(unlock_page);
839
485bb99b
RD
840/**
841 * end_page_writeback - end writeback against a page
842 * @page: the page
1da177e4
LT
843 */
844void end_page_writeback(struct page *page)
845{
888cf2db
MG
846 /*
847 * TestClearPageReclaim could be used here but it is an atomic
848 * operation and overkill in this particular case. Failing to
849 * shuffle a page marked for immediate reclaim is too mild to
850 * justify taking an atomic operation penalty at the end of
851 * ever page writeback.
852 */
853 if (PageReclaim(page)) {
854 ClearPageReclaim(page);
ac6aadb2 855 rotate_reclaimable_page(page);
888cf2db 856 }
ac6aadb2
MS
857
858 if (!test_clear_page_writeback(page))
859 BUG();
860
4e857c58 861 smp_mb__after_atomic();
1da177e4
LT
862 wake_up_page(page, PG_writeback);
863}
864EXPORT_SYMBOL(end_page_writeback);
865
57d99845
MW
866/*
867 * After completing I/O on a page, call this routine to update the page
868 * flags appropriately
869 */
870void page_endio(struct page *page, int rw, int err)
871{
872 if (rw == READ) {
873 if (!err) {
874 SetPageUptodate(page);
875 } else {
876 ClearPageUptodate(page);
877 SetPageError(page);
878 }
879 unlock_page(page);
880 } else { /* rw == WRITE */
881 if (err) {
882 SetPageError(page);
883 if (page->mapping)
884 mapping_set_error(page->mapping, err);
885 }
886 end_page_writeback(page);
887 }
888}
889EXPORT_SYMBOL_GPL(page_endio);
890
485bb99b
RD
891/**
892 * __lock_page - get a lock on the page, assuming we need to sleep to get it
893 * @page: the page to lock
1da177e4 894 */
920c7a5d 895void __lock_page(struct page *page)
1da177e4 896{
48c935ad
KS
897 struct page *page_head = compound_head(page);
898 DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
1da177e4 899
48c935ad 900 __wait_on_bit_lock(page_waitqueue(page_head), &wait, bit_wait_io,
1da177e4
LT
901 TASK_UNINTERRUPTIBLE);
902}
903EXPORT_SYMBOL(__lock_page);
904
b5606c2d 905int __lock_page_killable(struct page *page)
2687a356 906{
48c935ad
KS
907 struct page *page_head = compound_head(page);
908 DEFINE_WAIT_BIT(wait, &page_head->flags, PG_locked);
2687a356 909
48c935ad 910 return __wait_on_bit_lock(page_waitqueue(page_head), &wait,
74316201 911 bit_wait_io, TASK_KILLABLE);
2687a356 912}
18bc0bbd 913EXPORT_SYMBOL_GPL(__lock_page_killable);
2687a356 914
9a95f3cf
PC
915/*
916 * Return values:
917 * 1 - page is locked; mmap_sem is still held.
918 * 0 - page is not locked.
919 * mmap_sem has been released (up_read()), unless flags had both
920 * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
921 * which case mmap_sem is still held.
922 *
923 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
924 * with the page locked and the mmap_sem unperturbed.
925 */
d065bd81
ML
926int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
927 unsigned int flags)
928{
37b23e05
KM
929 if (flags & FAULT_FLAG_ALLOW_RETRY) {
930 /*
931 * CAUTION! In this case, mmap_sem is not released
932 * even though return 0.
933 */
934 if (flags & FAULT_FLAG_RETRY_NOWAIT)
935 return 0;
936
937 up_read(&mm->mmap_sem);
938 if (flags & FAULT_FLAG_KILLABLE)
939 wait_on_page_locked_killable(page);
940 else
318b275f 941 wait_on_page_locked(page);
d065bd81 942 return 0;
37b23e05
KM
943 } else {
944 if (flags & FAULT_FLAG_KILLABLE) {
945 int ret;
946
947 ret = __lock_page_killable(page);
948 if (ret) {
949 up_read(&mm->mmap_sem);
950 return 0;
951 }
952 } else
953 __lock_page(page);
954 return 1;
d065bd81
ML
955 }
956}
957
e7b563bb
JW
958/**
959 * page_cache_next_hole - find the next hole (not-present entry)
960 * @mapping: mapping
961 * @index: index
962 * @max_scan: maximum range to search
963 *
964 * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
965 * lowest indexed hole.
966 *
967 * Returns: the index of the hole if found, otherwise returns an index
968 * outside of the set specified (in which case 'return - index >=
969 * max_scan' will be true). In rare cases of index wrap-around, 0 will
970 * be returned.
971 *
972 * page_cache_next_hole may be called under rcu_read_lock. However,
973 * like radix_tree_gang_lookup, this will not atomically search a
974 * snapshot of the tree at a single point in time. For example, if a
975 * hole is created at index 5, then subsequently a hole is created at
976 * index 10, page_cache_next_hole covering both indexes may return 10
977 * if called under rcu_read_lock.
978 */
979pgoff_t page_cache_next_hole(struct address_space *mapping,
980 pgoff_t index, unsigned long max_scan)
981{
982 unsigned long i;
983
984 for (i = 0; i < max_scan; i++) {
0cd6144a
JW
985 struct page *page;
986
987 page = radix_tree_lookup(&mapping->page_tree, index);
988 if (!page || radix_tree_exceptional_entry(page))
e7b563bb
JW
989 break;
990 index++;
991 if (index == 0)
992 break;
993 }
994
995 return index;
996}
997EXPORT_SYMBOL(page_cache_next_hole);
998
999/**
1000 * page_cache_prev_hole - find the prev hole (not-present entry)
1001 * @mapping: mapping
1002 * @index: index
1003 * @max_scan: maximum range to search
1004 *
1005 * Search backwards in the range [max(index-max_scan+1, 0), index] for
1006 * the first hole.
1007 *
1008 * Returns: the index of the hole if found, otherwise returns an index
1009 * outside of the set specified (in which case 'index - return >=
1010 * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
1011 * will be returned.
1012 *
1013 * page_cache_prev_hole may be called under rcu_read_lock. However,
1014 * like radix_tree_gang_lookup, this will not atomically search a
1015 * snapshot of the tree at a single point in time. For example, if a
1016 * hole is created at index 10, then subsequently a hole is created at
1017 * index 5, page_cache_prev_hole covering both indexes may return 5 if
1018 * called under rcu_read_lock.
1019 */
1020pgoff_t page_cache_prev_hole(struct address_space *mapping,
1021 pgoff_t index, unsigned long max_scan)
1022{
1023 unsigned long i;
1024
1025 for (i = 0; i < max_scan; i++) {
0cd6144a
JW
1026 struct page *page;
1027
1028 page = radix_tree_lookup(&mapping->page_tree, index);
1029 if (!page || radix_tree_exceptional_entry(page))
e7b563bb
JW
1030 break;
1031 index--;
1032 if (index == ULONG_MAX)
1033 break;
1034 }
1035
1036 return index;
1037}
1038EXPORT_SYMBOL(page_cache_prev_hole);
1039
485bb99b 1040/**
0cd6144a 1041 * find_get_entry - find and get a page cache entry
485bb99b 1042 * @mapping: the address_space to search
0cd6144a
JW
1043 * @offset: the page cache index
1044 *
1045 * Looks up the page cache slot at @mapping & @offset. If there is a
1046 * page cache page, it is returned with an increased refcount.
485bb99b 1047 *
139b6a6f
JW
1048 * If the slot holds a shadow entry of a previously evicted page, or a
1049 * swap entry from shmem/tmpfs, it is returned.
0cd6144a
JW
1050 *
1051 * Otherwise, %NULL is returned.
1da177e4 1052 */
0cd6144a 1053struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
1da177e4 1054{
a60637c8 1055 void **pagep;
1da177e4
LT
1056 struct page *page;
1057
a60637c8
NP
1058 rcu_read_lock();
1059repeat:
1060 page = NULL;
1061 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
1062 if (pagep) {
1063 page = radix_tree_deref_slot(pagep);
27d20fdd
NP
1064 if (unlikely(!page))
1065 goto out;
a2c16d6c 1066 if (radix_tree_exception(page)) {
8079b1c8
HD
1067 if (radix_tree_deref_retry(page))
1068 goto repeat;
1069 /*
139b6a6f
JW
1070 * A shadow entry of a recently evicted page,
1071 * or a swap entry from shmem/tmpfs. Return
1072 * it without attempting to raise page count.
8079b1c8
HD
1073 */
1074 goto out;
a2c16d6c 1075 }
a60637c8
NP
1076 if (!page_cache_get_speculative(page))
1077 goto repeat;
1078
1079 /*
1080 * Has the page moved?
1081 * This is part of the lockless pagecache protocol. See
1082 * include/linux/pagemap.h for details.
1083 */
1084 if (unlikely(page != *pagep)) {
09cbfeaf 1085 put_page(page);
a60637c8
NP
1086 goto repeat;
1087 }
1088 }
27d20fdd 1089out:
a60637c8
NP
1090 rcu_read_unlock();
1091
1da177e4
LT
1092 return page;
1093}
0cd6144a 1094EXPORT_SYMBOL(find_get_entry);
1da177e4 1095
0cd6144a
JW
1096/**
1097 * find_lock_entry - locate, pin and lock a page cache entry
1098 * @mapping: the address_space to search
1099 * @offset: the page cache index
1100 *
1101 * Looks up the page cache slot at @mapping & @offset. If there is a
1102 * page cache page, it is returned locked and with an increased
1103 * refcount.
1104 *
139b6a6f
JW
1105 * If the slot holds a shadow entry of a previously evicted page, or a
1106 * swap entry from shmem/tmpfs, it is returned.
0cd6144a
JW
1107 *
1108 * Otherwise, %NULL is returned.
1109 *
1110 * find_lock_entry() may sleep.
1111 */
1112struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
1da177e4
LT
1113{
1114 struct page *page;
1115
1da177e4 1116repeat:
0cd6144a 1117 page = find_get_entry(mapping, offset);
a2c16d6c 1118 if (page && !radix_tree_exception(page)) {
a60637c8
NP
1119 lock_page(page);
1120 /* Has the page been truncated? */
1121 if (unlikely(page->mapping != mapping)) {
1122 unlock_page(page);
09cbfeaf 1123 put_page(page);
a60637c8 1124 goto repeat;
1da177e4 1125 }
309381fe 1126 VM_BUG_ON_PAGE(page->index != offset, page);
1da177e4 1127 }
1da177e4
LT
1128 return page;
1129}
0cd6144a
JW
1130EXPORT_SYMBOL(find_lock_entry);
1131
1132/**
2457aec6 1133 * pagecache_get_page - find and get a page reference
0cd6144a
JW
1134 * @mapping: the address_space to search
1135 * @offset: the page index
2457aec6 1136 * @fgp_flags: PCG flags
45f87de5 1137 * @gfp_mask: gfp mask to use for the page cache data page allocation
0cd6144a 1138 *
2457aec6 1139 * Looks up the page cache slot at @mapping & @offset.
1da177e4 1140 *
75325189 1141 * PCG flags modify how the page is returned.
0cd6144a 1142 *
2457aec6
MG
1143 * FGP_ACCESSED: the page will be marked accessed
1144 * FGP_LOCK: Page is return locked
1145 * FGP_CREAT: If page is not present then a new page is allocated using
45f87de5
MH
1146 * @gfp_mask and added to the page cache and the VM's LRU
1147 * list. The page is returned locked and with an increased
1148 * refcount. Otherwise, %NULL is returned.
1da177e4 1149 *
2457aec6
MG
1150 * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even
1151 * if the GFP flags specified for FGP_CREAT are atomic.
1da177e4 1152 *
2457aec6 1153 * If there is a page cache page, it is returned with an increased refcount.
1da177e4 1154 */
2457aec6 1155struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
45f87de5 1156 int fgp_flags, gfp_t gfp_mask)
1da177e4 1157{
eb2be189 1158 struct page *page;
2457aec6 1159
1da177e4 1160repeat:
2457aec6
MG
1161 page = find_get_entry(mapping, offset);
1162 if (radix_tree_exceptional_entry(page))
1163 page = NULL;
1164 if (!page)
1165 goto no_page;
1166
1167 if (fgp_flags & FGP_LOCK) {
1168 if (fgp_flags & FGP_NOWAIT) {
1169 if (!trylock_page(page)) {
09cbfeaf 1170 put_page(page);
2457aec6
MG
1171 return NULL;
1172 }
1173 } else {
1174 lock_page(page);
1175 }
1176
1177 /* Has the page been truncated? */
1178 if (unlikely(page->mapping != mapping)) {
1179 unlock_page(page);
09cbfeaf 1180 put_page(page);
2457aec6
MG
1181 goto repeat;
1182 }
1183 VM_BUG_ON_PAGE(page->index != offset, page);
1184 }
1185
1186 if (page && (fgp_flags & FGP_ACCESSED))
1187 mark_page_accessed(page);
1188
1189no_page:
1190 if (!page && (fgp_flags & FGP_CREAT)) {
1191 int err;
1192 if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
45f87de5
MH
1193 gfp_mask |= __GFP_WRITE;
1194 if (fgp_flags & FGP_NOFS)
1195 gfp_mask &= ~__GFP_FS;
2457aec6 1196
45f87de5 1197 page = __page_cache_alloc(gfp_mask);
eb2be189
NP
1198 if (!page)
1199 return NULL;
2457aec6
MG
1200
1201 if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK)))
1202 fgp_flags |= FGP_LOCK;
1203
eb39d618 1204 /* Init accessed so avoid atomic mark_page_accessed later */
2457aec6 1205 if (fgp_flags & FGP_ACCESSED)
eb39d618 1206 __SetPageReferenced(page);
2457aec6 1207
45f87de5
MH
1208 err = add_to_page_cache_lru(page, mapping, offset,
1209 gfp_mask & GFP_RECLAIM_MASK);
eb2be189 1210 if (unlikely(err)) {
09cbfeaf 1211 put_page(page);
eb2be189
NP
1212 page = NULL;
1213 if (err == -EEXIST)
1214 goto repeat;
1da177e4 1215 }
1da177e4 1216 }
2457aec6 1217
1da177e4
LT
1218 return page;
1219}
2457aec6 1220EXPORT_SYMBOL(pagecache_get_page);
1da177e4 1221
0cd6144a
JW
1222/**
1223 * find_get_entries - gang pagecache lookup
1224 * @mapping: The address_space to search
1225 * @start: The starting page cache index
1226 * @nr_entries: The maximum number of entries
1227 * @entries: Where the resulting entries are placed
1228 * @indices: The cache indices corresponding to the entries in @entries
1229 *
1230 * find_get_entries() will search for and return a group of up to
1231 * @nr_entries entries in the mapping. The entries are placed at
1232 * @entries. find_get_entries() takes a reference against any actual
1233 * pages it returns.
1234 *
1235 * The search returns a group of mapping-contiguous page cache entries
1236 * with ascending indexes. There may be holes in the indices due to
1237 * not-present pages.
1238 *
139b6a6f
JW
1239 * Any shadow entries of evicted pages, or swap entries from
1240 * shmem/tmpfs, are included in the returned array.
0cd6144a
JW
1241 *
1242 * find_get_entries() returns the number of pages and shadow entries
1243 * which were found.
1244 */
1245unsigned find_get_entries(struct address_space *mapping,
1246 pgoff_t start, unsigned int nr_entries,
1247 struct page **entries, pgoff_t *indices)
1248{
1249 void **slot;
1250 unsigned int ret = 0;
1251 struct radix_tree_iter iter;
1252
1253 if (!nr_entries)
1254 return 0;
1255
1256 rcu_read_lock();
0cd6144a
JW
1257 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1258 struct page *page;
1259repeat:
1260 page = radix_tree_deref_slot(slot);
1261 if (unlikely(!page))
1262 continue;
1263 if (radix_tree_exception(page)) {
2cf938aa
MW
1264 if (radix_tree_deref_retry(page)) {
1265 slot = radix_tree_iter_retry(&iter);
1266 continue;
1267 }
0cd6144a 1268 /*
f9fe48be
RZ
1269 * A shadow entry of a recently evicted page, a swap
1270 * entry from shmem/tmpfs or a DAX entry. Return it
1271 * without attempting to raise page count.
0cd6144a
JW
1272 */
1273 goto export;
1274 }
1275 if (!page_cache_get_speculative(page))
1276 goto repeat;
1277
1278 /* Has the page moved? */
1279 if (unlikely(page != *slot)) {
09cbfeaf 1280 put_page(page);
0cd6144a
JW
1281 goto repeat;
1282 }
1283export:
1284 indices[ret] = iter.index;
1285 entries[ret] = page;
1286 if (++ret == nr_entries)
1287 break;
1288 }
1289 rcu_read_unlock();
1290 return ret;
1291}
1292
1da177e4
LT
1293/**
1294 * find_get_pages - gang pagecache lookup
1295 * @mapping: The address_space to search
1296 * @start: The starting page index
1297 * @nr_pages: The maximum number of pages
1298 * @pages: Where the resulting pages are placed
1299 *
1300 * find_get_pages() will search for and return a group of up to
1301 * @nr_pages pages in the mapping. The pages are placed at @pages.
1302 * find_get_pages() takes a reference against the returned pages.
1303 *
1304 * The search returns a group of mapping-contiguous pages with ascending
1305 * indexes. There may be holes in the indices due to not-present pages.
1306 *
1307 * find_get_pages() returns the number of pages which were found.
1308 */
1309unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
1310 unsigned int nr_pages, struct page **pages)
1311{
0fc9d104
KK
1312 struct radix_tree_iter iter;
1313 void **slot;
1314 unsigned ret = 0;
1315
1316 if (unlikely(!nr_pages))
1317 return 0;
a60637c8
NP
1318
1319 rcu_read_lock();
0fc9d104 1320 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
a60637c8
NP
1321 struct page *page;
1322repeat:
0fc9d104 1323 page = radix_tree_deref_slot(slot);
a60637c8
NP
1324 if (unlikely(!page))
1325 continue;
9d8aa4ea 1326
a2c16d6c 1327 if (radix_tree_exception(page)) {
8079b1c8 1328 if (radix_tree_deref_retry(page)) {
2cf938aa
MW
1329 slot = radix_tree_iter_retry(&iter);
1330 continue;
8079b1c8 1331 }
a2c16d6c 1332 /*
139b6a6f
JW
1333 * A shadow entry of a recently evicted page,
1334 * or a swap entry from shmem/tmpfs. Skip
1335 * over it.
a2c16d6c 1336 */
8079b1c8 1337 continue;
27d20fdd 1338 }
a60637c8
NP
1339
1340 if (!page_cache_get_speculative(page))
1341 goto repeat;
1342
1343 /* Has the page moved? */
0fc9d104 1344 if (unlikely(page != *slot)) {
09cbfeaf 1345 put_page(page);
a60637c8
NP
1346 goto repeat;
1347 }
1da177e4 1348
a60637c8 1349 pages[ret] = page;
0fc9d104
KK
1350 if (++ret == nr_pages)
1351 break;
a60637c8 1352 }
5b280c0c 1353
a60637c8 1354 rcu_read_unlock();
1da177e4
LT
1355 return ret;
1356}
1357
ebf43500
JA
1358/**
1359 * find_get_pages_contig - gang contiguous pagecache lookup
1360 * @mapping: The address_space to search
1361 * @index: The starting page index
1362 * @nr_pages: The maximum number of pages
1363 * @pages: Where the resulting pages are placed
1364 *
1365 * find_get_pages_contig() works exactly like find_get_pages(), except
1366 * that the returned number of pages are guaranteed to be contiguous.
1367 *
1368 * find_get_pages_contig() returns the number of pages which were found.
1369 */
1370unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
1371 unsigned int nr_pages, struct page **pages)
1372{
0fc9d104
KK
1373 struct radix_tree_iter iter;
1374 void **slot;
1375 unsigned int ret = 0;
1376
1377 if (unlikely(!nr_pages))
1378 return 0;
a60637c8
NP
1379
1380 rcu_read_lock();
0fc9d104 1381 radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
a60637c8
NP
1382 struct page *page;
1383repeat:
0fc9d104
KK
1384 page = radix_tree_deref_slot(slot);
1385 /* The hole, there no reason to continue */
a60637c8 1386 if (unlikely(!page))
0fc9d104 1387 break;
9d8aa4ea 1388
a2c16d6c 1389 if (radix_tree_exception(page)) {
8079b1c8 1390 if (radix_tree_deref_retry(page)) {
2cf938aa
MW
1391 slot = radix_tree_iter_retry(&iter);
1392 continue;
8079b1c8 1393 }
a2c16d6c 1394 /*
139b6a6f
JW
1395 * A shadow entry of a recently evicted page,
1396 * or a swap entry from shmem/tmpfs. Stop
1397 * looking for contiguous pages.
a2c16d6c 1398 */
8079b1c8 1399 break;
a2c16d6c 1400 }
ebf43500 1401
a60637c8
NP
1402 if (!page_cache_get_speculative(page))
1403 goto repeat;
1404
1405 /* Has the page moved? */
0fc9d104 1406 if (unlikely(page != *slot)) {
09cbfeaf 1407 put_page(page);
a60637c8
NP
1408 goto repeat;
1409 }
1410
9cbb4cb2
NP
1411 /*
1412 * must check mapping and index after taking the ref.
1413 * otherwise we can get both false positives and false
1414 * negatives, which is just confusing to the caller.
1415 */
0fc9d104 1416 if (page->mapping == NULL || page->index != iter.index) {
09cbfeaf 1417 put_page(page);
9cbb4cb2
NP
1418 break;
1419 }
1420
a60637c8 1421 pages[ret] = page;
0fc9d104
KK
1422 if (++ret == nr_pages)
1423 break;
ebf43500 1424 }
a60637c8
NP
1425 rcu_read_unlock();
1426 return ret;
ebf43500 1427}
ef71c15c 1428EXPORT_SYMBOL(find_get_pages_contig);
ebf43500 1429
485bb99b
RD
1430/**
1431 * find_get_pages_tag - find and return pages that match @tag
1432 * @mapping: the address_space to search
1433 * @index: the starting page index
1434 * @tag: the tag index
1435 * @nr_pages: the maximum number of pages
1436 * @pages: where the resulting pages are placed
1437 *
1da177e4 1438 * Like find_get_pages, except we only return pages which are tagged with
485bb99b 1439 * @tag. We update @index to index the next page for the traversal.
1da177e4
LT
1440 */
1441unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
1442 int tag, unsigned int nr_pages, struct page **pages)
1443{
0fc9d104
KK
1444 struct radix_tree_iter iter;
1445 void **slot;
1446 unsigned ret = 0;
1447
1448 if (unlikely(!nr_pages))
1449 return 0;
a60637c8
NP
1450
1451 rcu_read_lock();
0fc9d104
KK
1452 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1453 &iter, *index, tag) {
a60637c8
NP
1454 struct page *page;
1455repeat:
0fc9d104 1456 page = radix_tree_deref_slot(slot);
a60637c8
NP
1457 if (unlikely(!page))
1458 continue;
9d8aa4ea 1459
a2c16d6c 1460 if (radix_tree_exception(page)) {
8079b1c8 1461 if (radix_tree_deref_retry(page)) {
2cf938aa
MW
1462 slot = radix_tree_iter_retry(&iter);
1463 continue;
8079b1c8 1464 }
a2c16d6c 1465 /*
139b6a6f
JW
1466 * A shadow entry of a recently evicted page.
1467 *
1468 * Those entries should never be tagged, but
1469 * this tree walk is lockless and the tags are
1470 * looked up in bulk, one radix tree node at a
1471 * time, so there is a sizable window for page
1472 * reclaim to evict a page we saw tagged.
1473 *
1474 * Skip over it.
a2c16d6c 1475 */
139b6a6f 1476 continue;
a2c16d6c 1477 }
a60637c8
NP
1478
1479 if (!page_cache_get_speculative(page))
1480 goto repeat;
1481
1482 /* Has the page moved? */
0fc9d104 1483 if (unlikely(page != *slot)) {
09cbfeaf 1484 put_page(page);
a60637c8
NP
1485 goto repeat;
1486 }
1487
1488 pages[ret] = page;
0fc9d104
KK
1489 if (++ret == nr_pages)
1490 break;
a60637c8 1491 }
5b280c0c 1492
a60637c8 1493 rcu_read_unlock();
1da177e4 1494
1da177e4
LT
1495 if (ret)
1496 *index = pages[ret - 1]->index + 1;
a60637c8 1497
1da177e4
LT
1498 return ret;
1499}
ef71c15c 1500EXPORT_SYMBOL(find_get_pages_tag);
1da177e4 1501
7e7f7749
RZ
1502/**
1503 * find_get_entries_tag - find and return entries that match @tag
1504 * @mapping: the address_space to search
1505 * @start: the starting page cache index
1506 * @tag: the tag index
1507 * @nr_entries: the maximum number of entries
1508 * @entries: where the resulting entries are placed
1509 * @indices: the cache indices corresponding to the entries in @entries
1510 *
1511 * Like find_get_entries, except we only return entries which are tagged with
1512 * @tag.
1513 */
1514unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
1515 int tag, unsigned int nr_entries,
1516 struct page **entries, pgoff_t *indices)
1517{
1518 void **slot;
1519 unsigned int ret = 0;
1520 struct radix_tree_iter iter;
1521
1522 if (!nr_entries)
1523 return 0;
1524
1525 rcu_read_lock();
7e7f7749
RZ
1526 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1527 &iter, start, tag) {
1528 struct page *page;
1529repeat:
1530 page = radix_tree_deref_slot(slot);
1531 if (unlikely(!page))
1532 continue;
1533 if (radix_tree_exception(page)) {
1534 if (radix_tree_deref_retry(page)) {
2cf938aa
MW
1535 slot = radix_tree_iter_retry(&iter);
1536 continue;
7e7f7749
RZ
1537 }
1538
1539 /*
1540 * A shadow entry of a recently evicted page, a swap
1541 * entry from shmem/tmpfs or a DAX entry. Return it
1542 * without attempting to raise page count.
1543 */
1544 goto export;
1545 }
1546 if (!page_cache_get_speculative(page))
1547 goto repeat;
1548
1549 /* Has the page moved? */
1550 if (unlikely(page != *slot)) {
09cbfeaf 1551 put_page(page);
7e7f7749
RZ
1552 goto repeat;
1553 }
1554export:
1555 indices[ret] = iter.index;
1556 entries[ret] = page;
1557 if (++ret == nr_entries)
1558 break;
1559 }
1560 rcu_read_unlock();
1561 return ret;
1562}
1563EXPORT_SYMBOL(find_get_entries_tag);
1564
76d42bd9
WF
1565/*
1566 * CD/DVDs are error prone. When a medium error occurs, the driver may fail
1567 * a _large_ part of the i/o request. Imagine the worst scenario:
1568 *
1569 * ---R__________________________________________B__________
1570 * ^ reading here ^ bad block(assume 4k)
1571 *
1572 * read(R) => miss => readahead(R...B) => media error => frustrating retries
1573 * => failing the whole request => read(R) => read(R+1) =>
1574 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
1575 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
1576 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
1577 *
1578 * It is going insane. Fix it by quickly scaling down the readahead size.
1579 */
1580static void shrink_readahead_size_eio(struct file *filp,
1581 struct file_ra_state *ra)
1582{
76d42bd9 1583 ra->ra_pages /= 4;
76d42bd9
WF
1584}
1585
485bb99b 1586/**
36e78914 1587 * do_generic_file_read - generic file read routine
485bb99b
RD
1588 * @filp: the file to read
1589 * @ppos: current file position
6e58e79d
AV
1590 * @iter: data destination
1591 * @written: already copied
485bb99b 1592 *
1da177e4 1593 * This is a generic file read routine, and uses the
485bb99b 1594 * mapping->a_ops->readpage() function for the actual low-level stuff.
1da177e4
LT
1595 *
1596 * This is really ugly. But the goto's actually try to clarify some
1597 * of the logic when it comes to error handling etc.
1da177e4 1598 */
6e58e79d
AV
1599static ssize_t do_generic_file_read(struct file *filp, loff_t *ppos,
1600 struct iov_iter *iter, ssize_t written)
1da177e4 1601{
36e78914 1602 struct address_space *mapping = filp->f_mapping;
1da177e4 1603 struct inode *inode = mapping->host;
36e78914 1604 struct file_ra_state *ra = &filp->f_ra;
57f6b96c
FW
1605 pgoff_t index;
1606 pgoff_t last_index;
1607 pgoff_t prev_index;
1608 unsigned long offset; /* offset into pagecache page */
ec0f1637 1609 unsigned int prev_offset;
6e58e79d 1610 int error = 0;
1da177e4 1611
09cbfeaf
KS
1612 index = *ppos >> PAGE_SHIFT;
1613 prev_index = ra->prev_pos >> PAGE_SHIFT;
1614 prev_offset = ra->prev_pos & (PAGE_SIZE-1);
1615 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT;
1616 offset = *ppos & ~PAGE_MASK;
1da177e4 1617
1da177e4
LT
1618 for (;;) {
1619 struct page *page;
57f6b96c 1620 pgoff_t end_index;
a32ea1e1 1621 loff_t isize;
1da177e4
LT
1622 unsigned long nr, ret;
1623
1da177e4 1624 cond_resched();
1da177e4
LT
1625find_page:
1626 page = find_get_page(mapping, index);
3ea89ee8 1627 if (!page) {
cf914a7d 1628 page_cache_sync_readahead(mapping,
7ff81078 1629 ra, filp,
3ea89ee8
FW
1630 index, last_index - index);
1631 page = find_get_page(mapping, index);
1632 if (unlikely(page == NULL))
1633 goto no_cached_page;
1634 }
1635 if (PageReadahead(page)) {
cf914a7d 1636 page_cache_async_readahead(mapping,
7ff81078 1637 ra, filp, page,
3ea89ee8 1638 index, last_index - index);
1da177e4 1639 }
8ab22b9a 1640 if (!PageUptodate(page)) {
ebded027
MG
1641 /*
1642 * See comment in do_read_cache_page on why
1643 * wait_on_page_locked is used to avoid unnecessarily
1644 * serialisations and why it's safe.
1645 */
1646 wait_on_page_locked_killable(page);
1647 if (PageUptodate(page))
1648 goto page_ok;
1649
09cbfeaf 1650 if (inode->i_blkbits == PAGE_SHIFT ||
8ab22b9a
HH
1651 !mapping->a_ops->is_partially_uptodate)
1652 goto page_not_up_to_date;
529ae9aa 1653 if (!trylock_page(page))
8ab22b9a 1654 goto page_not_up_to_date;
8d056cb9
DH
1655 /* Did it get truncated before we got the lock? */
1656 if (!page->mapping)
1657 goto page_not_up_to_date_locked;
8ab22b9a 1658 if (!mapping->a_ops->is_partially_uptodate(page,
6e58e79d 1659 offset, iter->count))
8ab22b9a
HH
1660 goto page_not_up_to_date_locked;
1661 unlock_page(page);
1662 }
1da177e4 1663page_ok:
a32ea1e1
N
1664 /*
1665 * i_size must be checked after we know the page is Uptodate.
1666 *
1667 * Checking i_size after the check allows us to calculate
1668 * the correct value for "nr", which means the zero-filled
1669 * part of the page is not copied back to userspace (unless
1670 * another truncate extends the file - this is desired though).
1671 */
1672
1673 isize = i_size_read(inode);
09cbfeaf 1674 end_index = (isize - 1) >> PAGE_SHIFT;
a32ea1e1 1675 if (unlikely(!isize || index > end_index)) {
09cbfeaf 1676 put_page(page);
a32ea1e1
N
1677 goto out;
1678 }
1679
1680 /* nr is the maximum number of bytes to copy from this page */
09cbfeaf 1681 nr = PAGE_SIZE;
a32ea1e1 1682 if (index == end_index) {
09cbfeaf 1683 nr = ((isize - 1) & ~PAGE_MASK) + 1;
a32ea1e1 1684 if (nr <= offset) {
09cbfeaf 1685 put_page(page);
a32ea1e1
N
1686 goto out;
1687 }
1688 }
1689 nr = nr - offset;
1da177e4
LT
1690
1691 /* If users can be writing to this page using arbitrary
1692 * virtual addresses, take care about potential aliasing
1693 * before reading the page on the kernel side.
1694 */
1695 if (mapping_writably_mapped(mapping))
1696 flush_dcache_page(page);
1697
1698 /*
ec0f1637
JK
1699 * When a sequential read accesses a page several times,
1700 * only mark it as accessed the first time.
1da177e4 1701 */
ec0f1637 1702 if (prev_index != index || offset != prev_offset)
1da177e4
LT
1703 mark_page_accessed(page);
1704 prev_index = index;
1705
1706 /*
1707 * Ok, we have the page, and it's up-to-date, so
1708 * now we can copy it to user space...
1da177e4 1709 */
6e58e79d
AV
1710
1711 ret = copy_page_to_iter(page, offset, nr, iter);
1da177e4 1712 offset += ret;
09cbfeaf
KS
1713 index += offset >> PAGE_SHIFT;
1714 offset &= ~PAGE_MASK;
6ce745ed 1715 prev_offset = offset;
1da177e4 1716
09cbfeaf 1717 put_page(page);
6e58e79d
AV
1718 written += ret;
1719 if (!iov_iter_count(iter))
1720 goto out;
1721 if (ret < nr) {
1722 error = -EFAULT;
1723 goto out;
1724 }
1725 continue;
1da177e4
LT
1726
1727page_not_up_to_date:
1728 /* Get exclusive access to the page ... */
85462323
ON
1729 error = lock_page_killable(page);
1730 if (unlikely(error))
1731 goto readpage_error;
1da177e4 1732
8ab22b9a 1733page_not_up_to_date_locked:
da6052f7 1734 /* Did it get truncated before we got the lock? */
1da177e4
LT
1735 if (!page->mapping) {
1736 unlock_page(page);
09cbfeaf 1737 put_page(page);
1da177e4
LT
1738 continue;
1739 }
1740
1741 /* Did somebody else fill it already? */
1742 if (PageUptodate(page)) {
1743 unlock_page(page);
1744 goto page_ok;
1745 }
1746
1747readpage:
91803b49
JM
1748 /*
1749 * A previous I/O error may have been due to temporary
1750 * failures, eg. multipath errors.
1751 * PG_error will be set again if readpage fails.
1752 */
1753 ClearPageError(page);
1da177e4
LT
1754 /* Start the actual read. The read will unlock the page. */
1755 error = mapping->a_ops->readpage(filp, page);
1756
994fc28c
ZB
1757 if (unlikely(error)) {
1758 if (error == AOP_TRUNCATED_PAGE) {
09cbfeaf 1759 put_page(page);
6e58e79d 1760 error = 0;
994fc28c
ZB
1761 goto find_page;
1762 }
1da177e4 1763 goto readpage_error;
994fc28c 1764 }
1da177e4
LT
1765
1766 if (!PageUptodate(page)) {
85462323
ON
1767 error = lock_page_killable(page);
1768 if (unlikely(error))
1769 goto readpage_error;
1da177e4
LT
1770 if (!PageUptodate(page)) {
1771 if (page->mapping == NULL) {
1772 /*
2ecdc82e 1773 * invalidate_mapping_pages got it
1da177e4
LT
1774 */
1775 unlock_page(page);
09cbfeaf 1776 put_page(page);
1da177e4
LT
1777 goto find_page;
1778 }
1779 unlock_page(page);
7ff81078 1780 shrink_readahead_size_eio(filp, ra);
85462323
ON
1781 error = -EIO;
1782 goto readpage_error;
1da177e4
LT
1783 }
1784 unlock_page(page);
1785 }
1786
1da177e4
LT
1787 goto page_ok;
1788
1789readpage_error:
1790 /* UHHUH! A synchronous read error occurred. Report it */
09cbfeaf 1791 put_page(page);
1da177e4
LT
1792 goto out;
1793
1794no_cached_page:
1795 /*
1796 * Ok, it wasn't cached, so we need to create a new
1797 * page..
1798 */
eb2be189
NP
1799 page = page_cache_alloc_cold(mapping);
1800 if (!page) {
6e58e79d 1801 error = -ENOMEM;
eb2be189 1802 goto out;
1da177e4 1803 }
6afdb859 1804 error = add_to_page_cache_lru(page, mapping, index,
c62d2555 1805 mapping_gfp_constraint(mapping, GFP_KERNEL));
1da177e4 1806 if (error) {
09cbfeaf 1807 put_page(page);
6e58e79d
AV
1808 if (error == -EEXIST) {
1809 error = 0;
1da177e4 1810 goto find_page;
6e58e79d 1811 }
1da177e4
LT
1812 goto out;
1813 }
1da177e4
LT
1814 goto readpage;
1815 }
1816
1817out:
7ff81078 1818 ra->prev_pos = prev_index;
09cbfeaf 1819 ra->prev_pos <<= PAGE_SHIFT;
7ff81078 1820 ra->prev_pos |= prev_offset;
1da177e4 1821
09cbfeaf 1822 *ppos = ((loff_t)index << PAGE_SHIFT) + offset;
0c6aa263 1823 file_accessed(filp);
6e58e79d 1824 return written ? written : error;
1da177e4
LT
1825}
1826
485bb99b 1827/**
6abd2322 1828 * generic_file_read_iter - generic filesystem read routine
485bb99b 1829 * @iocb: kernel I/O control block
6abd2322 1830 * @iter: destination for the data read
485bb99b 1831 *
6abd2322 1832 * This is the "read_iter()" routine for all filesystems
1da177e4
LT
1833 * that can use the page cache directly.
1834 */
1835ssize_t
ed978a81 1836generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
1da177e4 1837{
ed978a81 1838 struct file *file = iocb->ki_filp;
cb66a7a1 1839 ssize_t retval = 0;
e7080a43
NS
1840 size_t count = iov_iter_count(iter);
1841
1842 if (!count)
1843 goto out; /* skip atime */
1da177e4 1844
2ba48ce5 1845 if (iocb->ki_flags & IOCB_DIRECT) {
ed978a81
AV
1846 struct address_space *mapping = file->f_mapping;
1847 struct inode *inode = mapping->host;
543ade1f 1848 loff_t size;
1da177e4 1849
1da177e4 1850 size = i_size_read(inode);
c64fb5c7
CH
1851 retval = filemap_write_and_wait_range(mapping, iocb->ki_pos,
1852 iocb->ki_pos + count - 1);
9fe55eea 1853 if (!retval) {
ed978a81 1854 struct iov_iter data = *iter;
c8b8e32d 1855 retval = mapping->a_ops->direct_IO(iocb, &data);
9fe55eea 1856 }
d8d3d94b 1857
9fe55eea 1858 if (retval > 0) {
c64fb5c7 1859 iocb->ki_pos += retval;
ed978a81 1860 iov_iter_advance(iter, retval);
9fe55eea 1861 }
66f998f6 1862
9fe55eea
SW
1863 /*
1864 * Btrfs can have a short DIO read if we encounter
1865 * compressed extents, so if there was an error, or if
1866 * we've already read everything we wanted to, or if
1867 * there was a short read because we hit EOF, go ahead
1868 * and return. Otherwise fallthrough to buffered io for
fbbbad4b
MW
1869 * the rest of the read. Buffered reads will not work for
1870 * DAX files, so don't bother trying.
9fe55eea 1871 */
c64fb5c7 1872 if (retval < 0 || !iov_iter_count(iter) || iocb->ki_pos >= size ||
fbbbad4b 1873 IS_DAX(inode)) {
ed978a81 1874 file_accessed(file);
9fe55eea 1875 goto out;
0e0bcae3 1876 }
1da177e4
LT
1877 }
1878
c64fb5c7 1879 retval = do_generic_file_read(file, &iocb->ki_pos, iter, retval);
1da177e4
LT
1880out:
1881 return retval;
1882}
ed978a81 1883EXPORT_SYMBOL(generic_file_read_iter);
1da177e4 1884
1da177e4 1885#ifdef CONFIG_MMU
485bb99b
RD
1886/**
1887 * page_cache_read - adds requested page to the page cache if not already there
1888 * @file: file to read
1889 * @offset: page index
62eb320a 1890 * @gfp_mask: memory allocation flags
485bb99b 1891 *
1da177e4
LT
1892 * This adds the requested page to the page cache if it isn't already there,
1893 * and schedules an I/O to read in its contents from disk.
1894 */
c20cd45e 1895static int page_cache_read(struct file *file, pgoff_t offset, gfp_t gfp_mask)
1da177e4
LT
1896{
1897 struct address_space *mapping = file->f_mapping;
99dadfdd 1898 struct page *page;
994fc28c 1899 int ret;
1da177e4 1900
994fc28c 1901 do {
c20cd45e 1902 page = __page_cache_alloc(gfp_mask|__GFP_COLD);
994fc28c
ZB
1903 if (!page)
1904 return -ENOMEM;
1905
c20cd45e 1906 ret = add_to_page_cache_lru(page, mapping, offset, gfp_mask & GFP_KERNEL);
994fc28c
ZB
1907 if (ret == 0)
1908 ret = mapping->a_ops->readpage(file, page);
1909 else if (ret == -EEXIST)
1910 ret = 0; /* losing race to add is OK */
1da177e4 1911
09cbfeaf 1912 put_page(page);
1da177e4 1913
994fc28c 1914 } while (ret == AOP_TRUNCATED_PAGE);
99dadfdd 1915
994fc28c 1916 return ret;
1da177e4
LT
1917}
1918
1919#define MMAP_LOTSAMISS (100)
1920
ef00e08e
LT
1921/*
1922 * Synchronous readahead happens when we don't even find
1923 * a page in the page cache at all.
1924 */
1925static void do_sync_mmap_readahead(struct vm_area_struct *vma,
1926 struct file_ra_state *ra,
1927 struct file *file,
1928 pgoff_t offset)
1929{
ef00e08e
LT
1930 struct address_space *mapping = file->f_mapping;
1931
1932 /* If we don't want any read-ahead, don't bother */
64363aad 1933 if (vma->vm_flags & VM_RAND_READ)
ef00e08e 1934 return;
275b12bf
WF
1935 if (!ra->ra_pages)
1936 return;
ef00e08e 1937
64363aad 1938 if (vma->vm_flags & VM_SEQ_READ) {
7ffc59b4
WF
1939 page_cache_sync_readahead(mapping, ra, file, offset,
1940 ra->ra_pages);
ef00e08e
LT
1941 return;
1942 }
1943
207d04ba
AK
1944 /* Avoid banging the cache line if not needed */
1945 if (ra->mmap_miss < MMAP_LOTSAMISS * 10)
ef00e08e
LT
1946 ra->mmap_miss++;
1947
1948 /*
1949 * Do we miss much more than hit in this file? If so,
1950 * stop bothering with read-ahead. It will only hurt.
1951 */
1952 if (ra->mmap_miss > MMAP_LOTSAMISS)
1953 return;
1954
d30a1100
WF
1955 /*
1956 * mmap read-around
1957 */
600e19af
RG
1958 ra->start = max_t(long, 0, offset - ra->ra_pages / 2);
1959 ra->size = ra->ra_pages;
1960 ra->async_size = ra->ra_pages / 4;
275b12bf 1961 ra_submit(ra, mapping, file);
ef00e08e
LT
1962}
1963
1964/*
1965 * Asynchronous readahead happens when we find the page and PG_readahead,
1966 * so we want to possibly extend the readahead further..
1967 */
1968static void do_async_mmap_readahead(struct vm_area_struct *vma,
1969 struct file_ra_state *ra,
1970 struct file *file,
1971 struct page *page,
1972 pgoff_t offset)
1973{
1974 struct address_space *mapping = file->f_mapping;
1975
1976 /* If we don't want any read-ahead, don't bother */
64363aad 1977 if (vma->vm_flags & VM_RAND_READ)
ef00e08e
LT
1978 return;
1979 if (ra->mmap_miss > 0)
1980 ra->mmap_miss--;
1981 if (PageReadahead(page))
2fad6f5d
WF
1982 page_cache_async_readahead(mapping, ra, file,
1983 page, offset, ra->ra_pages);
ef00e08e
LT
1984}
1985
485bb99b 1986/**
54cb8821 1987 * filemap_fault - read in file data for page fault handling
d0217ac0
NP
1988 * @vma: vma in which the fault was taken
1989 * @vmf: struct vm_fault containing details of the fault
485bb99b 1990 *
54cb8821 1991 * filemap_fault() is invoked via the vma operations vector for a
1da177e4
LT
1992 * mapped memory region to read in file data during a page fault.
1993 *
1994 * The goto's are kind of ugly, but this streamlines the normal case of having
1995 * it in the page cache, and handles the special cases reasonably without
1996 * having a lot of duplicated code.
9a95f3cf
PC
1997 *
1998 * vma->vm_mm->mmap_sem must be held on entry.
1999 *
2000 * If our return value has VM_FAULT_RETRY set, it's because
2001 * lock_page_or_retry() returned 0.
2002 * The mmap_sem has usually been released in this case.
2003 * See __lock_page_or_retry() for the exception.
2004 *
2005 * If our return value does not have VM_FAULT_RETRY set, the mmap_sem
2006 * has not been released.
2007 *
2008 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
1da177e4 2009 */
d0217ac0 2010int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1da177e4
LT
2011{
2012 int error;
54cb8821 2013 struct file *file = vma->vm_file;
1da177e4
LT
2014 struct address_space *mapping = file->f_mapping;
2015 struct file_ra_state *ra = &file->f_ra;
2016 struct inode *inode = mapping->host;
ef00e08e 2017 pgoff_t offset = vmf->pgoff;
1da177e4 2018 struct page *page;
99e3e53f 2019 loff_t size;
83c54070 2020 int ret = 0;
1da177e4 2021
09cbfeaf
KS
2022 size = round_up(i_size_read(inode), PAGE_SIZE);
2023 if (offset >= size >> PAGE_SHIFT)
5307cc1a 2024 return VM_FAULT_SIGBUS;
1da177e4 2025
1da177e4 2026 /*
49426420 2027 * Do we have something in the page cache already?
1da177e4 2028 */
ef00e08e 2029 page = find_get_page(mapping, offset);
45cac65b 2030 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
1da177e4 2031 /*
ef00e08e
LT
2032 * We found the page, so try async readahead before
2033 * waiting for the lock.
1da177e4 2034 */
ef00e08e 2035 do_async_mmap_readahead(vma, ra, file, page, offset);
45cac65b 2036 } else if (!page) {
ef00e08e
LT
2037 /* No page in the page cache at all */
2038 do_sync_mmap_readahead(vma, ra, file, offset);
2039 count_vm_event(PGMAJFAULT);
456f998e 2040 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
ef00e08e
LT
2041 ret = VM_FAULT_MAJOR;
2042retry_find:
b522c94d 2043 page = find_get_page(mapping, offset);
1da177e4
LT
2044 if (!page)
2045 goto no_cached_page;
2046 }
2047
d88c0922 2048 if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) {
09cbfeaf 2049 put_page(page);
d065bd81 2050 return ret | VM_FAULT_RETRY;
d88c0922 2051 }
b522c94d
ML
2052
2053 /* Did it get truncated? */
2054 if (unlikely(page->mapping != mapping)) {
2055 unlock_page(page);
2056 put_page(page);
2057 goto retry_find;
2058 }
309381fe 2059 VM_BUG_ON_PAGE(page->index != offset, page);
b522c94d 2060
1da177e4 2061 /*
d00806b1
NP
2062 * We have a locked page in the page cache, now we need to check
2063 * that it's up-to-date. If not, it is going to be due to an error.
1da177e4 2064 */
d00806b1 2065 if (unlikely(!PageUptodate(page)))
1da177e4
LT
2066 goto page_not_uptodate;
2067
ef00e08e
LT
2068 /*
2069 * Found the page and have a reference on it.
2070 * We must recheck i_size under page lock.
2071 */
09cbfeaf
KS
2072 size = round_up(i_size_read(inode), PAGE_SIZE);
2073 if (unlikely(offset >= size >> PAGE_SHIFT)) {
d00806b1 2074 unlock_page(page);
09cbfeaf 2075 put_page(page);
5307cc1a 2076 return VM_FAULT_SIGBUS;
d00806b1
NP
2077 }
2078
d0217ac0 2079 vmf->page = page;
83c54070 2080 return ret | VM_FAULT_LOCKED;
1da177e4 2081
1da177e4
LT
2082no_cached_page:
2083 /*
2084 * We're only likely to ever get here if MADV_RANDOM is in
2085 * effect.
2086 */
c20cd45e 2087 error = page_cache_read(file, offset, vmf->gfp_mask);
1da177e4
LT
2088
2089 /*
2090 * The page we want has now been added to the page cache.
2091 * In the unlikely event that someone removed it in the
2092 * meantime, we'll just come back here and read it again.
2093 */
2094 if (error >= 0)
2095 goto retry_find;
2096
2097 /*
2098 * An error return from page_cache_read can result if the
2099 * system is low on memory, or a problem occurs while trying
2100 * to schedule I/O.
2101 */
2102 if (error == -ENOMEM)
d0217ac0
NP
2103 return VM_FAULT_OOM;
2104 return VM_FAULT_SIGBUS;
1da177e4
LT
2105
2106page_not_uptodate:
1da177e4
LT
2107 /*
2108 * Umm, take care of errors if the page isn't up-to-date.
2109 * Try to re-read it _once_. We do this synchronously,
2110 * because there really aren't any performance issues here
2111 * and we need to check for errors.
2112 */
1da177e4 2113 ClearPageError(page);
994fc28c 2114 error = mapping->a_ops->readpage(file, page);
3ef0f720
MS
2115 if (!error) {
2116 wait_on_page_locked(page);
2117 if (!PageUptodate(page))
2118 error = -EIO;
2119 }
09cbfeaf 2120 put_page(page);
d00806b1
NP
2121
2122 if (!error || error == AOP_TRUNCATED_PAGE)
994fc28c 2123 goto retry_find;
1da177e4 2124
d00806b1 2125 /* Things didn't work out. Return zero to tell the mm layer so. */
76d42bd9 2126 shrink_readahead_size_eio(file, ra);
d0217ac0 2127 return VM_FAULT_SIGBUS;
54cb8821
NP
2128}
2129EXPORT_SYMBOL(filemap_fault);
2130
f1820361
KS
2131void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
2132{
2133 struct radix_tree_iter iter;
2134 void **slot;
2135 struct file *file = vma->vm_file;
2136 struct address_space *mapping = file->f_mapping;
2137 loff_t size;
2138 struct page *page;
2139 unsigned long address = (unsigned long) vmf->virtual_address;
2140 unsigned long addr;
2141 pte_t *pte;
2142
2143 rcu_read_lock();
2144 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, vmf->pgoff) {
2145 if (iter.index > vmf->max_pgoff)
2146 break;
2147repeat:
2148 page = radix_tree_deref_slot(slot);
2149 if (unlikely(!page))
2150 goto next;
2151 if (radix_tree_exception(page)) {
2cf938aa
MW
2152 if (radix_tree_deref_retry(page)) {
2153 slot = radix_tree_iter_retry(&iter);
2154 continue;
2155 }
2156 goto next;
f1820361
KS
2157 }
2158
2159 if (!page_cache_get_speculative(page))
2160 goto repeat;
2161
2162 /* Has the page moved? */
2163 if (unlikely(page != *slot)) {
09cbfeaf 2164 put_page(page);
f1820361
KS
2165 goto repeat;
2166 }
2167
2168 if (!PageUptodate(page) ||
2169 PageReadahead(page) ||
2170 PageHWPoison(page))
2171 goto skip;
2172 if (!trylock_page(page))
2173 goto skip;
2174
2175 if (page->mapping != mapping || !PageUptodate(page))
2176 goto unlock;
2177
09cbfeaf
KS
2178 size = round_up(i_size_read(mapping->host), PAGE_SIZE);
2179 if (page->index >= size >> PAGE_SHIFT)
f1820361
KS
2180 goto unlock;
2181
2182 pte = vmf->pte + page->index - vmf->pgoff;
2183 if (!pte_none(*pte))
2184 goto unlock;
2185
2186 if (file->f_ra.mmap_miss > 0)
2187 file->f_ra.mmap_miss--;
2188 addr = address + (page->index - vmf->pgoff) * PAGE_SIZE;
5c0a85fa 2189 do_set_pte(vma, addr, page, pte, false, false, true);
f1820361
KS
2190 unlock_page(page);
2191 goto next;
2192unlock:
2193 unlock_page(page);
2194skip:
09cbfeaf 2195 put_page(page);
f1820361
KS
2196next:
2197 if (iter.index == vmf->max_pgoff)
2198 break;
2199 }
2200 rcu_read_unlock();
2201}
2202EXPORT_SYMBOL(filemap_map_pages);
2203
4fcf1c62
JK
2204int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
2205{
2206 struct page *page = vmf->page;
496ad9aa 2207 struct inode *inode = file_inode(vma->vm_file);
4fcf1c62
JK
2208 int ret = VM_FAULT_LOCKED;
2209
14da9200 2210 sb_start_pagefault(inode->i_sb);
4fcf1c62
JK
2211 file_update_time(vma->vm_file);
2212 lock_page(page);
2213 if (page->mapping != inode->i_mapping) {
2214 unlock_page(page);
2215 ret = VM_FAULT_NOPAGE;
2216 goto out;
2217 }
14da9200
JK
2218 /*
2219 * We mark the page dirty already here so that when freeze is in
2220 * progress, we are guaranteed that writeback during freezing will
2221 * see the dirty page and writeprotect it again.
2222 */
2223 set_page_dirty(page);
1d1d1a76 2224 wait_for_stable_page(page);
4fcf1c62 2225out:
14da9200 2226 sb_end_pagefault(inode->i_sb);
4fcf1c62
JK
2227 return ret;
2228}
2229EXPORT_SYMBOL(filemap_page_mkwrite);
2230
f0f37e2f 2231const struct vm_operations_struct generic_file_vm_ops = {
54cb8821 2232 .fault = filemap_fault,
f1820361 2233 .map_pages = filemap_map_pages,
4fcf1c62 2234 .page_mkwrite = filemap_page_mkwrite,
1da177e4
LT
2235};
2236
2237/* This is used for a general mmap of a disk file */
2238
2239int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2240{
2241 struct address_space *mapping = file->f_mapping;
2242
2243 if (!mapping->a_ops->readpage)
2244 return -ENOEXEC;
2245 file_accessed(file);
2246 vma->vm_ops = &generic_file_vm_ops;
2247 return 0;
2248}
1da177e4
LT
2249
2250/*
2251 * This is for filesystems which do not implement ->writepage.
2252 */
2253int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
2254{
2255 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2256 return -EINVAL;
2257 return generic_file_mmap(file, vma);
2258}
2259#else
2260int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2261{
2262 return -ENOSYS;
2263}
2264int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
2265{
2266 return -ENOSYS;
2267}
2268#endif /* CONFIG_MMU */
2269
2270EXPORT_SYMBOL(generic_file_mmap);
2271EXPORT_SYMBOL(generic_file_readonly_mmap);
2272
67f9fd91
SL
2273static struct page *wait_on_page_read(struct page *page)
2274{
2275 if (!IS_ERR(page)) {
2276 wait_on_page_locked(page);
2277 if (!PageUptodate(page)) {
09cbfeaf 2278 put_page(page);
67f9fd91
SL
2279 page = ERR_PTR(-EIO);
2280 }
2281 }
2282 return page;
2283}
2284
32b63529 2285static struct page *do_read_cache_page(struct address_space *mapping,
57f6b96c 2286 pgoff_t index,
5e5358e7 2287 int (*filler)(void *, struct page *),
0531b2aa
LT
2288 void *data,
2289 gfp_t gfp)
1da177e4 2290{
eb2be189 2291 struct page *page;
1da177e4
LT
2292 int err;
2293repeat:
2294 page = find_get_page(mapping, index);
2295 if (!page) {
0531b2aa 2296 page = __page_cache_alloc(gfp | __GFP_COLD);
eb2be189
NP
2297 if (!page)
2298 return ERR_PTR(-ENOMEM);
e6f67b8c 2299 err = add_to_page_cache_lru(page, mapping, index, gfp);
eb2be189 2300 if (unlikely(err)) {
09cbfeaf 2301 put_page(page);
eb2be189
NP
2302 if (err == -EEXIST)
2303 goto repeat;
1da177e4 2304 /* Presumably ENOMEM for radix tree node */
1da177e4
LT
2305 return ERR_PTR(err);
2306 }
32b63529
MG
2307
2308filler:
1da177e4
LT
2309 err = filler(data, page);
2310 if (err < 0) {
09cbfeaf 2311 put_page(page);
32b63529 2312 return ERR_PTR(err);
1da177e4 2313 }
1da177e4 2314
32b63529
MG
2315 page = wait_on_page_read(page);
2316 if (IS_ERR(page))
2317 return page;
2318 goto out;
2319 }
1da177e4
LT
2320 if (PageUptodate(page))
2321 goto out;
2322
ebded027
MG
2323 /*
2324 * Page is not up to date and may be locked due one of the following
2325 * case a: Page is being filled and the page lock is held
2326 * case b: Read/write error clearing the page uptodate status
2327 * case c: Truncation in progress (page locked)
2328 * case d: Reclaim in progress
2329 *
2330 * Case a, the page will be up to date when the page is unlocked.
2331 * There is no need to serialise on the page lock here as the page
2332 * is pinned so the lock gives no additional protection. Even if the
2333 * the page is truncated, the data is still valid if PageUptodate as
2334 * it's a race vs truncate race.
2335 * Case b, the page will not be up to date
2336 * Case c, the page may be truncated but in itself, the data may still
2337 * be valid after IO completes as it's a read vs truncate race. The
2338 * operation must restart if the page is not uptodate on unlock but
2339 * otherwise serialising on page lock to stabilise the mapping gives
2340 * no additional guarantees to the caller as the page lock is
2341 * released before return.
2342 * Case d, similar to truncation. If reclaim holds the page lock, it
2343 * will be a race with remove_mapping that determines if the mapping
2344 * is valid on unlock but otherwise the data is valid and there is
2345 * no need to serialise with page lock.
2346 *
2347 * As the page lock gives no additional guarantee, we optimistically
2348 * wait on the page to be unlocked and check if it's up to date and
2349 * use the page if it is. Otherwise, the page lock is required to
2350 * distinguish between the different cases. The motivation is that we
2351 * avoid spurious serialisations and wakeups when multiple processes
2352 * wait on the same page for IO to complete.
2353 */
2354 wait_on_page_locked(page);
2355 if (PageUptodate(page))
2356 goto out;
2357
2358 /* Distinguish between all the cases under the safety of the lock */
1da177e4 2359 lock_page(page);
ebded027
MG
2360
2361 /* Case c or d, restart the operation */
1da177e4
LT
2362 if (!page->mapping) {
2363 unlock_page(page);
09cbfeaf 2364 put_page(page);
32b63529 2365 goto repeat;
1da177e4 2366 }
ebded027
MG
2367
2368 /* Someone else locked and filled the page in a very small window */
1da177e4
LT
2369 if (PageUptodate(page)) {
2370 unlock_page(page);
2371 goto out;
2372 }
32b63529
MG
2373 goto filler;
2374
c855ff37 2375out:
6fe6900e
NP
2376 mark_page_accessed(page);
2377 return page;
2378}
0531b2aa
LT
2379
2380/**
67f9fd91 2381 * read_cache_page - read into page cache, fill it if needed
0531b2aa
LT
2382 * @mapping: the page's address_space
2383 * @index: the page index
2384 * @filler: function to perform the read
5e5358e7 2385 * @data: first arg to filler(data, page) function, often left as NULL
0531b2aa 2386 *
0531b2aa 2387 * Read into the page cache. If a page already exists, and PageUptodate() is
67f9fd91 2388 * not set, try to fill the page and wait for it to become unlocked.
0531b2aa
LT
2389 *
2390 * If the page does not get brought uptodate, return -EIO.
2391 */
67f9fd91 2392struct page *read_cache_page(struct address_space *mapping,
0531b2aa 2393 pgoff_t index,
5e5358e7 2394 int (*filler)(void *, struct page *),
0531b2aa
LT
2395 void *data)
2396{
2397 return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
2398}
67f9fd91 2399EXPORT_SYMBOL(read_cache_page);
0531b2aa
LT
2400
2401/**
2402 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
2403 * @mapping: the page's address_space
2404 * @index: the page index
2405 * @gfp: the page allocator flags to use if allocating
2406 *
2407 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
e6f67b8c 2408 * any new page allocations done using the specified allocation flags.
0531b2aa
LT
2409 *
2410 * If the page does not get brought uptodate, return -EIO.
2411 */
2412struct page *read_cache_page_gfp(struct address_space *mapping,
2413 pgoff_t index,
2414 gfp_t gfp)
2415{
2416 filler_t *filler = (filler_t *)mapping->a_ops->readpage;
2417
67f9fd91 2418 return do_read_cache_page(mapping, index, filler, NULL, gfp);
0531b2aa
LT
2419}
2420EXPORT_SYMBOL(read_cache_page_gfp);
2421
1da177e4
LT
2422/*
2423 * Performs necessary checks before doing a write
2424 *
485bb99b 2425 * Can adjust writing position or amount of bytes to write.
1da177e4
LT
2426 * Returns appropriate error code that caller should return or
2427 * zero in case that write should be allowed.
2428 */
3309dd04 2429inline ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
1da177e4 2430{
3309dd04 2431 struct file *file = iocb->ki_filp;
1da177e4 2432 struct inode *inode = file->f_mapping->host;
59e99e5b 2433 unsigned long limit = rlimit(RLIMIT_FSIZE);
3309dd04 2434 loff_t pos;
1da177e4 2435
3309dd04
AV
2436 if (!iov_iter_count(from))
2437 return 0;
1da177e4 2438
0fa6b005 2439 /* FIXME: this is for backwards compatibility with 2.4 */
2ba48ce5 2440 if (iocb->ki_flags & IOCB_APPEND)
3309dd04 2441 iocb->ki_pos = i_size_read(inode);
1da177e4 2442
3309dd04 2443 pos = iocb->ki_pos;
1da177e4 2444
0fa6b005 2445 if (limit != RLIM_INFINITY) {
3309dd04 2446 if (iocb->ki_pos >= limit) {
0fa6b005
AV
2447 send_sig(SIGXFSZ, current, 0);
2448 return -EFBIG;
1da177e4 2449 }
3309dd04 2450 iov_iter_truncate(from, limit - (unsigned long)pos);
1da177e4
LT
2451 }
2452
2453 /*
2454 * LFS rule
2455 */
3309dd04 2456 if (unlikely(pos + iov_iter_count(from) > MAX_NON_LFS &&
1da177e4 2457 !(file->f_flags & O_LARGEFILE))) {
3309dd04 2458 if (pos >= MAX_NON_LFS)
1da177e4 2459 return -EFBIG;
3309dd04 2460 iov_iter_truncate(from, MAX_NON_LFS - (unsigned long)pos);
1da177e4
LT
2461 }
2462
2463 /*
2464 * Are we about to exceed the fs block limit ?
2465 *
2466 * If we have written data it becomes a short write. If we have
2467 * exceeded without writing data we send a signal and return EFBIG.
2468 * Linus frestrict idea will clean these up nicely..
2469 */
3309dd04
AV
2470 if (unlikely(pos >= inode->i_sb->s_maxbytes))
2471 return -EFBIG;
1da177e4 2472
3309dd04
AV
2473 iov_iter_truncate(from, inode->i_sb->s_maxbytes - pos);
2474 return iov_iter_count(from);
1da177e4
LT
2475}
2476EXPORT_SYMBOL(generic_write_checks);
2477
afddba49
NP
2478int pagecache_write_begin(struct file *file, struct address_space *mapping,
2479 loff_t pos, unsigned len, unsigned flags,
2480 struct page **pagep, void **fsdata)
2481{
2482 const struct address_space_operations *aops = mapping->a_ops;
2483
4e02ed4b 2484 return aops->write_begin(file, mapping, pos, len, flags,
afddba49 2485 pagep, fsdata);
afddba49
NP
2486}
2487EXPORT_SYMBOL(pagecache_write_begin);
2488
2489int pagecache_write_end(struct file *file, struct address_space *mapping,
2490 loff_t pos, unsigned len, unsigned copied,
2491 struct page *page, void *fsdata)
2492{
2493 const struct address_space_operations *aops = mapping->a_ops;
afddba49 2494
4e02ed4b 2495 return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
afddba49
NP
2496}
2497EXPORT_SYMBOL(pagecache_write_end);
2498
1da177e4 2499ssize_t
1af5bb49 2500generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
1da177e4
LT
2501{
2502 struct file *file = iocb->ki_filp;
2503 struct address_space *mapping = file->f_mapping;
2504 struct inode *inode = mapping->host;
1af5bb49 2505 loff_t pos = iocb->ki_pos;
1da177e4 2506 ssize_t written;
a969e903
CH
2507 size_t write_len;
2508 pgoff_t end;
26978b8b 2509 struct iov_iter data;
1da177e4 2510
0c949334 2511 write_len = iov_iter_count(from);
09cbfeaf 2512 end = (pos + write_len - 1) >> PAGE_SHIFT;
a969e903 2513
48b47c56 2514 written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);
a969e903
CH
2515 if (written)
2516 goto out;
2517
2518 /*
2519 * After a write we want buffered reads to be sure to go to disk to get
2520 * the new data. We invalidate clean cached page from the region we're
2521 * about to write. We do this *before* the write so that we can return
6ccfa806 2522 * without clobbering -EIOCBQUEUED from ->direct_IO().
a969e903
CH
2523 */
2524 if (mapping->nrpages) {
2525 written = invalidate_inode_pages2_range(mapping,
09cbfeaf 2526 pos >> PAGE_SHIFT, end);
6ccfa806
HH
2527 /*
2528 * If a page can not be invalidated, return 0 to fall back
2529 * to buffered write.
2530 */
2531 if (written) {
2532 if (written == -EBUSY)
2533 return 0;
a969e903 2534 goto out;
6ccfa806 2535 }
a969e903
CH
2536 }
2537
26978b8b 2538 data = *from;
c8b8e32d 2539 written = mapping->a_ops->direct_IO(iocb, &data);
a969e903
CH
2540
2541 /*
2542 * Finally, try again to invalidate clean pages which might have been
2543 * cached by non-direct readahead, or faulted in by get_user_pages()
2544 * if the source of the write was an mmap'ed region of the file
2545 * we're writing. Either one is a pretty crazy thing to do,
2546 * so we don't support it 100%. If this invalidation
2547 * fails, tough, the write still worked...
2548 */
2549 if (mapping->nrpages) {
2550 invalidate_inode_pages2_range(mapping,
09cbfeaf 2551 pos >> PAGE_SHIFT, end);
a969e903
CH
2552 }
2553
1da177e4 2554 if (written > 0) {
0116651c 2555 pos += written;
f8579f86 2556 iov_iter_advance(from, written);
0116651c
NK
2557 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
2558 i_size_write(inode, pos);
1da177e4
LT
2559 mark_inode_dirty(inode);
2560 }
5cb6c6c7 2561 iocb->ki_pos = pos;
1da177e4 2562 }
a969e903 2563out:
1da177e4
LT
2564 return written;
2565}
2566EXPORT_SYMBOL(generic_file_direct_write);
2567
eb2be189
NP
2568/*
2569 * Find or create a page at the given pagecache position. Return the locked
2570 * page. This function is specifically for buffered writes.
2571 */
54566b2c
NP
2572struct page *grab_cache_page_write_begin(struct address_space *mapping,
2573 pgoff_t index, unsigned flags)
eb2be189 2574{
eb2be189 2575 struct page *page;
bbddabe2 2576 int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT;
0faa70cb 2577
54566b2c 2578 if (flags & AOP_FLAG_NOFS)
2457aec6
MG
2579 fgp_flags |= FGP_NOFS;
2580
2581 page = pagecache_get_page(mapping, index, fgp_flags,
45f87de5 2582 mapping_gfp_mask(mapping));
c585a267 2583 if (page)
2457aec6 2584 wait_for_stable_page(page);
eb2be189 2585
eb2be189
NP
2586 return page;
2587}
54566b2c 2588EXPORT_SYMBOL(grab_cache_page_write_begin);
eb2be189 2589
3b93f911 2590ssize_t generic_perform_write(struct file *file,
afddba49
NP
2591 struct iov_iter *i, loff_t pos)
2592{
2593 struct address_space *mapping = file->f_mapping;
2594 const struct address_space_operations *a_ops = mapping->a_ops;
2595 long status = 0;
2596 ssize_t written = 0;
674b892e
NP
2597 unsigned int flags = 0;
2598
2599 /*
2600 * Copies from kernel address space cannot fail (NFSD is a big user).
2601 */
777eda2c 2602 if (!iter_is_iovec(i))
674b892e 2603 flags |= AOP_FLAG_UNINTERRUPTIBLE;
afddba49
NP
2604
2605 do {
2606 struct page *page;
afddba49
NP
2607 unsigned long offset; /* Offset into pagecache page */
2608 unsigned long bytes; /* Bytes to write to page */
2609 size_t copied; /* Bytes copied from user */
2610 void *fsdata;
2611
09cbfeaf
KS
2612 offset = (pos & (PAGE_SIZE - 1));
2613 bytes = min_t(unsigned long, PAGE_SIZE - offset,
afddba49
NP
2614 iov_iter_count(i));
2615
2616again:
00a3d660
LT
2617 /*
2618 * Bring in the user page that we will copy from _first_.
2619 * Otherwise there's a nasty deadlock on copying from the
2620 * same page as we're writing to, without it being marked
2621 * up-to-date.
2622 *
2623 * Not only is this an optimisation, but it is also required
2624 * to check that the address is actually valid, when atomic
2625 * usercopies are used, below.
2626 */
2627 if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
2628 status = -EFAULT;
2629 break;
2630 }
2631
296291cd
JK
2632 if (fatal_signal_pending(current)) {
2633 status = -EINTR;
2634 break;
2635 }
2636
674b892e 2637 status = a_ops->write_begin(file, mapping, pos, bytes, flags,
afddba49 2638 &page, &fsdata);
2457aec6 2639 if (unlikely(status < 0))
afddba49
NP
2640 break;
2641
931e80e4 2642 if (mapping_writably_mapped(mapping))
2643 flush_dcache_page(page);
00a3d660 2644
afddba49 2645 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
afddba49
NP
2646 flush_dcache_page(page);
2647
2648 status = a_ops->write_end(file, mapping, pos, bytes, copied,
2649 page, fsdata);
2650 if (unlikely(status < 0))
2651 break;
2652 copied = status;
2653
2654 cond_resched();
2655
124d3b70 2656 iov_iter_advance(i, copied);
afddba49
NP
2657 if (unlikely(copied == 0)) {
2658 /*
2659 * If we were unable to copy any data at all, we must
2660 * fall back to a single segment length write.
2661 *
2662 * If we didn't fallback here, we could livelock
2663 * because not all segments in the iov can be copied at
2664 * once without a pagefault.
2665 */
09cbfeaf 2666 bytes = min_t(unsigned long, PAGE_SIZE - offset,
afddba49
NP
2667 iov_iter_single_seg_count(i));
2668 goto again;
2669 }
afddba49
NP
2670 pos += copied;
2671 written += copied;
2672
2673 balance_dirty_pages_ratelimited(mapping);
afddba49
NP
2674 } while (iov_iter_count(i));
2675
2676 return written ? written : status;
2677}
3b93f911 2678EXPORT_SYMBOL(generic_perform_write);
1da177e4 2679
e4dd9de3 2680/**
8174202b 2681 * __generic_file_write_iter - write data to a file
e4dd9de3 2682 * @iocb: IO state structure (file, offset, etc.)
8174202b 2683 * @from: iov_iter with data to write
e4dd9de3
JK
2684 *
2685 * This function does all the work needed for actually writing data to a
2686 * file. It does all basic checks, removes SUID from the file, updates
2687 * modification times and calls proper subroutines depending on whether we
2688 * do direct IO or a standard buffered write.
2689 *
2690 * It expects i_mutex to be grabbed unless we work on a block device or similar
2691 * object which does not need locking at all.
2692 *
2693 * This function does *not* take care of syncing data in case of O_SYNC write.
2694 * A caller has to handle it. This is mainly due to the fact that we want to
2695 * avoid syncing under i_mutex.
2696 */
8174202b 2697ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1da177e4
LT
2698{
2699 struct file *file = iocb->ki_filp;
fb5527e6 2700 struct address_space * mapping = file->f_mapping;
1da177e4 2701 struct inode *inode = mapping->host;
3b93f911 2702 ssize_t written = 0;
1da177e4 2703 ssize_t err;
3b93f911 2704 ssize_t status;
1da177e4 2705
1da177e4 2706 /* We can write back this queue in page reclaim */
de1414a6 2707 current->backing_dev_info = inode_to_bdi(inode);
5fa8e0a1 2708 err = file_remove_privs(file);
1da177e4
LT
2709 if (err)
2710 goto out;
2711
c3b2da31
JB
2712 err = file_update_time(file);
2713 if (err)
2714 goto out;
1da177e4 2715
2ba48ce5 2716 if (iocb->ki_flags & IOCB_DIRECT) {
0b8def9d 2717 loff_t pos, endbyte;
fb5527e6 2718
1af5bb49 2719 written = generic_file_direct_write(iocb, from);
1da177e4 2720 /*
fbbbad4b
MW
2721 * If the write stopped short of completing, fall back to
2722 * buffered writes. Some filesystems do this for writes to
2723 * holes, for example. For DAX files, a buffered write will
2724 * not succeed (even if it did, DAX does not handle dirty
2725 * page-cache pages correctly).
1da177e4 2726 */
0b8def9d 2727 if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
fbbbad4b
MW
2728 goto out;
2729
0b8def9d 2730 status = generic_perform_write(file, from, pos = iocb->ki_pos);
fb5527e6 2731 /*
3b93f911 2732 * If generic_perform_write() returned a synchronous error
fb5527e6
JM
2733 * then we want to return the number of bytes which were
2734 * direct-written, or the error code if that was zero. Note
2735 * that this differs from normal direct-io semantics, which
2736 * will return -EFOO even if some bytes were written.
2737 */
60bb4529 2738 if (unlikely(status < 0)) {
3b93f911 2739 err = status;
fb5527e6
JM
2740 goto out;
2741 }
fb5527e6
JM
2742 /*
2743 * We need to ensure that the page cache pages are written to
2744 * disk and invalidated to preserve the expected O_DIRECT
2745 * semantics.
2746 */
3b93f911 2747 endbyte = pos + status - 1;
0b8def9d 2748 err = filemap_write_and_wait_range(mapping, pos, endbyte);
fb5527e6 2749 if (err == 0) {
0b8def9d 2750 iocb->ki_pos = endbyte + 1;
3b93f911 2751 written += status;
fb5527e6 2752 invalidate_mapping_pages(mapping,
09cbfeaf
KS
2753 pos >> PAGE_SHIFT,
2754 endbyte >> PAGE_SHIFT);
fb5527e6
JM
2755 } else {
2756 /*
2757 * We don't know how much we wrote, so just return
2758 * the number of bytes which were direct-written
2759 */
2760 }
2761 } else {
0b8def9d
AV
2762 written = generic_perform_write(file, from, iocb->ki_pos);
2763 if (likely(written > 0))
2764 iocb->ki_pos += written;
fb5527e6 2765 }
1da177e4
LT
2766out:
2767 current->backing_dev_info = NULL;
2768 return written ? written : err;
2769}
8174202b 2770EXPORT_SYMBOL(__generic_file_write_iter);
e4dd9de3 2771
e4dd9de3 2772/**
8174202b 2773 * generic_file_write_iter - write data to a file
e4dd9de3 2774 * @iocb: IO state structure
8174202b 2775 * @from: iov_iter with data to write
e4dd9de3 2776 *
8174202b 2777 * This is a wrapper around __generic_file_write_iter() to be used by most
e4dd9de3
JK
2778 * filesystems. It takes care of syncing the file in case of O_SYNC file
2779 * and acquires i_mutex as needed.
2780 */
8174202b 2781ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1da177e4
LT
2782{
2783 struct file *file = iocb->ki_filp;
148f948b 2784 struct inode *inode = file->f_mapping->host;
1da177e4 2785 ssize_t ret;
1da177e4 2786
5955102c 2787 inode_lock(inode);
3309dd04
AV
2788 ret = generic_write_checks(iocb, from);
2789 if (ret > 0)
5f380c7f 2790 ret = __generic_file_write_iter(iocb, from);
5955102c 2791 inode_unlock(inode);
1da177e4 2792
e2592217
CH
2793 if (ret > 0)
2794 ret = generic_write_sync(iocb, ret);
1da177e4
LT
2795 return ret;
2796}
8174202b 2797EXPORT_SYMBOL(generic_file_write_iter);
1da177e4 2798
cf9a2ae8
DH
2799/**
2800 * try_to_release_page() - release old fs-specific metadata on a page
2801 *
2802 * @page: the page which the kernel is trying to free
2803 * @gfp_mask: memory allocation flags (and I/O mode)
2804 *
2805 * The address_space is to try to release any data against the page
2806 * (presumably at page->private). If the release was successful, return `1'.
2807 * Otherwise return zero.
2808 *
266cf658
DH
2809 * This may also be called if PG_fscache is set on a page, indicating that the
2810 * page is known to the local caching routines.
2811 *
cf9a2ae8 2812 * The @gfp_mask argument specifies whether I/O may be performed to release
71baba4b 2813 * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS).
cf9a2ae8 2814 *
cf9a2ae8
DH
2815 */
2816int try_to_release_page(struct page *page, gfp_t gfp_mask)
2817{
2818 struct address_space * const mapping = page->mapping;
2819
2820 BUG_ON(!PageLocked(page));
2821 if (PageWriteback(page))
2822 return 0;
2823
2824 if (mapping && mapping->a_ops->releasepage)
2825 return mapping->a_ops->releasepage(page, gfp_mask);
2826 return try_to_free_buffers(page);
2827}
2828
2829EXPORT_SYMBOL(try_to_release_page);