]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - mm/memory.c
mm/page_alloc.c: remove unnecessary end_bitidx for [set|get]_pfnblock_flags_mask()
[mirror_ubuntu-jammy-kernel.git] / mm / memory.c
CommitLineData
457c8996 1// SPDX-License-Identifier: GPL-2.0-only
1da177e4
LT
2/*
3 * linux/mm/memory.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 */
7
8/*
9 * demand-loading started 01.12.91 - seems it is high on the list of
10 * things wanted, and it should be easy to implement. - Linus
11 */
12
13/*
14 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
15 * pages started 02.12.91, seems to work. - Linus.
16 *
17 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
18 * would have taken more than the 6M I have free, but it worked well as
19 * far as I could see.
20 *
21 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
22 */
23
24/*
25 * Real VM (paging to/from disk) started 18.12.91. Much more work and
26 * thought has to go into this. Oh, well..
27 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
28 * Found it. Everything seems to work now.
29 * 20.12.91 - Ok, making the swap-device changeable like the root.
30 */
31
32/*
33 * 05.04.94 - Multi-page memory management added for v1.1.
166f61b9 34 * Idea by Alex Bligh (alex@cconcepts.co.uk)
1da177e4
LT
35 *
36 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
37 * (Gerhard.Wichert@pdb.siemens.de)
38 *
39 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
40 */
41
42#include <linux/kernel_stat.h>
43#include <linux/mm.h>
6e84f315 44#include <linux/sched/mm.h>
f7ccbae4 45#include <linux/sched/coredump.h>
6a3827d7 46#include <linux/sched/numa_balancing.h>
29930025 47#include <linux/sched/task.h>
1da177e4
LT
48#include <linux/hugetlb.h>
49#include <linux/mman.h>
50#include <linux/swap.h>
51#include <linux/highmem.h>
52#include <linux/pagemap.h>
5042db43 53#include <linux/memremap.h>
9a840895 54#include <linux/ksm.h>
1da177e4 55#include <linux/rmap.h>
b95f1b31 56#include <linux/export.h>
0ff92245 57#include <linux/delayacct.h>
1da177e4 58#include <linux/init.h>
01c8f1c4 59#include <linux/pfn_t.h>
edc79b2a 60#include <linux/writeback.h>
8a9f3ccd 61#include <linux/memcontrol.h>
cddb8a5c 62#include <linux/mmu_notifier.h>
3dc14741
HD
63#include <linux/swapops.h>
64#include <linux/elf.h>
5a0e3ad6 65#include <linux/gfp.h>
4daae3b4 66#include <linux/migrate.h>
2fbc57c5 67#include <linux/string.h>
0abdd7a8 68#include <linux/dma-debug.h>
1592eef0 69#include <linux/debugfs.h>
6b251fc9 70#include <linux/userfaultfd_k.h>
bc2466e4 71#include <linux/dax.h>
6b31d595 72#include <linux/oom.h>
98fa15f3 73#include <linux/numa.h>
1da177e4 74
b3d1411b
JFG
75#include <trace/events/kmem.h>
76
6952b61d 77#include <asm/io.h>
33a709b2 78#include <asm/mmu_context.h>
1da177e4 79#include <asm/pgalloc.h>
7c0f6ba6 80#include <linux/uaccess.h>
1da177e4
LT
81#include <asm/tlb.h>
82#include <asm/tlbflush.h>
1da177e4 83
42b77728
JB
84#include "internal.h"
85
af27d940 86#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
90572890 87#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
75980e97
PZ
88#endif
89
d41dee36 90#ifndef CONFIG_NEED_MULTIPLE_NODES
1da177e4
LT
91/* use the per-pgdat data instead for discontigmem - mbligh */
92unsigned long max_mapnr;
1da177e4 93EXPORT_SYMBOL(max_mapnr);
166f61b9
TH
94
95struct page *mem_map;
1da177e4
LT
96EXPORT_SYMBOL(mem_map);
97#endif
98
1da177e4
LT
99/*
100 * A number of key systems in x86 including ioremap() rely on the assumption
101 * that high_memory defines the upper bound on direct map memory, then end
102 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
103 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
104 * and ZONE_HIGHMEM.
105 */
166f61b9 106void *high_memory;
1da177e4 107EXPORT_SYMBOL(high_memory);
1da177e4 108
32a93233
IM
109/*
110 * Randomize the address space (stacks, mmaps, brk, etc.).
111 *
112 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
113 * as ancient (libc5 based) binaries can segfault. )
114 */
115int randomize_va_space __read_mostly =
116#ifdef CONFIG_COMPAT_BRK
117 1;
118#else
119 2;
120#endif
a62eaf15 121
83d116c5
JH
122#ifndef arch_faults_on_old_pte
123static inline bool arch_faults_on_old_pte(void)
124{
125 /*
126 * Those arches which don't have hw access flag feature need to
127 * implement their own helper. By default, "true" means pagefault
128 * will be hit on old pte.
129 */
130 return true;
131}
132#endif
133
a62eaf15
AK
134static int __init disable_randmaps(char *s)
135{
136 randomize_va_space = 0;
9b41046c 137 return 1;
a62eaf15
AK
138}
139__setup("norandmaps", disable_randmaps);
140
62eede62 141unsigned long zero_pfn __read_mostly;
0b70068e
AB
142EXPORT_SYMBOL(zero_pfn);
143
166f61b9
TH
144unsigned long highest_memmap_pfn __read_mostly;
145
a13ea5b7
HD
146/*
147 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
148 */
149static int __init init_zero_pfn(void)
150{
151 zero_pfn = page_to_pfn(ZERO_PAGE(0));
152 return 0;
153}
154core_initcall(init_zero_pfn);
a62eaf15 155
e4dcad20 156void mm_trace_rss_stat(struct mm_struct *mm, int member, long count)
b3d1411b 157{
e4dcad20 158 trace_rss_stat(mm, member, count);
b3d1411b 159}
d559db08 160
34e55232
KH
161#if defined(SPLIT_RSS_COUNTING)
162
ea48cf78 163void sync_mm_rss(struct mm_struct *mm)
34e55232
KH
164{
165 int i;
166
167 for (i = 0; i < NR_MM_COUNTERS; i++) {
05af2e10
DR
168 if (current->rss_stat.count[i]) {
169 add_mm_counter(mm, i, current->rss_stat.count[i]);
170 current->rss_stat.count[i] = 0;
34e55232
KH
171 }
172 }
05af2e10 173 current->rss_stat.events = 0;
34e55232
KH
174}
175
176static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
177{
178 struct task_struct *task = current;
179
180 if (likely(task->mm == mm))
181 task->rss_stat.count[member] += val;
182 else
183 add_mm_counter(mm, member, val);
184}
185#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
186#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)
187
188/* sync counter once per 64 page faults */
189#define TASK_RSS_EVENTS_THRESH (64)
190static void check_sync_rss_stat(struct task_struct *task)
191{
192 if (unlikely(task != current))
193 return;
194 if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
ea48cf78 195 sync_mm_rss(task->mm);
34e55232 196}
9547d01b 197#else /* SPLIT_RSS_COUNTING */
34e55232
KH
198
199#define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)
200#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member)
201
202static void check_sync_rss_stat(struct task_struct *task)
203{
204}
205
9547d01b
PZ
206#endif /* SPLIT_RSS_COUNTING */
207
1da177e4
LT
208/*
209 * Note: this doesn't free the actual pages themselves. That
210 * has been handled earlier when unmapping all the memory regions.
211 */
9e1b32ca
BH
212static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
213 unsigned long addr)
1da177e4 214{
2f569afd 215 pgtable_t token = pmd_pgtable(*pmd);
e0da382c 216 pmd_clear(pmd);
9e1b32ca 217 pte_free_tlb(tlb, token, addr);
c4812909 218 mm_dec_nr_ptes(tlb->mm);
1da177e4
LT
219}
220
e0da382c
HD
221static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
222 unsigned long addr, unsigned long end,
223 unsigned long floor, unsigned long ceiling)
1da177e4
LT
224{
225 pmd_t *pmd;
226 unsigned long next;
e0da382c 227 unsigned long start;
1da177e4 228
e0da382c 229 start = addr;
1da177e4 230 pmd = pmd_offset(pud, addr);
1da177e4
LT
231 do {
232 next = pmd_addr_end(addr, end);
233 if (pmd_none_or_clear_bad(pmd))
234 continue;
9e1b32ca 235 free_pte_range(tlb, pmd, addr);
1da177e4
LT
236 } while (pmd++, addr = next, addr != end);
237
e0da382c
HD
238 start &= PUD_MASK;
239 if (start < floor)
240 return;
241 if (ceiling) {
242 ceiling &= PUD_MASK;
243 if (!ceiling)
244 return;
1da177e4 245 }
e0da382c
HD
246 if (end - 1 > ceiling - 1)
247 return;
248
249 pmd = pmd_offset(pud, start);
250 pud_clear(pud);
9e1b32ca 251 pmd_free_tlb(tlb, pmd, start);
dc6c9a35 252 mm_dec_nr_pmds(tlb->mm);
1da177e4
LT
253}
254
c2febafc 255static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
e0da382c
HD
256 unsigned long addr, unsigned long end,
257 unsigned long floor, unsigned long ceiling)
1da177e4
LT
258{
259 pud_t *pud;
260 unsigned long next;
e0da382c 261 unsigned long start;
1da177e4 262
e0da382c 263 start = addr;
c2febafc 264 pud = pud_offset(p4d, addr);
1da177e4
LT
265 do {
266 next = pud_addr_end(addr, end);
267 if (pud_none_or_clear_bad(pud))
268 continue;
e0da382c 269 free_pmd_range(tlb, pud, addr, next, floor, ceiling);
1da177e4
LT
270 } while (pud++, addr = next, addr != end);
271
c2febafc
KS
272 start &= P4D_MASK;
273 if (start < floor)
274 return;
275 if (ceiling) {
276 ceiling &= P4D_MASK;
277 if (!ceiling)
278 return;
279 }
280 if (end - 1 > ceiling - 1)
281 return;
282
283 pud = pud_offset(p4d, start);
284 p4d_clear(p4d);
285 pud_free_tlb(tlb, pud, start);
b4e98d9a 286 mm_dec_nr_puds(tlb->mm);
c2febafc
KS
287}
288
289static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
290 unsigned long addr, unsigned long end,
291 unsigned long floor, unsigned long ceiling)
292{
293 p4d_t *p4d;
294 unsigned long next;
295 unsigned long start;
296
297 start = addr;
298 p4d = p4d_offset(pgd, addr);
299 do {
300 next = p4d_addr_end(addr, end);
301 if (p4d_none_or_clear_bad(p4d))
302 continue;
303 free_pud_range(tlb, p4d, addr, next, floor, ceiling);
304 } while (p4d++, addr = next, addr != end);
305
e0da382c
HD
306 start &= PGDIR_MASK;
307 if (start < floor)
308 return;
309 if (ceiling) {
310 ceiling &= PGDIR_MASK;
311 if (!ceiling)
312 return;
1da177e4 313 }
e0da382c
HD
314 if (end - 1 > ceiling - 1)
315 return;
316
c2febafc 317 p4d = p4d_offset(pgd, start);
e0da382c 318 pgd_clear(pgd);
c2febafc 319 p4d_free_tlb(tlb, p4d, start);
1da177e4
LT
320}
321
322/*
e0da382c 323 * This function frees user-level page tables of a process.
1da177e4 324 */
42b77728 325void free_pgd_range(struct mmu_gather *tlb,
e0da382c
HD
326 unsigned long addr, unsigned long end,
327 unsigned long floor, unsigned long ceiling)
1da177e4
LT
328{
329 pgd_t *pgd;
330 unsigned long next;
e0da382c
HD
331
332 /*
333 * The next few lines have given us lots of grief...
334 *
335 * Why are we testing PMD* at this top level? Because often
336 * there will be no work to do at all, and we'd prefer not to
337 * go all the way down to the bottom just to discover that.
338 *
339 * Why all these "- 1"s? Because 0 represents both the bottom
340 * of the address space and the top of it (using -1 for the
341 * top wouldn't help much: the masks would do the wrong thing).
342 * The rule is that addr 0 and floor 0 refer to the bottom of
343 * the address space, but end 0 and ceiling 0 refer to the top
344 * Comparisons need to use "end - 1" and "ceiling - 1" (though
345 * that end 0 case should be mythical).
346 *
347 * Wherever addr is brought up or ceiling brought down, we must
348 * be careful to reject "the opposite 0" before it confuses the
349 * subsequent tests. But what about where end is brought down
350 * by PMD_SIZE below? no, end can't go down to 0 there.
351 *
352 * Whereas we round start (addr) and ceiling down, by different
353 * masks at different levels, in order to test whether a table
354 * now has no other vmas using it, so can be freed, we don't
355 * bother to round floor or end up - the tests don't need that.
356 */
1da177e4 357
e0da382c
HD
358 addr &= PMD_MASK;
359 if (addr < floor) {
360 addr += PMD_SIZE;
361 if (!addr)
362 return;
363 }
364 if (ceiling) {
365 ceiling &= PMD_MASK;
366 if (!ceiling)
367 return;
368 }
369 if (end - 1 > ceiling - 1)
370 end -= PMD_SIZE;
371 if (addr > end - 1)
372 return;
07e32661
AK
373 /*
374 * We add page table cache pages with PAGE_SIZE,
375 * (see pte_free_tlb()), flush the tlb if we need
376 */
ed6a7935 377 tlb_change_page_size(tlb, PAGE_SIZE);
42b77728 378 pgd = pgd_offset(tlb->mm, addr);
1da177e4
LT
379 do {
380 next = pgd_addr_end(addr, end);
381 if (pgd_none_or_clear_bad(pgd))
382 continue;
c2febafc 383 free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
1da177e4 384 } while (pgd++, addr = next, addr != end);
e0da382c
HD
385}
386
42b77728 387void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
3bf5ee95 388 unsigned long floor, unsigned long ceiling)
e0da382c
HD
389{
390 while (vma) {
391 struct vm_area_struct *next = vma->vm_next;
392 unsigned long addr = vma->vm_start;
393
8f4f8c16 394 /*
25d9e2d1
NP
395 * Hide vma from rmap and truncate_pagecache before freeing
396 * pgtables
8f4f8c16 397 */
5beb4930 398 unlink_anon_vmas(vma);
8f4f8c16
HD
399 unlink_file_vma(vma);
400
9da61aef 401 if (is_vm_hugetlb_page(vma)) {
3bf5ee95 402 hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
166f61b9 403 floor, next ? next->vm_start : ceiling);
3bf5ee95
HD
404 } else {
405 /*
406 * Optimization: gather nearby vmas into one call down
407 */
408 while (next && next->vm_start <= vma->vm_end + PMD_SIZE
4866920b 409 && !is_vm_hugetlb_page(next)) {
3bf5ee95
HD
410 vma = next;
411 next = vma->vm_next;
5beb4930 412 unlink_anon_vmas(vma);
8f4f8c16 413 unlink_file_vma(vma);
3bf5ee95
HD
414 }
415 free_pgd_range(tlb, addr, vma->vm_end,
166f61b9 416 floor, next ? next->vm_start : ceiling);
3bf5ee95 417 }
e0da382c
HD
418 vma = next;
419 }
1da177e4
LT
420}
421
4cf58924 422int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
1da177e4 423{
c4088ebd 424 spinlock_t *ptl;
4cf58924 425 pgtable_t new = pte_alloc_one(mm);
1bb3630e
HD
426 if (!new)
427 return -ENOMEM;
428
362a61ad
NP
429 /*
430 * Ensure all pte setup (eg. pte page lock and page clearing) are
431 * visible before the pte is made visible to other CPUs by being
432 * put into page tables.
433 *
434 * The other side of the story is the pointer chasing in the page
435 * table walking code (when walking the page table without locking;
436 * ie. most of the time). Fortunately, these data accesses consist
437 * of a chain of data-dependent loads, meaning most CPUs (alpha
438 * being the notable exception) will already guarantee loads are
439 * seen in-order. See the alpha page table accessors for the
bb7cdd38 440 * smp_rmb() barriers in page table walking code.
362a61ad
NP
441 */
442 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
443
c4088ebd 444 ptl = pmd_lock(mm, pmd);
8ac1f832 445 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
c4812909 446 mm_inc_nr_ptes(mm);
1da177e4 447 pmd_populate(mm, pmd, new);
2f569afd 448 new = NULL;
4b471e88 449 }
c4088ebd 450 spin_unlock(ptl);
2f569afd
MS
451 if (new)
452 pte_free(mm, new);
1bb3630e 453 return 0;
1da177e4
LT
454}
455
4cf58924 456int __pte_alloc_kernel(pmd_t *pmd)
1da177e4 457{
4cf58924 458 pte_t *new = pte_alloc_one_kernel(&init_mm);
1bb3630e
HD
459 if (!new)
460 return -ENOMEM;
461
362a61ad
NP
462 smp_wmb(); /* See comment in __pte_alloc */
463
1bb3630e 464 spin_lock(&init_mm.page_table_lock);
8ac1f832 465 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
1bb3630e 466 pmd_populate_kernel(&init_mm, pmd, new);
2f569afd 467 new = NULL;
4b471e88 468 }
1bb3630e 469 spin_unlock(&init_mm.page_table_lock);
2f569afd
MS
470 if (new)
471 pte_free_kernel(&init_mm, new);
1bb3630e 472 return 0;
1da177e4
LT
473}
474
d559db08
KH
475static inline void init_rss_vec(int *rss)
476{
477 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
478}
479
480static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
ae859762 481{
d559db08
KH
482 int i;
483
34e55232 484 if (current->mm == mm)
05af2e10 485 sync_mm_rss(mm);
d559db08
KH
486 for (i = 0; i < NR_MM_COUNTERS; i++)
487 if (rss[i])
488 add_mm_counter(mm, i, rss[i]);
ae859762
HD
489}
490
b5810039 491/*
6aab341e
LT
492 * This function is called to print an error when a bad pte
493 * is found. For example, we might have a PFN-mapped pte in
494 * a region that doesn't allow it.
b5810039
NP
495 *
496 * The calling function must still handle the error.
497 */
3dc14741
HD
498static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
499 pte_t pte, struct page *page)
b5810039 500{
3dc14741 501 pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
c2febafc
KS
502 p4d_t *p4d = p4d_offset(pgd, addr);
503 pud_t *pud = pud_offset(p4d, addr);
3dc14741
HD
504 pmd_t *pmd = pmd_offset(pud, addr);
505 struct address_space *mapping;
506 pgoff_t index;
d936cf9b
HD
507 static unsigned long resume;
508 static unsigned long nr_shown;
509 static unsigned long nr_unshown;
510
511 /*
512 * Allow a burst of 60 reports, then keep quiet for that minute;
513 * or allow a steady drip of one report per second.
514 */
515 if (nr_shown == 60) {
516 if (time_before(jiffies, resume)) {
517 nr_unshown++;
518 return;
519 }
520 if (nr_unshown) {
1170532b
JP
521 pr_alert("BUG: Bad page map: %lu messages suppressed\n",
522 nr_unshown);
d936cf9b
HD
523 nr_unshown = 0;
524 }
525 nr_shown = 0;
526 }
527 if (nr_shown++ == 0)
528 resume = jiffies + 60 * HZ;
3dc14741
HD
529
530 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
531 index = linear_page_index(vma, addr);
532
1170532b
JP
533 pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
534 current->comm,
535 (long long)pte_val(pte), (long long)pmd_val(*pmd));
718a3821 536 if (page)
f0b791a3 537 dump_page(page, "bad pte");
6aa9b8b2 538 pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
1170532b 539 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
d75f773c 540 pr_alert("file:%pD fault:%ps mmap:%ps readpage:%ps\n",
2682582a
KK
541 vma->vm_file,
542 vma->vm_ops ? vma->vm_ops->fault : NULL,
543 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
544 mapping ? mapping->a_ops->readpage : NULL);
b5810039 545 dump_stack();
373d4d09 546 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
b5810039
NP
547}
548
ee498ed7 549/*
7e675137 550 * vm_normal_page -- This function gets the "struct page" associated with a pte.
6aab341e 551 *
7e675137
NP
552 * "Special" mappings do not wish to be associated with a "struct page" (either
553 * it doesn't exist, or it exists but they don't want to touch it). In this
554 * case, NULL is returned here. "Normal" mappings do have a struct page.
b379d790 555 *
7e675137
NP
556 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
557 * pte bit, in which case this function is trivial. Secondly, an architecture
558 * may not have a spare pte bit, which requires a more complicated scheme,
559 * described below.
560 *
561 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
562 * special mapping (even if there are underlying and valid "struct pages").
563 * COWed pages of a VM_PFNMAP are always normal.
6aab341e 564 *
b379d790
JH
565 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
566 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
7e675137
NP
567 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
568 * mapping will always honor the rule
6aab341e
LT
569 *
570 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
571 *
7e675137
NP
572 * And for normal mappings this is false.
573 *
574 * This restricts such mappings to be a linear translation from virtual address
575 * to pfn. To get around this restriction, we allow arbitrary mappings so long
576 * as the vma is not a COW mapping; in that case, we know that all ptes are
577 * special (because none can have been COWed).
b379d790 578 *
b379d790 579 *
7e675137 580 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
b379d790
JH
581 *
582 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
583 * page" backing, however the difference is that _all_ pages with a struct
584 * page (that is, those where pfn_valid is true) are refcounted and considered
585 * normal pages by the VM. The disadvantage is that pages are refcounted
586 * (which can be slower and simply not an option for some PFNMAP users). The
587 * advantage is that we don't have to follow the strict linearity rule of
588 * PFNMAP mappings in order to support COWable mappings.
589 *
ee498ed7 590 */
25b2995a
CH
591struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
592 pte_t pte)
ee498ed7 593{
22b31eec 594 unsigned long pfn = pte_pfn(pte);
7e675137 595
00b3a331 596 if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
b38af472 597 if (likely(!pte_special(pte)))
22b31eec 598 goto check_pfn;
667a0a06
DV
599 if (vma->vm_ops && vma->vm_ops->find_special_page)
600 return vma->vm_ops->find_special_page(vma, addr);
a13ea5b7
HD
601 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
602 return NULL;
df6ad698
JG
603 if (is_zero_pfn(pfn))
604 return NULL;
e1fb4a08
DJ
605 if (pte_devmap(pte))
606 return NULL;
607
df6ad698 608 print_bad_pte(vma, addr, pte, NULL);
7e675137
NP
609 return NULL;
610 }
611
00b3a331 612 /* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */
7e675137 613
b379d790
JH
614 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
615 if (vma->vm_flags & VM_MIXEDMAP) {
616 if (!pfn_valid(pfn))
617 return NULL;
618 goto out;
619 } else {
7e675137
NP
620 unsigned long off;
621 off = (addr - vma->vm_start) >> PAGE_SHIFT;
b379d790
JH
622 if (pfn == vma->vm_pgoff + off)
623 return NULL;
624 if (!is_cow_mapping(vma->vm_flags))
625 return NULL;
626 }
6aab341e
LT
627 }
628
b38af472
HD
629 if (is_zero_pfn(pfn))
630 return NULL;
00b3a331 631
22b31eec
HD
632check_pfn:
633 if (unlikely(pfn > highest_memmap_pfn)) {
634 print_bad_pte(vma, addr, pte, NULL);
635 return NULL;
636 }
6aab341e
LT
637
638 /*
7e675137 639 * NOTE! We still have PageReserved() pages in the page tables.
7e675137 640 * eg. VDSO mappings can cause them to exist.
6aab341e 641 */
b379d790 642out:
6aab341e 643 return pfn_to_page(pfn);
ee498ed7
HD
644}
645
28093f9f
GS
646#ifdef CONFIG_TRANSPARENT_HUGEPAGE
647struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
648 pmd_t pmd)
649{
650 unsigned long pfn = pmd_pfn(pmd);
651
652 /*
653 * There is no pmd_special() but there may be special pmds, e.g.
654 * in a direct-access (dax) mapping, so let's just replicate the
00b3a331 655 * !CONFIG_ARCH_HAS_PTE_SPECIAL case from vm_normal_page() here.
28093f9f
GS
656 */
657 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
658 if (vma->vm_flags & VM_MIXEDMAP) {
659 if (!pfn_valid(pfn))
660 return NULL;
661 goto out;
662 } else {
663 unsigned long off;
664 off = (addr - vma->vm_start) >> PAGE_SHIFT;
665 if (pfn == vma->vm_pgoff + off)
666 return NULL;
667 if (!is_cow_mapping(vma->vm_flags))
668 return NULL;
669 }
670 }
671
e1fb4a08
DJ
672 if (pmd_devmap(pmd))
673 return NULL;
3cde287b 674 if (is_huge_zero_pmd(pmd))
28093f9f
GS
675 return NULL;
676 if (unlikely(pfn > highest_memmap_pfn))
677 return NULL;
678
679 /*
680 * NOTE! We still have PageReserved() pages in the page tables.
681 * eg. VDSO mappings can cause them to exist.
682 */
683out:
684 return pfn_to_page(pfn);
685}
686#endif
687
1da177e4
LT
688/*
689 * copy one vm_area from one task to the other. Assumes the page tables
690 * already present in the new task to be cleared in the whole range
691 * covered by this vma.
1da177e4
LT
692 */
693
570a335b 694static inline unsigned long
1da177e4 695copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
b5810039 696 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
8c103762 697 unsigned long addr, int *rss)
1da177e4 698{
b5810039 699 unsigned long vm_flags = vma->vm_flags;
1da177e4
LT
700 pte_t pte = *src_pte;
701 struct page *page;
1da177e4
LT
702
703 /* pte contains position in swap or file, so copy. */
704 if (unlikely(!pte_present(pte))) {
0661a336
KS
705 swp_entry_t entry = pte_to_swp_entry(pte);
706
707 if (likely(!non_swap_entry(entry))) {
708 if (swap_duplicate(entry) < 0)
709 return entry.val;
710
711 /* make sure dst_mm is on swapoff's mmlist. */
712 if (unlikely(list_empty(&dst_mm->mmlist))) {
713 spin_lock(&mmlist_lock);
714 if (list_empty(&dst_mm->mmlist))
715 list_add(&dst_mm->mmlist,
716 &src_mm->mmlist);
717 spin_unlock(&mmlist_lock);
718 }
719 rss[MM_SWAPENTS]++;
720 } else if (is_migration_entry(entry)) {
721 page = migration_entry_to_page(entry);
722
eca56ff9 723 rss[mm_counter(page)]++;
0661a336
KS
724
725 if (is_write_migration_entry(entry) &&
726 is_cow_mapping(vm_flags)) {
727 /*
728 * COW mappings require pages in both
729 * parent and child to be set to read.
730 */
731 make_migration_entry_read(&entry);
732 pte = swp_entry_to_pte(entry);
733 if (pte_swp_soft_dirty(*src_pte))
734 pte = pte_swp_mksoft_dirty(pte);
f45ec5ff
PX
735 if (pte_swp_uffd_wp(*src_pte))
736 pte = pte_swp_mkuffd_wp(pte);
0661a336 737 set_pte_at(src_mm, addr, src_pte, pte);
0697212a 738 }
5042db43
JG
739 } else if (is_device_private_entry(entry)) {
740 page = device_private_entry_to_page(entry);
741
742 /*
743 * Update rss count even for unaddressable pages, as
744 * they should treated just like normal pages in this
745 * respect.
746 *
747 * We will likely want to have some new rss counters
748 * for unaddressable pages, at some point. But for now
749 * keep things as they are.
750 */
751 get_page(page);
752 rss[mm_counter(page)]++;
753 page_dup_rmap(page, false);
754
755 /*
756 * We do not preserve soft-dirty information, because so
757 * far, checkpoint/restore is the only feature that
758 * requires that. And checkpoint/restore does not work
759 * when a device driver is involved (you cannot easily
760 * save and restore device driver state).
761 */
762 if (is_write_device_private_entry(entry) &&
763 is_cow_mapping(vm_flags)) {
764 make_device_private_entry_read(&entry);
765 pte = swp_entry_to_pte(entry);
f45ec5ff
PX
766 if (pte_swp_uffd_wp(*src_pte))
767 pte = pte_swp_mkuffd_wp(pte);
5042db43
JG
768 set_pte_at(src_mm, addr, src_pte, pte);
769 }
1da177e4 770 }
ae859762 771 goto out_set_pte;
1da177e4
LT
772 }
773
1da177e4
LT
774 /*
775 * If it's a COW mapping, write protect it both
776 * in the parent and the child
777 */
1b2de5d0 778 if (is_cow_mapping(vm_flags) && pte_write(pte)) {
1da177e4 779 ptep_set_wrprotect(src_mm, addr, src_pte);
3dc90795 780 pte = pte_wrprotect(pte);
1da177e4
LT
781 }
782
783 /*
784 * If it's a shared mapping, mark it clean in
785 * the child
786 */
787 if (vm_flags & VM_SHARED)
788 pte = pte_mkclean(pte);
789 pte = pte_mkold(pte);
6aab341e 790
b569a176
PX
791 /*
792 * Make sure the _PAGE_UFFD_WP bit is cleared if the new VMA
793 * does not have the VM_UFFD_WP, which means that the uffd
794 * fork event is not enabled.
795 */
796 if (!(vm_flags & VM_UFFD_WP))
797 pte = pte_clear_uffd_wp(pte);
798
6aab341e
LT
799 page = vm_normal_page(vma, addr, pte);
800 if (page) {
801 get_page(page);
53f9263b 802 page_dup_rmap(page, false);
eca56ff9 803 rss[mm_counter(page)]++;
6aab341e 804 }
ae859762
HD
805
806out_set_pte:
807 set_pte_at(dst_mm, addr, dst_pte, pte);
570a335b 808 return 0;
1da177e4
LT
809}
810
21bda264 811static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
71e3aac0
AA
812 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
813 unsigned long addr, unsigned long end)
1da177e4 814{
c36987e2 815 pte_t *orig_src_pte, *orig_dst_pte;
1da177e4 816 pte_t *src_pte, *dst_pte;
c74df32c 817 spinlock_t *src_ptl, *dst_ptl;
e040f218 818 int progress = 0;
d559db08 819 int rss[NR_MM_COUNTERS];
570a335b 820 swp_entry_t entry = (swp_entry_t){0};
1da177e4
LT
821
822again:
d559db08
KH
823 init_rss_vec(rss);
824
c74df32c 825 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
1da177e4
LT
826 if (!dst_pte)
827 return -ENOMEM;
ece0e2b6 828 src_pte = pte_offset_map(src_pmd, addr);
4c21e2f2 829 src_ptl = pte_lockptr(src_mm, src_pmd);
f20dc5f7 830 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
c36987e2
DN
831 orig_src_pte = src_pte;
832 orig_dst_pte = dst_pte;
6606c3e0 833 arch_enter_lazy_mmu_mode();
1da177e4 834
1da177e4
LT
835 do {
836 /*
837 * We are holding two locks at this point - either of them
838 * could generate latencies in another task on another CPU.
839 */
e040f218
HD
840 if (progress >= 32) {
841 progress = 0;
842 if (need_resched() ||
95c354fe 843 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
e040f218
HD
844 break;
845 }
1da177e4
LT
846 if (pte_none(*src_pte)) {
847 progress++;
848 continue;
849 }
570a335b
HD
850 entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
851 vma, addr, rss);
852 if (entry.val)
853 break;
1da177e4
LT
854 progress += 8;
855 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
1da177e4 856
6606c3e0 857 arch_leave_lazy_mmu_mode();
c74df32c 858 spin_unlock(src_ptl);
ece0e2b6 859 pte_unmap(orig_src_pte);
d559db08 860 add_mm_rss_vec(dst_mm, rss);
c36987e2 861 pte_unmap_unlock(orig_dst_pte, dst_ptl);
c74df32c 862 cond_resched();
570a335b
HD
863
864 if (entry.val) {
865 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0)
866 return -ENOMEM;
867 progress = 0;
868 }
1da177e4
LT
869 if (addr != end)
870 goto again;
871 return 0;
872}
873
874static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
875 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
876 unsigned long addr, unsigned long end)
877{
878 pmd_t *src_pmd, *dst_pmd;
879 unsigned long next;
880
881 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
882 if (!dst_pmd)
883 return -ENOMEM;
884 src_pmd = pmd_offset(src_pud, addr);
885 do {
886 next = pmd_addr_end(addr, end);
84c3fc4e
ZY
887 if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
888 || pmd_devmap(*src_pmd)) {
71e3aac0 889 int err;
a00cc7d9 890 VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, vma);
71e3aac0
AA
891 err = copy_huge_pmd(dst_mm, src_mm,
892 dst_pmd, src_pmd, addr, vma);
893 if (err == -ENOMEM)
894 return -ENOMEM;
895 if (!err)
896 continue;
897 /* fall through */
898 }
1da177e4
LT
899 if (pmd_none_or_clear_bad(src_pmd))
900 continue;
901 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
902 vma, addr, next))
903 return -ENOMEM;
904 } while (dst_pmd++, src_pmd++, addr = next, addr != end);
905 return 0;
906}
907
908static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
c2febafc 909 p4d_t *dst_p4d, p4d_t *src_p4d, struct vm_area_struct *vma,
1da177e4
LT
910 unsigned long addr, unsigned long end)
911{
912 pud_t *src_pud, *dst_pud;
913 unsigned long next;
914
c2febafc 915 dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
1da177e4
LT
916 if (!dst_pud)
917 return -ENOMEM;
c2febafc 918 src_pud = pud_offset(src_p4d, addr);
1da177e4
LT
919 do {
920 next = pud_addr_end(addr, end);
a00cc7d9
MW
921 if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
922 int err;
923
924 VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, vma);
925 err = copy_huge_pud(dst_mm, src_mm,
926 dst_pud, src_pud, addr, vma);
927 if (err == -ENOMEM)
928 return -ENOMEM;
929 if (!err)
930 continue;
931 /* fall through */
932 }
1da177e4
LT
933 if (pud_none_or_clear_bad(src_pud))
934 continue;
935 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
936 vma, addr, next))
937 return -ENOMEM;
938 } while (dst_pud++, src_pud++, addr = next, addr != end);
939 return 0;
940}
941
c2febafc
KS
942static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
943 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
944 unsigned long addr, unsigned long end)
945{
946 p4d_t *src_p4d, *dst_p4d;
947 unsigned long next;
948
949 dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
950 if (!dst_p4d)
951 return -ENOMEM;
952 src_p4d = p4d_offset(src_pgd, addr);
953 do {
954 next = p4d_addr_end(addr, end);
955 if (p4d_none_or_clear_bad(src_p4d))
956 continue;
957 if (copy_pud_range(dst_mm, src_mm, dst_p4d, src_p4d,
958 vma, addr, next))
959 return -ENOMEM;
960 } while (dst_p4d++, src_p4d++, addr = next, addr != end);
961 return 0;
962}
963
1da177e4
LT
964int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
965 struct vm_area_struct *vma)
966{
967 pgd_t *src_pgd, *dst_pgd;
968 unsigned long next;
969 unsigned long addr = vma->vm_start;
970 unsigned long end = vma->vm_end;
ac46d4f3 971 struct mmu_notifier_range range;
2ec74c3e 972 bool is_cow;
cddb8a5c 973 int ret;
1da177e4 974
d992895b
NP
975 /*
976 * Don't copy ptes where a page fault will fill them correctly.
977 * Fork becomes much lighter when there are big shared or private
978 * readonly mappings. The tradeoff is that copy_page_range is more
979 * efficient than faulting.
980 */
0661a336
KS
981 if (!(vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) &&
982 !vma->anon_vma)
983 return 0;
d992895b 984
1da177e4
LT
985 if (is_vm_hugetlb_page(vma))
986 return copy_hugetlb_page_range(dst_mm, src_mm, vma);
987
b3b9c293 988 if (unlikely(vma->vm_flags & VM_PFNMAP)) {
2ab64037 989 /*
990 * We do not free on error cases below as remove_vma
991 * gets called on error from higher level routine
992 */
5180da41 993 ret = track_pfn_copy(vma);
2ab64037 994 if (ret)
995 return ret;
996 }
997
cddb8a5c
AA
998 /*
999 * We need to invalidate the secondary MMU mappings only when
1000 * there could be a permission downgrade on the ptes of the
1001 * parent mm. And a permission downgrade will only happen if
1002 * is_cow_mapping() returns true.
1003 */
2ec74c3e 1004 is_cow = is_cow_mapping(vma->vm_flags);
ac46d4f3
JG
1005
1006 if (is_cow) {
7269f999
JG
1007 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
1008 0, vma, src_mm, addr, end);
ac46d4f3
JG
1009 mmu_notifier_invalidate_range_start(&range);
1010 }
cddb8a5c
AA
1011
1012 ret = 0;
1da177e4
LT
1013 dst_pgd = pgd_offset(dst_mm, addr);
1014 src_pgd = pgd_offset(src_mm, addr);
1015 do {
1016 next = pgd_addr_end(addr, end);
1017 if (pgd_none_or_clear_bad(src_pgd))
1018 continue;
c2febafc 1019 if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd,
cddb8a5c
AA
1020 vma, addr, next))) {
1021 ret = -ENOMEM;
1022 break;
1023 }
1da177e4 1024 } while (dst_pgd++, src_pgd++, addr = next, addr != end);
cddb8a5c 1025
2ec74c3e 1026 if (is_cow)
ac46d4f3 1027 mmu_notifier_invalidate_range_end(&range);
cddb8a5c 1028 return ret;
1da177e4
LT
1029}
1030
51c6f666 1031static unsigned long zap_pte_range(struct mmu_gather *tlb,
b5810039 1032 struct vm_area_struct *vma, pmd_t *pmd,
1da177e4 1033 unsigned long addr, unsigned long end,
97a89413 1034 struct zap_details *details)
1da177e4 1035{
b5810039 1036 struct mm_struct *mm = tlb->mm;
d16dfc55 1037 int force_flush = 0;
d559db08 1038 int rss[NR_MM_COUNTERS];
97a89413 1039 spinlock_t *ptl;
5f1a1907 1040 pte_t *start_pte;
97a89413 1041 pte_t *pte;
8a5f14a2 1042 swp_entry_t entry;
d559db08 1043
ed6a7935 1044 tlb_change_page_size(tlb, PAGE_SIZE);
d16dfc55 1045again:
e303297e 1046 init_rss_vec(rss);
5f1a1907
SR
1047 start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
1048 pte = start_pte;
3ea27719 1049 flush_tlb_batched_pending(mm);
6606c3e0 1050 arch_enter_lazy_mmu_mode();
1da177e4
LT
1051 do {
1052 pte_t ptent = *pte;
166f61b9 1053 if (pte_none(ptent))
1da177e4 1054 continue;
6f5e6b9e 1055
7b167b68
MK
1056 if (need_resched())
1057 break;
1058
1da177e4 1059 if (pte_present(ptent)) {
ee498ed7 1060 struct page *page;
51c6f666 1061
25b2995a 1062 page = vm_normal_page(vma, addr, ptent);
1da177e4
LT
1063 if (unlikely(details) && page) {
1064 /*
1065 * unmap_shared_mapping_pages() wants to
1066 * invalidate cache without truncating:
1067 * unmap shared but keep private pages.
1068 */
1069 if (details->check_mapping &&
800d8c63 1070 details->check_mapping != page_rmapping(page))
1da177e4 1071 continue;
1da177e4 1072 }
b5810039 1073 ptent = ptep_get_and_clear_full(mm, addr, pte,
a600388d 1074 tlb->fullmm);
1da177e4
LT
1075 tlb_remove_tlb_entry(tlb, pte, addr);
1076 if (unlikely(!page))
1077 continue;
eca56ff9
JM
1078
1079 if (!PageAnon(page)) {
1cf35d47
LT
1080 if (pte_dirty(ptent)) {
1081 force_flush = 1;
6237bcd9 1082 set_page_dirty(page);
1cf35d47 1083 }
4917e5d0 1084 if (pte_young(ptent) &&
64363aad 1085 likely(!(vma->vm_flags & VM_SEQ_READ)))
bf3f3bc5 1086 mark_page_accessed(page);
6237bcd9 1087 }
eca56ff9 1088 rss[mm_counter(page)]--;
d281ee61 1089 page_remove_rmap(page, false);
3dc14741
HD
1090 if (unlikely(page_mapcount(page) < 0))
1091 print_bad_pte(vma, addr, ptent, page);
e9d55e15 1092 if (unlikely(__tlb_remove_page(tlb, page))) {
1cf35d47 1093 force_flush = 1;
ce9ec37b 1094 addr += PAGE_SIZE;
d16dfc55 1095 break;
1cf35d47 1096 }
1da177e4
LT
1097 continue;
1098 }
5042db43
JG
1099
1100 entry = pte_to_swp_entry(ptent);
463b7a17 1101 if (is_device_private_entry(entry)) {
5042db43
JG
1102 struct page *page = device_private_entry_to_page(entry);
1103
1104 if (unlikely(details && details->check_mapping)) {
1105 /*
1106 * unmap_shared_mapping_pages() wants to
1107 * invalidate cache without truncating:
1108 * unmap shared but keep private pages.
1109 */
1110 if (details->check_mapping !=
1111 page_rmapping(page))
1112 continue;
1113 }
1114
1115 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
1116 rss[mm_counter(page)]--;
1117 page_remove_rmap(page, false);
1118 put_page(page);
1119 continue;
1120 }
1121
3e8715fd
KS
1122 /* If details->check_mapping, we leave swap entries. */
1123 if (unlikely(details))
1da177e4 1124 continue;
b084d435 1125
8a5f14a2
KS
1126 if (!non_swap_entry(entry))
1127 rss[MM_SWAPENTS]--;
1128 else if (is_migration_entry(entry)) {
1129 struct page *page;
9f9f1acd 1130
8a5f14a2 1131 page = migration_entry_to_page(entry);
eca56ff9 1132 rss[mm_counter(page)]--;
b084d435 1133 }
8a5f14a2
KS
1134 if (unlikely(!free_swap_and_cache(entry)))
1135 print_bad_pte(vma, addr, ptent, NULL);
9888a1ca 1136 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
97a89413 1137 } while (pte++, addr += PAGE_SIZE, addr != end);
ae859762 1138
d559db08 1139 add_mm_rss_vec(mm, rss);
6606c3e0 1140 arch_leave_lazy_mmu_mode();
51c6f666 1141
1cf35d47 1142 /* Do the actual TLB flush before dropping ptl */
fb7332a9 1143 if (force_flush)
1cf35d47 1144 tlb_flush_mmu_tlbonly(tlb);
1cf35d47
LT
1145 pte_unmap_unlock(start_pte, ptl);
1146
1147 /*
1148 * If we forced a TLB flush (either due to running out of
1149 * batch buffers or because we needed to flush dirty TLB
1150 * entries before releasing the ptl), free the batched
1151 * memory too. Restart if we didn't do everything.
1152 */
1153 if (force_flush) {
1154 force_flush = 0;
fa0aafb8 1155 tlb_flush_mmu(tlb);
7b167b68
MK
1156 }
1157
1158 if (addr != end) {
1159 cond_resched();
1160 goto again;
d16dfc55
PZ
1161 }
1162
51c6f666 1163 return addr;
1da177e4
LT
1164}
1165
51c6f666 1166static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
b5810039 1167 struct vm_area_struct *vma, pud_t *pud,
1da177e4 1168 unsigned long addr, unsigned long end,
97a89413 1169 struct zap_details *details)
1da177e4
LT
1170{
1171 pmd_t *pmd;
1172 unsigned long next;
1173
1174 pmd = pmd_offset(pud, addr);
1175 do {
1176 next = pmd_addr_end(addr, end);
84c3fc4e 1177 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
53406ed1 1178 if (next - addr != HPAGE_PMD_SIZE)
fd60775a 1179 __split_huge_pmd(vma, pmd, addr, false, NULL);
53406ed1 1180 else if (zap_huge_pmd(tlb, vma, pmd, addr))
1a5a9906 1181 goto next;
71e3aac0
AA
1182 /* fall through */
1183 }
1a5a9906
AA
1184 /*
1185 * Here there can be other concurrent MADV_DONTNEED or
1186 * trans huge page faults running, and if the pmd is
1187 * none or trans huge it can change under us. This is
c1e8d7c6 1188 * because MADV_DONTNEED holds the mmap_lock in read
1a5a9906
AA
1189 * mode.
1190 */
1191 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1192 goto next;
97a89413 1193 next = zap_pte_range(tlb, vma, pmd, addr, next, details);
1a5a9906 1194next:
97a89413
PZ
1195 cond_resched();
1196 } while (pmd++, addr = next, addr != end);
51c6f666
RH
1197
1198 return addr;
1da177e4
LT
1199}
1200
51c6f666 1201static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
c2febafc 1202 struct vm_area_struct *vma, p4d_t *p4d,
1da177e4 1203 unsigned long addr, unsigned long end,
97a89413 1204 struct zap_details *details)
1da177e4
LT
1205{
1206 pud_t *pud;
1207 unsigned long next;
1208
c2febafc 1209 pud = pud_offset(p4d, addr);
1da177e4
LT
1210 do {
1211 next = pud_addr_end(addr, end);
a00cc7d9
MW
1212 if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
1213 if (next - addr != HPAGE_PUD_SIZE) {
42fc5414 1214 mmap_assert_locked(tlb->mm);
a00cc7d9
MW
1215 split_huge_pud(vma, pud, addr);
1216 } else if (zap_huge_pud(tlb, vma, pud, addr))
1217 goto next;
1218 /* fall through */
1219 }
97a89413 1220 if (pud_none_or_clear_bad(pud))
1da177e4 1221 continue;
97a89413 1222 next = zap_pmd_range(tlb, vma, pud, addr, next, details);
a00cc7d9
MW
1223next:
1224 cond_resched();
97a89413 1225 } while (pud++, addr = next, addr != end);
51c6f666
RH
1226
1227 return addr;
1da177e4
LT
1228}
1229
c2febafc
KS
1230static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
1231 struct vm_area_struct *vma, pgd_t *pgd,
1232 unsigned long addr, unsigned long end,
1233 struct zap_details *details)
1234{
1235 p4d_t *p4d;
1236 unsigned long next;
1237
1238 p4d = p4d_offset(pgd, addr);
1239 do {
1240 next = p4d_addr_end(addr, end);
1241 if (p4d_none_or_clear_bad(p4d))
1242 continue;
1243 next = zap_pud_range(tlb, vma, p4d, addr, next, details);
1244 } while (p4d++, addr = next, addr != end);
1245
1246 return addr;
1247}
1248
aac45363 1249void unmap_page_range(struct mmu_gather *tlb,
038c7aa1
AV
1250 struct vm_area_struct *vma,
1251 unsigned long addr, unsigned long end,
1252 struct zap_details *details)
1da177e4
LT
1253{
1254 pgd_t *pgd;
1255 unsigned long next;
1256
1da177e4
LT
1257 BUG_ON(addr >= end);
1258 tlb_start_vma(tlb, vma);
1259 pgd = pgd_offset(vma->vm_mm, addr);
1260 do {
1261 next = pgd_addr_end(addr, end);
97a89413 1262 if (pgd_none_or_clear_bad(pgd))
1da177e4 1263 continue;
c2febafc 1264 next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
97a89413 1265 } while (pgd++, addr = next, addr != end);
1da177e4
LT
1266 tlb_end_vma(tlb, vma);
1267}
51c6f666 1268
f5cc4eef
AV
1269
1270static void unmap_single_vma(struct mmu_gather *tlb,
1271 struct vm_area_struct *vma, unsigned long start_addr,
4f74d2c8 1272 unsigned long end_addr,
f5cc4eef
AV
1273 struct zap_details *details)
1274{
1275 unsigned long start = max(vma->vm_start, start_addr);
1276 unsigned long end;
1277
1278 if (start >= vma->vm_end)
1279 return;
1280 end = min(vma->vm_end, end_addr);
1281 if (end <= vma->vm_start)
1282 return;
1283
cbc91f71
SD
1284 if (vma->vm_file)
1285 uprobe_munmap(vma, start, end);
1286
b3b9c293 1287 if (unlikely(vma->vm_flags & VM_PFNMAP))
5180da41 1288 untrack_pfn(vma, 0, 0);
f5cc4eef
AV
1289
1290 if (start != end) {
1291 if (unlikely(is_vm_hugetlb_page(vma))) {
1292 /*
1293 * It is undesirable to test vma->vm_file as it
1294 * should be non-null for valid hugetlb area.
1295 * However, vm_file will be NULL in the error
7aa6b4ad 1296 * cleanup path of mmap_region. When
f5cc4eef 1297 * hugetlbfs ->mmap method fails,
7aa6b4ad 1298 * mmap_region() nullifies vma->vm_file
f5cc4eef
AV
1299 * before calling this function to clean up.
1300 * Since no pte has actually been setup, it is
1301 * safe to do nothing in this case.
1302 */
24669e58 1303 if (vma->vm_file) {
83cde9e8 1304 i_mmap_lock_write(vma->vm_file->f_mapping);
d833352a 1305 __unmap_hugepage_range_final(tlb, vma, start, end, NULL);
83cde9e8 1306 i_mmap_unlock_write(vma->vm_file->f_mapping);
24669e58 1307 }
f5cc4eef
AV
1308 } else
1309 unmap_page_range(tlb, vma, start, end, details);
1310 }
1da177e4
LT
1311}
1312
1da177e4
LT
1313/**
1314 * unmap_vmas - unmap a range of memory covered by a list of vma's
0164f69d 1315 * @tlb: address of the caller's struct mmu_gather
1da177e4
LT
1316 * @vma: the starting vma
1317 * @start_addr: virtual address at which to start unmapping
1318 * @end_addr: virtual address at which to end unmapping
1da177e4 1319 *
508034a3 1320 * Unmap all pages in the vma list.
1da177e4 1321 *
1da177e4
LT
1322 * Only addresses between `start' and `end' will be unmapped.
1323 *
1324 * The VMA list must be sorted in ascending virtual address order.
1325 *
1326 * unmap_vmas() assumes that the caller will flush the whole unmapped address
1327 * range after unmap_vmas() returns. So the only responsibility here is to
1328 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
1329 * drops the lock and schedules.
1330 */
6e8bb019 1331void unmap_vmas(struct mmu_gather *tlb,
1da177e4 1332 struct vm_area_struct *vma, unsigned long start_addr,
4f74d2c8 1333 unsigned long end_addr)
1da177e4 1334{
ac46d4f3 1335 struct mmu_notifier_range range;
1da177e4 1336
6f4f13e8
JG
1337 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
1338 start_addr, end_addr);
ac46d4f3 1339 mmu_notifier_invalidate_range_start(&range);
f5cc4eef 1340 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
4f74d2c8 1341 unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
ac46d4f3 1342 mmu_notifier_invalidate_range_end(&range);
1da177e4
LT
1343}
1344
1345/**
1346 * zap_page_range - remove user pages in a given range
1347 * @vma: vm_area_struct holding the applicable pages
eb4546bb 1348 * @start: starting address of pages to zap
1da177e4 1349 * @size: number of bytes to zap
f5cc4eef
AV
1350 *
1351 * Caller must protect the VMA list
1da177e4 1352 */
7e027b14 1353void zap_page_range(struct vm_area_struct *vma, unsigned long start,
ecf1385d 1354 unsigned long size)
1da177e4 1355{
ac46d4f3 1356 struct mmu_notifier_range range;
d16dfc55 1357 struct mmu_gather tlb;
1da177e4 1358
1da177e4 1359 lru_add_drain();
7269f999 1360 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
6f4f13e8 1361 start, start + size);
ac46d4f3
JG
1362 tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end);
1363 update_hiwater_rss(vma->vm_mm);
1364 mmu_notifier_invalidate_range_start(&range);
1365 for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next)
1366 unmap_single_vma(&tlb, vma, start, range.end, NULL);
1367 mmu_notifier_invalidate_range_end(&range);
1368 tlb_finish_mmu(&tlb, start, range.end);
1da177e4
LT
1369}
1370
f5cc4eef
AV
1371/**
1372 * zap_page_range_single - remove user pages in a given range
1373 * @vma: vm_area_struct holding the applicable pages
1374 * @address: starting address of pages to zap
1375 * @size: number of bytes to zap
8a5f14a2 1376 * @details: details of shared cache invalidation
f5cc4eef
AV
1377 *
1378 * The range must fit into one VMA.
1da177e4 1379 */
f5cc4eef 1380static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
1da177e4
LT
1381 unsigned long size, struct zap_details *details)
1382{
ac46d4f3 1383 struct mmu_notifier_range range;
d16dfc55 1384 struct mmu_gather tlb;
1da177e4 1385
1da177e4 1386 lru_add_drain();
7269f999 1387 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
6f4f13e8 1388 address, address + size);
ac46d4f3
JG
1389 tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end);
1390 update_hiwater_rss(vma->vm_mm);
1391 mmu_notifier_invalidate_range_start(&range);
1392 unmap_single_vma(&tlb, vma, address, range.end, details);
1393 mmu_notifier_invalidate_range_end(&range);
1394 tlb_finish_mmu(&tlb, address, range.end);
1da177e4
LT
1395}
1396
c627f9cc
JS
1397/**
1398 * zap_vma_ptes - remove ptes mapping the vma
1399 * @vma: vm_area_struct holding ptes to be zapped
1400 * @address: starting address of pages to zap
1401 * @size: number of bytes to zap
1402 *
1403 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
1404 *
1405 * The entire address range must be fully contained within the vma.
1406 *
c627f9cc 1407 */
27d036e3 1408void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
c627f9cc
JS
1409 unsigned long size)
1410{
1411 if (address < vma->vm_start || address + size > vma->vm_end ||
1412 !(vma->vm_flags & VM_PFNMAP))
27d036e3
LR
1413 return;
1414
f5cc4eef 1415 zap_page_range_single(vma, address, size, NULL);
c627f9cc
JS
1416}
1417EXPORT_SYMBOL_GPL(zap_vma_ptes);
1418
8cd3984d 1419static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
c9cfcddf 1420{
c2febafc
KS
1421 pgd_t *pgd;
1422 p4d_t *p4d;
1423 pud_t *pud;
1424 pmd_t *pmd;
1425
1426 pgd = pgd_offset(mm, addr);
1427 p4d = p4d_alloc(mm, pgd, addr);
1428 if (!p4d)
1429 return NULL;
1430 pud = pud_alloc(mm, p4d, addr);
1431 if (!pud)
1432 return NULL;
1433 pmd = pmd_alloc(mm, pud, addr);
1434 if (!pmd)
1435 return NULL;
1436
1437 VM_BUG_ON(pmd_trans_huge(*pmd));
8cd3984d
AR
1438 return pmd;
1439}
1440
1441pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1442 spinlock_t **ptl)
1443{
1444 pmd_t *pmd = walk_to_pmd(mm, addr);
1445
1446 if (!pmd)
1447 return NULL;
c2febafc 1448 return pte_alloc_map_lock(mm, pmd, addr, ptl);
c9cfcddf
LT
1449}
1450
8efd6f5b
AR
1451static int validate_page_before_insert(struct page *page)
1452{
1453 if (PageAnon(page) || PageSlab(page) || page_has_type(page))
1454 return -EINVAL;
1455 flush_dcache_page(page);
1456 return 0;
1457}
1458
1459static int insert_page_into_pte_locked(struct mm_struct *mm, pte_t *pte,
1460 unsigned long addr, struct page *page, pgprot_t prot)
1461{
1462 if (!pte_none(*pte))
1463 return -EBUSY;
1464 /* Ok, finally just insert the thing.. */
1465 get_page(page);
1466 inc_mm_counter_fast(mm, mm_counter_file(page));
1467 page_add_file_rmap(page, false);
1468 set_pte_at(mm, addr, pte, mk_pte(page, prot));
1469 return 0;
1470}
1471
238f58d8
LT
1472/*
1473 * This is the old fallback for page remapping.
1474 *
1475 * For historical reasons, it only allows reserved pages. Only
1476 * old drivers should use this, and they needed to mark their
1477 * pages reserved for the old functions anyway.
1478 */
423bad60
NP
1479static int insert_page(struct vm_area_struct *vma, unsigned long addr,
1480 struct page *page, pgprot_t prot)
238f58d8 1481{
423bad60 1482 struct mm_struct *mm = vma->vm_mm;
238f58d8 1483 int retval;
c9cfcddf 1484 pte_t *pte;
8a9f3ccd
BS
1485 spinlock_t *ptl;
1486
8efd6f5b
AR
1487 retval = validate_page_before_insert(page);
1488 if (retval)
5b4e655e 1489 goto out;
238f58d8 1490 retval = -ENOMEM;
c9cfcddf 1491 pte = get_locked_pte(mm, addr, &ptl);
238f58d8 1492 if (!pte)
5b4e655e 1493 goto out;
8efd6f5b 1494 retval = insert_page_into_pte_locked(mm, pte, addr, page, prot);
238f58d8
LT
1495 pte_unmap_unlock(pte, ptl);
1496out:
1497 return retval;
1498}
1499
8cd3984d 1500#ifdef pte_index
7f70c2a6 1501static int insert_page_in_batch_locked(struct mm_struct *mm, pte_t *pte,
8cd3984d
AR
1502 unsigned long addr, struct page *page, pgprot_t prot)
1503{
1504 int err;
1505
1506 if (!page_count(page))
1507 return -EINVAL;
1508 err = validate_page_before_insert(page);
7f70c2a6
AR
1509 if (err)
1510 return err;
1511 return insert_page_into_pte_locked(mm, pte, addr, page, prot);
8cd3984d
AR
1512}
1513
1514/* insert_pages() amortizes the cost of spinlock operations
1515 * when inserting pages in a loop. Arch *must* define pte_index.
1516 */
1517static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
1518 struct page **pages, unsigned long *num, pgprot_t prot)
1519{
1520 pmd_t *pmd = NULL;
7f70c2a6
AR
1521 pte_t *start_pte, *pte;
1522 spinlock_t *pte_lock;
8cd3984d
AR
1523 struct mm_struct *const mm = vma->vm_mm;
1524 unsigned long curr_page_idx = 0;
1525 unsigned long remaining_pages_total = *num;
1526 unsigned long pages_to_write_in_pmd;
1527 int ret;
1528more:
1529 ret = -EFAULT;
1530 pmd = walk_to_pmd(mm, addr);
1531 if (!pmd)
1532 goto out;
1533
1534 pages_to_write_in_pmd = min_t(unsigned long,
1535 remaining_pages_total, PTRS_PER_PTE - pte_index(addr));
1536
1537 /* Allocate the PTE if necessary; takes PMD lock once only. */
1538 ret = -ENOMEM;
1539 if (pte_alloc(mm, pmd))
1540 goto out;
8cd3984d
AR
1541
1542 while (pages_to_write_in_pmd) {
1543 int pte_idx = 0;
1544 const int batch_size = min_t(int, pages_to_write_in_pmd, 8);
1545
7f70c2a6
AR
1546 start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
1547 for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
1548 int err = insert_page_in_batch_locked(mm, pte,
8cd3984d
AR
1549 addr, pages[curr_page_idx], prot);
1550 if (unlikely(err)) {
7f70c2a6 1551 pte_unmap_unlock(start_pte, pte_lock);
8cd3984d
AR
1552 ret = err;
1553 remaining_pages_total -= pte_idx;
1554 goto out;
1555 }
1556 addr += PAGE_SIZE;
1557 ++curr_page_idx;
1558 }
7f70c2a6 1559 pte_unmap_unlock(start_pte, pte_lock);
8cd3984d
AR
1560 pages_to_write_in_pmd -= batch_size;
1561 remaining_pages_total -= batch_size;
1562 }
1563 if (remaining_pages_total)
1564 goto more;
1565 ret = 0;
1566out:
1567 *num = remaining_pages_total;
1568 return ret;
1569}
1570#endif /* ifdef pte_index */
1571
1572/**
1573 * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
1574 * @vma: user vma to map to
1575 * @addr: target start user address of these pages
1576 * @pages: source kernel pages
1577 * @num: in: number of pages to map. out: number of pages that were *not*
1578 * mapped. (0 means all pages were successfully mapped).
1579 *
1580 * Preferred over vm_insert_page() when inserting multiple pages.
1581 *
1582 * In case of error, we may have mapped a subset of the provided
1583 * pages. It is the caller's responsibility to account for this case.
1584 *
1585 * The same restrictions apply as in vm_insert_page().
1586 */
1587int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
1588 struct page **pages, unsigned long *num)
1589{
1590#ifdef pte_index
1591 const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;
1592
1593 if (addr < vma->vm_start || end_addr >= vma->vm_end)
1594 return -EFAULT;
1595 if (!(vma->vm_flags & VM_MIXEDMAP)) {
d8ed45c5 1596 BUG_ON(mmap_read_trylock(vma->vm_mm));
8cd3984d
AR
1597 BUG_ON(vma->vm_flags & VM_PFNMAP);
1598 vma->vm_flags |= VM_MIXEDMAP;
1599 }
1600 /* Defer page refcount checking till we're about to map that page. */
1601 return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
1602#else
1603 unsigned long idx = 0, pgcount = *num;
45779b03 1604 int err = -EINVAL;
8cd3984d
AR
1605
1606 for (; idx < pgcount; ++idx) {
1607 err = vm_insert_page(vma, addr + (PAGE_SIZE * idx), pages[idx]);
1608 if (err)
1609 break;
1610 }
1611 *num = pgcount - idx;
1612 return err;
1613#endif /* ifdef pte_index */
1614}
1615EXPORT_SYMBOL(vm_insert_pages);
1616
bfa5bf6d
REB
1617/**
1618 * vm_insert_page - insert single page into user vma
1619 * @vma: user vma to map to
1620 * @addr: target user address of this page
1621 * @page: source kernel page
1622 *
a145dd41
LT
1623 * This allows drivers to insert individual pages they've allocated
1624 * into a user vma.
1625 *
1626 * The page has to be a nice clean _individual_ kernel allocation.
1627 * If you allocate a compound page, you need to have marked it as
1628 * such (__GFP_COMP), or manually just split the page up yourself
8dfcc9ba 1629 * (see split_page()).
a145dd41
LT
1630 *
1631 * NOTE! Traditionally this was done with "remap_pfn_range()" which
1632 * took an arbitrary page protection parameter. This doesn't allow
1633 * that. Your vma protection will have to be set up correctly, which
1634 * means that if you want a shared writable mapping, you'd better
1635 * ask for a shared writable mapping!
1636 *
1637 * The page does not need to be reserved.
4b6e1e37
KK
1638 *
1639 * Usually this function is called from f_op->mmap() handler
c1e8d7c6 1640 * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
4b6e1e37
KK
1641 * Caller must set VM_MIXEDMAP on vma if it wants to call this
1642 * function from other places, for example from page-fault handler.
a862f68a
MR
1643 *
1644 * Return: %0 on success, negative error code otherwise.
a145dd41 1645 */
423bad60
NP
1646int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
1647 struct page *page)
a145dd41
LT
1648{
1649 if (addr < vma->vm_start || addr >= vma->vm_end)
1650 return -EFAULT;
1651 if (!page_count(page))
1652 return -EINVAL;
4b6e1e37 1653 if (!(vma->vm_flags & VM_MIXEDMAP)) {
d8ed45c5 1654 BUG_ON(mmap_read_trylock(vma->vm_mm));
4b6e1e37
KK
1655 BUG_ON(vma->vm_flags & VM_PFNMAP);
1656 vma->vm_flags |= VM_MIXEDMAP;
1657 }
423bad60 1658 return insert_page(vma, addr, page, vma->vm_page_prot);
a145dd41 1659}
e3c3374f 1660EXPORT_SYMBOL(vm_insert_page);
a145dd41 1661
a667d745
SJ
1662/*
1663 * __vm_map_pages - maps range of kernel pages into user vma
1664 * @vma: user vma to map to
1665 * @pages: pointer to array of source kernel pages
1666 * @num: number of pages in page array
1667 * @offset: user's requested vm_pgoff
1668 *
1669 * This allows drivers to map range of kernel pages into a user vma.
1670 *
1671 * Return: 0 on success and error code otherwise.
1672 */
1673static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
1674 unsigned long num, unsigned long offset)
1675{
1676 unsigned long count = vma_pages(vma);
1677 unsigned long uaddr = vma->vm_start;
1678 int ret, i;
1679
1680 /* Fail if the user requested offset is beyond the end of the object */
96756fcb 1681 if (offset >= num)
a667d745
SJ
1682 return -ENXIO;
1683
1684 /* Fail if the user requested size exceeds available object size */
1685 if (count > num - offset)
1686 return -ENXIO;
1687
1688 for (i = 0; i < count; i++) {
1689 ret = vm_insert_page(vma, uaddr, pages[offset + i]);
1690 if (ret < 0)
1691 return ret;
1692 uaddr += PAGE_SIZE;
1693 }
1694
1695 return 0;
1696}
1697
1698/**
1699 * vm_map_pages - maps range of kernel pages starts with non zero offset
1700 * @vma: user vma to map to
1701 * @pages: pointer to array of source kernel pages
1702 * @num: number of pages in page array
1703 *
1704 * Maps an object consisting of @num pages, catering for the user's
1705 * requested vm_pgoff
1706 *
1707 * If we fail to insert any page into the vma, the function will return
1708 * immediately leaving any previously inserted pages present. Callers
1709 * from the mmap handler may immediately return the error as their caller
1710 * will destroy the vma, removing any successfully inserted pages. Other
1711 * callers should make their own arrangements for calling unmap_region().
1712 *
1713 * Context: Process context. Called by mmap handlers.
1714 * Return: 0 on success and error code otherwise.
1715 */
1716int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
1717 unsigned long num)
1718{
1719 return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
1720}
1721EXPORT_SYMBOL(vm_map_pages);
1722
1723/**
1724 * vm_map_pages_zero - map range of kernel pages starts with zero offset
1725 * @vma: user vma to map to
1726 * @pages: pointer to array of source kernel pages
1727 * @num: number of pages in page array
1728 *
1729 * Similar to vm_map_pages(), except that it explicitly sets the offset
1730 * to 0. This function is intended for the drivers that did not consider
1731 * vm_pgoff.
1732 *
1733 * Context: Process context. Called by mmap handlers.
1734 * Return: 0 on success and error code otherwise.
1735 */
1736int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
1737 unsigned long num)
1738{
1739 return __vm_map_pages(vma, pages, num, 0);
1740}
1741EXPORT_SYMBOL(vm_map_pages_zero);
1742
9b5a8e00 1743static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
b2770da6 1744 pfn_t pfn, pgprot_t prot, bool mkwrite)
423bad60
NP
1745{
1746 struct mm_struct *mm = vma->vm_mm;
423bad60
NP
1747 pte_t *pte, entry;
1748 spinlock_t *ptl;
1749
423bad60
NP
1750 pte = get_locked_pte(mm, addr, &ptl);
1751 if (!pte)
9b5a8e00 1752 return VM_FAULT_OOM;
b2770da6
RZ
1753 if (!pte_none(*pte)) {
1754 if (mkwrite) {
1755 /*
1756 * For read faults on private mappings the PFN passed
1757 * in may not match the PFN we have mapped if the
1758 * mapped PFN is a writeable COW page. In the mkwrite
1759 * case we are creating a writable PTE for a shared
f2c57d91
JK
1760 * mapping and we expect the PFNs to match. If they
1761 * don't match, we are likely racing with block
1762 * allocation and mapping invalidation so just skip the
1763 * update.
b2770da6 1764 */
f2c57d91
JK
1765 if (pte_pfn(*pte) != pfn_t_to_pfn(pfn)) {
1766 WARN_ON_ONCE(!is_zero_pfn(pte_pfn(*pte)));
b2770da6 1767 goto out_unlock;
f2c57d91 1768 }
cae85cb8
JK
1769 entry = pte_mkyoung(*pte);
1770 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1771 if (ptep_set_access_flags(vma, addr, pte, entry, 1))
1772 update_mmu_cache(vma, addr, pte);
1773 }
1774 goto out_unlock;
b2770da6 1775 }
423bad60
NP
1776
1777 /* Ok, finally just insert the thing.. */
01c8f1c4
DW
1778 if (pfn_t_devmap(pfn))
1779 entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
1780 else
1781 entry = pte_mkspecial(pfn_t_pte(pfn, prot));
b2770da6 1782
b2770da6
RZ
1783 if (mkwrite) {
1784 entry = pte_mkyoung(entry);
1785 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1786 }
1787
423bad60 1788 set_pte_at(mm, addr, pte, entry);
4b3073e1 1789 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
423bad60 1790
423bad60
NP
1791out_unlock:
1792 pte_unmap_unlock(pte, ptl);
9b5a8e00 1793 return VM_FAULT_NOPAGE;
423bad60
NP
1794}
1795
f5e6d1d5
MW
1796/**
1797 * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
1798 * @vma: user vma to map to
1799 * @addr: target user address of this page
1800 * @pfn: source kernel pfn
1801 * @pgprot: pgprot flags for the inserted page
1802 *
1803 * This is exactly like vmf_insert_pfn(), except that it allows drivers to
1804 * to override pgprot on a per-page basis.
1805 *
1806 * This only makes sense for IO mappings, and it makes no sense for
1807 * COW mappings. In general, using multiple vmas is preferable;
ae2b01f3 1808 * vmf_insert_pfn_prot should only be used if using multiple VMAs is
f5e6d1d5
MW
1809 * impractical.
1810 *
574c5b3d
TH
1811 * See vmf_insert_mixed_prot() for a discussion of the implication of using
1812 * a value of @pgprot different from that of @vma->vm_page_prot.
1813 *
ae2b01f3 1814 * Context: Process context. May allocate using %GFP_KERNEL.
f5e6d1d5
MW
1815 * Return: vm_fault_t value.
1816 */
1817vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
1818 unsigned long pfn, pgprot_t pgprot)
1819{
6d958546
MW
1820 /*
1821 * Technically, architectures with pte_special can avoid all these
1822 * restrictions (same for remap_pfn_range). However we would like
1823 * consistency in testing and feature parity among all, so we should
1824 * try to keep these invariants in place for everybody.
1825 */
1826 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
1827 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
1828 (VM_PFNMAP|VM_MIXEDMAP));
1829 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
1830 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
1831
1832 if (addr < vma->vm_start || addr >= vma->vm_end)
1833 return VM_FAULT_SIGBUS;
1834
1835 if (!pfn_modify_allowed(pfn, pgprot))
1836 return VM_FAULT_SIGBUS;
1837
1838 track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
1839
9b5a8e00 1840 return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
6d958546 1841 false);
f5e6d1d5
MW
1842}
1843EXPORT_SYMBOL(vmf_insert_pfn_prot);
e0dc0d8f 1844
ae2b01f3
MW
1845/**
1846 * vmf_insert_pfn - insert single pfn into user vma
1847 * @vma: user vma to map to
1848 * @addr: target user address of this page
1849 * @pfn: source kernel pfn
1850 *
1851 * Similar to vm_insert_page, this allows drivers to insert individual pages
1852 * they've allocated into a user vma. Same comments apply.
1853 *
1854 * This function should only be called from a vm_ops->fault handler, and
1855 * in that case the handler should return the result of this function.
1856 *
1857 * vma cannot be a COW mapping.
1858 *
1859 * As this is called only for pages that do not currently exist, we
1860 * do not need to flush old virtual caches or the TLB.
1861 *
1862 * Context: Process context. May allocate using %GFP_KERNEL.
1863 * Return: vm_fault_t value.
1864 */
1865vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1866 unsigned long pfn)
1867{
1868 return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
1869}
1870EXPORT_SYMBOL(vmf_insert_pfn);
1871
785a3fab
DW
1872static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn)
1873{
1874 /* these checks mirror the abort conditions in vm_normal_page */
1875 if (vma->vm_flags & VM_MIXEDMAP)
1876 return true;
1877 if (pfn_t_devmap(pfn))
1878 return true;
1879 if (pfn_t_special(pfn))
1880 return true;
1881 if (is_zero_pfn(pfn_t_to_pfn(pfn)))
1882 return true;
1883 return false;
1884}
1885
79f3aa5b 1886static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
574c5b3d
TH
1887 unsigned long addr, pfn_t pfn, pgprot_t pgprot,
1888 bool mkwrite)
423bad60 1889{
79f3aa5b 1890 int err;
87744ab3 1891
785a3fab 1892 BUG_ON(!vm_mixed_ok(vma, pfn));
e0dc0d8f 1893
423bad60 1894 if (addr < vma->vm_start || addr >= vma->vm_end)
79f3aa5b 1895 return VM_FAULT_SIGBUS;
308a047c
BP
1896
1897 track_pfn_insert(vma, &pgprot, pfn);
e0dc0d8f 1898
42e4089c 1899 if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
79f3aa5b 1900 return VM_FAULT_SIGBUS;
42e4089c 1901
423bad60
NP
1902 /*
1903 * If we don't have pte special, then we have to use the pfn_valid()
1904 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
1905 * refcount the page if pfn_valid is true (hence insert_page rather
62eede62
HD
1906 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
1907 * without pte special, it would there be refcounted as a normal page.
423bad60 1908 */
00b3a331
LD
1909 if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
1910 !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
423bad60
NP
1911 struct page *page;
1912
03fc2da6
DW
1913 /*
1914 * At this point we are committed to insert_page()
1915 * regardless of whether the caller specified flags that
1916 * result in pfn_t_has_page() == false.
1917 */
1918 page = pfn_to_page(pfn_t_to_pfn(pfn));
79f3aa5b
MW
1919 err = insert_page(vma, addr, page, pgprot);
1920 } else {
9b5a8e00 1921 return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
423bad60 1922 }
b2770da6 1923
5d747637
MW
1924 if (err == -ENOMEM)
1925 return VM_FAULT_OOM;
1926 if (err < 0 && err != -EBUSY)
1927 return VM_FAULT_SIGBUS;
1928
1929 return VM_FAULT_NOPAGE;
e0dc0d8f 1930}
79f3aa5b 1931
574c5b3d
TH
1932/**
1933 * vmf_insert_mixed_prot - insert single pfn into user vma with specified pgprot
1934 * @vma: user vma to map to
1935 * @addr: target user address of this page
1936 * @pfn: source kernel pfn
1937 * @pgprot: pgprot flags for the inserted page
1938 *
1939 * This is exactly like vmf_insert_mixed(), except that it allows drivers to
1940 * to override pgprot on a per-page basis.
1941 *
1942 * Typically this function should be used by drivers to set caching- and
1943 * encryption bits different than those of @vma->vm_page_prot, because
1944 * the caching- or encryption mode may not be known at mmap() time.
1945 * This is ok as long as @vma->vm_page_prot is not used by the core vm
1946 * to set caching and encryption bits for those vmas (except for COW pages).
1947 * This is ensured by core vm only modifying these page table entries using
1948 * functions that don't touch caching- or encryption bits, using pte_modify()
1949 * if needed. (See for example mprotect()).
1950 * Also when new page-table entries are created, this is only done using the
1951 * fault() callback, and never using the value of vma->vm_page_prot,
1952 * except for page-table entries that point to anonymous pages as the result
1953 * of COW.
1954 *
1955 * Context: Process context. May allocate using %GFP_KERNEL.
1956 * Return: vm_fault_t value.
1957 */
1958vm_fault_t vmf_insert_mixed_prot(struct vm_area_struct *vma, unsigned long addr,
1959 pfn_t pfn, pgprot_t pgprot)
1960{
1961 return __vm_insert_mixed(vma, addr, pfn, pgprot, false);
1962}
5379e4dd 1963EXPORT_SYMBOL(vmf_insert_mixed_prot);
574c5b3d 1964
79f3aa5b
MW
1965vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1966 pfn_t pfn)
1967{
574c5b3d 1968 return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, false);
79f3aa5b 1969}
5d747637 1970EXPORT_SYMBOL(vmf_insert_mixed);
e0dc0d8f 1971
ab77dab4
SJ
1972/*
1973 * If the insertion of PTE failed because someone else already added a
1974 * different entry in the mean time, we treat that as success as we assume
1975 * the same entry was actually inserted.
1976 */
ab77dab4
SJ
1977vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
1978 unsigned long addr, pfn_t pfn)
b2770da6 1979{
574c5b3d 1980 return __vm_insert_mixed(vma, addr, pfn, vma->vm_page_prot, true);
b2770da6 1981}
ab77dab4 1982EXPORT_SYMBOL(vmf_insert_mixed_mkwrite);
b2770da6 1983
1da177e4
LT
1984/*
1985 * maps a range of physical memory into the requested pages. the old
1986 * mappings are removed. any references to nonexistent pages results
1987 * in null mappings (currently treated as "copy-on-access")
1988 */
1989static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
1990 unsigned long addr, unsigned long end,
1991 unsigned long pfn, pgprot_t prot)
1992{
1993 pte_t *pte;
c74df32c 1994 spinlock_t *ptl;
42e4089c 1995 int err = 0;
1da177e4 1996
c74df32c 1997 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
1da177e4
LT
1998 if (!pte)
1999 return -ENOMEM;
6606c3e0 2000 arch_enter_lazy_mmu_mode();
1da177e4
LT
2001 do {
2002 BUG_ON(!pte_none(*pte));
42e4089c
AK
2003 if (!pfn_modify_allowed(pfn, prot)) {
2004 err = -EACCES;
2005 break;
2006 }
7e675137 2007 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
1da177e4
LT
2008 pfn++;
2009 } while (pte++, addr += PAGE_SIZE, addr != end);
6606c3e0 2010 arch_leave_lazy_mmu_mode();
c74df32c 2011 pte_unmap_unlock(pte - 1, ptl);
42e4089c 2012 return err;
1da177e4
LT
2013}
2014
2015static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
2016 unsigned long addr, unsigned long end,
2017 unsigned long pfn, pgprot_t prot)
2018{
2019 pmd_t *pmd;
2020 unsigned long next;
42e4089c 2021 int err;
1da177e4
LT
2022
2023 pfn -= addr >> PAGE_SHIFT;
2024 pmd = pmd_alloc(mm, pud, addr);
2025 if (!pmd)
2026 return -ENOMEM;
f66055ab 2027 VM_BUG_ON(pmd_trans_huge(*pmd));
1da177e4
LT
2028 do {
2029 next = pmd_addr_end(addr, end);
42e4089c
AK
2030 err = remap_pte_range(mm, pmd, addr, next,
2031 pfn + (addr >> PAGE_SHIFT), prot);
2032 if (err)
2033 return err;
1da177e4
LT
2034 } while (pmd++, addr = next, addr != end);
2035 return 0;
2036}
2037
c2febafc 2038static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
1da177e4
LT
2039 unsigned long addr, unsigned long end,
2040 unsigned long pfn, pgprot_t prot)
2041{
2042 pud_t *pud;
2043 unsigned long next;
42e4089c 2044 int err;
1da177e4
LT
2045
2046 pfn -= addr >> PAGE_SHIFT;
c2febafc 2047 pud = pud_alloc(mm, p4d, addr);
1da177e4
LT
2048 if (!pud)
2049 return -ENOMEM;
2050 do {
2051 next = pud_addr_end(addr, end);
42e4089c
AK
2052 err = remap_pmd_range(mm, pud, addr, next,
2053 pfn + (addr >> PAGE_SHIFT), prot);
2054 if (err)
2055 return err;
1da177e4
LT
2056 } while (pud++, addr = next, addr != end);
2057 return 0;
2058}
2059
c2febafc
KS
2060static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2061 unsigned long addr, unsigned long end,
2062 unsigned long pfn, pgprot_t prot)
2063{
2064 p4d_t *p4d;
2065 unsigned long next;
42e4089c 2066 int err;
c2febafc
KS
2067
2068 pfn -= addr >> PAGE_SHIFT;
2069 p4d = p4d_alloc(mm, pgd, addr);
2070 if (!p4d)
2071 return -ENOMEM;
2072 do {
2073 next = p4d_addr_end(addr, end);
42e4089c
AK
2074 err = remap_pud_range(mm, p4d, addr, next,
2075 pfn + (addr >> PAGE_SHIFT), prot);
2076 if (err)
2077 return err;
c2febafc
KS
2078 } while (p4d++, addr = next, addr != end);
2079 return 0;
2080}
2081
bfa5bf6d
REB
2082/**
2083 * remap_pfn_range - remap kernel memory to userspace
2084 * @vma: user vma to map to
0c4123e3 2085 * @addr: target page aligned user address to start at
86a76331 2086 * @pfn: page frame number of kernel physical memory address
552657b7 2087 * @size: size of mapping area
bfa5bf6d
REB
2088 * @prot: page protection flags for this mapping
2089 *
a862f68a
MR
2090 * Note: this is only safe if the mm semaphore is held when called.
2091 *
2092 * Return: %0 on success, negative error code otherwise.
bfa5bf6d 2093 */
1da177e4
LT
2094int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
2095 unsigned long pfn, unsigned long size, pgprot_t prot)
2096{
2097 pgd_t *pgd;
2098 unsigned long next;
2d15cab8 2099 unsigned long end = addr + PAGE_ALIGN(size);
1da177e4 2100 struct mm_struct *mm = vma->vm_mm;
d5957d2f 2101 unsigned long remap_pfn = pfn;
1da177e4
LT
2102 int err;
2103
0c4123e3
AZ
2104 if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
2105 return -EINVAL;
2106
1da177e4
LT
2107 /*
2108 * Physically remapped pages are special. Tell the
2109 * rest of the world about it:
2110 * VM_IO tells people not to look at these pages
2111 * (accesses can have side effects).
6aab341e
LT
2112 * VM_PFNMAP tells the core MM that the base pages are just
2113 * raw PFN mappings, and do not have a "struct page" associated
2114 * with them.
314e51b9
KK
2115 * VM_DONTEXPAND
2116 * Disable vma merging and expanding with mremap().
2117 * VM_DONTDUMP
2118 * Omit vma from core dump, even when VM_IO turned off.
fb155c16
LT
2119 *
2120 * There's a horrible special case to handle copy-on-write
2121 * behaviour that some programs depend on. We mark the "original"
2122 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
b3b9c293 2123 * See vm_normal_page() for details.
1da177e4 2124 */
b3b9c293
KK
2125 if (is_cow_mapping(vma->vm_flags)) {
2126 if (addr != vma->vm_start || end != vma->vm_end)
2127 return -EINVAL;
fb155c16 2128 vma->vm_pgoff = pfn;
b3b9c293
KK
2129 }
2130
d5957d2f 2131 err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size));
b3b9c293 2132 if (err)
3c8bb73a 2133 return -EINVAL;
fb155c16 2134
314e51b9 2135 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1da177e4
LT
2136
2137 BUG_ON(addr >= end);
2138 pfn -= addr >> PAGE_SHIFT;
2139 pgd = pgd_offset(mm, addr);
2140 flush_cache_range(vma, addr, end);
1da177e4
LT
2141 do {
2142 next = pgd_addr_end(addr, end);
c2febafc 2143 err = remap_p4d_range(mm, pgd, addr, next,
1da177e4
LT
2144 pfn + (addr >> PAGE_SHIFT), prot);
2145 if (err)
2146 break;
2147 } while (pgd++, addr = next, addr != end);
2ab64037 2148
2149 if (err)
d5957d2f 2150 untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size));
2ab64037 2151
1da177e4
LT
2152 return err;
2153}
2154EXPORT_SYMBOL(remap_pfn_range);
2155
b4cbb197
LT
2156/**
2157 * vm_iomap_memory - remap memory to userspace
2158 * @vma: user vma to map to
abd69b9e 2159 * @start: start of the physical memory to be mapped
b4cbb197
LT
2160 * @len: size of area
2161 *
2162 * This is a simplified io_remap_pfn_range() for common driver use. The
2163 * driver just needs to give us the physical memory range to be mapped,
2164 * we'll figure out the rest from the vma information.
2165 *
2166 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
2167 * whatever write-combining details or similar.
a862f68a
MR
2168 *
2169 * Return: %0 on success, negative error code otherwise.
b4cbb197
LT
2170 */
2171int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
2172{
2173 unsigned long vm_len, pfn, pages;
2174
2175 /* Check that the physical memory area passed in looks valid */
2176 if (start + len < start)
2177 return -EINVAL;
2178 /*
2179 * You *really* shouldn't map things that aren't page-aligned,
2180 * but we've historically allowed it because IO memory might
2181 * just have smaller alignment.
2182 */
2183 len += start & ~PAGE_MASK;
2184 pfn = start >> PAGE_SHIFT;
2185 pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
2186 if (pfn + pages < pfn)
2187 return -EINVAL;
2188
2189 /* We start the mapping 'vm_pgoff' pages into the area */
2190 if (vma->vm_pgoff > pages)
2191 return -EINVAL;
2192 pfn += vma->vm_pgoff;
2193 pages -= vma->vm_pgoff;
2194
2195 /* Can we fit all of the mapping? */
2196 vm_len = vma->vm_end - vma->vm_start;
2197 if (vm_len >> PAGE_SHIFT > pages)
2198 return -EINVAL;
2199
2200 /* Ok, let it rip */
2201 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
2202}
2203EXPORT_SYMBOL(vm_iomap_memory);
2204
aee16b3c
JF
2205static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
2206 unsigned long addr, unsigned long end,
be1db475 2207 pte_fn_t fn, void *data, bool create)
aee16b3c
JF
2208{
2209 pte_t *pte;
be1db475 2210 int err = 0;
3f649ab7 2211 spinlock_t *ptl;
aee16b3c 2212
be1db475
DA
2213 if (create) {
2214 pte = (mm == &init_mm) ?
2215 pte_alloc_kernel(pmd, addr) :
2216 pte_alloc_map_lock(mm, pmd, addr, &ptl);
2217 if (!pte)
2218 return -ENOMEM;
2219 } else {
2220 pte = (mm == &init_mm) ?
2221 pte_offset_kernel(pmd, addr) :
2222 pte_offset_map_lock(mm, pmd, addr, &ptl);
2223 }
aee16b3c
JF
2224
2225 BUG_ON(pmd_huge(*pmd));
2226
38e0edb1
JF
2227 arch_enter_lazy_mmu_mode();
2228
aee16b3c 2229 do {
be1db475
DA
2230 if (create || !pte_none(*pte)) {
2231 err = fn(pte++, addr, data);
2232 if (err)
2233 break;
2234 }
c36987e2 2235 } while (addr += PAGE_SIZE, addr != end);
aee16b3c 2236
38e0edb1
JF
2237 arch_leave_lazy_mmu_mode();
2238
aee16b3c
JF
2239 if (mm != &init_mm)
2240 pte_unmap_unlock(pte-1, ptl);
2241 return err;
2242}
2243
2244static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
2245 unsigned long addr, unsigned long end,
be1db475 2246 pte_fn_t fn, void *data, bool create)
aee16b3c
JF
2247{
2248 pmd_t *pmd;
2249 unsigned long next;
be1db475 2250 int err = 0;
aee16b3c 2251
ceb86879
AK
2252 BUG_ON(pud_huge(*pud));
2253
be1db475
DA
2254 if (create) {
2255 pmd = pmd_alloc(mm, pud, addr);
2256 if (!pmd)
2257 return -ENOMEM;
2258 } else {
2259 pmd = pmd_offset(pud, addr);
2260 }
aee16b3c
JF
2261 do {
2262 next = pmd_addr_end(addr, end);
be1db475
DA
2263 if (create || !pmd_none_or_clear_bad(pmd)) {
2264 err = apply_to_pte_range(mm, pmd, addr, next, fn, data,
2265 create);
2266 if (err)
2267 break;
2268 }
aee16b3c
JF
2269 } while (pmd++, addr = next, addr != end);
2270 return err;
2271}
2272
c2febafc 2273static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
aee16b3c 2274 unsigned long addr, unsigned long end,
be1db475 2275 pte_fn_t fn, void *data, bool create)
aee16b3c
JF
2276{
2277 pud_t *pud;
2278 unsigned long next;
be1db475 2279 int err = 0;
aee16b3c 2280
be1db475
DA
2281 if (create) {
2282 pud = pud_alloc(mm, p4d, addr);
2283 if (!pud)
2284 return -ENOMEM;
2285 } else {
2286 pud = pud_offset(p4d, addr);
2287 }
aee16b3c
JF
2288 do {
2289 next = pud_addr_end(addr, end);
be1db475
DA
2290 if (create || !pud_none_or_clear_bad(pud)) {
2291 err = apply_to_pmd_range(mm, pud, addr, next, fn, data,
2292 create);
2293 if (err)
2294 break;
2295 }
aee16b3c
JF
2296 } while (pud++, addr = next, addr != end);
2297 return err;
2298}
2299
c2febafc
KS
2300static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2301 unsigned long addr, unsigned long end,
be1db475 2302 pte_fn_t fn, void *data, bool create)
c2febafc
KS
2303{
2304 p4d_t *p4d;
2305 unsigned long next;
be1db475 2306 int err = 0;
c2febafc 2307
be1db475
DA
2308 if (create) {
2309 p4d = p4d_alloc(mm, pgd, addr);
2310 if (!p4d)
2311 return -ENOMEM;
2312 } else {
2313 p4d = p4d_offset(pgd, addr);
2314 }
c2febafc
KS
2315 do {
2316 next = p4d_addr_end(addr, end);
be1db475
DA
2317 if (create || !p4d_none_or_clear_bad(p4d)) {
2318 err = apply_to_pud_range(mm, p4d, addr, next, fn, data,
2319 create);
2320 if (err)
2321 break;
2322 }
c2febafc
KS
2323 } while (p4d++, addr = next, addr != end);
2324 return err;
2325}
2326
be1db475
DA
2327static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2328 unsigned long size, pte_fn_t fn,
2329 void *data, bool create)
aee16b3c
JF
2330{
2331 pgd_t *pgd;
2332 unsigned long next;
57250a5b 2333 unsigned long end = addr + size;
be1db475 2334 int err = 0;
aee16b3c 2335
9cb65bc3
MP
2336 if (WARN_ON(addr >= end))
2337 return -EINVAL;
2338
aee16b3c
JF
2339 pgd = pgd_offset(mm, addr);
2340 do {
2341 next = pgd_addr_end(addr, end);
be1db475
DA
2342 if (!create && pgd_none_or_clear_bad(pgd))
2343 continue;
2344 err = apply_to_p4d_range(mm, pgd, addr, next, fn, data, create);
aee16b3c
JF
2345 if (err)
2346 break;
2347 } while (pgd++, addr = next, addr != end);
57250a5b 2348
aee16b3c
JF
2349 return err;
2350}
be1db475
DA
2351
2352/*
2353 * Scan a region of virtual memory, filling in page tables as necessary
2354 * and calling a provided function on each leaf page table.
2355 */
2356int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2357 unsigned long size, pte_fn_t fn, void *data)
2358{
2359 return __apply_to_page_range(mm, addr, size, fn, data, true);
2360}
aee16b3c
JF
2361EXPORT_SYMBOL_GPL(apply_to_page_range);
2362
be1db475
DA
2363/*
2364 * Scan a region of virtual memory, calling a provided function on
2365 * each leaf page table where it exists.
2366 *
2367 * Unlike apply_to_page_range, this does _not_ fill in page tables
2368 * where they are absent.
2369 */
2370int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
2371 unsigned long size, pte_fn_t fn, void *data)
2372{
2373 return __apply_to_page_range(mm, addr, size, fn, data, false);
2374}
2375EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
2376
8f4e2101 2377/*
9b4bdd2f
KS
2378 * handle_pte_fault chooses page fault handler according to an entry which was
2379 * read non-atomically. Before making any commitment, on those architectures
2380 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
2381 * parts, do_swap_page must check under lock before unmapping the pte and
2382 * proceeding (but do_wp_page is only called after already making such a check;
a335b2e1 2383 * and do_anonymous_page can safely check later on).
8f4e2101 2384 */
4c21e2f2 2385static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
8f4e2101
HD
2386 pte_t *page_table, pte_t orig_pte)
2387{
2388 int same = 1;
923717cb 2389#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
8f4e2101 2390 if (sizeof(pte_t) > sizeof(unsigned long)) {
4c21e2f2
HD
2391 spinlock_t *ptl = pte_lockptr(mm, pmd);
2392 spin_lock(ptl);
8f4e2101 2393 same = pte_same(*page_table, orig_pte);
4c21e2f2 2394 spin_unlock(ptl);
8f4e2101
HD
2395 }
2396#endif
2397 pte_unmap(page_table);
2398 return same;
2399}
2400
83d116c5
JH
2401static inline bool cow_user_page(struct page *dst, struct page *src,
2402 struct vm_fault *vmf)
6aab341e 2403{
83d116c5
JH
2404 bool ret;
2405 void *kaddr;
2406 void __user *uaddr;
c3e5ea6e 2407 bool locked = false;
83d116c5
JH
2408 struct vm_area_struct *vma = vmf->vma;
2409 struct mm_struct *mm = vma->vm_mm;
2410 unsigned long addr = vmf->address;
2411
0abdd7a8
DW
2412 debug_dma_assert_idle(src);
2413
83d116c5
JH
2414 if (likely(src)) {
2415 copy_user_highpage(dst, src, addr, vma);
2416 return true;
2417 }
2418
6aab341e
LT
2419 /*
2420 * If the source page was a PFN mapping, we don't have
2421 * a "struct page" for it. We do a best-effort copy by
2422 * just copying from the original user address. If that
2423 * fails, we just zero-fill it. Live with it.
2424 */
83d116c5
JH
2425 kaddr = kmap_atomic(dst);
2426 uaddr = (void __user *)(addr & PAGE_MASK);
2427
2428 /*
2429 * On architectures with software "accessed" bits, we would
2430 * take a double page fault, so mark it accessed here.
2431 */
c3e5ea6e 2432 if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) {
83d116c5 2433 pte_t entry;
5d2a2dbb 2434
83d116c5 2435 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
c3e5ea6e 2436 locked = true;
83d116c5
JH
2437 if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
2438 /*
2439 * Other thread has already handled the fault
7df67697 2440 * and update local tlb only
83d116c5 2441 */
7df67697 2442 update_mmu_tlb(vma, addr, vmf->pte);
83d116c5
JH
2443 ret = false;
2444 goto pte_unlock;
2445 }
2446
2447 entry = pte_mkyoung(vmf->orig_pte);
2448 if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
2449 update_mmu_cache(vma, addr, vmf->pte);
2450 }
2451
2452 /*
2453 * This really shouldn't fail, because the page is there
2454 * in the page tables. But it might just be unreadable,
2455 * in which case we just give up and fill the result with
2456 * zeroes.
2457 */
2458 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
c3e5ea6e
KS
2459 if (locked)
2460 goto warn;
2461
2462 /* Re-validate under PTL if the page is still mapped */
2463 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
2464 locked = true;
2465 if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) {
7df67697
BM
2466 /* The PTE changed under us, update local tlb */
2467 update_mmu_tlb(vma, addr, vmf->pte);
c3e5ea6e
KS
2468 ret = false;
2469 goto pte_unlock;
2470 }
2471
5d2a2dbb 2472 /*
985ba004 2473 * The same page can be mapped back since last copy attempt.
c3e5ea6e 2474 * Try to copy again under PTL.
5d2a2dbb 2475 */
c3e5ea6e
KS
2476 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
2477 /*
2478 * Give a warn in case there can be some obscure
2479 * use-case
2480 */
2481warn:
2482 WARN_ON_ONCE(1);
2483 clear_page(kaddr);
2484 }
83d116c5
JH
2485 }
2486
2487 ret = true;
2488
2489pte_unlock:
c3e5ea6e 2490 if (locked)
83d116c5
JH
2491 pte_unmap_unlock(vmf->pte, vmf->ptl);
2492 kunmap_atomic(kaddr);
2493 flush_dcache_page(dst);
2494
2495 return ret;
6aab341e
LT
2496}
2497
c20cd45e
MH
2498static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
2499{
2500 struct file *vm_file = vma->vm_file;
2501
2502 if (vm_file)
2503 return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
2504
2505 /*
2506 * Special mappings (e.g. VDSO) do not have any file so fake
2507 * a default GFP_KERNEL for them.
2508 */
2509 return GFP_KERNEL;
2510}
2511
fb09a464
KS
2512/*
2513 * Notify the address space that the page is about to become writable so that
2514 * it can prohibit this or wait for the page to get into an appropriate state.
2515 *
2516 * We do this without the lock held, so that it can sleep if it needs to.
2517 */
2b740303 2518static vm_fault_t do_page_mkwrite(struct vm_fault *vmf)
fb09a464 2519{
2b740303 2520 vm_fault_t ret;
38b8cb7f
JK
2521 struct page *page = vmf->page;
2522 unsigned int old_flags = vmf->flags;
fb09a464 2523
38b8cb7f 2524 vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
fb09a464 2525
dc617f29
DW
2526 if (vmf->vma->vm_file &&
2527 IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
2528 return VM_FAULT_SIGBUS;
2529
11bac800 2530 ret = vmf->vma->vm_ops->page_mkwrite(vmf);
38b8cb7f
JK
2531 /* Restore original flags so that caller is not surprised */
2532 vmf->flags = old_flags;
fb09a464
KS
2533 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
2534 return ret;
2535 if (unlikely(!(ret & VM_FAULT_LOCKED))) {
2536 lock_page(page);
2537 if (!page->mapping) {
2538 unlock_page(page);
2539 return 0; /* retry */
2540 }
2541 ret |= VM_FAULT_LOCKED;
2542 } else
2543 VM_BUG_ON_PAGE(!PageLocked(page), page);
2544 return ret;
2545}
2546
97ba0c2b
JK
2547/*
2548 * Handle dirtying of a page in shared file mapping on a write fault.
2549 *
2550 * The function expects the page to be locked and unlocks it.
2551 */
89b15332 2552static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
97ba0c2b 2553{
89b15332 2554 struct vm_area_struct *vma = vmf->vma;
97ba0c2b 2555 struct address_space *mapping;
89b15332 2556 struct page *page = vmf->page;
97ba0c2b
JK
2557 bool dirtied;
2558 bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
2559
2560 dirtied = set_page_dirty(page);
2561 VM_BUG_ON_PAGE(PageAnon(page), page);
2562 /*
2563 * Take a local copy of the address_space - page.mapping may be zeroed
2564 * by truncate after unlock_page(). The address_space itself remains
2565 * pinned by vma->vm_file's reference. We rely on unlock_page()'s
2566 * release semantics to prevent the compiler from undoing this copying.
2567 */
2568 mapping = page_rmapping(page);
2569 unlock_page(page);
2570
89b15332
JW
2571 if (!page_mkwrite)
2572 file_update_time(vma->vm_file);
2573
2574 /*
2575 * Throttle page dirtying rate down to writeback speed.
2576 *
2577 * mapping may be NULL here because some device drivers do not
2578 * set page.mapping but still dirty their pages
2579 *
c1e8d7c6 2580 * Drop the mmap_lock before waiting on IO, if we can. The file
89b15332
JW
2581 * is pinning the mapping, as per above.
2582 */
97ba0c2b 2583 if ((dirtied || page_mkwrite) && mapping) {
89b15332
JW
2584 struct file *fpin;
2585
2586 fpin = maybe_unlock_mmap_for_io(vmf, NULL);
97ba0c2b 2587 balance_dirty_pages_ratelimited(mapping);
89b15332
JW
2588 if (fpin) {
2589 fput(fpin);
2590 return VM_FAULT_RETRY;
2591 }
97ba0c2b
JK
2592 }
2593
89b15332 2594 return 0;
97ba0c2b
JK
2595}
2596
4e047f89
SR
2597/*
2598 * Handle write page faults for pages that can be reused in the current vma
2599 *
2600 * This can happen either due to the mapping being with the VM_SHARED flag,
2601 * or due to us being the last reference standing to the page. In either
2602 * case, all we need to do here is to mark the page as writable and update
2603 * any related book-keeping.
2604 */
997dd98d 2605static inline void wp_page_reuse(struct vm_fault *vmf)
82b0f8c3 2606 __releases(vmf->ptl)
4e047f89 2607{
82b0f8c3 2608 struct vm_area_struct *vma = vmf->vma;
a41b70d6 2609 struct page *page = vmf->page;
4e047f89
SR
2610 pte_t entry;
2611 /*
2612 * Clear the pages cpupid information as the existing
2613 * information potentially belongs to a now completely
2614 * unrelated process.
2615 */
2616 if (page)
2617 page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
2618
2994302b
JK
2619 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
2620 entry = pte_mkyoung(vmf->orig_pte);
4e047f89 2621 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
82b0f8c3
JK
2622 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
2623 update_mmu_cache(vma, vmf->address, vmf->pte);
2624 pte_unmap_unlock(vmf->pte, vmf->ptl);
4e047f89
SR
2625}
2626
2f38ab2c
SR
2627/*
2628 * Handle the case of a page which we actually need to copy to a new page.
2629 *
c1e8d7c6 2630 * Called with mmap_lock locked and the old page referenced, but
2f38ab2c
SR
2631 * without the ptl held.
2632 *
2633 * High level logic flow:
2634 *
2635 * - Allocate a page, copy the content of the old page to the new one.
2636 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
2637 * - Take the PTL. If the pte changed, bail out and release the allocated page
2638 * - If the pte is still the way we remember it, update the page table and all
2639 * relevant references. This includes dropping the reference the page-table
2640 * held to the old page, as well as updating the rmap.
2641 * - In any case, unlock the PTL and drop the reference we took to the old page.
2642 */
2b740303 2643static vm_fault_t wp_page_copy(struct vm_fault *vmf)
2f38ab2c 2644{
82b0f8c3 2645 struct vm_area_struct *vma = vmf->vma;
bae473a4 2646 struct mm_struct *mm = vma->vm_mm;
a41b70d6 2647 struct page *old_page = vmf->page;
2f38ab2c 2648 struct page *new_page = NULL;
2f38ab2c
SR
2649 pte_t entry;
2650 int page_copied = 0;
ac46d4f3 2651 struct mmu_notifier_range range;
2f38ab2c
SR
2652
2653 if (unlikely(anon_vma_prepare(vma)))
2654 goto oom;
2655
2994302b 2656 if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
82b0f8c3
JK
2657 new_page = alloc_zeroed_user_highpage_movable(vma,
2658 vmf->address);
2f38ab2c
SR
2659 if (!new_page)
2660 goto oom;
2661 } else {
bae473a4 2662 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
82b0f8c3 2663 vmf->address);
2f38ab2c
SR
2664 if (!new_page)
2665 goto oom;
83d116c5
JH
2666
2667 if (!cow_user_page(new_page, old_page, vmf)) {
2668 /*
2669 * COW failed, if the fault was solved by other,
2670 * it's fine. If not, userspace would re-fault on
2671 * the same address and we will handle the fault
2672 * from the second attempt.
2673 */
2674 put_page(new_page);
2675 if (old_page)
2676 put_page(old_page);
2677 return 0;
2678 }
2f38ab2c 2679 }
2f38ab2c 2680
d9eb1ea2 2681 if (mem_cgroup_charge(new_page, mm, GFP_KERNEL))
2f38ab2c 2682 goto oom_free_new;
9d82c694 2683 cgroup_throttle_swaprate(new_page, GFP_KERNEL);
2f38ab2c 2684
eb3c24f3
MG
2685 __SetPageUptodate(new_page);
2686
7269f999 2687 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
6f4f13e8 2688 vmf->address & PAGE_MASK,
ac46d4f3
JG
2689 (vmf->address & PAGE_MASK) + PAGE_SIZE);
2690 mmu_notifier_invalidate_range_start(&range);
2f38ab2c
SR
2691
2692 /*
2693 * Re-check the pte - we dropped the lock
2694 */
82b0f8c3 2695 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
2994302b 2696 if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
2f38ab2c
SR
2697 if (old_page) {
2698 if (!PageAnon(old_page)) {
eca56ff9
JM
2699 dec_mm_counter_fast(mm,
2700 mm_counter_file(old_page));
2f38ab2c
SR
2701 inc_mm_counter_fast(mm, MM_ANONPAGES);
2702 }
2703 } else {
2704 inc_mm_counter_fast(mm, MM_ANONPAGES);
2705 }
2994302b 2706 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
2f38ab2c 2707 entry = mk_pte(new_page, vma->vm_page_prot);
44bf431b 2708 entry = pte_sw_mkyoung(entry);
2f38ab2c
SR
2709 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2710 /*
2711 * Clear the pte entry and flush it first, before updating the
2712 * pte with the new entry. This will avoid a race condition
2713 * seen in the presence of one thread doing SMC and another
2714 * thread doing COW.
2715 */
82b0f8c3
JK
2716 ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
2717 page_add_new_anon_rmap(new_page, vma, vmf->address, false);
2f38ab2c
SR
2718 lru_cache_add_active_or_unevictable(new_page, vma);
2719 /*
2720 * We call the notify macro here because, when using secondary
2721 * mmu page tables (such as kvm shadow page tables), we want the
2722 * new page to be mapped directly into the secondary page table.
2723 */
82b0f8c3
JK
2724 set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
2725 update_mmu_cache(vma, vmf->address, vmf->pte);
2f38ab2c
SR
2726 if (old_page) {
2727 /*
2728 * Only after switching the pte to the new page may
2729 * we remove the mapcount here. Otherwise another
2730 * process may come and find the rmap count decremented
2731 * before the pte is switched to the new page, and
2732 * "reuse" the old page writing into it while our pte
2733 * here still points into it and can be read by other
2734 * threads.
2735 *
2736 * The critical issue is to order this
2737 * page_remove_rmap with the ptp_clear_flush above.
2738 * Those stores are ordered by (if nothing else,)
2739 * the barrier present in the atomic_add_negative
2740 * in page_remove_rmap.
2741 *
2742 * Then the TLB flush in ptep_clear_flush ensures that
2743 * no process can access the old page before the
2744 * decremented mapcount is visible. And the old page
2745 * cannot be reused until after the decremented
2746 * mapcount is visible. So transitively, TLBs to
2747 * old page will be flushed before it can be reused.
2748 */
d281ee61 2749 page_remove_rmap(old_page, false);
2f38ab2c
SR
2750 }
2751
2752 /* Free the old page.. */
2753 new_page = old_page;
2754 page_copied = 1;
2755 } else {
7df67697 2756 update_mmu_tlb(vma, vmf->address, vmf->pte);
2f38ab2c
SR
2757 }
2758
2759 if (new_page)
09cbfeaf 2760 put_page(new_page);
2f38ab2c 2761
82b0f8c3 2762 pte_unmap_unlock(vmf->pte, vmf->ptl);
4645b9fe
JG
2763 /*
2764 * No need to double call mmu_notifier->invalidate_range() callback as
2765 * the above ptep_clear_flush_notify() did already call it.
2766 */
ac46d4f3 2767 mmu_notifier_invalidate_range_only_end(&range);
2f38ab2c
SR
2768 if (old_page) {
2769 /*
2770 * Don't let another task, with possibly unlocked vma,
2771 * keep the mlocked page.
2772 */
2773 if (page_copied && (vma->vm_flags & VM_LOCKED)) {
2774 lock_page(old_page); /* LRU manipulation */
e90309c9
KS
2775 if (PageMlocked(old_page))
2776 munlock_vma_page(old_page);
2f38ab2c
SR
2777 unlock_page(old_page);
2778 }
09cbfeaf 2779 put_page(old_page);
2f38ab2c
SR
2780 }
2781 return page_copied ? VM_FAULT_WRITE : 0;
2782oom_free_new:
09cbfeaf 2783 put_page(new_page);
2f38ab2c
SR
2784oom:
2785 if (old_page)
09cbfeaf 2786 put_page(old_page);
2f38ab2c
SR
2787 return VM_FAULT_OOM;
2788}
2789
66a6197c
JK
2790/**
2791 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
2792 * writeable once the page is prepared
2793 *
2794 * @vmf: structure describing the fault
2795 *
2796 * This function handles all that is needed to finish a write page fault in a
2797 * shared mapping due to PTE being read-only once the mapped page is prepared.
a862f68a 2798 * It handles locking of PTE and modifying it.
66a6197c
JK
2799 *
2800 * The function expects the page to be locked or other protection against
2801 * concurrent faults / writeback (such as DAX radix tree locks).
a862f68a
MR
2802 *
2803 * Return: %VM_FAULT_WRITE on success, %0 when PTE got changed before
2804 * we acquired PTE lock.
66a6197c 2805 */
2b740303 2806vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf)
66a6197c
JK
2807{
2808 WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
2809 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
2810 &vmf->ptl);
2811 /*
2812 * We might have raced with another page fault while we released the
2813 * pte_offset_map_lock.
2814 */
2815 if (!pte_same(*vmf->pte, vmf->orig_pte)) {
7df67697 2816 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
66a6197c 2817 pte_unmap_unlock(vmf->pte, vmf->ptl);
a19e2553 2818 return VM_FAULT_NOPAGE;
66a6197c
JK
2819 }
2820 wp_page_reuse(vmf);
a19e2553 2821 return 0;
66a6197c
JK
2822}
2823
dd906184
BH
2824/*
2825 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
2826 * mapping
2827 */
2b740303 2828static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
dd906184 2829{
82b0f8c3 2830 struct vm_area_struct *vma = vmf->vma;
bae473a4 2831
dd906184 2832 if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
2b740303 2833 vm_fault_t ret;
dd906184 2834
82b0f8c3 2835 pte_unmap_unlock(vmf->pte, vmf->ptl);
fe82221f 2836 vmf->flags |= FAULT_FLAG_MKWRITE;
11bac800 2837 ret = vma->vm_ops->pfn_mkwrite(vmf);
2f89dc12 2838 if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
dd906184 2839 return ret;
66a6197c 2840 return finish_mkwrite_fault(vmf);
dd906184 2841 }
997dd98d
JK
2842 wp_page_reuse(vmf);
2843 return VM_FAULT_WRITE;
dd906184
BH
2844}
2845
2b740303 2846static vm_fault_t wp_page_shared(struct vm_fault *vmf)
82b0f8c3 2847 __releases(vmf->ptl)
93e478d4 2848{
82b0f8c3 2849 struct vm_area_struct *vma = vmf->vma;
89b15332 2850 vm_fault_t ret = VM_FAULT_WRITE;
93e478d4 2851
a41b70d6 2852 get_page(vmf->page);
93e478d4 2853
93e478d4 2854 if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
2b740303 2855 vm_fault_t tmp;
93e478d4 2856
82b0f8c3 2857 pte_unmap_unlock(vmf->pte, vmf->ptl);
38b8cb7f 2858 tmp = do_page_mkwrite(vmf);
93e478d4
SR
2859 if (unlikely(!tmp || (tmp &
2860 (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
a41b70d6 2861 put_page(vmf->page);
93e478d4
SR
2862 return tmp;
2863 }
66a6197c 2864 tmp = finish_mkwrite_fault(vmf);
a19e2553 2865 if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
a41b70d6 2866 unlock_page(vmf->page);
a41b70d6 2867 put_page(vmf->page);
66a6197c 2868 return tmp;
93e478d4 2869 }
66a6197c
JK
2870 } else {
2871 wp_page_reuse(vmf);
997dd98d 2872 lock_page(vmf->page);
93e478d4 2873 }
89b15332 2874 ret |= fault_dirty_shared_page(vmf);
997dd98d 2875 put_page(vmf->page);
93e478d4 2876
89b15332 2877 return ret;
93e478d4
SR
2878}
2879
1da177e4
LT
2880/*
2881 * This routine handles present pages, when users try to write
2882 * to a shared page. It is done by copying the page to a new address
2883 * and decrementing the shared-page counter for the old page.
2884 *
1da177e4
LT
2885 * Note that this routine assumes that the protection checks have been
2886 * done by the caller (the low-level page fault routine in most cases).
2887 * Thus we can safely just mark it writable once we've done any necessary
2888 * COW.
2889 *
2890 * We also mark the page dirty at this point even though the page will
2891 * change only once the write actually happens. This avoids a few races,
2892 * and potentially makes it more efficient.
2893 *
c1e8d7c6 2894 * We enter with non-exclusive mmap_lock (to exclude vma changes,
8f4e2101 2895 * but allow concurrent faults), with pte both mapped and locked.
c1e8d7c6 2896 * We return with mmap_lock still held, but pte unmapped and unlocked.
1da177e4 2897 */
2b740303 2898static vm_fault_t do_wp_page(struct vm_fault *vmf)
82b0f8c3 2899 __releases(vmf->ptl)
1da177e4 2900{
82b0f8c3 2901 struct vm_area_struct *vma = vmf->vma;
1da177e4 2902
292924b2 2903 if (userfaultfd_pte_wp(vma, *vmf->pte)) {
529b930b
AA
2904 pte_unmap_unlock(vmf->pte, vmf->ptl);
2905 return handle_userfault(vmf, VM_UFFD_WP);
2906 }
2907
a41b70d6
JK
2908 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
2909 if (!vmf->page) {
251b97f5 2910 /*
64e45507
PF
2911 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
2912 * VM_PFNMAP VMA.
251b97f5
PZ
2913 *
2914 * We should not cow pages in a shared writeable mapping.
dd906184 2915 * Just mark the pages writable and/or call ops->pfn_mkwrite.
251b97f5
PZ
2916 */
2917 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
2918 (VM_WRITE|VM_SHARED))
2994302b 2919 return wp_pfn_shared(vmf);
2f38ab2c 2920
82b0f8c3 2921 pte_unmap_unlock(vmf->pte, vmf->ptl);
a41b70d6 2922 return wp_page_copy(vmf);
251b97f5 2923 }
1da177e4 2924
d08b3851 2925 /*
ee6a6457
PZ
2926 * Take out anonymous pages first, anonymous shared vmas are
2927 * not dirty accountable.
d08b3851 2928 */
52d1e606 2929 if (PageAnon(vmf->page)) {
ba3c4ce6 2930 int total_map_swapcount;
52d1e606
KT
2931 if (PageKsm(vmf->page) && (PageSwapCache(vmf->page) ||
2932 page_count(vmf->page) != 1))
2933 goto copy;
a41b70d6
JK
2934 if (!trylock_page(vmf->page)) {
2935 get_page(vmf->page);
82b0f8c3 2936 pte_unmap_unlock(vmf->pte, vmf->ptl);
a41b70d6 2937 lock_page(vmf->page);
82b0f8c3
JK
2938 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
2939 vmf->address, &vmf->ptl);
2994302b 2940 if (!pte_same(*vmf->pte, vmf->orig_pte)) {
7df67697 2941 update_mmu_tlb(vma, vmf->address, vmf->pte);
a41b70d6 2942 unlock_page(vmf->page);
82b0f8c3 2943 pte_unmap_unlock(vmf->pte, vmf->ptl);
a41b70d6 2944 put_page(vmf->page);
28766805 2945 return 0;
ab967d86 2946 }
a41b70d6 2947 put_page(vmf->page);
ee6a6457 2948 }
52d1e606
KT
2949 if (PageKsm(vmf->page)) {
2950 bool reused = reuse_ksm_page(vmf->page, vmf->vma,
2951 vmf->address);
2952 unlock_page(vmf->page);
2953 if (!reused)
2954 goto copy;
2955 wp_page_reuse(vmf);
2956 return VM_FAULT_WRITE;
2957 }
ba3c4ce6
HY
2958 if (reuse_swap_page(vmf->page, &total_map_swapcount)) {
2959 if (total_map_swapcount == 1) {
6d0a07ed
AA
2960 /*
2961 * The page is all ours. Move it to
2962 * our anon_vma so the rmap code will
2963 * not search our parent or siblings.
2964 * Protected against the rmap code by
2965 * the page lock.
2966 */
a41b70d6 2967 page_move_anon_rmap(vmf->page, vma);
6d0a07ed 2968 }
a41b70d6 2969 unlock_page(vmf->page);
997dd98d
JK
2970 wp_page_reuse(vmf);
2971 return VM_FAULT_WRITE;
b009c024 2972 }
a41b70d6 2973 unlock_page(vmf->page);
ee6a6457 2974 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
d08b3851 2975 (VM_WRITE|VM_SHARED))) {
a41b70d6 2976 return wp_page_shared(vmf);
1da177e4 2977 }
52d1e606 2978copy:
1da177e4
LT
2979 /*
2980 * Ok, we need to copy. Oh, well..
2981 */
a41b70d6 2982 get_page(vmf->page);
28766805 2983
82b0f8c3 2984 pte_unmap_unlock(vmf->pte, vmf->ptl);
a41b70d6 2985 return wp_page_copy(vmf);
1da177e4
LT
2986}
2987
97a89413 2988static void unmap_mapping_range_vma(struct vm_area_struct *vma,
1da177e4
LT
2989 unsigned long start_addr, unsigned long end_addr,
2990 struct zap_details *details)
2991{
f5cc4eef 2992 zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
1da177e4
LT
2993}
2994
f808c13f 2995static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
1da177e4
LT
2996 struct zap_details *details)
2997{
2998 struct vm_area_struct *vma;
1da177e4
LT
2999 pgoff_t vba, vea, zba, zea;
3000
6b2dbba8 3001 vma_interval_tree_foreach(vma, root,
1da177e4 3002 details->first_index, details->last_index) {
1da177e4
LT
3003
3004 vba = vma->vm_pgoff;
d6e93217 3005 vea = vba + vma_pages(vma) - 1;
1da177e4
LT
3006 zba = details->first_index;
3007 if (zba < vba)
3008 zba = vba;
3009 zea = details->last_index;
3010 if (zea > vea)
3011 zea = vea;
3012
97a89413 3013 unmap_mapping_range_vma(vma,
1da177e4
LT
3014 ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
3015 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
97a89413 3016 details);
1da177e4
LT
3017 }
3018}
3019
977fbdcd
MW
3020/**
3021 * unmap_mapping_pages() - Unmap pages from processes.
3022 * @mapping: The address space containing pages to be unmapped.
3023 * @start: Index of first page to be unmapped.
3024 * @nr: Number of pages to be unmapped. 0 to unmap to end of file.
3025 * @even_cows: Whether to unmap even private COWed pages.
3026 *
3027 * Unmap the pages in this address space from any userspace process which
3028 * has them mmaped. Generally, you want to remove COWed pages as well when
3029 * a file is being truncated, but not when invalidating pages from the page
3030 * cache.
3031 */
3032void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
3033 pgoff_t nr, bool even_cows)
3034{
3035 struct zap_details details = { };
3036
3037 details.check_mapping = even_cows ? NULL : mapping;
3038 details.first_index = start;
3039 details.last_index = start + nr - 1;
3040 if (details.last_index < details.first_index)
3041 details.last_index = ULONG_MAX;
3042
3043 i_mmap_lock_write(mapping);
3044 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
3045 unmap_mapping_range_tree(&mapping->i_mmap, &details);
3046 i_mmap_unlock_write(mapping);
3047}
3048
1da177e4 3049/**
8a5f14a2 3050 * unmap_mapping_range - unmap the portion of all mmaps in the specified
977fbdcd 3051 * address_space corresponding to the specified byte range in the underlying
8a5f14a2
KS
3052 * file.
3053 *
3d41088f 3054 * @mapping: the address space containing mmaps to be unmapped.
1da177e4
LT
3055 * @holebegin: byte in first page to unmap, relative to the start of
3056 * the underlying file. This will be rounded down to a PAGE_SIZE
25d9e2d1 3057 * boundary. Note that this is different from truncate_pagecache(), which
1da177e4
LT
3058 * must keep the partial page. In contrast, we must get rid of
3059 * partial pages.
3060 * @holelen: size of prospective hole in bytes. This will be rounded
3061 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
3062 * end of the file.
3063 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
3064 * but 0 when invalidating pagecache, don't throw away private data.
3065 */
3066void unmap_mapping_range(struct address_space *mapping,
3067 loff_t const holebegin, loff_t const holelen, int even_cows)
3068{
1da177e4
LT
3069 pgoff_t hba = holebegin >> PAGE_SHIFT;
3070 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
3071
3072 /* Check for overflow. */
3073 if (sizeof(holelen) > sizeof(hlen)) {
3074 long long holeend =
3075 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
3076 if (holeend & ~(long long)ULONG_MAX)
3077 hlen = ULONG_MAX - hba + 1;
3078 }
3079
977fbdcd 3080 unmap_mapping_pages(mapping, hba, hlen, even_cows);
1da177e4
LT
3081}
3082EXPORT_SYMBOL(unmap_mapping_range);
3083
1da177e4 3084/*
c1e8d7c6 3085 * We enter with non-exclusive mmap_lock (to exclude vma changes,
8f4e2101 3086 * but allow concurrent faults), and pte mapped but not yet locked.
9a95f3cf
PC
3087 * We return with pte unmapped and unlocked.
3088 *
c1e8d7c6 3089 * We return with the mmap_lock locked or unlocked in the same cases
9a95f3cf 3090 * as does filemap_fault().
1da177e4 3091 */
2b740303 3092vm_fault_t do_swap_page(struct vm_fault *vmf)
1da177e4 3093{
82b0f8c3 3094 struct vm_area_struct *vma = vmf->vma;
eaf649eb 3095 struct page *page = NULL, *swapcache;
65500d23 3096 swp_entry_t entry;
1da177e4 3097 pte_t pte;
d065bd81 3098 int locked;
ad8c2ee8 3099 int exclusive = 0;
2b740303 3100 vm_fault_t ret = 0;
1da177e4 3101
eaf649eb 3102 if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte))
8f4e2101 3103 goto out;
65500d23 3104
2994302b 3105 entry = pte_to_swp_entry(vmf->orig_pte);
d1737fdb
AK
3106 if (unlikely(non_swap_entry(entry))) {
3107 if (is_migration_entry(entry)) {
82b0f8c3
JK
3108 migration_entry_wait(vma->vm_mm, vmf->pmd,
3109 vmf->address);
5042db43 3110 } else if (is_device_private_entry(entry)) {
897e6365
CH
3111 vmf->page = device_private_entry_to_page(entry);
3112 ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
d1737fdb
AK
3113 } else if (is_hwpoison_entry(entry)) {
3114 ret = VM_FAULT_HWPOISON;
3115 } else {
2994302b 3116 print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
d99be1a8 3117 ret = VM_FAULT_SIGBUS;
d1737fdb 3118 }
0697212a
CL
3119 goto out;
3120 }
0bcac06f
MK
3121
3122
0ff92245 3123 delayacct_set_flag(DELAYACCT_PF_SWAPIN);
eaf649eb
MK
3124 page = lookup_swap_cache(entry, vma, vmf->address);
3125 swapcache = page;
f8020772 3126
1da177e4 3127 if (!page) {
0bcac06f
MK
3128 struct swap_info_struct *si = swp_swap_info(entry);
3129
aa8d22a1 3130 if (si->flags & SWP_SYNCHRONOUS_IO &&
eb085574 3131 __swap_count(entry) == 1) {
0bcac06f 3132 /* skip swapcache */
e9e9b7ec
MK
3133 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
3134 vmf->address);
0bcac06f 3135 if (page) {
4c6355b2
JW
3136 int err;
3137
0bcac06f
MK
3138 __SetPageLocked(page);
3139 __SetPageSwapBacked(page);
3140 set_page_private(page, entry.val);
4c6355b2
JW
3141
3142 /* Tell memcg to use swap ownership records */
3143 SetPageSwapCache(page);
3144 err = mem_cgroup_charge(page, vma->vm_mm,
d9eb1ea2 3145 GFP_KERNEL);
4c6355b2 3146 ClearPageSwapCache(page);
545b1b07
MH
3147 if (err) {
3148 ret = VM_FAULT_OOM;
4c6355b2 3149 goto out_page;
545b1b07 3150 }
4c6355b2 3151
0076f029
JK
3152 /*
3153 * XXX: Move to lru_cache_add() when it
3154 * supports new vs putback
3155 */
3156 spin_lock_irq(&page_pgdat(page)->lru_lock);
3157 lru_note_cost_page(page);
3158 spin_unlock_irq(&page_pgdat(page)->lru_lock);
3159
6058eaec 3160 lru_cache_add(page);
0bcac06f
MK
3161 swap_readpage(page, true);
3162 }
aa8d22a1 3163 } else {
e9e9b7ec
MK
3164 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
3165 vmf);
aa8d22a1 3166 swapcache = page;
0bcac06f
MK
3167 }
3168
1da177e4
LT
3169 if (!page) {
3170 /*
8f4e2101
HD
3171 * Back out if somebody else faulted in this pte
3172 * while we released the pte lock.
1da177e4 3173 */
82b0f8c3
JK
3174 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
3175 vmf->address, &vmf->ptl);
2994302b 3176 if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
1da177e4 3177 ret = VM_FAULT_OOM;
0ff92245 3178 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
65500d23 3179 goto unlock;
1da177e4
LT
3180 }
3181
3182 /* Had to read the page from swap area: Major fault */
3183 ret = VM_FAULT_MAJOR;
f8891e5e 3184 count_vm_event(PGMAJFAULT);
2262185c 3185 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
d1737fdb 3186 } else if (PageHWPoison(page)) {
71f72525
WF
3187 /*
3188 * hwpoisoned dirty swapcache pages are kept for killing
3189 * owner processes (which may be unknown at hwpoison time)
3190 */
d1737fdb
AK
3191 ret = VM_FAULT_HWPOISON;
3192 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
4779cb31 3193 goto out_release;
1da177e4
LT
3194 }
3195
82b0f8c3 3196 locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
e709ffd6 3197
073e587e 3198 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
d065bd81
ML
3199 if (!locked) {
3200 ret |= VM_FAULT_RETRY;
3201 goto out_release;
3202 }
073e587e 3203
4969c119 3204 /*
31c4a3d3
HD
3205 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not
3206 * release the swapcache from under us. The page pin, and pte_same
3207 * test below, are not enough to exclude that. Even if it is still
3208 * swapcache, we need to check that the page's swap has not changed.
4969c119 3209 */
0bcac06f
MK
3210 if (unlikely((!PageSwapCache(page) ||
3211 page_private(page) != entry.val)) && swapcache)
4969c119
AA
3212 goto out_page;
3213
82b0f8c3 3214 page = ksm_might_need_to_copy(page, vma, vmf->address);
cbf86cfe
HD
3215 if (unlikely(!page)) {
3216 ret = VM_FAULT_OOM;
3217 page = swapcache;
cbf86cfe 3218 goto out_page;
5ad64688
HD
3219 }
3220
9d82c694 3221 cgroup_throttle_swaprate(page, GFP_KERNEL);
8a9f3ccd 3222
1da177e4 3223 /*
8f4e2101 3224 * Back out if somebody else already faulted in this pte.
1da177e4 3225 */
82b0f8c3
JK
3226 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3227 &vmf->ptl);
2994302b 3228 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
b8107480 3229 goto out_nomap;
b8107480
KK
3230
3231 if (unlikely(!PageUptodate(page))) {
3232 ret = VM_FAULT_SIGBUS;
3233 goto out_nomap;
1da177e4
LT
3234 }
3235
8c7c6e34
KH
3236 /*
3237 * The page isn't present yet, go ahead with the fault.
3238 *
3239 * Be careful about the sequence of operations here.
3240 * To get its accounting right, reuse_swap_page() must be called
3241 * while the page is counted on swap but not yet in mapcount i.e.
3242 * before page_add_anon_rmap() and swap_free(); try_to_free_swap()
3243 * must be called after the swap_free(), or it will never succeed.
8c7c6e34 3244 */
1da177e4 3245
bae473a4
KS
3246 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
3247 dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
1da177e4 3248 pte = mk_pte(page, vma->vm_page_prot);
82b0f8c3 3249 if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
1da177e4 3250 pte = maybe_mkwrite(pte_mkdirty(pte), vma);
82b0f8c3 3251 vmf->flags &= ~FAULT_FLAG_WRITE;
9a5b489b 3252 ret |= VM_FAULT_WRITE;
d281ee61 3253 exclusive = RMAP_EXCLUSIVE;
1da177e4 3254 }
1da177e4 3255 flush_icache_page(vma, page);
2994302b 3256 if (pte_swp_soft_dirty(vmf->orig_pte))
179ef71c 3257 pte = pte_mksoft_dirty(pte);
f45ec5ff
PX
3258 if (pte_swp_uffd_wp(vmf->orig_pte)) {
3259 pte = pte_mkuffd_wp(pte);
3260 pte = pte_wrprotect(pte);
3261 }
82b0f8c3 3262 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
ca827d55 3263 arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte);
2994302b 3264 vmf->orig_pte = pte;
0bcac06f
MK
3265
3266 /* ksm created a completely new copy */
3267 if (unlikely(page != swapcache && swapcache)) {
82b0f8c3 3268 page_add_new_anon_rmap(page, vma, vmf->address, false);
00501b53 3269 lru_cache_add_active_or_unevictable(page, vma);
0bcac06f
MK
3270 } else {
3271 do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
0bcac06f 3272 activate_page(page);
00501b53 3273 }
1da177e4 3274
c475a8ab 3275 swap_free(entry);
5ccc5aba
VD
3276 if (mem_cgroup_swap_full(page) ||
3277 (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
a2c43eed 3278 try_to_free_swap(page);
c475a8ab 3279 unlock_page(page);
0bcac06f 3280 if (page != swapcache && swapcache) {
4969c119
AA
3281 /*
3282 * Hold the lock to avoid the swap entry to be reused
3283 * until we take the PT lock for the pte_same() check
3284 * (to avoid false positives from pte_same). For
3285 * further safety release the lock after the swap_free
3286 * so that the swap count won't change under a
3287 * parallel locked swapcache.
3288 */
3289 unlock_page(swapcache);
09cbfeaf 3290 put_page(swapcache);
4969c119 3291 }
c475a8ab 3292
82b0f8c3 3293 if (vmf->flags & FAULT_FLAG_WRITE) {
2994302b 3294 ret |= do_wp_page(vmf);
61469f1d
HD
3295 if (ret & VM_FAULT_ERROR)
3296 ret &= VM_FAULT_ERROR;
1da177e4
LT
3297 goto out;
3298 }
3299
3300 /* No need to invalidate - it was non-present before */
82b0f8c3 3301 update_mmu_cache(vma, vmf->address, vmf->pte);
65500d23 3302unlock:
82b0f8c3 3303 pte_unmap_unlock(vmf->pte, vmf->ptl);
1da177e4
LT
3304out:
3305 return ret;
b8107480 3306out_nomap:
82b0f8c3 3307 pte_unmap_unlock(vmf->pte, vmf->ptl);
bc43f75c 3308out_page:
b8107480 3309 unlock_page(page);
4779cb31 3310out_release:
09cbfeaf 3311 put_page(page);
0bcac06f 3312 if (page != swapcache && swapcache) {
4969c119 3313 unlock_page(swapcache);
09cbfeaf 3314 put_page(swapcache);
4969c119 3315 }
65500d23 3316 return ret;
1da177e4
LT
3317}
3318
3319/*
c1e8d7c6 3320 * We enter with non-exclusive mmap_lock (to exclude vma changes,
8f4e2101 3321 * but allow concurrent faults), and pte mapped but not yet locked.
c1e8d7c6 3322 * We return with mmap_lock still held, but pte unmapped and unlocked.
1da177e4 3323 */
2b740303 3324static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
1da177e4 3325{
82b0f8c3 3326 struct vm_area_struct *vma = vmf->vma;
8f4e2101 3327 struct page *page;
2b740303 3328 vm_fault_t ret = 0;
1da177e4 3329 pte_t entry;
1da177e4 3330
6b7339f4
KS
3331 /* File mapping without ->vm_ops ? */
3332 if (vma->vm_flags & VM_SHARED)
3333 return VM_FAULT_SIGBUS;
3334
7267ec00
KS
3335 /*
3336 * Use pte_alloc() instead of pte_alloc_map(). We can't run
3337 * pte_offset_map() on pmds where a huge pmd might be created
3338 * from a different thread.
3339 *
3e4e28c5 3340 * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
7267ec00
KS
3341 * parallel threads are excluded by other means.
3342 *
3e4e28c5 3343 * Here we only have mmap_read_lock(mm).
7267ec00 3344 */
4cf58924 3345 if (pte_alloc(vma->vm_mm, vmf->pmd))
7267ec00
KS
3346 return VM_FAULT_OOM;
3347
3348 /* See the comment in pte_alloc_one_map() */
82b0f8c3 3349 if (unlikely(pmd_trans_unstable(vmf->pmd)))
7267ec00
KS
3350 return 0;
3351
11ac5524 3352 /* Use the zero-page for reads */
82b0f8c3 3353 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
bae473a4 3354 !mm_forbids_zeropage(vma->vm_mm)) {
82b0f8c3 3355 entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
62eede62 3356 vma->vm_page_prot));
82b0f8c3
JK
3357 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
3358 vmf->address, &vmf->ptl);
7df67697
BM
3359 if (!pte_none(*vmf->pte)) {
3360 update_mmu_tlb(vma, vmf->address, vmf->pte);
a13ea5b7 3361 goto unlock;
7df67697 3362 }
6b31d595
MH
3363 ret = check_stable_address_space(vma->vm_mm);
3364 if (ret)
3365 goto unlock;
6b251fc9
AA
3366 /* Deliver the page fault to userland, check inside PT lock */
3367 if (userfaultfd_missing(vma)) {
82b0f8c3
JK
3368 pte_unmap_unlock(vmf->pte, vmf->ptl);
3369 return handle_userfault(vmf, VM_UFFD_MISSING);
6b251fc9 3370 }
a13ea5b7
HD
3371 goto setpte;
3372 }
3373
557ed1fa 3374 /* Allocate our own private page. */
557ed1fa
NP
3375 if (unlikely(anon_vma_prepare(vma)))
3376 goto oom;
82b0f8c3 3377 page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
557ed1fa
NP
3378 if (!page)
3379 goto oom;
eb3c24f3 3380
d9eb1ea2 3381 if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
eb3c24f3 3382 goto oom_free_page;
9d82c694 3383 cgroup_throttle_swaprate(page, GFP_KERNEL);
eb3c24f3 3384
52f37629
MK
3385 /*
3386 * The memory barrier inside __SetPageUptodate makes sure that
f4f5329d 3387 * preceding stores to the page contents become visible before
52f37629
MK
3388 * the set_pte_at() write.
3389 */
0ed361de 3390 __SetPageUptodate(page);
8f4e2101 3391
557ed1fa 3392 entry = mk_pte(page, vma->vm_page_prot);
44bf431b 3393 entry = pte_sw_mkyoung(entry);
1ac0cb5d
HD
3394 if (vma->vm_flags & VM_WRITE)
3395 entry = pte_mkwrite(pte_mkdirty(entry));
1da177e4 3396
82b0f8c3
JK
3397 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3398 &vmf->ptl);
7df67697
BM
3399 if (!pte_none(*vmf->pte)) {
3400 update_mmu_cache(vma, vmf->address, vmf->pte);
557ed1fa 3401 goto release;
7df67697 3402 }
9ba69294 3403
6b31d595
MH
3404 ret = check_stable_address_space(vma->vm_mm);
3405 if (ret)
3406 goto release;
3407
6b251fc9
AA
3408 /* Deliver the page fault to userland, check inside PT lock */
3409 if (userfaultfd_missing(vma)) {
82b0f8c3 3410 pte_unmap_unlock(vmf->pte, vmf->ptl);
09cbfeaf 3411 put_page(page);
82b0f8c3 3412 return handle_userfault(vmf, VM_UFFD_MISSING);
6b251fc9
AA
3413 }
3414
bae473a4 3415 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
82b0f8c3 3416 page_add_new_anon_rmap(page, vma, vmf->address, false);
00501b53 3417 lru_cache_add_active_or_unevictable(page, vma);
a13ea5b7 3418setpte:
82b0f8c3 3419 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
1da177e4
LT
3420
3421 /* No need to invalidate - it was non-present before */
82b0f8c3 3422 update_mmu_cache(vma, vmf->address, vmf->pte);
65500d23 3423unlock:
82b0f8c3 3424 pte_unmap_unlock(vmf->pte, vmf->ptl);
6b31d595 3425 return ret;
8f4e2101 3426release:
09cbfeaf 3427 put_page(page);
8f4e2101 3428 goto unlock;
8a9f3ccd 3429oom_free_page:
09cbfeaf 3430 put_page(page);
65500d23 3431oom:
1da177e4
LT
3432 return VM_FAULT_OOM;
3433}
3434
9a95f3cf 3435/*
c1e8d7c6 3436 * The mmap_lock must have been held on entry, and may have been
9a95f3cf
PC
3437 * released depending on flags and vma->vm_ops->fault() return value.
3438 * See filemap_fault() and __lock_page_retry().
3439 */
2b740303 3440static vm_fault_t __do_fault(struct vm_fault *vmf)
7eae74af 3441{
82b0f8c3 3442 struct vm_area_struct *vma = vmf->vma;
2b740303 3443 vm_fault_t ret;
7eae74af 3444
63f3655f
MH
3445 /*
3446 * Preallocate pte before we take page_lock because this might lead to
3447 * deadlocks for memcg reclaim which waits for pages under writeback:
3448 * lock_page(A)
3449 * SetPageWriteback(A)
3450 * unlock_page(A)
3451 * lock_page(B)
3452 * lock_page(B)
3453 * pte_alloc_pne
3454 * shrink_page_list
3455 * wait_on_page_writeback(A)
3456 * SetPageWriteback(B)
3457 * unlock_page(B)
3458 * # flush A, B to clear the writeback
3459 */
3460 if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
3461 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
3462 if (!vmf->prealloc_pte)
3463 return VM_FAULT_OOM;
3464 smp_wmb(); /* See comment in __pte_alloc() */
3465 }
3466
11bac800 3467 ret = vma->vm_ops->fault(vmf);
3917048d 3468 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
b1aa812b 3469 VM_FAULT_DONE_COW)))
bc2466e4 3470 return ret;
7eae74af 3471
667240e0 3472 if (unlikely(PageHWPoison(vmf->page))) {
7eae74af 3473 if (ret & VM_FAULT_LOCKED)
667240e0
JK
3474 unlock_page(vmf->page);
3475 put_page(vmf->page);
936ca80d 3476 vmf->page = NULL;
7eae74af
KS
3477 return VM_FAULT_HWPOISON;
3478 }
3479
3480 if (unlikely(!(ret & VM_FAULT_LOCKED)))
667240e0 3481 lock_page(vmf->page);
7eae74af 3482 else
667240e0 3483 VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
7eae74af 3484
7eae74af
KS
3485 return ret;
3486}
3487
d0f0931d
RZ
3488/*
3489 * The ordering of these checks is important for pmds with _PAGE_DEVMAP set.
3490 * If we check pmd_trans_unstable() first we will trip the bad_pmd() check
3491 * inside of pmd_none_or_trans_huge_or_clear_bad(). This will end up correctly
3492 * returning 1 but not before it spams dmesg with the pmd_clear_bad() output.
3493 */
3494static int pmd_devmap_trans_unstable(pmd_t *pmd)
3495{
3496 return pmd_devmap(*pmd) || pmd_trans_unstable(pmd);
3497}
3498
2b740303 3499static vm_fault_t pte_alloc_one_map(struct vm_fault *vmf)
7267ec00 3500{
82b0f8c3 3501 struct vm_area_struct *vma = vmf->vma;
7267ec00 3502
82b0f8c3 3503 if (!pmd_none(*vmf->pmd))
7267ec00 3504 goto map_pte;
82b0f8c3
JK
3505 if (vmf->prealloc_pte) {
3506 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
3507 if (unlikely(!pmd_none(*vmf->pmd))) {
3508 spin_unlock(vmf->ptl);
7267ec00
KS
3509 goto map_pte;
3510 }
3511
c4812909 3512 mm_inc_nr_ptes(vma->vm_mm);
82b0f8c3
JK
3513 pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
3514 spin_unlock(vmf->ptl);
7f2b6ce8 3515 vmf->prealloc_pte = NULL;
4cf58924 3516 } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd))) {
7267ec00
KS
3517 return VM_FAULT_OOM;
3518 }
3519map_pte:
3520 /*
3521 * If a huge pmd materialized under us just retry later. Use
d0f0931d
RZ
3522 * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead of
3523 * pmd_trans_huge() to ensure the pmd didn't become pmd_trans_huge
3524 * under us and then back to pmd_none, as a result of MADV_DONTNEED
3525 * running immediately after a huge pmd fault in a different thread of
3526 * this mm, in turn leading to a misleading pmd_trans_huge() retval.
3527 * All we have to ensure is that it is a regular pmd that we can walk
3528 * with pte_offset_map() and we can do that through an atomic read in
3529 * C, which is what pmd_trans_unstable() provides.
7267ec00 3530 */
d0f0931d 3531 if (pmd_devmap_trans_unstable(vmf->pmd))
7267ec00
KS
3532 return VM_FAULT_NOPAGE;
3533
d0f0931d
RZ
3534 /*
3535 * At this point we know that our vmf->pmd points to a page of ptes
3536 * and it cannot become pmd_none(), pmd_devmap() or pmd_trans_huge()
3537 * for the duration of the fault. If a racing MADV_DONTNEED runs and
3538 * we zap the ptes pointed to by our vmf->pmd, the vmf->ptl will still
3539 * be valid and we will re-check to make sure the vmf->pte isn't
3540 * pte_none() under vmf->ptl protection when we return to
3541 * alloc_set_pte().
3542 */
82b0f8c3
JK
3543 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3544 &vmf->ptl);
7267ec00
KS
3545 return 0;
3546}
3547
396bcc52 3548#ifdef CONFIG_TRANSPARENT_HUGEPAGE
82b0f8c3 3549static void deposit_prealloc_pte(struct vm_fault *vmf)
953c66c2 3550{
82b0f8c3 3551 struct vm_area_struct *vma = vmf->vma;
953c66c2 3552
82b0f8c3 3553 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
953c66c2
AK
3554 /*
3555 * We are going to consume the prealloc table,
3556 * count that as nr_ptes.
3557 */
c4812909 3558 mm_inc_nr_ptes(vma->vm_mm);
7f2b6ce8 3559 vmf->prealloc_pte = NULL;
953c66c2
AK
3560}
3561
2b740303 3562static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
10102459 3563{
82b0f8c3
JK
3564 struct vm_area_struct *vma = vmf->vma;
3565 bool write = vmf->flags & FAULT_FLAG_WRITE;
3566 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
10102459 3567 pmd_t entry;
2b740303
SJ
3568 int i;
3569 vm_fault_t ret;
10102459
KS
3570
3571 if (!transhuge_vma_suitable(vma, haddr))
3572 return VM_FAULT_FALLBACK;
3573
3574 ret = VM_FAULT_FALLBACK;
3575 page = compound_head(page);
3576
953c66c2
AK
3577 /*
3578 * Archs like ppc64 need additonal space to store information
3579 * related to pte entry. Use the preallocated table for that.
3580 */
82b0f8c3 3581 if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
4cf58924 3582 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
82b0f8c3 3583 if (!vmf->prealloc_pte)
953c66c2
AK
3584 return VM_FAULT_OOM;
3585 smp_wmb(); /* See comment in __pte_alloc() */
3586 }
3587
82b0f8c3
JK
3588 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
3589 if (unlikely(!pmd_none(*vmf->pmd)))
10102459
KS
3590 goto out;
3591
3592 for (i = 0; i < HPAGE_PMD_NR; i++)
3593 flush_icache_page(vma, page + i);
3594
3595 entry = mk_huge_pmd(page, vma->vm_page_prot);
3596 if (write)
f55e1014 3597 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
10102459 3598
fadae295 3599 add_mm_counter(vma->vm_mm, mm_counter_file(page), HPAGE_PMD_NR);
10102459 3600 page_add_file_rmap(page, true);
953c66c2
AK
3601 /*
3602 * deposit and withdraw with pmd lock held
3603 */
3604 if (arch_needs_pgtable_deposit())
82b0f8c3 3605 deposit_prealloc_pte(vmf);
10102459 3606
82b0f8c3 3607 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
10102459 3608
82b0f8c3 3609 update_mmu_cache_pmd(vma, haddr, vmf->pmd);
10102459
KS
3610
3611 /* fault is handled */
3612 ret = 0;
95ecedcd 3613 count_vm_event(THP_FILE_MAPPED);
10102459 3614out:
82b0f8c3 3615 spin_unlock(vmf->ptl);
10102459
KS
3616 return ret;
3617}
3618#else
2b740303 3619static vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page)
10102459
KS
3620{
3621 BUILD_BUG();
3622 return 0;
3623}
3624#endif
3625
8c6e50b0 3626/**
7267ec00
KS
3627 * alloc_set_pte - setup new PTE entry for given page and add reverse page
3628 * mapping. If needed, the fucntion allocates page table or use pre-allocated.
8c6e50b0 3629 *
82b0f8c3 3630 * @vmf: fault environment
8c6e50b0 3631 * @page: page to map
8c6e50b0 3632 *
82b0f8c3
JK
3633 * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on
3634 * return.
8c6e50b0
KS
3635 *
3636 * Target users are page handler itself and implementations of
3637 * vm_ops->map_pages.
a862f68a
MR
3638 *
3639 * Return: %0 on success, %VM_FAULT_ code in case of error.
8c6e50b0 3640 */
9d82c694 3641vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page)
3bb97794 3642{
82b0f8c3
JK
3643 struct vm_area_struct *vma = vmf->vma;
3644 bool write = vmf->flags & FAULT_FLAG_WRITE;
3bb97794 3645 pte_t entry;
2b740303 3646 vm_fault_t ret;
10102459 3647
396bcc52 3648 if (pmd_none(*vmf->pmd) && PageTransCompound(page)) {
82b0f8c3 3649 ret = do_set_pmd(vmf, page);
10102459 3650 if (ret != VM_FAULT_FALLBACK)
b0b9b3df 3651 return ret;
10102459 3652 }
3bb97794 3653
82b0f8c3
JK
3654 if (!vmf->pte) {
3655 ret = pte_alloc_one_map(vmf);
7267ec00 3656 if (ret)
b0b9b3df 3657 return ret;
7267ec00
KS
3658 }
3659
3660 /* Re-check under ptl */
7df67697
BM
3661 if (unlikely(!pte_none(*vmf->pte))) {
3662 update_mmu_tlb(vma, vmf->address, vmf->pte);
b0b9b3df 3663 return VM_FAULT_NOPAGE;
7df67697 3664 }
7267ec00 3665
3bb97794
KS
3666 flush_icache_page(vma, page);
3667 entry = mk_pte(page, vma->vm_page_prot);
44bf431b 3668 entry = pte_sw_mkyoung(entry);
3bb97794
KS
3669 if (write)
3670 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
bae473a4
KS
3671 /* copy-on-write page */
3672 if (write && !(vma->vm_flags & VM_SHARED)) {
3bb97794 3673 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
82b0f8c3 3674 page_add_new_anon_rmap(page, vma, vmf->address, false);
7267ec00 3675 lru_cache_add_active_or_unevictable(page, vma);
3bb97794 3676 } else {
eca56ff9 3677 inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
dd78fedd 3678 page_add_file_rmap(page, false);
3bb97794 3679 }
82b0f8c3 3680 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
3bb97794
KS
3681
3682 /* no need to invalidate: a not-present page won't be cached */
82b0f8c3 3683 update_mmu_cache(vma, vmf->address, vmf->pte);
7267ec00 3684
b0b9b3df 3685 return 0;
3bb97794
KS
3686}
3687
9118c0cb
JK
3688
3689/**
3690 * finish_fault - finish page fault once we have prepared the page to fault
3691 *
3692 * @vmf: structure describing the fault
3693 *
3694 * This function handles all that is needed to finish a page fault once the
3695 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
3696 * given page, adds reverse page mapping, handles memcg charges and LRU
a862f68a 3697 * addition.
9118c0cb
JK
3698 *
3699 * The function expects the page to be locked and on success it consumes a
3700 * reference of a page being mapped (for the PTE which maps it).
a862f68a
MR
3701 *
3702 * Return: %0 on success, %VM_FAULT_ code in case of error.
9118c0cb 3703 */
2b740303 3704vm_fault_t finish_fault(struct vm_fault *vmf)
9118c0cb
JK
3705{
3706 struct page *page;
2b740303 3707 vm_fault_t ret = 0;
9118c0cb
JK
3708
3709 /* Did we COW the page? */
3710 if ((vmf->flags & FAULT_FLAG_WRITE) &&
3711 !(vmf->vma->vm_flags & VM_SHARED))
3712 page = vmf->cow_page;
3713 else
3714 page = vmf->page;
6b31d595
MH
3715
3716 /*
3717 * check even for read faults because we might have lost our CoWed
3718 * page
3719 */
3720 if (!(vmf->vma->vm_flags & VM_SHARED))
3721 ret = check_stable_address_space(vmf->vma->vm_mm);
3722 if (!ret)
9d82c694 3723 ret = alloc_set_pte(vmf, page);
9118c0cb
JK
3724 if (vmf->pte)
3725 pte_unmap_unlock(vmf->pte, vmf->ptl);
3726 return ret;
3727}
3728
3a91053a
KS
3729static unsigned long fault_around_bytes __read_mostly =
3730 rounddown_pow_of_two(65536);
a9b0f861 3731
a9b0f861
KS
3732#ifdef CONFIG_DEBUG_FS
3733static int fault_around_bytes_get(void *data, u64 *val)
1592eef0 3734{
a9b0f861 3735 *val = fault_around_bytes;
1592eef0
KS
3736 return 0;
3737}
3738
b4903d6e 3739/*
da391d64
WK
3740 * fault_around_bytes must be rounded down to the nearest page order as it's
3741 * what do_fault_around() expects to see.
b4903d6e 3742 */
a9b0f861 3743static int fault_around_bytes_set(void *data, u64 val)
1592eef0 3744{
a9b0f861 3745 if (val / PAGE_SIZE > PTRS_PER_PTE)
1592eef0 3746 return -EINVAL;
b4903d6e
AR
3747 if (val > PAGE_SIZE)
3748 fault_around_bytes = rounddown_pow_of_two(val);
3749 else
3750 fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */
1592eef0
KS
3751 return 0;
3752}
0a1345f8 3753DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
a9b0f861 3754 fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
1592eef0
KS
3755
3756static int __init fault_around_debugfs(void)
3757{
d9f7979c
GKH
3758 debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
3759 &fault_around_bytes_fops);
1592eef0
KS
3760 return 0;
3761}
3762late_initcall(fault_around_debugfs);
1592eef0 3763#endif
8c6e50b0 3764
1fdb412b
KS
3765/*
3766 * do_fault_around() tries to map few pages around the fault address. The hope
3767 * is that the pages will be needed soon and this will lower the number of
3768 * faults to handle.
3769 *
3770 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
3771 * not ready to be mapped: not up-to-date, locked, etc.
3772 *
3773 * This function is called with the page table lock taken. In the split ptlock
3774 * case the page table lock only protects only those entries which belong to
3775 * the page table corresponding to the fault address.
3776 *
3777 * This function doesn't cross the VMA boundaries, in order to call map_pages()
3778 * only once.
3779 *
da391d64
WK
3780 * fault_around_bytes defines how many bytes we'll try to map.
3781 * do_fault_around() expects it to be set to a power of two less than or equal
3782 * to PTRS_PER_PTE.
1fdb412b 3783 *
da391d64
WK
3784 * The virtual address of the area that we map is naturally aligned to
3785 * fault_around_bytes rounded down to the machine page size
3786 * (and therefore to page order). This way it's easier to guarantee
3787 * that we don't cross page table boundaries.
1fdb412b 3788 */
2b740303 3789static vm_fault_t do_fault_around(struct vm_fault *vmf)
8c6e50b0 3790{
82b0f8c3 3791 unsigned long address = vmf->address, nr_pages, mask;
0721ec8b 3792 pgoff_t start_pgoff = vmf->pgoff;
bae473a4 3793 pgoff_t end_pgoff;
2b740303
SJ
3794 int off;
3795 vm_fault_t ret = 0;
8c6e50b0 3796
4db0c3c2 3797 nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
aecd6f44
KS
3798 mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;
3799
82b0f8c3
JK
3800 vmf->address = max(address & mask, vmf->vma->vm_start);
3801 off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
bae473a4 3802 start_pgoff -= off;
8c6e50b0
KS
3803
3804 /*
da391d64
WK
3805 * end_pgoff is either the end of the page table, the end of
3806 * the vma or nr_pages from start_pgoff, depending what is nearest.
8c6e50b0 3807 */
bae473a4 3808 end_pgoff = start_pgoff -
82b0f8c3 3809 ((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
8c6e50b0 3810 PTRS_PER_PTE - 1;
82b0f8c3 3811 end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
bae473a4 3812 start_pgoff + nr_pages - 1);
8c6e50b0 3813
82b0f8c3 3814 if (pmd_none(*vmf->pmd)) {
4cf58924 3815 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
82b0f8c3 3816 if (!vmf->prealloc_pte)
c5f88bd2 3817 goto out;
7267ec00 3818 smp_wmb(); /* See comment in __pte_alloc() */
8c6e50b0
KS
3819 }
3820
82b0f8c3 3821 vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff);
7267ec00 3822
7267ec00 3823 /* Huge page is mapped? Page fault is solved */
82b0f8c3 3824 if (pmd_trans_huge(*vmf->pmd)) {
7267ec00
KS
3825 ret = VM_FAULT_NOPAGE;
3826 goto out;
3827 }
3828
3829 /* ->map_pages() haven't done anything useful. Cold page cache? */
82b0f8c3 3830 if (!vmf->pte)
7267ec00
KS
3831 goto out;
3832
3833 /* check if the page fault is solved */
82b0f8c3
JK
3834 vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT);
3835 if (!pte_none(*vmf->pte))
7267ec00 3836 ret = VM_FAULT_NOPAGE;
82b0f8c3 3837 pte_unmap_unlock(vmf->pte, vmf->ptl);
bae473a4 3838out:
82b0f8c3
JK
3839 vmf->address = address;
3840 vmf->pte = NULL;
7267ec00 3841 return ret;
8c6e50b0
KS
3842}
3843
2b740303 3844static vm_fault_t do_read_fault(struct vm_fault *vmf)
e655fb29 3845{
82b0f8c3 3846 struct vm_area_struct *vma = vmf->vma;
2b740303 3847 vm_fault_t ret = 0;
8c6e50b0
KS
3848
3849 /*
3850 * Let's call ->map_pages() first and use ->fault() as fallback
3851 * if page by the offset is not ready to be mapped (cold cache or
3852 * something).
3853 */
9b4bdd2f 3854 if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
0721ec8b 3855 ret = do_fault_around(vmf);
7267ec00
KS
3856 if (ret)
3857 return ret;
8c6e50b0 3858 }
e655fb29 3859
936ca80d 3860 ret = __do_fault(vmf);
e655fb29
KS
3861 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3862 return ret;
3863
9118c0cb 3864 ret |= finish_fault(vmf);
936ca80d 3865 unlock_page(vmf->page);
7267ec00 3866 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
936ca80d 3867 put_page(vmf->page);
e655fb29
KS
3868 return ret;
3869}
3870
2b740303 3871static vm_fault_t do_cow_fault(struct vm_fault *vmf)
ec47c3b9 3872{
82b0f8c3 3873 struct vm_area_struct *vma = vmf->vma;
2b740303 3874 vm_fault_t ret;
ec47c3b9
KS
3875
3876 if (unlikely(anon_vma_prepare(vma)))
3877 return VM_FAULT_OOM;
3878
936ca80d
JK
3879 vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
3880 if (!vmf->cow_page)
ec47c3b9
KS
3881 return VM_FAULT_OOM;
3882
d9eb1ea2 3883 if (mem_cgroup_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL)) {
936ca80d 3884 put_page(vmf->cow_page);
ec47c3b9
KS
3885 return VM_FAULT_OOM;
3886 }
9d82c694 3887 cgroup_throttle_swaprate(vmf->cow_page, GFP_KERNEL);
ec47c3b9 3888
936ca80d 3889 ret = __do_fault(vmf);
ec47c3b9
KS
3890 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3891 goto uncharge_out;
3917048d
JK
3892 if (ret & VM_FAULT_DONE_COW)
3893 return ret;
ec47c3b9 3894
b1aa812b 3895 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
936ca80d 3896 __SetPageUptodate(vmf->cow_page);
ec47c3b9 3897
9118c0cb 3898 ret |= finish_fault(vmf);
b1aa812b
JK
3899 unlock_page(vmf->page);
3900 put_page(vmf->page);
7267ec00
KS
3901 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3902 goto uncharge_out;
ec47c3b9
KS
3903 return ret;
3904uncharge_out:
936ca80d 3905 put_page(vmf->cow_page);
ec47c3b9
KS
3906 return ret;
3907}
3908
2b740303 3909static vm_fault_t do_shared_fault(struct vm_fault *vmf)
1da177e4 3910{
82b0f8c3 3911 struct vm_area_struct *vma = vmf->vma;
2b740303 3912 vm_fault_t ret, tmp;
1d65f86d 3913
936ca80d 3914 ret = __do_fault(vmf);
7eae74af 3915 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
f0c6d4d2 3916 return ret;
1da177e4
LT
3917
3918 /*
f0c6d4d2
KS
3919 * Check if the backing address space wants to know that the page is
3920 * about to become writable
1da177e4 3921 */
fb09a464 3922 if (vma->vm_ops->page_mkwrite) {
936ca80d 3923 unlock_page(vmf->page);
38b8cb7f 3924 tmp = do_page_mkwrite(vmf);
fb09a464
KS
3925 if (unlikely(!tmp ||
3926 (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
936ca80d 3927 put_page(vmf->page);
fb09a464 3928 return tmp;
4294621f 3929 }
fb09a464
KS
3930 }
3931
9118c0cb 3932 ret |= finish_fault(vmf);
7267ec00
KS
3933 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
3934 VM_FAULT_RETRY))) {
936ca80d
JK
3935 unlock_page(vmf->page);
3936 put_page(vmf->page);
f0c6d4d2 3937 return ret;
1da177e4 3938 }
b827e496 3939
89b15332 3940 ret |= fault_dirty_shared_page(vmf);
1d65f86d 3941 return ret;
54cb8821 3942}
d00806b1 3943
9a95f3cf 3944/*
c1e8d7c6 3945 * We enter with non-exclusive mmap_lock (to exclude vma changes,
9a95f3cf 3946 * but allow concurrent faults).
c1e8d7c6 3947 * The mmap_lock may have been released depending on flags and our
9a95f3cf 3948 * return value. See filemap_fault() and __lock_page_or_retry().
c1e8d7c6 3949 * If mmap_lock is released, vma may become invalid (for example
fc8efd2d 3950 * by other thread calling munmap()).
9a95f3cf 3951 */
2b740303 3952static vm_fault_t do_fault(struct vm_fault *vmf)
54cb8821 3953{
82b0f8c3 3954 struct vm_area_struct *vma = vmf->vma;
fc8efd2d 3955 struct mm_struct *vm_mm = vma->vm_mm;
2b740303 3956 vm_fault_t ret;
54cb8821 3957
ff09d7ec
AK
3958 /*
3959 * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
3960 */
3961 if (!vma->vm_ops->fault) {
3962 /*
3963 * If we find a migration pmd entry or a none pmd entry, which
3964 * should never happen, return SIGBUS
3965 */
3966 if (unlikely(!pmd_present(*vmf->pmd)))
3967 ret = VM_FAULT_SIGBUS;
3968 else {
3969 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm,
3970 vmf->pmd,
3971 vmf->address,
3972 &vmf->ptl);
3973 /*
3974 * Make sure this is not a temporary clearing of pte
3975 * by holding ptl and checking again. A R/M/W update
3976 * of pte involves: take ptl, clearing the pte so that
3977 * we don't have concurrent modification by hardware
3978 * followed by an update.
3979 */
3980 if (unlikely(pte_none(*vmf->pte)))
3981 ret = VM_FAULT_SIGBUS;
3982 else
3983 ret = VM_FAULT_NOPAGE;
3984
3985 pte_unmap_unlock(vmf->pte, vmf->ptl);
3986 }
3987 } else if (!(vmf->flags & FAULT_FLAG_WRITE))
b0b9b3df
HD
3988 ret = do_read_fault(vmf);
3989 else if (!(vma->vm_flags & VM_SHARED))
3990 ret = do_cow_fault(vmf);
3991 else
3992 ret = do_shared_fault(vmf);
3993
3994 /* preallocated pagetable is unused: free it */
3995 if (vmf->prealloc_pte) {
fc8efd2d 3996 pte_free(vm_mm, vmf->prealloc_pte);
7f2b6ce8 3997 vmf->prealloc_pte = NULL;
b0b9b3df
HD
3998 }
3999 return ret;
54cb8821
NP
4000}
4001
b19a9939 4002static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
04bb2f94
RR
4003 unsigned long addr, int page_nid,
4004 int *flags)
9532fec1
MG
4005{
4006 get_page(page);
4007
4008 count_vm_numa_event(NUMA_HINT_FAULTS);
04bb2f94 4009 if (page_nid == numa_node_id()) {
9532fec1 4010 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
04bb2f94
RR
4011 *flags |= TNF_FAULT_LOCAL;
4012 }
9532fec1
MG
4013
4014 return mpol_misplaced(page, vma, addr);
4015}
4016
2b740303 4017static vm_fault_t do_numa_page(struct vm_fault *vmf)
d10e63f2 4018{
82b0f8c3 4019 struct vm_area_struct *vma = vmf->vma;
4daae3b4 4020 struct page *page = NULL;
98fa15f3 4021 int page_nid = NUMA_NO_NODE;
90572890 4022 int last_cpupid;
cbee9f88 4023 int target_nid;
b8593bfd 4024 bool migrated = false;
04a86453 4025 pte_t pte, old_pte;
288bc549 4026 bool was_writable = pte_savedwrite(vmf->orig_pte);
6688cc05 4027 int flags = 0;
d10e63f2
MG
4028
4029 /*
166f61b9
TH
4030 * The "pte" at this point cannot be used safely without
4031 * validation through pte_unmap_same(). It's of NUMA type but
4032 * the pfn may be screwed if the read is non atomic.
166f61b9 4033 */
82b0f8c3
JK
4034 vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
4035 spin_lock(vmf->ptl);
cee216a6 4036 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
82b0f8c3 4037 pte_unmap_unlock(vmf->pte, vmf->ptl);
4daae3b4
MG
4038 goto out;
4039 }
4040
cee216a6
AK
4041 /*
4042 * Make it present again, Depending on how arch implementes non
4043 * accessible ptes, some can allow access by kernel mode.
4044 */
04a86453
AK
4045 old_pte = ptep_modify_prot_start(vma, vmf->address, vmf->pte);
4046 pte = pte_modify(old_pte, vma->vm_page_prot);
4d942466 4047 pte = pte_mkyoung(pte);
b191f9b1
MG
4048 if (was_writable)
4049 pte = pte_mkwrite(pte);
04a86453 4050 ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
82b0f8c3 4051 update_mmu_cache(vma, vmf->address, vmf->pte);
d10e63f2 4052
82b0f8c3 4053 page = vm_normal_page(vma, vmf->address, pte);
d10e63f2 4054 if (!page) {
82b0f8c3 4055 pte_unmap_unlock(vmf->pte, vmf->ptl);
d10e63f2
MG
4056 return 0;
4057 }
4058
e81c4802
KS
4059 /* TODO: handle PTE-mapped THP */
4060 if (PageCompound(page)) {
82b0f8c3 4061 pte_unmap_unlock(vmf->pte, vmf->ptl);
e81c4802
KS
4062 return 0;
4063 }
4064
6688cc05 4065 /*
bea66fbd
MG
4066 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
4067 * much anyway since they can be in shared cache state. This misses
4068 * the case where a mapping is writable but the process never writes
4069 * to it but pte_write gets cleared during protection updates and
4070 * pte_dirty has unpredictable behaviour between PTE scan updates,
4071 * background writeback, dirty balancing and application behaviour.
6688cc05 4072 */
d59dc7bc 4073 if (!pte_write(pte))
6688cc05
PZ
4074 flags |= TNF_NO_GROUP;
4075
dabe1d99
RR
4076 /*
4077 * Flag if the page is shared between multiple address spaces. This
4078 * is later used when determining whether to group tasks together
4079 */
4080 if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
4081 flags |= TNF_SHARED;
4082
90572890 4083 last_cpupid = page_cpupid_last(page);
8191acbd 4084 page_nid = page_to_nid(page);
82b0f8c3 4085 target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
bae473a4 4086 &flags);
82b0f8c3 4087 pte_unmap_unlock(vmf->pte, vmf->ptl);
98fa15f3 4088 if (target_nid == NUMA_NO_NODE) {
4daae3b4
MG
4089 put_page(page);
4090 goto out;
4091 }
4092
4093 /* Migrate to the requested node */
1bc115d8 4094 migrated = migrate_misplaced_page(page, vma, target_nid);
6688cc05 4095 if (migrated) {
8191acbd 4096 page_nid = target_nid;
6688cc05 4097 flags |= TNF_MIGRATED;
074c2381
MG
4098 } else
4099 flags |= TNF_MIGRATE_FAIL;
4daae3b4
MG
4100
4101out:
98fa15f3 4102 if (page_nid != NUMA_NO_NODE)
6688cc05 4103 task_numa_fault(last_cpupid, page_nid, 1, flags);
d10e63f2
MG
4104 return 0;
4105}
4106
2b740303 4107static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
b96375f7 4108{
f4200391 4109 if (vma_is_anonymous(vmf->vma))
82b0f8c3 4110 return do_huge_pmd_anonymous_page(vmf);
a2d58167 4111 if (vmf->vma->vm_ops->huge_fault)
c791ace1 4112 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
b96375f7
MW
4113 return VM_FAULT_FALLBACK;
4114}
4115
183f24aa 4116/* `inline' is required to avoid gcc 4.1.2 build error */
2b740303 4117static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd)
b96375f7 4118{
529b930b 4119 if (vma_is_anonymous(vmf->vma)) {
292924b2 4120 if (userfaultfd_huge_pmd_wp(vmf->vma, orig_pmd))
529b930b 4121 return handle_userfault(vmf, VM_UFFD_WP);
82b0f8c3 4122 return do_huge_pmd_wp_page(vmf, orig_pmd);
529b930b 4123 }
327e9fd4
THV
4124 if (vmf->vma->vm_ops->huge_fault) {
4125 vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
4126
4127 if (!(ret & VM_FAULT_FALLBACK))
4128 return ret;
4129 }
af9e4d5f 4130
327e9fd4 4131 /* COW or write-notify handled on pte level: split pmd. */
82b0f8c3 4132 __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
af9e4d5f 4133
b96375f7
MW
4134 return VM_FAULT_FALLBACK;
4135}
4136
2b740303 4137static vm_fault_t create_huge_pud(struct vm_fault *vmf)
a00cc7d9 4138{
327e9fd4
THV
4139#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
4140 defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
a00cc7d9
MW
4141 /* No support for anonymous transparent PUD pages yet */
4142 if (vma_is_anonymous(vmf->vma))
327e9fd4
THV
4143 goto split;
4144 if (vmf->vma->vm_ops->huge_fault) {
4145 vm_fault_t ret = vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
4146
4147 if (!(ret & VM_FAULT_FALLBACK))
4148 return ret;
4149 }
4150split:
4151 /* COW or write-notify not handled on PUD level: split pud.*/
4152 __split_huge_pud(vmf->vma, vmf->pud, vmf->address);
a00cc7d9
MW
4153#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
4154 return VM_FAULT_FALLBACK;
4155}
4156
2b740303 4157static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
a00cc7d9
MW
4158{
4159#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4160 /* No support for anonymous transparent PUD pages yet */
4161 if (vma_is_anonymous(vmf->vma))
4162 return VM_FAULT_FALLBACK;
4163 if (vmf->vma->vm_ops->huge_fault)
c791ace1 4164 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
a00cc7d9
MW
4165#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
4166 return VM_FAULT_FALLBACK;
4167}
4168
1da177e4
LT
4169/*
4170 * These routines also need to handle stuff like marking pages dirty
4171 * and/or accessed for architectures that don't do it in hardware (most
4172 * RISC architectures). The early dirtying is also good on the i386.
4173 *
4174 * There is also a hook called "update_mmu_cache()" that architectures
4175 * with external mmu caches can use to update those (ie the Sparc or
4176 * PowerPC hashed page tables that act as extended TLBs).
4177 *
c1e8d7c6 4178 * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
7267ec00 4179 * concurrent faults).
9a95f3cf 4180 *
c1e8d7c6 4181 * The mmap_lock may have been released depending on flags and our return value.
7267ec00 4182 * See filemap_fault() and __lock_page_or_retry().
1da177e4 4183 */
2b740303 4184static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
1da177e4
LT
4185{
4186 pte_t entry;
4187
82b0f8c3 4188 if (unlikely(pmd_none(*vmf->pmd))) {
7267ec00
KS
4189 /*
4190 * Leave __pte_alloc() until later: because vm_ops->fault may
4191 * want to allocate huge page, and if we expose page table
4192 * for an instant, it will be difficult to retract from
4193 * concurrent faults and from rmap lookups.
4194 */
82b0f8c3 4195 vmf->pte = NULL;
7267ec00
KS
4196 } else {
4197 /* See comment in pte_alloc_one_map() */
d0f0931d 4198 if (pmd_devmap_trans_unstable(vmf->pmd))
7267ec00
KS
4199 return 0;
4200 /*
4201 * A regular pmd is established and it can't morph into a huge
4202 * pmd from under us anymore at this point because we hold the
c1e8d7c6 4203 * mmap_lock read mode and khugepaged takes it in write mode.
7267ec00
KS
4204 * So now it's safe to run pte_offset_map().
4205 */
82b0f8c3 4206 vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
2994302b 4207 vmf->orig_pte = *vmf->pte;
7267ec00
KS
4208
4209 /*
4210 * some architectures can have larger ptes than wordsize,
4211 * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
b03a0fe0
PM
4212 * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic
4213 * accesses. The code below just needs a consistent view
4214 * for the ifs and we later double check anyway with the
7267ec00
KS
4215 * ptl lock held. So here a barrier will do.
4216 */
4217 barrier();
2994302b 4218 if (pte_none(vmf->orig_pte)) {
82b0f8c3
JK
4219 pte_unmap(vmf->pte);
4220 vmf->pte = NULL;
65500d23 4221 }
1da177e4
LT
4222 }
4223
82b0f8c3
JK
4224 if (!vmf->pte) {
4225 if (vma_is_anonymous(vmf->vma))
4226 return do_anonymous_page(vmf);
7267ec00 4227 else
82b0f8c3 4228 return do_fault(vmf);
7267ec00
KS
4229 }
4230
2994302b
JK
4231 if (!pte_present(vmf->orig_pte))
4232 return do_swap_page(vmf);
7267ec00 4233
2994302b
JK
4234 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
4235 return do_numa_page(vmf);
d10e63f2 4236
82b0f8c3
JK
4237 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
4238 spin_lock(vmf->ptl);
2994302b 4239 entry = vmf->orig_pte;
7df67697
BM
4240 if (unlikely(!pte_same(*vmf->pte, entry))) {
4241 update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
8f4e2101 4242 goto unlock;
7df67697 4243 }
82b0f8c3 4244 if (vmf->flags & FAULT_FLAG_WRITE) {
f6f37321 4245 if (!pte_write(entry))
2994302b 4246 return do_wp_page(vmf);
1da177e4
LT
4247 entry = pte_mkdirty(entry);
4248 }
4249 entry = pte_mkyoung(entry);
82b0f8c3
JK
4250 if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
4251 vmf->flags & FAULT_FLAG_WRITE)) {
4252 update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
1a44e149
AA
4253 } else {
4254 /*
4255 * This is needed only for protection faults but the arch code
4256 * is not yet telling us if this is a protection fault or not.
4257 * This still avoids useless tlb flushes for .text page faults
4258 * with threads.
4259 */
82b0f8c3
JK
4260 if (vmf->flags & FAULT_FLAG_WRITE)
4261 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
1a44e149 4262 }
8f4e2101 4263unlock:
82b0f8c3 4264 pte_unmap_unlock(vmf->pte, vmf->ptl);
83c54070 4265 return 0;
1da177e4
LT
4266}
4267
4268/*
4269 * By the time we get here, we already hold the mm semaphore
9a95f3cf 4270 *
c1e8d7c6 4271 * The mmap_lock may have been released depending on flags and our
9a95f3cf 4272 * return value. See filemap_fault() and __lock_page_or_retry().
1da177e4 4273 */
2b740303
SJ
4274static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
4275 unsigned long address, unsigned int flags)
1da177e4 4276{
82b0f8c3 4277 struct vm_fault vmf = {
bae473a4 4278 .vma = vma,
1a29d85e 4279 .address = address & PAGE_MASK,
bae473a4 4280 .flags = flags,
0721ec8b 4281 .pgoff = linear_page_index(vma, address),
667240e0 4282 .gfp_mask = __get_fault_gfp_mask(vma),
bae473a4 4283 };
fde26bed 4284 unsigned int dirty = flags & FAULT_FLAG_WRITE;
dcddffd4 4285 struct mm_struct *mm = vma->vm_mm;
1da177e4 4286 pgd_t *pgd;
c2febafc 4287 p4d_t *p4d;
2b740303 4288 vm_fault_t ret;
1da177e4 4289
1da177e4 4290 pgd = pgd_offset(mm, address);
c2febafc
KS
4291 p4d = p4d_alloc(mm, pgd, address);
4292 if (!p4d)
4293 return VM_FAULT_OOM;
a00cc7d9 4294
c2febafc 4295 vmf.pud = pud_alloc(mm, p4d, address);
a00cc7d9 4296 if (!vmf.pud)
c74df32c 4297 return VM_FAULT_OOM;
625110b5 4298retry_pud:
7635d9cb 4299 if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {
a00cc7d9
MW
4300 ret = create_huge_pud(&vmf);
4301 if (!(ret & VM_FAULT_FALLBACK))
4302 return ret;
4303 } else {
4304 pud_t orig_pud = *vmf.pud;
4305
4306 barrier();
4307 if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
a00cc7d9 4308
a00cc7d9
MW
4309 /* NUMA case for anonymous PUDs would go here */
4310
f6f37321 4311 if (dirty && !pud_write(orig_pud)) {
a00cc7d9
MW
4312 ret = wp_huge_pud(&vmf, orig_pud);
4313 if (!(ret & VM_FAULT_FALLBACK))
4314 return ret;
4315 } else {
4316 huge_pud_set_accessed(&vmf, orig_pud);
4317 return 0;
4318 }
4319 }
4320 }
4321
4322 vmf.pmd = pmd_alloc(mm, vmf.pud, address);
82b0f8c3 4323 if (!vmf.pmd)
c74df32c 4324 return VM_FAULT_OOM;
625110b5
TH
4325
4326 /* Huge pud page fault raced with pmd_alloc? */
4327 if (pud_trans_unstable(vmf.pud))
4328 goto retry_pud;
4329
7635d9cb 4330 if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {
a2d58167 4331 ret = create_huge_pmd(&vmf);
c0292554
KS
4332 if (!(ret & VM_FAULT_FALLBACK))
4333 return ret;
71e3aac0 4334 } else {
82b0f8c3 4335 pmd_t orig_pmd = *vmf.pmd;
1f1d06c3 4336
71e3aac0 4337 barrier();
84c3fc4e
ZY
4338 if (unlikely(is_swap_pmd(orig_pmd))) {
4339 VM_BUG_ON(thp_migration_supported() &&
4340 !is_pmd_migration_entry(orig_pmd));
4341 if (is_pmd_migration_entry(orig_pmd))
4342 pmd_migration_entry_wait(mm, vmf.pmd);
4343 return 0;
4344 }
5c7fb56e 4345 if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) {
38e08854 4346 if (pmd_protnone(orig_pmd) && vma_is_accessible(vma))
82b0f8c3 4347 return do_huge_pmd_numa_page(&vmf, orig_pmd);
d10e63f2 4348
f6f37321 4349 if (dirty && !pmd_write(orig_pmd)) {
82b0f8c3 4350 ret = wp_huge_pmd(&vmf, orig_pmd);
9845cbbd
KS
4351 if (!(ret & VM_FAULT_FALLBACK))
4352 return ret;
a1dd450b 4353 } else {
82b0f8c3 4354 huge_pmd_set_accessed(&vmf, orig_pmd);
9845cbbd 4355 return 0;
1f1d06c3 4356 }
71e3aac0
AA
4357 }
4358 }
4359
82b0f8c3 4360 return handle_pte_fault(&vmf);
1da177e4
LT
4361}
4362
9a95f3cf
PC
4363/*
4364 * By the time we get here, we already hold the mm semaphore
4365 *
c1e8d7c6 4366 * The mmap_lock may have been released depending on flags and our
9a95f3cf
PC
4367 * return value. See filemap_fault() and __lock_page_or_retry().
4368 */
2b740303 4369vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
dcddffd4 4370 unsigned int flags)
519e5247 4371{
2b740303 4372 vm_fault_t ret;
519e5247
JW
4373
4374 __set_current_state(TASK_RUNNING);
4375
4376 count_vm_event(PGFAULT);
2262185c 4377 count_memcg_event_mm(vma->vm_mm, PGFAULT);
519e5247
JW
4378
4379 /* do counter updates before entering really critical section. */
4380 check_sync_rss_stat(current);
4381
de0c799b
LD
4382 if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
4383 flags & FAULT_FLAG_INSTRUCTION,
4384 flags & FAULT_FLAG_REMOTE))
4385 return VM_FAULT_SIGSEGV;
4386
519e5247
JW
4387 /*
4388 * Enable the memcg OOM handling for faults triggered in user
4389 * space. Kernel faults are handled more gracefully.
4390 */
4391 if (flags & FAULT_FLAG_USER)
29ef680a 4392 mem_cgroup_enter_user_fault();
519e5247 4393
bae473a4
KS
4394 if (unlikely(is_vm_hugetlb_page(vma)))
4395 ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
4396 else
4397 ret = __handle_mm_fault(vma, address, flags);
519e5247 4398
49426420 4399 if (flags & FAULT_FLAG_USER) {
29ef680a 4400 mem_cgroup_exit_user_fault();
166f61b9
TH
4401 /*
4402 * The task may have entered a memcg OOM situation but
4403 * if the allocation error was handled gracefully (no
4404 * VM_FAULT_OOM), there is no need to kill anything.
4405 * Just clean up the OOM state peacefully.
4406 */
4407 if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
4408 mem_cgroup_oom_synchronize(false);
49426420 4409 }
3812c8c8 4410
519e5247
JW
4411 return ret;
4412}
e1d6d01a 4413EXPORT_SYMBOL_GPL(handle_mm_fault);
519e5247 4414
90eceff1
KS
4415#ifndef __PAGETABLE_P4D_FOLDED
4416/*
4417 * Allocate p4d page table.
4418 * We've already handled the fast-path in-line.
4419 */
4420int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
4421{
4422 p4d_t *new = p4d_alloc_one(mm, address);
4423 if (!new)
4424 return -ENOMEM;
4425
4426 smp_wmb(); /* See comment in __pte_alloc */
4427
4428 spin_lock(&mm->page_table_lock);
4429 if (pgd_present(*pgd)) /* Another has populated it */
4430 p4d_free(mm, new);
4431 else
4432 pgd_populate(mm, pgd, new);
4433 spin_unlock(&mm->page_table_lock);
4434 return 0;
4435}
4436#endif /* __PAGETABLE_P4D_FOLDED */
4437
1da177e4
LT
4438#ifndef __PAGETABLE_PUD_FOLDED
4439/*
4440 * Allocate page upper directory.
872fec16 4441 * We've already handled the fast-path in-line.
1da177e4 4442 */
c2febafc 4443int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
1da177e4 4444{
c74df32c
HD
4445 pud_t *new = pud_alloc_one(mm, address);
4446 if (!new)
1bb3630e 4447 return -ENOMEM;
1da177e4 4448
362a61ad
NP
4449 smp_wmb(); /* See comment in __pte_alloc */
4450
872fec16 4451 spin_lock(&mm->page_table_lock);
b4e98d9a
KS
4452 if (!p4d_present(*p4d)) {
4453 mm_inc_nr_puds(mm);
c2febafc 4454 p4d_populate(mm, p4d, new);
b4e98d9a 4455 } else /* Another has populated it */
5e541973 4456 pud_free(mm, new);
c74df32c 4457 spin_unlock(&mm->page_table_lock);
1bb3630e 4458 return 0;
1da177e4
LT
4459}
4460#endif /* __PAGETABLE_PUD_FOLDED */
4461
4462#ifndef __PAGETABLE_PMD_FOLDED
4463/*
4464 * Allocate page middle directory.
872fec16 4465 * We've already handled the fast-path in-line.
1da177e4 4466 */
1bb3630e 4467int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1da177e4 4468{
a00cc7d9 4469 spinlock_t *ptl;
c74df32c
HD
4470 pmd_t *new = pmd_alloc_one(mm, address);
4471 if (!new)
1bb3630e 4472 return -ENOMEM;
1da177e4 4473
362a61ad
NP
4474 smp_wmb(); /* See comment in __pte_alloc */
4475
a00cc7d9 4476 ptl = pud_lock(mm, pud);
dc6c9a35
KS
4477 if (!pud_present(*pud)) {
4478 mm_inc_nr_pmds(mm);
1bb3630e 4479 pud_populate(mm, pud, new);
dc6c9a35 4480 } else /* Another has populated it */
5e541973 4481 pmd_free(mm, new);
a00cc7d9 4482 spin_unlock(ptl);
1bb3630e 4483 return 0;
e0f39591 4484}
1da177e4
LT
4485#endif /* __PAGETABLE_PMD_FOLDED */
4486
09796395 4487static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
ac46d4f3 4488 struct mmu_notifier_range *range,
a4d1a885 4489 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
f8ad0f49
JW
4490{
4491 pgd_t *pgd;
c2febafc 4492 p4d_t *p4d;
f8ad0f49
JW
4493 pud_t *pud;
4494 pmd_t *pmd;
4495 pte_t *ptep;
4496
4497 pgd = pgd_offset(mm, address);
4498 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
4499 goto out;
4500
c2febafc
KS
4501 p4d = p4d_offset(pgd, address);
4502 if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
4503 goto out;
4504
4505 pud = pud_offset(p4d, address);
f8ad0f49
JW
4506 if (pud_none(*pud) || unlikely(pud_bad(*pud)))
4507 goto out;
4508
4509 pmd = pmd_offset(pud, address);
f66055ab 4510 VM_BUG_ON(pmd_trans_huge(*pmd));
f8ad0f49 4511
09796395
RZ
4512 if (pmd_huge(*pmd)) {
4513 if (!pmdpp)
4514 goto out;
4515
ac46d4f3 4516 if (range) {
7269f999 4517 mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0,
6f4f13e8
JG
4518 NULL, mm, address & PMD_MASK,
4519 (address & PMD_MASK) + PMD_SIZE);
ac46d4f3 4520 mmu_notifier_invalidate_range_start(range);
a4d1a885 4521 }
09796395
RZ
4522 *ptlp = pmd_lock(mm, pmd);
4523 if (pmd_huge(*pmd)) {
4524 *pmdpp = pmd;
4525 return 0;
4526 }
4527 spin_unlock(*ptlp);
ac46d4f3
JG
4528 if (range)
4529 mmu_notifier_invalidate_range_end(range);
09796395
RZ
4530 }
4531
4532 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
f8ad0f49
JW
4533 goto out;
4534
ac46d4f3 4535 if (range) {
7269f999 4536 mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0, NULL, mm,
6f4f13e8
JG
4537 address & PAGE_MASK,
4538 (address & PAGE_MASK) + PAGE_SIZE);
ac46d4f3 4539 mmu_notifier_invalidate_range_start(range);
a4d1a885 4540 }
f8ad0f49 4541 ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
f8ad0f49
JW
4542 if (!pte_present(*ptep))
4543 goto unlock;
4544 *ptepp = ptep;
4545 return 0;
4546unlock:
4547 pte_unmap_unlock(ptep, *ptlp);
ac46d4f3
JG
4548 if (range)
4549 mmu_notifier_invalidate_range_end(range);
f8ad0f49
JW
4550out:
4551 return -EINVAL;
4552}
4553
f729c8c9
RZ
4554static inline int follow_pte(struct mm_struct *mm, unsigned long address,
4555 pte_t **ptepp, spinlock_t **ptlp)
1b36ba81
NK
4556{
4557 int res;
4558
4559 /* (void) is needed to make gcc happy */
4560 (void) __cond_lock(*ptlp,
ac46d4f3 4561 !(res = __follow_pte_pmd(mm, address, NULL,
a4d1a885 4562 ptepp, NULL, ptlp)));
09796395
RZ
4563 return res;
4564}
4565
4566int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
ac46d4f3
JG
4567 struct mmu_notifier_range *range,
4568 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
09796395
RZ
4569{
4570 int res;
4571
4572 /* (void) is needed to make gcc happy */
4573 (void) __cond_lock(*ptlp,
ac46d4f3 4574 !(res = __follow_pte_pmd(mm, address, range,
a4d1a885 4575 ptepp, pmdpp, ptlp)));
1b36ba81
NK
4576 return res;
4577}
09796395 4578EXPORT_SYMBOL(follow_pte_pmd);
1b36ba81 4579
3b6748e2
JW
4580/**
4581 * follow_pfn - look up PFN at a user virtual address
4582 * @vma: memory mapping
4583 * @address: user virtual address
4584 * @pfn: location to store found PFN
4585 *
4586 * Only IO mappings and raw PFN mappings are allowed.
4587 *
a862f68a 4588 * Return: zero and the pfn at @pfn on success, -ve otherwise.
3b6748e2
JW
4589 */
4590int follow_pfn(struct vm_area_struct *vma, unsigned long address,
4591 unsigned long *pfn)
4592{
4593 int ret = -EINVAL;
4594 spinlock_t *ptl;
4595 pte_t *ptep;
4596
4597 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
4598 return ret;
4599
4600 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
4601 if (ret)
4602 return ret;
4603 *pfn = pte_pfn(*ptep);
4604 pte_unmap_unlock(ptep, ptl);
4605 return 0;
4606}
4607EXPORT_SYMBOL(follow_pfn);
4608
28b2ee20 4609#ifdef CONFIG_HAVE_IOREMAP_PROT
d87fe660 4610int follow_phys(struct vm_area_struct *vma,
4611 unsigned long address, unsigned int flags,
4612 unsigned long *prot, resource_size_t *phys)
28b2ee20 4613{
03668a4d 4614 int ret = -EINVAL;
28b2ee20
RR
4615 pte_t *ptep, pte;
4616 spinlock_t *ptl;
28b2ee20 4617
d87fe660 4618 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
4619 goto out;
28b2ee20 4620
03668a4d 4621 if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
d87fe660 4622 goto out;
28b2ee20 4623 pte = *ptep;
03668a4d 4624
f6f37321 4625 if ((flags & FOLL_WRITE) && !pte_write(pte))
28b2ee20 4626 goto unlock;
28b2ee20
RR
4627
4628 *prot = pgprot_val(pte_pgprot(pte));
03668a4d 4629 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
28b2ee20 4630
03668a4d 4631 ret = 0;
28b2ee20
RR
4632unlock:
4633 pte_unmap_unlock(ptep, ptl);
4634out:
d87fe660 4635 return ret;
28b2ee20
RR
4636}
4637
4638int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
4639 void *buf, int len, int write)
4640{
4641 resource_size_t phys_addr;
4642 unsigned long prot = 0;
2bc7273b 4643 void __iomem *maddr;
28b2ee20
RR
4644 int offset = addr & (PAGE_SIZE-1);
4645
d87fe660 4646 if (follow_phys(vma, addr, write, &prot, &phys_addr))
28b2ee20
RR
4647 return -EINVAL;
4648
9cb12d7b 4649 maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
24eee1e4 4650 if (!maddr)
4651 return -ENOMEM;
4652
28b2ee20
RR
4653 if (write)
4654 memcpy_toio(maddr + offset, buf, len);
4655 else
4656 memcpy_fromio(buf, maddr + offset, len);
4657 iounmap(maddr);
4658
4659 return len;
4660}
5a73633e 4661EXPORT_SYMBOL_GPL(generic_access_phys);
28b2ee20
RR
4662#endif
4663
0ec76a11 4664/*
206cb636
SW
4665 * Access another process' address space as given in mm. If non-NULL, use the
4666 * given task for page fault accounting.
0ec76a11 4667 */
84d77d3f 4668int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
442486ec 4669 unsigned long addr, void *buf, int len, unsigned int gup_flags)
0ec76a11 4670{
0ec76a11 4671 struct vm_area_struct *vma;
0ec76a11 4672 void *old_buf = buf;
442486ec 4673 int write = gup_flags & FOLL_WRITE;
0ec76a11 4674
d8ed45c5 4675 if (mmap_read_lock_killable(mm))
1e426fe2
KK
4676 return 0;
4677
183ff22b 4678 /* ignore errors, just check how much was successfully transferred */
0ec76a11
DH
4679 while (len) {
4680 int bytes, ret, offset;
4681 void *maddr;
28b2ee20 4682 struct page *page = NULL;
0ec76a11 4683
1e987790 4684 ret = get_user_pages_remote(tsk, mm, addr, 1,
5b56d49f 4685 gup_flags, &page, &vma, NULL);
28b2ee20 4686 if (ret <= 0) {
dbffcd03
RR
4687#ifndef CONFIG_HAVE_IOREMAP_PROT
4688 break;
4689#else
28b2ee20
RR
4690 /*
4691 * Check if this is a VM_IO | VM_PFNMAP VMA, which
4692 * we can access using slightly different code.
4693 */
28b2ee20 4694 vma = find_vma(mm, addr);
fe936dfc 4695 if (!vma || vma->vm_start > addr)
28b2ee20
RR
4696 break;
4697 if (vma->vm_ops && vma->vm_ops->access)
4698 ret = vma->vm_ops->access(vma, addr, buf,
4699 len, write);
4700 if (ret <= 0)
28b2ee20
RR
4701 break;
4702 bytes = ret;
dbffcd03 4703#endif
0ec76a11 4704 } else {
28b2ee20
RR
4705 bytes = len;
4706 offset = addr & (PAGE_SIZE-1);
4707 if (bytes > PAGE_SIZE-offset)
4708 bytes = PAGE_SIZE-offset;
4709
4710 maddr = kmap(page);
4711 if (write) {
4712 copy_to_user_page(vma, page, addr,
4713 maddr + offset, buf, bytes);
4714 set_page_dirty_lock(page);
4715 } else {
4716 copy_from_user_page(vma, page, addr,
4717 buf, maddr + offset, bytes);
4718 }
4719 kunmap(page);
09cbfeaf 4720 put_page(page);
0ec76a11 4721 }
0ec76a11
DH
4722 len -= bytes;
4723 buf += bytes;
4724 addr += bytes;
4725 }
d8ed45c5 4726 mmap_read_unlock(mm);
0ec76a11
DH
4727
4728 return buf - old_buf;
4729}
03252919 4730
5ddd36b9 4731/**
ae91dbfc 4732 * access_remote_vm - access another process' address space
5ddd36b9
SW
4733 * @mm: the mm_struct of the target address space
4734 * @addr: start address to access
4735 * @buf: source or destination buffer
4736 * @len: number of bytes to transfer
6347e8d5 4737 * @gup_flags: flags modifying lookup behaviour
5ddd36b9
SW
4738 *
4739 * The caller must hold a reference on @mm.
a862f68a
MR
4740 *
4741 * Return: number of bytes copied from source to destination.
5ddd36b9
SW
4742 */
4743int access_remote_vm(struct mm_struct *mm, unsigned long addr,
6347e8d5 4744 void *buf, int len, unsigned int gup_flags)
5ddd36b9 4745{
6347e8d5 4746 return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
5ddd36b9
SW
4747}
4748
206cb636
SW
4749/*
4750 * Access another process' address space.
4751 * Source/target buffer must be kernel space,
4752 * Do not walk the page table directly, use get_user_pages
4753 */
4754int access_process_vm(struct task_struct *tsk, unsigned long addr,
f307ab6d 4755 void *buf, int len, unsigned int gup_flags)
206cb636
SW
4756{
4757 struct mm_struct *mm;
4758 int ret;
4759
4760 mm = get_task_mm(tsk);
4761 if (!mm)
4762 return 0;
4763
f307ab6d 4764 ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
442486ec 4765
206cb636
SW
4766 mmput(mm);
4767
4768 return ret;
4769}
fcd35857 4770EXPORT_SYMBOL_GPL(access_process_vm);
206cb636 4771
03252919
AK
4772/*
4773 * Print the name of a VMA.
4774 */
4775void print_vma_addr(char *prefix, unsigned long ip)
4776{
4777 struct mm_struct *mm = current->mm;
4778 struct vm_area_struct *vma;
4779
e8bff74a 4780 /*
0a7f682d 4781 * we might be running from an atomic context so we cannot sleep
e8bff74a 4782 */
d8ed45c5 4783 if (!mmap_read_trylock(mm))
e8bff74a
IM
4784 return;
4785
03252919
AK
4786 vma = find_vma(mm, ip);
4787 if (vma && vma->vm_file) {
4788 struct file *f = vma->vm_file;
0a7f682d 4789 char *buf = (char *)__get_free_page(GFP_NOWAIT);
03252919 4790 if (buf) {
2fbc57c5 4791 char *p;
03252919 4792
9bf39ab2 4793 p = file_path(f, buf, PAGE_SIZE);
03252919
AK
4794 if (IS_ERR(p))
4795 p = "?";
2fbc57c5 4796 printk("%s%s[%lx+%lx]", prefix, kbasename(p),
03252919
AK
4797 vma->vm_start,
4798 vma->vm_end - vma->vm_start);
4799 free_page((unsigned long)buf);
4800 }
4801 }
d8ed45c5 4802 mmap_read_unlock(mm);
03252919 4803}
3ee1afa3 4804
662bbcb2 4805#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
9ec23531 4806void __might_fault(const char *file, int line)
3ee1afa3 4807{
95156f00
PZ
4808 /*
4809 * Some code (nfs/sunrpc) uses socket ops on kernel memory while
c1e8d7c6 4810 * holding the mmap_lock, this is safe because kernel memory doesn't
95156f00
PZ
4811 * get paged out, therefore we'll never actually fault, and the
4812 * below annotations will generate false positives.
4813 */
db68ce10 4814 if (uaccess_kernel())
95156f00 4815 return;
9ec23531 4816 if (pagefault_disabled())
662bbcb2 4817 return;
9ec23531
DH
4818 __might_sleep(file, line, 0);
4819#if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
662bbcb2 4820 if (current->mm)
da1c55f1 4821 might_lock_read(&current->mm->mmap_lock);
9ec23531 4822#endif
3ee1afa3 4823}
9ec23531 4824EXPORT_SYMBOL(__might_fault);
3ee1afa3 4825#endif
47ad8475
AA
4826
4827#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
c6ddfb6c
HY
4828/*
4829 * Process all subpages of the specified huge page with the specified
4830 * operation. The target subpage will be processed last to keep its
4831 * cache lines hot.
4832 */
4833static inline void process_huge_page(
4834 unsigned long addr_hint, unsigned int pages_per_huge_page,
4835 void (*process_subpage)(unsigned long addr, int idx, void *arg),
4836 void *arg)
47ad8475 4837{
c79b57e4
HY
4838 int i, n, base, l;
4839 unsigned long addr = addr_hint &
4840 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
47ad8475 4841
c6ddfb6c 4842 /* Process target subpage last to keep its cache lines hot */
47ad8475 4843 might_sleep();
c79b57e4
HY
4844 n = (addr_hint - addr) / PAGE_SIZE;
4845 if (2 * n <= pages_per_huge_page) {
c6ddfb6c 4846 /* If target subpage in first half of huge page */
c79b57e4
HY
4847 base = 0;
4848 l = n;
c6ddfb6c 4849 /* Process subpages at the end of huge page */
c79b57e4
HY
4850 for (i = pages_per_huge_page - 1; i >= 2 * n; i--) {
4851 cond_resched();
c6ddfb6c 4852 process_subpage(addr + i * PAGE_SIZE, i, arg);
c79b57e4
HY
4853 }
4854 } else {
c6ddfb6c 4855 /* If target subpage in second half of huge page */
c79b57e4
HY
4856 base = pages_per_huge_page - 2 * (pages_per_huge_page - n);
4857 l = pages_per_huge_page - n;
c6ddfb6c 4858 /* Process subpages at the begin of huge page */
c79b57e4
HY
4859 for (i = 0; i < base; i++) {
4860 cond_resched();
c6ddfb6c 4861 process_subpage(addr + i * PAGE_SIZE, i, arg);
c79b57e4
HY
4862 }
4863 }
4864 /*
c6ddfb6c
HY
4865 * Process remaining subpages in left-right-left-right pattern
4866 * towards the target subpage
c79b57e4
HY
4867 */
4868 for (i = 0; i < l; i++) {
4869 int left_idx = base + i;
4870 int right_idx = base + 2 * l - 1 - i;
4871
4872 cond_resched();
c6ddfb6c 4873 process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
47ad8475 4874 cond_resched();
c6ddfb6c 4875 process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
47ad8475
AA
4876 }
4877}
4878
c6ddfb6c
HY
4879static void clear_gigantic_page(struct page *page,
4880 unsigned long addr,
4881 unsigned int pages_per_huge_page)
4882{
4883 int i;
4884 struct page *p = page;
4885
4886 might_sleep();
4887 for (i = 0; i < pages_per_huge_page;
4888 i++, p = mem_map_next(p, page, i)) {
4889 cond_resched();
4890 clear_user_highpage(p, addr + i * PAGE_SIZE);
4891 }
4892}
4893
4894static void clear_subpage(unsigned long addr, int idx, void *arg)
4895{
4896 struct page *page = arg;
4897
4898 clear_user_highpage(page + idx, addr);
4899}
4900
4901void clear_huge_page(struct page *page,
4902 unsigned long addr_hint, unsigned int pages_per_huge_page)
4903{
4904 unsigned long addr = addr_hint &
4905 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
4906
4907 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
4908 clear_gigantic_page(page, addr, pages_per_huge_page);
4909 return;
4910 }
4911
4912 process_huge_page(addr_hint, pages_per_huge_page, clear_subpage, page);
4913}
4914
47ad8475
AA
4915static void copy_user_gigantic_page(struct page *dst, struct page *src,
4916 unsigned long addr,
4917 struct vm_area_struct *vma,
4918 unsigned int pages_per_huge_page)
4919{
4920 int i;
4921 struct page *dst_base = dst;
4922 struct page *src_base = src;
4923
4924 for (i = 0; i < pages_per_huge_page; ) {
4925 cond_resched();
4926 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
4927
4928 i++;
4929 dst = mem_map_next(dst, dst_base, i);
4930 src = mem_map_next(src, src_base, i);
4931 }
4932}
4933
c9f4cd71
HY
4934struct copy_subpage_arg {
4935 struct page *dst;
4936 struct page *src;
4937 struct vm_area_struct *vma;
4938};
4939
4940static void copy_subpage(unsigned long addr, int idx, void *arg)
4941{
4942 struct copy_subpage_arg *copy_arg = arg;
4943
4944 copy_user_highpage(copy_arg->dst + idx, copy_arg->src + idx,
4945 addr, copy_arg->vma);
4946}
4947
47ad8475 4948void copy_user_huge_page(struct page *dst, struct page *src,
c9f4cd71 4949 unsigned long addr_hint, struct vm_area_struct *vma,
47ad8475
AA
4950 unsigned int pages_per_huge_page)
4951{
c9f4cd71
HY
4952 unsigned long addr = addr_hint &
4953 ~(((unsigned long)pages_per_huge_page << PAGE_SHIFT) - 1);
4954 struct copy_subpage_arg arg = {
4955 .dst = dst,
4956 .src = src,
4957 .vma = vma,
4958 };
47ad8475
AA
4959
4960 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
4961 copy_user_gigantic_page(dst, src, addr, vma,
4962 pages_per_huge_page);
4963 return;
4964 }
4965
c9f4cd71 4966 process_huge_page(addr_hint, pages_per_huge_page, copy_subpage, &arg);
47ad8475 4967}
fa4d75c1
MK
4968
4969long copy_huge_page_from_user(struct page *dst_page,
4970 const void __user *usr_src,
810a56b9
MK
4971 unsigned int pages_per_huge_page,
4972 bool allow_pagefault)
fa4d75c1
MK
4973{
4974 void *src = (void *)usr_src;
4975 void *page_kaddr;
4976 unsigned long i, rc = 0;
4977 unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;
4978
4979 for (i = 0; i < pages_per_huge_page; i++) {
810a56b9
MK
4980 if (allow_pagefault)
4981 page_kaddr = kmap(dst_page + i);
4982 else
4983 page_kaddr = kmap_atomic(dst_page + i);
fa4d75c1
MK
4984 rc = copy_from_user(page_kaddr,
4985 (const void __user *)(src + i * PAGE_SIZE),
4986 PAGE_SIZE);
810a56b9
MK
4987 if (allow_pagefault)
4988 kunmap(dst_page + i);
4989 else
4990 kunmap_atomic(page_kaddr);
fa4d75c1
MK
4991
4992 ret_val -= (PAGE_SIZE - rc);
4993 if (rc)
4994 break;
4995
4996 cond_resched();
4997 }
4998 return ret_val;
4999}
47ad8475 5000#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
49076ec2 5001
40b64acd 5002#if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
b35f1819
KS
5003
5004static struct kmem_cache *page_ptl_cachep;
5005
5006void __init ptlock_cache_init(void)
5007{
5008 page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
5009 SLAB_PANIC, NULL);
5010}
5011
539edb58 5012bool ptlock_alloc(struct page *page)
49076ec2
KS
5013{
5014 spinlock_t *ptl;
5015
b35f1819 5016 ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
49076ec2
KS
5017 if (!ptl)
5018 return false;
539edb58 5019 page->ptl = ptl;
49076ec2
KS
5020 return true;
5021}
5022
539edb58 5023void ptlock_free(struct page *page)
49076ec2 5024{
b35f1819 5025 kmem_cache_free(page_ptl_cachep, page->ptl);
49076ec2
KS
5026}
5027#endif