]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - mm/memory.c
netfilter: use nf_conntrack_helpers_register when possible
[mirror_ubuntu-artful-kernel.git] / mm / memory.c
CommitLineData
1da177e4
LT
1/*
2 * linux/mm/memory.c
3 *
4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5 */
6
7/*
8 * demand-loading started 01.12.91 - seems it is high on the list of
9 * things wanted, and it should be easy to implement. - Linus
10 */
11
12/*
13 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
14 * pages started 02.12.91, seems to work. - Linus.
15 *
16 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
17 * would have taken more than the 6M I have free, but it worked well as
18 * far as I could see.
19 *
20 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
21 */
22
23/*
24 * Real VM (paging to/from disk) started 18.12.91. Much more work and
25 * thought has to go into this. Oh, well..
26 * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
27 * Found it. Everything seems to work now.
28 * 20.12.91 - Ok, making the swap-device changeable like the root.
29 */
30
31/*
32 * 05.04.94 - Multi-page memory management added for v1.1.
166f61b9 33 * Idea by Alex Bligh (alex@cconcepts.co.uk)
1da177e4
LT
34 *
35 * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
36 * (Gerhard.Wichert@pdb.siemens.de)
37 *
38 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
39 */
40
41#include <linux/kernel_stat.h>
42#include <linux/mm.h>
6e84f315 43#include <linux/sched/mm.h>
f7ccbae4 44#include <linux/sched/coredump.h>
6a3827d7 45#include <linux/sched/numa_balancing.h>
29930025 46#include <linux/sched/task.h>
1da177e4
LT
47#include <linux/hugetlb.h>
48#include <linux/mman.h>
49#include <linux/swap.h>
50#include <linux/highmem.h>
51#include <linux/pagemap.h>
9a840895 52#include <linux/ksm.h>
1da177e4 53#include <linux/rmap.h>
b95f1b31 54#include <linux/export.h>
0ff92245 55#include <linux/delayacct.h>
1da177e4 56#include <linux/init.h>
01c8f1c4 57#include <linux/pfn_t.h>
edc79b2a 58#include <linux/writeback.h>
8a9f3ccd 59#include <linux/memcontrol.h>
cddb8a5c 60#include <linux/mmu_notifier.h>
3dc14741
HD
61#include <linux/kallsyms.h>
62#include <linux/swapops.h>
63#include <linux/elf.h>
5a0e3ad6 64#include <linux/gfp.h>
4daae3b4 65#include <linux/migrate.h>
2fbc57c5 66#include <linux/string.h>
0abdd7a8 67#include <linux/dma-debug.h>
1592eef0 68#include <linux/debugfs.h>
6b251fc9 69#include <linux/userfaultfd_k.h>
bc2466e4 70#include <linux/dax.h>
1da177e4 71
6952b61d 72#include <asm/io.h>
33a709b2 73#include <asm/mmu_context.h>
1da177e4 74#include <asm/pgalloc.h>
7c0f6ba6 75#include <linux/uaccess.h>
1da177e4
LT
76#include <asm/tlb.h>
77#include <asm/tlbflush.h>
78#include <asm/pgtable.h>
79
42b77728
JB
80#include "internal.h"
81
90572890
PZ
82#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
83#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
75980e97
PZ
84#endif
85
d41dee36 86#ifndef CONFIG_NEED_MULTIPLE_NODES
1da177e4
LT
87/* use the per-pgdat data instead for discontigmem - mbligh */
88unsigned long max_mapnr;
1da177e4 89EXPORT_SYMBOL(max_mapnr);
166f61b9
TH
90
91struct page *mem_map;
1da177e4
LT
92EXPORT_SYMBOL(mem_map);
93#endif
94
1da177e4
LT
95/*
96 * A number of key systems in x86 including ioremap() rely on the assumption
97 * that high_memory defines the upper bound on direct map memory, then end
98 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
99 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
100 * and ZONE_HIGHMEM.
101 */
166f61b9 102void *high_memory;
1da177e4 103EXPORT_SYMBOL(high_memory);
1da177e4 104
32a93233
IM
105/*
106 * Randomize the address space (stacks, mmaps, brk, etc.).
107 *
108 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
109 * as ancient (libc5 based) binaries can segfault. )
110 */
111int randomize_va_space __read_mostly =
112#ifdef CONFIG_COMPAT_BRK
113 1;
114#else
115 2;
116#endif
a62eaf15
AK
117
118static int __init disable_randmaps(char *s)
119{
120 randomize_va_space = 0;
9b41046c 121 return 1;
a62eaf15
AK
122}
123__setup("norandmaps", disable_randmaps);
124
62eede62 125unsigned long zero_pfn __read_mostly;
0b70068e
AB
126EXPORT_SYMBOL(zero_pfn);
127
166f61b9
TH
128unsigned long highest_memmap_pfn __read_mostly;
129
a13ea5b7
HD
130/*
131 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
132 */
133static int __init init_zero_pfn(void)
134{
135 zero_pfn = page_to_pfn(ZERO_PAGE(0));
136 return 0;
137}
138core_initcall(init_zero_pfn);
a62eaf15 139
d559db08 140
34e55232
KH
141#if defined(SPLIT_RSS_COUNTING)
142
ea48cf78 143void sync_mm_rss(struct mm_struct *mm)
34e55232
KH
144{
145 int i;
146
147 for (i = 0; i < NR_MM_COUNTERS; i++) {
05af2e10
DR
148 if (current->rss_stat.count[i]) {
149 add_mm_counter(mm, i, current->rss_stat.count[i]);
150 current->rss_stat.count[i] = 0;
34e55232
KH
151 }
152 }
05af2e10 153 current->rss_stat.events = 0;
34e55232
KH
154}
155
156static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
157{
158 struct task_struct *task = current;
159
160 if (likely(task->mm == mm))
161 task->rss_stat.count[member] += val;
162 else
163 add_mm_counter(mm, member, val);
164}
165#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
166#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)
167
168/* sync counter once per 64 page faults */
169#define TASK_RSS_EVENTS_THRESH (64)
170static void check_sync_rss_stat(struct task_struct *task)
171{
172 if (unlikely(task != current))
173 return;
174 if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
ea48cf78 175 sync_mm_rss(task->mm);
34e55232 176}
9547d01b 177#else /* SPLIT_RSS_COUNTING */
34e55232
KH
178
179#define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)
180#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member)
181
182static void check_sync_rss_stat(struct task_struct *task)
183{
184}
185
9547d01b
PZ
186#endif /* SPLIT_RSS_COUNTING */
187
188#ifdef HAVE_GENERIC_MMU_GATHER
189
ca1d6c7d 190static bool tlb_next_batch(struct mmu_gather *tlb)
9547d01b
PZ
191{
192 struct mmu_gather_batch *batch;
193
194 batch = tlb->active;
195 if (batch->next) {
196 tlb->active = batch->next;
ca1d6c7d 197 return true;
9547d01b
PZ
198 }
199
53a59fc6 200 if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
ca1d6c7d 201 return false;
53a59fc6 202
9547d01b
PZ
203 batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
204 if (!batch)
ca1d6c7d 205 return false;
9547d01b 206
53a59fc6 207 tlb->batch_count++;
9547d01b
PZ
208 batch->next = NULL;
209 batch->nr = 0;
210 batch->max = MAX_GATHER_BATCH;
211
212 tlb->active->next = batch;
213 tlb->active = batch;
214
ca1d6c7d 215 return true;
9547d01b
PZ
216}
217
218/* tlb_gather_mmu
219 * Called to initialize an (on-stack) mmu_gather structure for page-table
220 * tear-down from @mm. The @fullmm argument is used when @mm is without
221 * users and we're going to destroy the full address space (exit/execve).
222 */
2b047252 223void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)
9547d01b
PZ
224{
225 tlb->mm = mm;
226
2b047252
LT
227 /* Is it from 0 to ~0? */
228 tlb->fullmm = !(start | (end+1));
1de14c3c 229 tlb->need_flush_all = 0;
9547d01b
PZ
230 tlb->local.next = NULL;
231 tlb->local.nr = 0;
232 tlb->local.max = ARRAY_SIZE(tlb->__pages);
233 tlb->active = &tlb->local;
53a59fc6 234 tlb->batch_count = 0;
9547d01b
PZ
235
236#ifdef CONFIG_HAVE_RCU_TABLE_FREE
237 tlb->batch = NULL;
238#endif
e77b0852 239 tlb->page_size = 0;
fb7332a9
WD
240
241 __tlb_reset_range(tlb);
9547d01b
PZ
242}
243
1cf35d47 244static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
9547d01b 245{
721c21c1
WD
246 if (!tlb->end)
247 return;
248
9547d01b 249 tlb_flush(tlb);
34ee645e 250 mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);
9547d01b
PZ
251#ifdef CONFIG_HAVE_RCU_TABLE_FREE
252 tlb_table_flush(tlb);
34e55232 253#endif
fb7332a9 254 __tlb_reset_range(tlb);
1cf35d47
LT
255}
256
257static void tlb_flush_mmu_free(struct mmu_gather *tlb)
258{
259 struct mmu_gather_batch *batch;
34e55232 260
721c21c1 261 for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
9547d01b
PZ
262 free_pages_and_swap_cache(batch->pages, batch->nr);
263 batch->nr = 0;
264 }
265 tlb->active = &tlb->local;
266}
267
1cf35d47
LT
268void tlb_flush_mmu(struct mmu_gather *tlb)
269{
1cf35d47
LT
270 tlb_flush_mmu_tlbonly(tlb);
271 tlb_flush_mmu_free(tlb);
272}
273
9547d01b
PZ
274/* tlb_finish_mmu
275 * Called at the end of the shootdown operation to free up any resources
276 * that were required.
277 */
278void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
279{
280 struct mmu_gather_batch *batch, *next;
281
282 tlb_flush_mmu(tlb);
283
284 /* keep the page table cache within bounds */
285 check_pgt_cache();
286
287 for (batch = tlb->local.next; batch; batch = next) {
288 next = batch->next;
289 free_pages((unsigned long)batch, 0);
290 }
291 tlb->local.next = NULL;
292}
293
294/* __tlb_remove_page
295 * Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while
296 * handling the additional races in SMP caused by other CPUs caching valid
297 * mappings in their TLBs. Returns the number of free page slots left.
298 * When out of page slots we must call tlb_flush_mmu().
e9d55e15 299 *returns true if the caller should flush.
9547d01b 300 */
e77b0852 301bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
9547d01b
PZ
302{
303 struct mmu_gather_batch *batch;
304
fb7332a9 305 VM_BUG_ON(!tlb->end);
692a68c1 306 VM_WARN_ON(tlb->page_size != page_size);
e77b0852 307
9547d01b 308 batch = tlb->active;
692a68c1
AK
309 /*
310 * Add the page and check if we are full. If so
311 * force a flush.
312 */
313 batch->pages[batch->nr++] = page;
9547d01b
PZ
314 if (batch->nr == batch->max) {
315 if (!tlb_next_batch(tlb))
e9d55e15 316 return true;
0b43c3aa 317 batch = tlb->active;
9547d01b 318 }
309381fe 319 VM_BUG_ON_PAGE(batch->nr > batch->max, page);
9547d01b 320
e9d55e15 321 return false;
9547d01b
PZ
322}
323
324#endif /* HAVE_GENERIC_MMU_GATHER */
325
26723911
PZ
326#ifdef CONFIG_HAVE_RCU_TABLE_FREE
327
328/*
329 * See the comment near struct mmu_table_batch.
330 */
331
332static void tlb_remove_table_smp_sync(void *arg)
333{
334 /* Simply deliver the interrupt */
335}
336
337static void tlb_remove_table_one(void *table)
338{
339 /*
340 * This isn't an RCU grace period and hence the page-tables cannot be
341 * assumed to be actually RCU-freed.
342 *
343 * It is however sufficient for software page-table walkers that rely on
344 * IRQ disabling. See the comment near struct mmu_table_batch.
345 */
346 smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
347 __tlb_remove_table(table);
348}
349
350static void tlb_remove_table_rcu(struct rcu_head *head)
351{
352 struct mmu_table_batch *batch;
353 int i;
354
355 batch = container_of(head, struct mmu_table_batch, rcu);
356
357 for (i = 0; i < batch->nr; i++)
358 __tlb_remove_table(batch->tables[i]);
359
360 free_page((unsigned long)batch);
361}
362
363void tlb_table_flush(struct mmu_gather *tlb)
364{
365 struct mmu_table_batch **batch = &tlb->batch;
366
367 if (*batch) {
368 call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
369 *batch = NULL;
370 }
371}
372
373void tlb_remove_table(struct mmu_gather *tlb, void *table)
374{
375 struct mmu_table_batch **batch = &tlb->batch;
376
26723911
PZ
377 /*
378 * When there's less then two users of this mm there cannot be a
379 * concurrent page-table walk.
380 */
381 if (atomic_read(&tlb->mm->mm_users) < 2) {
382 __tlb_remove_table(table);
383 return;
384 }
385
386 if (*batch == NULL) {
387 *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
388 if (*batch == NULL) {
389 tlb_remove_table_one(table);
390 return;
391 }
392 (*batch)->nr = 0;
393 }
394 (*batch)->tables[(*batch)->nr++] = table;
395 if ((*batch)->nr == MAX_TABLE_BATCH)
396 tlb_table_flush(tlb);
397}
398
9547d01b 399#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
26723911 400
1da177e4
LT
401/*
402 * Note: this doesn't free the actual pages themselves. That
403 * has been handled earlier when unmapping all the memory regions.
404 */
9e1b32ca
BH
405static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
406 unsigned long addr)
1da177e4 407{
2f569afd 408 pgtable_t token = pmd_pgtable(*pmd);
e0da382c 409 pmd_clear(pmd);
9e1b32ca 410 pte_free_tlb(tlb, token, addr);
e1f56c89 411 atomic_long_dec(&tlb->mm->nr_ptes);
1da177e4
LT
412}
413
e0da382c
HD
414static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
415 unsigned long addr, unsigned long end,
416 unsigned long floor, unsigned long ceiling)
1da177e4
LT
417{
418 pmd_t *pmd;
419 unsigned long next;
e0da382c 420 unsigned long start;
1da177e4 421
e0da382c 422 start = addr;
1da177e4 423 pmd = pmd_offset(pud, addr);
1da177e4
LT
424 do {
425 next = pmd_addr_end(addr, end);
426 if (pmd_none_or_clear_bad(pmd))
427 continue;
9e1b32ca 428 free_pte_range(tlb, pmd, addr);
1da177e4
LT
429 } while (pmd++, addr = next, addr != end);
430
e0da382c
HD
431 start &= PUD_MASK;
432 if (start < floor)
433 return;
434 if (ceiling) {
435 ceiling &= PUD_MASK;
436 if (!ceiling)
437 return;
1da177e4 438 }
e0da382c
HD
439 if (end - 1 > ceiling - 1)
440 return;
441
442 pmd = pmd_offset(pud, start);
443 pud_clear(pud);
9e1b32ca 444 pmd_free_tlb(tlb, pmd, start);
dc6c9a35 445 mm_dec_nr_pmds(tlb->mm);
1da177e4
LT
446}
447
c2febafc 448static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
e0da382c
HD
449 unsigned long addr, unsigned long end,
450 unsigned long floor, unsigned long ceiling)
1da177e4
LT
451{
452 pud_t *pud;
453 unsigned long next;
e0da382c 454 unsigned long start;
1da177e4 455
e0da382c 456 start = addr;
c2febafc 457 pud = pud_offset(p4d, addr);
1da177e4
LT
458 do {
459 next = pud_addr_end(addr, end);
460 if (pud_none_or_clear_bad(pud))
461 continue;
e0da382c 462 free_pmd_range(tlb, pud, addr, next, floor, ceiling);
1da177e4
LT
463 } while (pud++, addr = next, addr != end);
464
c2febafc
KS
465 start &= P4D_MASK;
466 if (start < floor)
467 return;
468 if (ceiling) {
469 ceiling &= P4D_MASK;
470 if (!ceiling)
471 return;
472 }
473 if (end - 1 > ceiling - 1)
474 return;
475
476 pud = pud_offset(p4d, start);
477 p4d_clear(p4d);
478 pud_free_tlb(tlb, pud, start);
479}
480
481static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
482 unsigned long addr, unsigned long end,
483 unsigned long floor, unsigned long ceiling)
484{
485 p4d_t *p4d;
486 unsigned long next;
487 unsigned long start;
488
489 start = addr;
490 p4d = p4d_offset(pgd, addr);
491 do {
492 next = p4d_addr_end(addr, end);
493 if (p4d_none_or_clear_bad(p4d))
494 continue;
495 free_pud_range(tlb, p4d, addr, next, floor, ceiling);
496 } while (p4d++, addr = next, addr != end);
497
e0da382c
HD
498 start &= PGDIR_MASK;
499 if (start < floor)
500 return;
501 if (ceiling) {
502 ceiling &= PGDIR_MASK;
503 if (!ceiling)
504 return;
1da177e4 505 }
e0da382c
HD
506 if (end - 1 > ceiling - 1)
507 return;
508
c2febafc 509 p4d = p4d_offset(pgd, start);
e0da382c 510 pgd_clear(pgd);
c2febafc 511 p4d_free_tlb(tlb, p4d, start);
1da177e4
LT
512}
513
514/*
e0da382c 515 * This function frees user-level page tables of a process.
1da177e4 516 */
42b77728 517void free_pgd_range(struct mmu_gather *tlb,
e0da382c
HD
518 unsigned long addr, unsigned long end,
519 unsigned long floor, unsigned long ceiling)
1da177e4
LT
520{
521 pgd_t *pgd;
522 unsigned long next;
e0da382c
HD
523
524 /*
525 * The next few lines have given us lots of grief...
526 *
527 * Why are we testing PMD* at this top level? Because often
528 * there will be no work to do at all, and we'd prefer not to
529 * go all the way down to the bottom just to discover that.
530 *
531 * Why all these "- 1"s? Because 0 represents both the bottom
532 * of the address space and the top of it (using -1 for the
533 * top wouldn't help much: the masks would do the wrong thing).
534 * The rule is that addr 0 and floor 0 refer to the bottom of
535 * the address space, but end 0 and ceiling 0 refer to the top
536 * Comparisons need to use "end - 1" and "ceiling - 1" (though
537 * that end 0 case should be mythical).
538 *
539 * Wherever addr is brought up or ceiling brought down, we must
540 * be careful to reject "the opposite 0" before it confuses the
541 * subsequent tests. But what about where end is brought down
542 * by PMD_SIZE below? no, end can't go down to 0 there.
543 *
544 * Whereas we round start (addr) and ceiling down, by different
545 * masks at different levels, in order to test whether a table
546 * now has no other vmas using it, so can be freed, we don't
547 * bother to round floor or end up - the tests don't need that.
548 */
1da177e4 549
e0da382c
HD
550 addr &= PMD_MASK;
551 if (addr < floor) {
552 addr += PMD_SIZE;
553 if (!addr)
554 return;
555 }
556 if (ceiling) {
557 ceiling &= PMD_MASK;
558 if (!ceiling)
559 return;
560 }
561 if (end - 1 > ceiling - 1)
562 end -= PMD_SIZE;
563 if (addr > end - 1)
564 return;
07e32661
AK
565 /*
566 * We add page table cache pages with PAGE_SIZE,
567 * (see pte_free_tlb()), flush the tlb if we need
568 */
569 tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
42b77728 570 pgd = pgd_offset(tlb->mm, addr);
1da177e4
LT
571 do {
572 next = pgd_addr_end(addr, end);
573 if (pgd_none_or_clear_bad(pgd))
574 continue;
c2febafc 575 free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
1da177e4 576 } while (pgd++, addr = next, addr != end);
e0da382c
HD
577}
578
42b77728 579void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
3bf5ee95 580 unsigned long floor, unsigned long ceiling)
e0da382c
HD
581{
582 while (vma) {
583 struct vm_area_struct *next = vma->vm_next;
584 unsigned long addr = vma->vm_start;
585
8f4f8c16 586 /*
25d9e2d1 587 * Hide vma from rmap and truncate_pagecache before freeing
588 * pgtables
8f4f8c16 589 */
5beb4930 590 unlink_anon_vmas(vma);
8f4f8c16
HD
591 unlink_file_vma(vma);
592
9da61aef 593 if (is_vm_hugetlb_page(vma)) {
3bf5ee95 594 hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
166f61b9 595 floor, next ? next->vm_start : ceiling);
3bf5ee95
HD
596 } else {
597 /*
598 * Optimization: gather nearby vmas into one call down
599 */
600 while (next && next->vm_start <= vma->vm_end + PMD_SIZE
4866920b 601 && !is_vm_hugetlb_page(next)) {
3bf5ee95
HD
602 vma = next;
603 next = vma->vm_next;
5beb4930 604 unlink_anon_vmas(vma);
8f4f8c16 605 unlink_file_vma(vma);
3bf5ee95
HD
606 }
607 free_pgd_range(tlb, addr, vma->vm_end,
166f61b9 608 floor, next ? next->vm_start : ceiling);
3bf5ee95 609 }
e0da382c
HD
610 vma = next;
611 }
1da177e4
LT
612}
613
3ed3a4f0 614int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
1da177e4 615{
c4088ebd 616 spinlock_t *ptl;
2f569afd 617 pgtable_t new = pte_alloc_one(mm, address);
1bb3630e
HD
618 if (!new)
619 return -ENOMEM;
620
362a61ad
NP
621 /*
622 * Ensure all pte setup (eg. pte page lock and page clearing) are
623 * visible before the pte is made visible to other CPUs by being
624 * put into page tables.
625 *
626 * The other side of the story is the pointer chasing in the page
627 * table walking code (when walking the page table without locking;
628 * ie. most of the time). Fortunately, these data accesses consist
629 * of a chain of data-dependent loads, meaning most CPUs (alpha
630 * being the notable exception) will already guarantee loads are
631 * seen in-order. See the alpha page table accessors for the
632 * smp_read_barrier_depends() barriers in page table walking code.
633 */
634 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
635
c4088ebd 636 ptl = pmd_lock(mm, pmd);
8ac1f832 637 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
e1f56c89 638 atomic_long_inc(&mm->nr_ptes);
1da177e4 639 pmd_populate(mm, pmd, new);
2f569afd 640 new = NULL;
4b471e88 641 }
c4088ebd 642 spin_unlock(ptl);
2f569afd
MS
643 if (new)
644 pte_free(mm, new);
1bb3630e 645 return 0;
1da177e4
LT
646}
647
1bb3630e 648int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
1da177e4 649{
1bb3630e
HD
650 pte_t *new = pte_alloc_one_kernel(&init_mm, address);
651 if (!new)
652 return -ENOMEM;
653
362a61ad
NP
654 smp_wmb(); /* See comment in __pte_alloc */
655
1bb3630e 656 spin_lock(&init_mm.page_table_lock);
8ac1f832 657 if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
1bb3630e 658 pmd_populate_kernel(&init_mm, pmd, new);
2f569afd 659 new = NULL;
4b471e88 660 }
1bb3630e 661 spin_unlock(&init_mm.page_table_lock);
2f569afd
MS
662 if (new)
663 pte_free_kernel(&init_mm, new);
1bb3630e 664 return 0;
1da177e4
LT
665}
666
d559db08
KH
667static inline void init_rss_vec(int *rss)
668{
669 memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
670}
671
672static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
ae859762 673{
d559db08
KH
674 int i;
675
34e55232 676 if (current->mm == mm)
05af2e10 677 sync_mm_rss(mm);
d559db08
KH
678 for (i = 0; i < NR_MM_COUNTERS; i++)
679 if (rss[i])
680 add_mm_counter(mm, i, rss[i]);
ae859762
HD
681}
682
b5810039 683/*
6aab341e
LT
684 * This function is called to print an error when a bad pte
685 * is found. For example, we might have a PFN-mapped pte in
686 * a region that doesn't allow it.
b5810039
NP
687 *
688 * The calling function must still handle the error.
689 */
3dc14741
HD
690static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
691 pte_t pte, struct page *page)
b5810039 692{
3dc14741 693 pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
c2febafc
KS
694 p4d_t *p4d = p4d_offset(pgd, addr);
695 pud_t *pud = pud_offset(p4d, addr);
3dc14741
HD
696 pmd_t *pmd = pmd_offset(pud, addr);
697 struct address_space *mapping;
698 pgoff_t index;
d936cf9b
HD
699 static unsigned long resume;
700 static unsigned long nr_shown;
701 static unsigned long nr_unshown;
702
703 /*
704 * Allow a burst of 60 reports, then keep quiet for that minute;
705 * or allow a steady drip of one report per second.
706 */
707 if (nr_shown == 60) {
708 if (time_before(jiffies, resume)) {
709 nr_unshown++;
710 return;
711 }
712 if (nr_unshown) {
1170532b
JP
713 pr_alert("BUG: Bad page map: %lu messages suppressed\n",
714 nr_unshown);
d936cf9b
HD
715 nr_unshown = 0;
716 }
717 nr_shown = 0;
718 }
719 if (nr_shown++ == 0)
720 resume = jiffies + 60 * HZ;
3dc14741
HD
721
722 mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
723 index = linear_page_index(vma, addr);
724
1170532b
JP
725 pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
726 current->comm,
727 (long long)pte_val(pte), (long long)pmd_val(*pmd));
718a3821 728 if (page)
f0b791a3 729 dump_page(page, "bad pte");
1170532b
JP
730 pr_alert("addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
731 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
3dc14741
HD
732 /*
733 * Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y
734 */
2682582a
KK
735 pr_alert("file:%pD fault:%pf mmap:%pf readpage:%pf\n",
736 vma->vm_file,
737 vma->vm_ops ? vma->vm_ops->fault : NULL,
738 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
739 mapping ? mapping->a_ops->readpage : NULL);
b5810039 740 dump_stack();
373d4d09 741 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
b5810039
NP
742}
743
ee498ed7 744/*
7e675137 745 * vm_normal_page -- This function gets the "struct page" associated with a pte.
6aab341e 746 *
7e675137
NP
747 * "Special" mappings do not wish to be associated with a "struct page" (either
748 * it doesn't exist, or it exists but they don't want to touch it). In this
749 * case, NULL is returned here. "Normal" mappings do have a struct page.
b379d790 750 *
7e675137
NP
751 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
752 * pte bit, in which case this function is trivial. Secondly, an architecture
753 * may not have a spare pte bit, which requires a more complicated scheme,
754 * described below.
755 *
756 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
757 * special mapping (even if there are underlying and valid "struct pages").
758 * COWed pages of a VM_PFNMAP are always normal.
6aab341e 759 *
b379d790
JH
760 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
761 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
7e675137
NP
762 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
763 * mapping will always honor the rule
6aab341e
LT
764 *
765 * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
766 *
7e675137
NP
767 * And for normal mappings this is false.
768 *
769 * This restricts such mappings to be a linear translation from virtual address
770 * to pfn. To get around this restriction, we allow arbitrary mappings so long
771 * as the vma is not a COW mapping; in that case, we know that all ptes are
772 * special (because none can have been COWed).
b379d790 773 *
b379d790 774 *
7e675137 775 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
b379d790
JH
776 *
777 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
778 * page" backing, however the difference is that _all_ pages with a struct
779 * page (that is, those where pfn_valid is true) are refcounted and considered
780 * normal pages by the VM. The disadvantage is that pages are refcounted
781 * (which can be slower and simply not an option for some PFNMAP users). The
782 * advantage is that we don't have to follow the strict linearity rule of
783 * PFNMAP mappings in order to support COWable mappings.
784 *
ee498ed7 785 */
7e675137
NP
786#ifdef __HAVE_ARCH_PTE_SPECIAL
787# define HAVE_PTE_SPECIAL 1
788#else
789# define HAVE_PTE_SPECIAL 0
790#endif
791struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
792 pte_t pte)
ee498ed7 793{
22b31eec 794 unsigned long pfn = pte_pfn(pte);
7e675137
NP
795
796 if (HAVE_PTE_SPECIAL) {
b38af472 797 if (likely(!pte_special(pte)))
22b31eec 798 goto check_pfn;
667a0a06
DV
799 if (vma->vm_ops && vma->vm_ops->find_special_page)
800 return vma->vm_ops->find_special_page(vma, addr);
a13ea5b7
HD
801 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
802 return NULL;
62eede62 803 if (!is_zero_pfn(pfn))
22b31eec 804 print_bad_pte(vma, addr, pte, NULL);
7e675137
NP
805 return NULL;
806 }
807
808 /* !HAVE_PTE_SPECIAL case follows: */
809
b379d790
JH
810 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
811 if (vma->vm_flags & VM_MIXEDMAP) {
812 if (!pfn_valid(pfn))
813 return NULL;
814 goto out;
815 } else {
7e675137
NP
816 unsigned long off;
817 off = (addr - vma->vm_start) >> PAGE_SHIFT;
b379d790
JH
818 if (pfn == vma->vm_pgoff + off)
819 return NULL;
820 if (!is_cow_mapping(vma->vm_flags))
821 return NULL;
822 }
6aab341e
LT
823 }
824
b38af472
HD
825 if (is_zero_pfn(pfn))
826 return NULL;
22b31eec
HD
827check_pfn:
828 if (unlikely(pfn > highest_memmap_pfn)) {
829 print_bad_pte(vma, addr, pte, NULL);
830 return NULL;
831 }
6aab341e
LT
832
833 /*
7e675137 834 * NOTE! We still have PageReserved() pages in the page tables.
7e675137 835 * eg. VDSO mappings can cause them to exist.
6aab341e 836 */
b379d790 837out:
6aab341e 838 return pfn_to_page(pfn);
ee498ed7
HD
839}
840
28093f9f
GS
841#ifdef CONFIG_TRANSPARENT_HUGEPAGE
842struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
843 pmd_t pmd)
844{
845 unsigned long pfn = pmd_pfn(pmd);
846
847 /*
848 * There is no pmd_special() but there may be special pmds, e.g.
849 * in a direct-access (dax) mapping, so let's just replicate the
850 * !HAVE_PTE_SPECIAL case from vm_normal_page() here.
851 */
852 if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
853 if (vma->vm_flags & VM_MIXEDMAP) {
854 if (!pfn_valid(pfn))
855 return NULL;
856 goto out;
857 } else {
858 unsigned long off;
859 off = (addr - vma->vm_start) >> PAGE_SHIFT;
860 if (pfn == vma->vm_pgoff + off)
861 return NULL;
862 if (!is_cow_mapping(vma->vm_flags))
863 return NULL;
864 }
865 }
866
867 if (is_zero_pfn(pfn))
868 return NULL;
869 if (unlikely(pfn > highest_memmap_pfn))
870 return NULL;
871
872 /*
873 * NOTE! We still have PageReserved() pages in the page tables.
874 * eg. VDSO mappings can cause them to exist.
875 */
876out:
877 return pfn_to_page(pfn);
878}
879#endif
880
1da177e4
LT
881/*
882 * copy one vm_area from one task to the other. Assumes the page tables
883 * already present in the new task to be cleared in the whole range
884 * covered by this vma.
1da177e4
LT
885 */
886
570a335b 887static inline unsigned long
1da177e4 888copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
b5810039 889 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
8c103762 890 unsigned long addr, int *rss)
1da177e4 891{
b5810039 892 unsigned long vm_flags = vma->vm_flags;
1da177e4
LT
893 pte_t pte = *src_pte;
894 struct page *page;
1da177e4
LT
895
896 /* pte contains position in swap or file, so copy. */
897 if (unlikely(!pte_present(pte))) {
0661a336
KS
898 swp_entry_t entry = pte_to_swp_entry(pte);
899
900 if (likely(!non_swap_entry(entry))) {
901 if (swap_duplicate(entry) < 0)
902 return entry.val;
903
904 /* make sure dst_mm is on swapoff's mmlist. */
905 if (unlikely(list_empty(&dst_mm->mmlist))) {
906 spin_lock(&mmlist_lock);
907 if (list_empty(&dst_mm->mmlist))
908 list_add(&dst_mm->mmlist,
909 &src_mm->mmlist);
910 spin_unlock(&mmlist_lock);
911 }
912 rss[MM_SWAPENTS]++;
913 } else if (is_migration_entry(entry)) {
914 page = migration_entry_to_page(entry);
915
eca56ff9 916 rss[mm_counter(page)]++;
0661a336
KS
917
918 if (is_write_migration_entry(entry) &&
919 is_cow_mapping(vm_flags)) {
920 /*
921 * COW mappings require pages in both
922 * parent and child to be set to read.
923 */
924 make_migration_entry_read(&entry);
925 pte = swp_entry_to_pte(entry);
926 if (pte_swp_soft_dirty(*src_pte))
927 pte = pte_swp_mksoft_dirty(pte);
928 set_pte_at(src_mm, addr, src_pte, pte);
0697212a 929 }
1da177e4 930 }
ae859762 931 goto out_set_pte;
1da177e4
LT
932 }
933
1da177e4
LT
934 /*
935 * If it's a COW mapping, write protect it both
936 * in the parent and the child
937 */
67121172 938 if (is_cow_mapping(vm_flags)) {
1da177e4 939 ptep_set_wrprotect(src_mm, addr, src_pte);
3dc90795 940 pte = pte_wrprotect(pte);
1da177e4
LT
941 }
942
943 /*
944 * If it's a shared mapping, mark it clean in
945 * the child
946 */
947 if (vm_flags & VM_SHARED)
948 pte = pte_mkclean(pte);
949 pte = pte_mkold(pte);
6aab341e
LT
950
951 page = vm_normal_page(vma, addr, pte);
952 if (page) {
953 get_page(page);
53f9263b 954 page_dup_rmap(page, false);
eca56ff9 955 rss[mm_counter(page)]++;
6aab341e 956 }
ae859762
HD
957
958out_set_pte:
959 set_pte_at(dst_mm, addr, dst_pte, pte);
570a335b 960 return 0;
1da177e4
LT
961}
962
21bda264 963static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
71e3aac0
AA
964 pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
965 unsigned long addr, unsigned long end)
1da177e4 966{
c36987e2 967 pte_t *orig_src_pte, *orig_dst_pte;
1da177e4 968 pte_t *src_pte, *dst_pte;
c74df32c 969 spinlock_t *src_ptl, *dst_ptl;
e040f218 970 int progress = 0;
d559db08 971 int rss[NR_MM_COUNTERS];
570a335b 972 swp_entry_t entry = (swp_entry_t){0};
1da177e4
LT
973
974again:
d559db08
KH
975 init_rss_vec(rss);
976
c74df32c 977 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
1da177e4
LT
978 if (!dst_pte)
979 return -ENOMEM;
ece0e2b6 980 src_pte = pte_offset_map(src_pmd, addr);
4c21e2f2 981 src_ptl = pte_lockptr(src_mm, src_pmd);
f20dc5f7 982 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
c36987e2
DN
983 orig_src_pte = src_pte;
984 orig_dst_pte = dst_pte;
6606c3e0 985 arch_enter_lazy_mmu_mode();
1da177e4 986
1da177e4
LT
987 do {
988 /*
989 * We are holding two locks at this point - either of them
990 * could generate latencies in another task on another CPU.
991 */
e040f218
HD
992 if (progress >= 32) {
993 progress = 0;
994 if (need_resched() ||
95c354fe 995 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
e040f218
HD
996 break;
997 }
1da177e4
LT
998 if (pte_none(*src_pte)) {
999 progress++;
1000 continue;
1001 }
570a335b
HD
1002 entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
1003 vma, addr, rss);
1004 if (entry.val)
1005 break;
1da177e4
LT
1006 progress += 8;
1007 } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
1da177e4 1008
6606c3e0 1009 arch_leave_lazy_mmu_mode();
c74df32c 1010 spin_unlock(src_ptl);
ece0e2b6 1011 pte_unmap(orig_src_pte);
d559db08 1012 add_mm_rss_vec(dst_mm, rss);
c36987e2 1013 pte_unmap_unlock(orig_dst_pte, dst_ptl);
c74df32c 1014 cond_resched();
570a335b
HD
1015
1016 if (entry.val) {
1017 if (add_swap_count_continuation(entry, GFP_KERNEL) < 0)
1018 return -ENOMEM;
1019 progress = 0;
1020 }
1da177e4
LT
1021 if (addr != end)
1022 goto again;
1023 return 0;
1024}
1025
1026static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1027 pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
1028 unsigned long addr, unsigned long end)
1029{
1030 pmd_t *src_pmd, *dst_pmd;
1031 unsigned long next;
1032
1033 dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
1034 if (!dst_pmd)
1035 return -ENOMEM;
1036 src_pmd = pmd_offset(src_pud, addr);
1037 do {
1038 next = pmd_addr_end(addr, end);
5c7fb56e 1039 if (pmd_trans_huge(*src_pmd) || pmd_devmap(*src_pmd)) {
71e3aac0 1040 int err;
a00cc7d9 1041 VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, vma);
71e3aac0
AA
1042 err = copy_huge_pmd(dst_mm, src_mm,
1043 dst_pmd, src_pmd, addr, vma);
1044 if (err == -ENOMEM)
1045 return -ENOMEM;
1046 if (!err)
1047 continue;
1048 /* fall through */
1049 }
1da177e4
LT
1050 if (pmd_none_or_clear_bad(src_pmd))
1051 continue;
1052 if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
1053 vma, addr, next))
1054 return -ENOMEM;
1055 } while (dst_pmd++, src_pmd++, addr = next, addr != end);
1056 return 0;
1057}
1058
1059static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
c2febafc 1060 p4d_t *dst_p4d, p4d_t *src_p4d, struct vm_area_struct *vma,
1da177e4
LT
1061 unsigned long addr, unsigned long end)
1062{
1063 pud_t *src_pud, *dst_pud;
1064 unsigned long next;
1065
c2febafc 1066 dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
1da177e4
LT
1067 if (!dst_pud)
1068 return -ENOMEM;
c2febafc 1069 src_pud = pud_offset(src_p4d, addr);
1da177e4
LT
1070 do {
1071 next = pud_addr_end(addr, end);
a00cc7d9
MW
1072 if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
1073 int err;
1074
1075 VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, vma);
1076 err = copy_huge_pud(dst_mm, src_mm,
1077 dst_pud, src_pud, addr, vma);
1078 if (err == -ENOMEM)
1079 return -ENOMEM;
1080 if (!err)
1081 continue;
1082 /* fall through */
1083 }
1da177e4
LT
1084 if (pud_none_or_clear_bad(src_pud))
1085 continue;
1086 if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
1087 vma, addr, next))
1088 return -ENOMEM;
1089 } while (dst_pud++, src_pud++, addr = next, addr != end);
1090 return 0;
1091}
1092
c2febafc
KS
1093static inline int copy_p4d_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1094 pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
1095 unsigned long addr, unsigned long end)
1096{
1097 p4d_t *src_p4d, *dst_p4d;
1098 unsigned long next;
1099
1100 dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
1101 if (!dst_p4d)
1102 return -ENOMEM;
1103 src_p4d = p4d_offset(src_pgd, addr);
1104 do {
1105 next = p4d_addr_end(addr, end);
1106 if (p4d_none_or_clear_bad(src_p4d))
1107 continue;
1108 if (copy_pud_range(dst_mm, src_mm, dst_p4d, src_p4d,
1109 vma, addr, next))
1110 return -ENOMEM;
1111 } while (dst_p4d++, src_p4d++, addr = next, addr != end);
1112 return 0;
1113}
1114
1da177e4
LT
1115int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1116 struct vm_area_struct *vma)
1117{
1118 pgd_t *src_pgd, *dst_pgd;
1119 unsigned long next;
1120 unsigned long addr = vma->vm_start;
1121 unsigned long end = vma->vm_end;
2ec74c3e
SG
1122 unsigned long mmun_start; /* For mmu_notifiers */
1123 unsigned long mmun_end; /* For mmu_notifiers */
1124 bool is_cow;
cddb8a5c 1125 int ret;
1da177e4 1126
d992895b
NP
1127 /*
1128 * Don't copy ptes where a page fault will fill them correctly.
1129 * Fork becomes much lighter when there are big shared or private
1130 * readonly mappings. The tradeoff is that copy_page_range is more
1131 * efficient than faulting.
1132 */
0661a336
KS
1133 if (!(vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) &&
1134 !vma->anon_vma)
1135 return 0;
d992895b 1136
1da177e4
LT
1137 if (is_vm_hugetlb_page(vma))
1138 return copy_hugetlb_page_range(dst_mm, src_mm, vma);
1139
b3b9c293 1140 if (unlikely(vma->vm_flags & VM_PFNMAP)) {
2ab64037 1141 /*
1142 * We do not free on error cases below as remove_vma
1143 * gets called on error from higher level routine
1144 */
5180da41 1145 ret = track_pfn_copy(vma);
2ab64037 1146 if (ret)
1147 return ret;
1148 }
1149
cddb8a5c
AA
1150 /*
1151 * We need to invalidate the secondary MMU mappings only when
1152 * there could be a permission downgrade on the ptes of the
1153 * parent mm. And a permission downgrade will only happen if
1154 * is_cow_mapping() returns true.
1155 */
2ec74c3e
SG
1156 is_cow = is_cow_mapping(vma->vm_flags);
1157 mmun_start = addr;
1158 mmun_end = end;
1159 if (is_cow)
1160 mmu_notifier_invalidate_range_start(src_mm, mmun_start,
1161 mmun_end);
cddb8a5c
AA
1162
1163 ret = 0;
1da177e4
LT
1164 dst_pgd = pgd_offset(dst_mm, addr);
1165 src_pgd = pgd_offset(src_mm, addr);
1166 do {
1167 next = pgd_addr_end(addr, end);
1168 if (pgd_none_or_clear_bad(src_pgd))
1169 continue;
c2febafc 1170 if (unlikely(copy_p4d_range(dst_mm, src_mm, dst_pgd, src_pgd,
cddb8a5c
AA
1171 vma, addr, next))) {
1172 ret = -ENOMEM;
1173 break;
1174 }
1da177e4 1175 } while (dst_pgd++, src_pgd++, addr = next, addr != end);
cddb8a5c 1176
2ec74c3e
SG
1177 if (is_cow)
1178 mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end);
cddb8a5c 1179 return ret;
1da177e4
LT
1180}
1181
51c6f666 1182static unsigned long zap_pte_range(struct mmu_gather *tlb,
b5810039 1183 struct vm_area_struct *vma, pmd_t *pmd,
1da177e4 1184 unsigned long addr, unsigned long end,
97a89413 1185 struct zap_details *details)
1da177e4 1186{
b5810039 1187 struct mm_struct *mm = tlb->mm;
d16dfc55 1188 int force_flush = 0;
d559db08 1189 int rss[NR_MM_COUNTERS];
97a89413 1190 spinlock_t *ptl;
5f1a1907 1191 pte_t *start_pte;
97a89413 1192 pte_t *pte;
8a5f14a2 1193 swp_entry_t entry;
d559db08 1194
07e32661 1195 tlb_remove_check_page_size_change(tlb, PAGE_SIZE);
d16dfc55 1196again:
e303297e 1197 init_rss_vec(rss);
5f1a1907
SR
1198 start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
1199 pte = start_pte;
6606c3e0 1200 arch_enter_lazy_mmu_mode();
1da177e4
LT
1201 do {
1202 pte_t ptent = *pte;
166f61b9 1203 if (pte_none(ptent))
1da177e4 1204 continue;
6f5e6b9e 1205
1da177e4 1206 if (pte_present(ptent)) {
ee498ed7 1207 struct page *page;
51c6f666 1208
6aab341e 1209 page = vm_normal_page(vma, addr, ptent);
1da177e4
LT
1210 if (unlikely(details) && page) {
1211 /*
1212 * unmap_shared_mapping_pages() wants to
1213 * invalidate cache without truncating:
1214 * unmap shared but keep private pages.
1215 */
1216 if (details->check_mapping &&
800d8c63 1217 details->check_mapping != page_rmapping(page))
1da177e4 1218 continue;
1da177e4 1219 }
b5810039 1220 ptent = ptep_get_and_clear_full(mm, addr, pte,
a600388d 1221 tlb->fullmm);
1da177e4
LT
1222 tlb_remove_tlb_entry(tlb, pte, addr);
1223 if (unlikely(!page))
1224 continue;
eca56ff9
JM
1225
1226 if (!PageAnon(page)) {
1cf35d47
LT
1227 if (pte_dirty(ptent)) {
1228 force_flush = 1;
6237bcd9 1229 set_page_dirty(page);
1cf35d47 1230 }
4917e5d0 1231 if (pte_young(ptent) &&
64363aad 1232 likely(!(vma->vm_flags & VM_SEQ_READ)))
bf3f3bc5 1233 mark_page_accessed(page);
6237bcd9 1234 }
eca56ff9 1235 rss[mm_counter(page)]--;
d281ee61 1236 page_remove_rmap(page, false);
3dc14741
HD
1237 if (unlikely(page_mapcount(page) < 0))
1238 print_bad_pte(vma, addr, ptent, page);
e9d55e15 1239 if (unlikely(__tlb_remove_page(tlb, page))) {
1cf35d47 1240 force_flush = 1;
ce9ec37b 1241 addr += PAGE_SIZE;
d16dfc55 1242 break;
1cf35d47 1243 }
1da177e4
LT
1244 continue;
1245 }
3e8715fd
KS
1246 /* If details->check_mapping, we leave swap entries. */
1247 if (unlikely(details))
1da177e4 1248 continue;
b084d435 1249
8a5f14a2
KS
1250 entry = pte_to_swp_entry(ptent);
1251 if (!non_swap_entry(entry))
1252 rss[MM_SWAPENTS]--;
1253 else if (is_migration_entry(entry)) {
1254 struct page *page;
9f9f1acd 1255
8a5f14a2 1256 page = migration_entry_to_page(entry);
eca56ff9 1257 rss[mm_counter(page)]--;
b084d435 1258 }
8a5f14a2
KS
1259 if (unlikely(!free_swap_and_cache(entry)))
1260 print_bad_pte(vma, addr, ptent, NULL);
9888a1ca 1261 pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
97a89413 1262 } while (pte++, addr += PAGE_SIZE, addr != end);
ae859762 1263
d559db08 1264 add_mm_rss_vec(mm, rss);
6606c3e0 1265 arch_leave_lazy_mmu_mode();
51c6f666 1266
1cf35d47 1267 /* Do the actual TLB flush before dropping ptl */
fb7332a9 1268 if (force_flush)
1cf35d47 1269 tlb_flush_mmu_tlbonly(tlb);
1cf35d47
LT
1270 pte_unmap_unlock(start_pte, ptl);
1271
1272 /*
1273 * If we forced a TLB flush (either due to running out of
1274 * batch buffers or because we needed to flush dirty TLB
1275 * entries before releasing the ptl), free the batched
1276 * memory too. Restart if we didn't do everything.
1277 */
1278 if (force_flush) {
1279 force_flush = 0;
1280 tlb_flush_mmu_free(tlb);
2b047252 1281 if (addr != end)
d16dfc55
PZ
1282 goto again;
1283 }
1284
51c6f666 1285 return addr;
1da177e4
LT
1286}
1287
51c6f666 1288static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
b5810039 1289 struct vm_area_struct *vma, pud_t *pud,
1da177e4 1290 unsigned long addr, unsigned long end,
97a89413 1291 struct zap_details *details)
1da177e4
LT
1292{
1293 pmd_t *pmd;
1294 unsigned long next;
1295
1296 pmd = pmd_offset(pud, addr);
1297 do {
1298 next = pmd_addr_end(addr, end);
5c7fb56e 1299 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
1a5a9906 1300 if (next - addr != HPAGE_PMD_SIZE) {
68428398
HD
1301 VM_BUG_ON_VMA(vma_is_anonymous(vma) &&
1302 !rwsem_is_locked(&tlb->mm->mmap_sem), vma);
fd60775a 1303 __split_huge_pmd(vma, pmd, addr, false, NULL);
f21760b1 1304 } else if (zap_huge_pmd(tlb, vma, pmd, addr))
1a5a9906 1305 goto next;
71e3aac0
AA
1306 /* fall through */
1307 }
1a5a9906
AA
1308 /*
1309 * Here there can be other concurrent MADV_DONTNEED or
1310 * trans huge page faults running, and if the pmd is
1311 * none or trans huge it can change under us. This is
1312 * because MADV_DONTNEED holds the mmap_sem in read
1313 * mode.
1314 */
1315 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1316 goto next;
97a89413 1317 next = zap_pte_range(tlb, vma, pmd, addr, next, details);
1a5a9906 1318next:
97a89413
PZ
1319 cond_resched();
1320 } while (pmd++, addr = next, addr != end);
51c6f666
RH
1321
1322 return addr;
1da177e4
LT
1323}
1324
51c6f666 1325static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
c2febafc 1326 struct vm_area_struct *vma, p4d_t *p4d,
1da177e4 1327 unsigned long addr, unsigned long end,
97a89413 1328 struct zap_details *details)
1da177e4
LT
1329{
1330 pud_t *pud;
1331 unsigned long next;
1332
c2febafc 1333 pud = pud_offset(p4d, addr);
1da177e4
LT
1334 do {
1335 next = pud_addr_end(addr, end);
a00cc7d9
MW
1336 if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
1337 if (next - addr != HPAGE_PUD_SIZE) {
1338 VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma);
1339 split_huge_pud(vma, pud, addr);
1340 } else if (zap_huge_pud(tlb, vma, pud, addr))
1341 goto next;
1342 /* fall through */
1343 }
97a89413 1344 if (pud_none_or_clear_bad(pud))
1da177e4 1345 continue;
97a89413 1346 next = zap_pmd_range(tlb, vma, pud, addr, next, details);
a00cc7d9
MW
1347next:
1348 cond_resched();
97a89413 1349 } while (pud++, addr = next, addr != end);
51c6f666
RH
1350
1351 return addr;
1da177e4
LT
1352}
1353
c2febafc
KS
1354static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
1355 struct vm_area_struct *vma, pgd_t *pgd,
1356 unsigned long addr, unsigned long end,
1357 struct zap_details *details)
1358{
1359 p4d_t *p4d;
1360 unsigned long next;
1361
1362 p4d = p4d_offset(pgd, addr);
1363 do {
1364 next = p4d_addr_end(addr, end);
1365 if (p4d_none_or_clear_bad(p4d))
1366 continue;
1367 next = zap_pud_range(tlb, vma, p4d, addr, next, details);
1368 } while (p4d++, addr = next, addr != end);
1369
1370 return addr;
1371}
1372
aac45363 1373void unmap_page_range(struct mmu_gather *tlb,
038c7aa1
AV
1374 struct vm_area_struct *vma,
1375 unsigned long addr, unsigned long end,
1376 struct zap_details *details)
1da177e4
LT
1377{
1378 pgd_t *pgd;
1379 unsigned long next;
1380
1da177e4
LT
1381 BUG_ON(addr >= end);
1382 tlb_start_vma(tlb, vma);
1383 pgd = pgd_offset(vma->vm_mm, addr);
1384 do {
1385 next = pgd_addr_end(addr, end);
97a89413 1386 if (pgd_none_or_clear_bad(pgd))
1da177e4 1387 continue;
c2febafc 1388 next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
97a89413 1389 } while (pgd++, addr = next, addr != end);
1da177e4
LT
1390 tlb_end_vma(tlb, vma);
1391}
51c6f666 1392
f5cc4eef
AV
1393
1394static void unmap_single_vma(struct mmu_gather *tlb,
1395 struct vm_area_struct *vma, unsigned long start_addr,
4f74d2c8 1396 unsigned long end_addr,
f5cc4eef
AV
1397 struct zap_details *details)
1398{
1399 unsigned long start = max(vma->vm_start, start_addr);
1400 unsigned long end;
1401
1402 if (start >= vma->vm_end)
1403 return;
1404 end = min(vma->vm_end, end_addr);
1405 if (end <= vma->vm_start)
1406 return;
1407
cbc91f71
SD
1408 if (vma->vm_file)
1409 uprobe_munmap(vma, start, end);
1410
b3b9c293 1411 if (unlikely(vma->vm_flags & VM_PFNMAP))
5180da41 1412 untrack_pfn(vma, 0, 0);
f5cc4eef
AV
1413
1414 if (start != end) {
1415 if (unlikely(is_vm_hugetlb_page(vma))) {
1416 /*
1417 * It is undesirable to test vma->vm_file as it
1418 * should be non-null for valid hugetlb area.
1419 * However, vm_file will be NULL in the error
7aa6b4ad 1420 * cleanup path of mmap_region. When
f5cc4eef 1421 * hugetlbfs ->mmap method fails,
7aa6b4ad 1422 * mmap_region() nullifies vma->vm_file
f5cc4eef
AV
1423 * before calling this function to clean up.
1424 * Since no pte has actually been setup, it is
1425 * safe to do nothing in this case.
1426 */
24669e58 1427 if (vma->vm_file) {
83cde9e8 1428 i_mmap_lock_write(vma->vm_file->f_mapping);
d833352a 1429 __unmap_hugepage_range_final(tlb, vma, start, end, NULL);
83cde9e8 1430 i_mmap_unlock_write(vma->vm_file->f_mapping);
24669e58 1431 }
f5cc4eef
AV
1432 } else
1433 unmap_page_range(tlb, vma, start, end, details);
1434 }
1da177e4
LT
1435}
1436
1da177e4
LT
1437/**
1438 * unmap_vmas - unmap a range of memory covered by a list of vma's
0164f69d 1439 * @tlb: address of the caller's struct mmu_gather
1da177e4
LT
1440 * @vma: the starting vma
1441 * @start_addr: virtual address at which to start unmapping
1442 * @end_addr: virtual address at which to end unmapping
1da177e4 1443 *
508034a3 1444 * Unmap all pages in the vma list.
1da177e4 1445 *
1da177e4
LT
1446 * Only addresses between `start' and `end' will be unmapped.
1447 *
1448 * The VMA list must be sorted in ascending virtual address order.
1449 *
1450 * unmap_vmas() assumes that the caller will flush the whole unmapped address
1451 * range after unmap_vmas() returns. So the only responsibility here is to
1452 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
1453 * drops the lock and schedules.
1454 */
6e8bb019 1455void unmap_vmas(struct mmu_gather *tlb,
1da177e4 1456 struct vm_area_struct *vma, unsigned long start_addr,
4f74d2c8 1457 unsigned long end_addr)
1da177e4 1458{
cddb8a5c 1459 struct mm_struct *mm = vma->vm_mm;
1da177e4 1460
cddb8a5c 1461 mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
f5cc4eef 1462 for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
4f74d2c8 1463 unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
cddb8a5c 1464 mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
1da177e4
LT
1465}
1466
1467/**
1468 * zap_page_range - remove user pages in a given range
1469 * @vma: vm_area_struct holding the applicable pages
eb4546bb 1470 * @start: starting address of pages to zap
1da177e4 1471 * @size: number of bytes to zap
f5cc4eef
AV
1472 *
1473 * Caller must protect the VMA list
1da177e4 1474 */
7e027b14 1475void zap_page_range(struct vm_area_struct *vma, unsigned long start,
ecf1385d 1476 unsigned long size)
1da177e4
LT
1477{
1478 struct mm_struct *mm = vma->vm_mm;
d16dfc55 1479 struct mmu_gather tlb;
7e027b14 1480 unsigned long end = start + size;
1da177e4 1481
1da177e4 1482 lru_add_drain();
2b047252 1483 tlb_gather_mmu(&tlb, mm, start, end);
365e9c87 1484 update_hiwater_rss(mm);
7e027b14
LT
1485 mmu_notifier_invalidate_range_start(mm, start, end);
1486 for ( ; vma && vma->vm_start < end; vma = vma->vm_next)
ecf1385d 1487 unmap_single_vma(&tlb, vma, start, end, NULL);
7e027b14
LT
1488 mmu_notifier_invalidate_range_end(mm, start, end);
1489 tlb_finish_mmu(&tlb, start, end);
1da177e4
LT
1490}
1491
f5cc4eef
AV
1492/**
1493 * zap_page_range_single - remove user pages in a given range
1494 * @vma: vm_area_struct holding the applicable pages
1495 * @address: starting address of pages to zap
1496 * @size: number of bytes to zap
8a5f14a2 1497 * @details: details of shared cache invalidation
f5cc4eef
AV
1498 *
1499 * The range must fit into one VMA.
1da177e4 1500 */
f5cc4eef 1501static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
1da177e4
LT
1502 unsigned long size, struct zap_details *details)
1503{
1504 struct mm_struct *mm = vma->vm_mm;
d16dfc55 1505 struct mmu_gather tlb;
1da177e4 1506 unsigned long end = address + size;
1da177e4 1507
1da177e4 1508 lru_add_drain();
2b047252 1509 tlb_gather_mmu(&tlb, mm, address, end);
365e9c87 1510 update_hiwater_rss(mm);
f5cc4eef 1511 mmu_notifier_invalidate_range_start(mm, address, end);
4f74d2c8 1512 unmap_single_vma(&tlb, vma, address, end, details);
f5cc4eef 1513 mmu_notifier_invalidate_range_end(mm, address, end);
d16dfc55 1514 tlb_finish_mmu(&tlb, address, end);
1da177e4
LT
1515}
1516
c627f9cc
JS
1517/**
1518 * zap_vma_ptes - remove ptes mapping the vma
1519 * @vma: vm_area_struct holding ptes to be zapped
1520 * @address: starting address of pages to zap
1521 * @size: number of bytes to zap
1522 *
1523 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
1524 *
1525 * The entire address range must be fully contained within the vma.
1526 *
1527 * Returns 0 if successful.
1528 */
1529int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1530 unsigned long size)
1531{
1532 if (address < vma->vm_start || address + size > vma->vm_end ||
1533 !(vma->vm_flags & VM_PFNMAP))
1534 return -1;
f5cc4eef 1535 zap_page_range_single(vma, address, size, NULL);
c627f9cc
JS
1536 return 0;
1537}
1538EXPORT_SYMBOL_GPL(zap_vma_ptes);
1539
25ca1d6c 1540pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
920c7a5d 1541 spinlock_t **ptl)
c9cfcddf 1542{
c2febafc
KS
1543 pgd_t *pgd;
1544 p4d_t *p4d;
1545 pud_t *pud;
1546 pmd_t *pmd;
1547
1548 pgd = pgd_offset(mm, addr);
1549 p4d = p4d_alloc(mm, pgd, addr);
1550 if (!p4d)
1551 return NULL;
1552 pud = pud_alloc(mm, p4d, addr);
1553 if (!pud)
1554 return NULL;
1555 pmd = pmd_alloc(mm, pud, addr);
1556 if (!pmd)
1557 return NULL;
1558
1559 VM_BUG_ON(pmd_trans_huge(*pmd));
1560 return pte_alloc_map_lock(mm, pmd, addr, ptl);
c9cfcddf
LT
1561}
1562
238f58d8
LT
1563/*
1564 * This is the old fallback for page remapping.
1565 *
1566 * For historical reasons, it only allows reserved pages. Only
1567 * old drivers should use this, and they needed to mark their
1568 * pages reserved for the old functions anyway.
1569 */
423bad60
NP
1570static int insert_page(struct vm_area_struct *vma, unsigned long addr,
1571 struct page *page, pgprot_t prot)
238f58d8 1572{
423bad60 1573 struct mm_struct *mm = vma->vm_mm;
238f58d8 1574 int retval;
c9cfcddf 1575 pte_t *pte;
8a9f3ccd
BS
1576 spinlock_t *ptl;
1577
238f58d8 1578 retval = -EINVAL;
a145dd41 1579 if (PageAnon(page))
5b4e655e 1580 goto out;
238f58d8
LT
1581 retval = -ENOMEM;
1582 flush_dcache_page(page);
c9cfcddf 1583 pte = get_locked_pte(mm, addr, &ptl);
238f58d8 1584 if (!pte)
5b4e655e 1585 goto out;
238f58d8
LT
1586 retval = -EBUSY;
1587 if (!pte_none(*pte))
1588 goto out_unlock;
1589
1590 /* Ok, finally just insert the thing.. */
1591 get_page(page);
eca56ff9 1592 inc_mm_counter_fast(mm, mm_counter_file(page));
dd78fedd 1593 page_add_file_rmap(page, false);
238f58d8
LT
1594 set_pte_at(mm, addr, pte, mk_pte(page, prot));
1595
1596 retval = 0;
8a9f3ccd
BS
1597 pte_unmap_unlock(pte, ptl);
1598 return retval;
238f58d8
LT
1599out_unlock:
1600 pte_unmap_unlock(pte, ptl);
1601out:
1602 return retval;
1603}
1604
bfa5bf6d
REB
1605/**
1606 * vm_insert_page - insert single page into user vma
1607 * @vma: user vma to map to
1608 * @addr: target user address of this page
1609 * @page: source kernel page
1610 *
a145dd41
LT
1611 * This allows drivers to insert individual pages they've allocated
1612 * into a user vma.
1613 *
1614 * The page has to be a nice clean _individual_ kernel allocation.
1615 * If you allocate a compound page, you need to have marked it as
1616 * such (__GFP_COMP), or manually just split the page up yourself
8dfcc9ba 1617 * (see split_page()).
a145dd41
LT
1618 *
1619 * NOTE! Traditionally this was done with "remap_pfn_range()" which
1620 * took an arbitrary page protection parameter. This doesn't allow
1621 * that. Your vma protection will have to be set up correctly, which
1622 * means that if you want a shared writable mapping, you'd better
1623 * ask for a shared writable mapping!
1624 *
1625 * The page does not need to be reserved.
4b6e1e37
KK
1626 *
1627 * Usually this function is called from f_op->mmap() handler
1628 * under mm->mmap_sem write-lock, so it can change vma->vm_flags.
1629 * Caller must set VM_MIXEDMAP on vma if it wants to call this
1630 * function from other places, for example from page-fault handler.
a145dd41 1631 */
423bad60
NP
1632int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
1633 struct page *page)
a145dd41
LT
1634{
1635 if (addr < vma->vm_start || addr >= vma->vm_end)
1636 return -EFAULT;
1637 if (!page_count(page))
1638 return -EINVAL;
4b6e1e37
KK
1639 if (!(vma->vm_flags & VM_MIXEDMAP)) {
1640 BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem));
1641 BUG_ON(vma->vm_flags & VM_PFNMAP);
1642 vma->vm_flags |= VM_MIXEDMAP;
1643 }
423bad60 1644 return insert_page(vma, addr, page, vma->vm_page_prot);
a145dd41 1645}
e3c3374f 1646EXPORT_SYMBOL(vm_insert_page);
a145dd41 1647
423bad60 1648static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
01c8f1c4 1649 pfn_t pfn, pgprot_t prot)
423bad60
NP
1650{
1651 struct mm_struct *mm = vma->vm_mm;
1652 int retval;
1653 pte_t *pte, entry;
1654 spinlock_t *ptl;
1655
1656 retval = -ENOMEM;
1657 pte = get_locked_pte(mm, addr, &ptl);
1658 if (!pte)
1659 goto out;
1660 retval = -EBUSY;
1661 if (!pte_none(*pte))
1662 goto out_unlock;
1663
1664 /* Ok, finally just insert the thing.. */
01c8f1c4
DW
1665 if (pfn_t_devmap(pfn))
1666 entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
1667 else
1668 entry = pte_mkspecial(pfn_t_pte(pfn, prot));
423bad60 1669 set_pte_at(mm, addr, pte, entry);
4b3073e1 1670 update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
423bad60
NP
1671
1672 retval = 0;
1673out_unlock:
1674 pte_unmap_unlock(pte, ptl);
1675out:
1676 return retval;
1677}
1678
e0dc0d8f
NP
1679/**
1680 * vm_insert_pfn - insert single pfn into user vma
1681 * @vma: user vma to map to
1682 * @addr: target user address of this page
1683 * @pfn: source kernel pfn
1684 *
c462f179 1685 * Similar to vm_insert_page, this allows drivers to insert individual pages
e0dc0d8f
NP
1686 * they've allocated into a user vma. Same comments apply.
1687 *
1688 * This function should only be called from a vm_ops->fault handler, and
1689 * in that case the handler should return NULL.
0d71d10a
NP
1690 *
1691 * vma cannot be a COW mapping.
1692 *
1693 * As this is called only for pages that do not currently exist, we
1694 * do not need to flush old virtual caches or the TLB.
e0dc0d8f
NP
1695 */
1696int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
423bad60 1697 unsigned long pfn)
1745cbc5
AL
1698{
1699 return vm_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
1700}
1701EXPORT_SYMBOL(vm_insert_pfn);
1702
1703/**
1704 * vm_insert_pfn_prot - insert single pfn into user vma with specified pgprot
1705 * @vma: user vma to map to
1706 * @addr: target user address of this page
1707 * @pfn: source kernel pfn
1708 * @pgprot: pgprot flags for the inserted page
1709 *
1710 * This is exactly like vm_insert_pfn, except that it allows drivers to
1711 * to override pgprot on a per-page basis.
1712 *
1713 * This only makes sense for IO mappings, and it makes no sense for
1714 * cow mappings. In general, using multiple vmas is preferable;
1715 * vm_insert_pfn_prot should only be used if using multiple VMAs is
1716 * impractical.
1717 */
1718int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
1719 unsigned long pfn, pgprot_t pgprot)
e0dc0d8f 1720{
2ab64037 1721 int ret;
7e675137
NP
1722 /*
1723 * Technically, architectures with pte_special can avoid all these
1724 * restrictions (same for remap_pfn_range). However we would like
1725 * consistency in testing and feature parity among all, so we should
1726 * try to keep these invariants in place for everybody.
1727 */
b379d790
JH
1728 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
1729 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
1730 (VM_PFNMAP|VM_MIXEDMAP));
1731 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
1732 BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
e0dc0d8f 1733
423bad60
NP
1734 if (addr < vma->vm_start || addr >= vma->vm_end)
1735 return -EFAULT;
308a047c
BP
1736
1737 track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV));
2ab64037 1738
01c8f1c4 1739 ret = insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot);
2ab64037 1740
2ab64037 1741 return ret;
423bad60 1742}
1745cbc5 1743EXPORT_SYMBOL(vm_insert_pfn_prot);
e0dc0d8f 1744
423bad60 1745int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
01c8f1c4 1746 pfn_t pfn)
423bad60 1747{
87744ab3
DW
1748 pgprot_t pgprot = vma->vm_page_prot;
1749
423bad60 1750 BUG_ON(!(vma->vm_flags & VM_MIXEDMAP));
e0dc0d8f 1751
423bad60
NP
1752 if (addr < vma->vm_start || addr >= vma->vm_end)
1753 return -EFAULT;
308a047c
BP
1754
1755 track_pfn_insert(vma, &pgprot, pfn);
e0dc0d8f 1756
423bad60
NP
1757 /*
1758 * If we don't have pte special, then we have to use the pfn_valid()
1759 * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
1760 * refcount the page if pfn_valid is true (hence insert_page rather
62eede62
HD
1761 * than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
1762 * without pte special, it would there be refcounted as a normal page.
423bad60 1763 */
03fc2da6 1764 if (!HAVE_PTE_SPECIAL && !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
423bad60
NP
1765 struct page *page;
1766
03fc2da6
DW
1767 /*
1768 * At this point we are committed to insert_page()
1769 * regardless of whether the caller specified flags that
1770 * result in pfn_t_has_page() == false.
1771 */
1772 page = pfn_to_page(pfn_t_to_pfn(pfn));
87744ab3 1773 return insert_page(vma, addr, page, pgprot);
423bad60 1774 }
87744ab3 1775 return insert_pfn(vma, addr, pfn, pgprot);
e0dc0d8f 1776}
423bad60 1777EXPORT_SYMBOL(vm_insert_mixed);
e0dc0d8f 1778
1da177e4
LT
1779/*
1780 * maps a range of physical memory into the requested pages. the old
1781 * mappings are removed. any references to nonexistent pages results
1782 * in null mappings (currently treated as "copy-on-access")
1783 */
1784static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
1785 unsigned long addr, unsigned long end,
1786 unsigned long pfn, pgprot_t prot)
1787{
1788 pte_t *pte;
c74df32c 1789 spinlock_t *ptl;
1da177e4 1790
c74df32c 1791 pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
1da177e4
LT
1792 if (!pte)
1793 return -ENOMEM;
6606c3e0 1794 arch_enter_lazy_mmu_mode();
1da177e4
LT
1795 do {
1796 BUG_ON(!pte_none(*pte));
7e675137 1797 set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
1da177e4
LT
1798 pfn++;
1799 } while (pte++, addr += PAGE_SIZE, addr != end);
6606c3e0 1800 arch_leave_lazy_mmu_mode();
c74df32c 1801 pte_unmap_unlock(pte - 1, ptl);
1da177e4
LT
1802 return 0;
1803}
1804
1805static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
1806 unsigned long addr, unsigned long end,
1807 unsigned long pfn, pgprot_t prot)
1808{
1809 pmd_t *pmd;
1810 unsigned long next;
1811
1812 pfn -= addr >> PAGE_SHIFT;
1813 pmd = pmd_alloc(mm, pud, addr);
1814 if (!pmd)
1815 return -ENOMEM;
f66055ab 1816 VM_BUG_ON(pmd_trans_huge(*pmd));
1da177e4
LT
1817 do {
1818 next = pmd_addr_end(addr, end);
1819 if (remap_pte_range(mm, pmd, addr, next,
1820 pfn + (addr >> PAGE_SHIFT), prot))
1821 return -ENOMEM;
1822 } while (pmd++, addr = next, addr != end);
1823 return 0;
1824}
1825
c2febafc 1826static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
1da177e4
LT
1827 unsigned long addr, unsigned long end,
1828 unsigned long pfn, pgprot_t prot)
1829{
1830 pud_t *pud;
1831 unsigned long next;
1832
1833 pfn -= addr >> PAGE_SHIFT;
c2febafc 1834 pud = pud_alloc(mm, p4d, addr);
1da177e4
LT
1835 if (!pud)
1836 return -ENOMEM;
1837 do {
1838 next = pud_addr_end(addr, end);
1839 if (remap_pmd_range(mm, pud, addr, next,
1840 pfn + (addr >> PAGE_SHIFT), prot))
1841 return -ENOMEM;
1842 } while (pud++, addr = next, addr != end);
1843 return 0;
1844}
1845
c2febafc
KS
1846static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
1847 unsigned long addr, unsigned long end,
1848 unsigned long pfn, pgprot_t prot)
1849{
1850 p4d_t *p4d;
1851 unsigned long next;
1852
1853 pfn -= addr >> PAGE_SHIFT;
1854 p4d = p4d_alloc(mm, pgd, addr);
1855 if (!p4d)
1856 return -ENOMEM;
1857 do {
1858 next = p4d_addr_end(addr, end);
1859 if (remap_pud_range(mm, p4d, addr, next,
1860 pfn + (addr >> PAGE_SHIFT), prot))
1861 return -ENOMEM;
1862 } while (p4d++, addr = next, addr != end);
1863 return 0;
1864}
1865
bfa5bf6d
REB
1866/**
1867 * remap_pfn_range - remap kernel memory to userspace
1868 * @vma: user vma to map to
1869 * @addr: target user address to start at
1870 * @pfn: physical address of kernel memory
1871 * @size: size of map area
1872 * @prot: page protection flags for this mapping
1873 *
1874 * Note: this is only safe if the mm semaphore is held when called.
1875 */
1da177e4
LT
1876int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1877 unsigned long pfn, unsigned long size, pgprot_t prot)
1878{
1879 pgd_t *pgd;
1880 unsigned long next;
2d15cab8 1881 unsigned long end = addr + PAGE_ALIGN(size);
1da177e4 1882 struct mm_struct *mm = vma->vm_mm;
d5957d2f 1883 unsigned long remap_pfn = pfn;
1da177e4
LT
1884 int err;
1885
1886 /*
1887 * Physically remapped pages are special. Tell the
1888 * rest of the world about it:
1889 * VM_IO tells people not to look at these pages
1890 * (accesses can have side effects).
6aab341e
LT
1891 * VM_PFNMAP tells the core MM that the base pages are just
1892 * raw PFN mappings, and do not have a "struct page" associated
1893 * with them.
314e51b9
KK
1894 * VM_DONTEXPAND
1895 * Disable vma merging and expanding with mremap().
1896 * VM_DONTDUMP
1897 * Omit vma from core dump, even when VM_IO turned off.
fb155c16
LT
1898 *
1899 * There's a horrible special case to handle copy-on-write
1900 * behaviour that some programs depend on. We mark the "original"
1901 * un-COW'ed pages by matching them up with "vma->vm_pgoff".
b3b9c293 1902 * See vm_normal_page() for details.
1da177e4 1903 */
b3b9c293
KK
1904 if (is_cow_mapping(vma->vm_flags)) {
1905 if (addr != vma->vm_start || end != vma->vm_end)
1906 return -EINVAL;
fb155c16 1907 vma->vm_pgoff = pfn;
b3b9c293
KK
1908 }
1909
d5957d2f 1910 err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size));
b3b9c293 1911 if (err)
3c8bb73a 1912 return -EINVAL;
fb155c16 1913
314e51b9 1914 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1da177e4
LT
1915
1916 BUG_ON(addr >= end);
1917 pfn -= addr >> PAGE_SHIFT;
1918 pgd = pgd_offset(mm, addr);
1919 flush_cache_range(vma, addr, end);
1da177e4
LT
1920 do {
1921 next = pgd_addr_end(addr, end);
c2febafc 1922 err = remap_p4d_range(mm, pgd, addr, next,
1da177e4
LT
1923 pfn + (addr >> PAGE_SHIFT), prot);
1924 if (err)
1925 break;
1926 } while (pgd++, addr = next, addr != end);
2ab64037 1927
1928 if (err)
d5957d2f 1929 untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size));
2ab64037 1930
1da177e4
LT
1931 return err;
1932}
1933EXPORT_SYMBOL(remap_pfn_range);
1934
b4cbb197
LT
1935/**
1936 * vm_iomap_memory - remap memory to userspace
1937 * @vma: user vma to map to
1938 * @start: start of area
1939 * @len: size of area
1940 *
1941 * This is a simplified io_remap_pfn_range() for common driver use. The
1942 * driver just needs to give us the physical memory range to be mapped,
1943 * we'll figure out the rest from the vma information.
1944 *
1945 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
1946 * whatever write-combining details or similar.
1947 */
1948int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
1949{
1950 unsigned long vm_len, pfn, pages;
1951
1952 /* Check that the physical memory area passed in looks valid */
1953 if (start + len < start)
1954 return -EINVAL;
1955 /*
1956 * You *really* shouldn't map things that aren't page-aligned,
1957 * but we've historically allowed it because IO memory might
1958 * just have smaller alignment.
1959 */
1960 len += start & ~PAGE_MASK;
1961 pfn = start >> PAGE_SHIFT;
1962 pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
1963 if (pfn + pages < pfn)
1964 return -EINVAL;
1965
1966 /* We start the mapping 'vm_pgoff' pages into the area */
1967 if (vma->vm_pgoff > pages)
1968 return -EINVAL;
1969 pfn += vma->vm_pgoff;
1970 pages -= vma->vm_pgoff;
1971
1972 /* Can we fit all of the mapping? */
1973 vm_len = vma->vm_end - vma->vm_start;
1974 if (vm_len >> PAGE_SHIFT > pages)
1975 return -EINVAL;
1976
1977 /* Ok, let it rip */
1978 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
1979}
1980EXPORT_SYMBOL(vm_iomap_memory);
1981
aee16b3c
JF
1982static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
1983 unsigned long addr, unsigned long end,
1984 pte_fn_t fn, void *data)
1985{
1986 pte_t *pte;
1987 int err;
2f569afd 1988 pgtable_t token;
94909914 1989 spinlock_t *uninitialized_var(ptl);
aee16b3c
JF
1990
1991 pte = (mm == &init_mm) ?
1992 pte_alloc_kernel(pmd, addr) :
1993 pte_alloc_map_lock(mm, pmd, addr, &ptl);
1994 if (!pte)
1995 return -ENOMEM;
1996
1997 BUG_ON(pmd_huge(*pmd));
1998
38e0edb1
JF
1999 arch_enter_lazy_mmu_mode();
2000
2f569afd 2001 token = pmd_pgtable(*pmd);
aee16b3c
JF
2002
2003 do {
c36987e2 2004 err = fn(pte++, token, addr, data);
aee16b3c
JF
2005 if (err)
2006 break;
c36987e2 2007 } while (addr += PAGE_SIZE, addr != end);
aee16b3c 2008
38e0edb1
JF
2009 arch_leave_lazy_mmu_mode();
2010
aee16b3c
JF
2011 if (mm != &init_mm)
2012 pte_unmap_unlock(pte-1, ptl);
2013 return err;
2014}
2015
2016static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
2017 unsigned long addr, unsigned long end,
2018 pte_fn_t fn, void *data)
2019{
2020 pmd_t *pmd;
2021 unsigned long next;
2022 int err;
2023
ceb86879
AK
2024 BUG_ON(pud_huge(*pud));
2025
aee16b3c
JF
2026 pmd = pmd_alloc(mm, pud, addr);
2027 if (!pmd)
2028 return -ENOMEM;
2029 do {
2030 next = pmd_addr_end(addr, end);
2031 err = apply_to_pte_range(mm, pmd, addr, next, fn, data);
2032 if (err)
2033 break;
2034 } while (pmd++, addr = next, addr != end);
2035 return err;
2036}
2037
c2febafc 2038static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
aee16b3c
JF
2039 unsigned long addr, unsigned long end,
2040 pte_fn_t fn, void *data)
2041{
2042 pud_t *pud;
2043 unsigned long next;
2044 int err;
2045
c2febafc 2046 pud = pud_alloc(mm, p4d, addr);
aee16b3c
JF
2047 if (!pud)
2048 return -ENOMEM;
2049 do {
2050 next = pud_addr_end(addr, end);
2051 err = apply_to_pmd_range(mm, pud, addr, next, fn, data);
2052 if (err)
2053 break;
2054 } while (pud++, addr = next, addr != end);
2055 return err;
2056}
2057
c2febafc
KS
2058static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
2059 unsigned long addr, unsigned long end,
2060 pte_fn_t fn, void *data)
2061{
2062 p4d_t *p4d;
2063 unsigned long next;
2064 int err;
2065
2066 p4d = p4d_alloc(mm, pgd, addr);
2067 if (!p4d)
2068 return -ENOMEM;
2069 do {
2070 next = p4d_addr_end(addr, end);
2071 err = apply_to_pud_range(mm, p4d, addr, next, fn, data);
2072 if (err)
2073 break;
2074 } while (p4d++, addr = next, addr != end);
2075 return err;
2076}
2077
aee16b3c
JF
2078/*
2079 * Scan a region of virtual memory, filling in page tables as necessary
2080 * and calling a provided function on each leaf page table.
2081 */
2082int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
2083 unsigned long size, pte_fn_t fn, void *data)
2084{
2085 pgd_t *pgd;
2086 unsigned long next;
57250a5b 2087 unsigned long end = addr + size;
aee16b3c
JF
2088 int err;
2089
9cb65bc3
MP
2090 if (WARN_ON(addr >= end))
2091 return -EINVAL;
2092
aee16b3c
JF
2093 pgd = pgd_offset(mm, addr);
2094 do {
2095 next = pgd_addr_end(addr, end);
c2febafc 2096 err = apply_to_p4d_range(mm, pgd, addr, next, fn, data);
aee16b3c
JF
2097 if (err)
2098 break;
2099 } while (pgd++, addr = next, addr != end);
57250a5b 2100
aee16b3c
JF
2101 return err;
2102}
2103EXPORT_SYMBOL_GPL(apply_to_page_range);
2104
8f4e2101 2105/*
9b4bdd2f
KS
2106 * handle_pte_fault chooses page fault handler according to an entry which was
2107 * read non-atomically. Before making any commitment, on those architectures
2108 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
2109 * parts, do_swap_page must check under lock before unmapping the pte and
2110 * proceeding (but do_wp_page is only called after already making such a check;
a335b2e1 2111 * and do_anonymous_page can safely check later on).
8f4e2101 2112 */
4c21e2f2 2113static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
8f4e2101
HD
2114 pte_t *page_table, pte_t orig_pte)
2115{
2116 int same = 1;
2117#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT)
2118 if (sizeof(pte_t) > sizeof(unsigned long)) {
4c21e2f2
HD
2119 spinlock_t *ptl = pte_lockptr(mm, pmd);
2120 spin_lock(ptl);
8f4e2101 2121 same = pte_same(*page_table, orig_pte);
4c21e2f2 2122 spin_unlock(ptl);
8f4e2101
HD
2123 }
2124#endif
2125 pte_unmap(page_table);
2126 return same;
2127}
2128
9de455b2 2129static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
6aab341e 2130{
0abdd7a8
DW
2131 debug_dma_assert_idle(src);
2132
6aab341e
LT
2133 /*
2134 * If the source page was a PFN mapping, we don't have
2135 * a "struct page" for it. We do a best-effort copy by
2136 * just copying from the original user address. If that
2137 * fails, we just zero-fill it. Live with it.
2138 */
2139 if (unlikely(!src)) {
9b04c5fe 2140 void *kaddr = kmap_atomic(dst);
5d2a2dbb
LT
2141 void __user *uaddr = (void __user *)(va & PAGE_MASK);
2142
2143 /*
2144 * This really shouldn't fail, because the page is there
2145 * in the page tables. But it might just be unreadable,
2146 * in which case we just give up and fill the result with
2147 * zeroes.
2148 */
2149 if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
3ecb01df 2150 clear_page(kaddr);
9b04c5fe 2151 kunmap_atomic(kaddr);
c4ec7b0d 2152 flush_dcache_page(dst);
0ed361de
NP
2153 } else
2154 copy_user_highpage(dst, src, va, vma);
6aab341e
LT
2155}
2156
c20cd45e
MH
2157static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
2158{
2159 struct file *vm_file = vma->vm_file;
2160
2161 if (vm_file)
2162 return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
2163
2164 /*
2165 * Special mappings (e.g. VDSO) do not have any file so fake
2166 * a default GFP_KERNEL for them.
2167 */
2168 return GFP_KERNEL;
2169}
2170
fb09a464
KS
2171/*
2172 * Notify the address space that the page is about to become writable so that
2173 * it can prohibit this or wait for the page to get into an appropriate state.
2174 *
2175 * We do this without the lock held, so that it can sleep if it needs to.
2176 */
38b8cb7f 2177static int do_page_mkwrite(struct vm_fault *vmf)
fb09a464 2178{
fb09a464 2179 int ret;
38b8cb7f
JK
2180 struct page *page = vmf->page;
2181 unsigned int old_flags = vmf->flags;
fb09a464 2182
38b8cb7f 2183 vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
fb09a464 2184
11bac800 2185 ret = vmf->vma->vm_ops->page_mkwrite(vmf);
38b8cb7f
JK
2186 /* Restore original flags so that caller is not surprised */
2187 vmf->flags = old_flags;
fb09a464
KS
2188 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
2189 return ret;
2190 if (unlikely(!(ret & VM_FAULT_LOCKED))) {
2191 lock_page(page);
2192 if (!page->mapping) {
2193 unlock_page(page);
2194 return 0; /* retry */
2195 }
2196 ret |= VM_FAULT_LOCKED;
2197 } else
2198 VM_BUG_ON_PAGE(!PageLocked(page), page);
2199 return ret;
2200}
2201
97ba0c2b
JK
2202/*
2203 * Handle dirtying of a page in shared file mapping on a write fault.
2204 *
2205 * The function expects the page to be locked and unlocks it.
2206 */
2207static void fault_dirty_shared_page(struct vm_area_struct *vma,
2208 struct page *page)
2209{
2210 struct address_space *mapping;
2211 bool dirtied;
2212 bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
2213
2214 dirtied = set_page_dirty(page);
2215 VM_BUG_ON_PAGE(PageAnon(page), page);
2216 /*
2217 * Take a local copy of the address_space - page.mapping may be zeroed
2218 * by truncate after unlock_page(). The address_space itself remains
2219 * pinned by vma->vm_file's reference. We rely on unlock_page()'s
2220 * release semantics to prevent the compiler from undoing this copying.
2221 */
2222 mapping = page_rmapping(page);
2223 unlock_page(page);
2224
2225 if ((dirtied || page_mkwrite) && mapping) {
2226 /*
2227 * Some device drivers do not set page.mapping
2228 * but still dirty their pages
2229 */
2230 balance_dirty_pages_ratelimited(mapping);
2231 }
2232
2233 if (!page_mkwrite)
2234 file_update_time(vma->vm_file);
2235}
2236
4e047f89
SR
2237/*
2238 * Handle write page faults for pages that can be reused in the current vma
2239 *
2240 * This can happen either due to the mapping being with the VM_SHARED flag,
2241 * or due to us being the last reference standing to the page. In either
2242 * case, all we need to do here is to mark the page as writable and update
2243 * any related book-keeping.
2244 */
997dd98d 2245static inline void wp_page_reuse(struct vm_fault *vmf)
82b0f8c3 2246 __releases(vmf->ptl)
4e047f89 2247{
82b0f8c3 2248 struct vm_area_struct *vma = vmf->vma;
a41b70d6 2249 struct page *page = vmf->page;
4e047f89
SR
2250 pte_t entry;
2251 /*
2252 * Clear the pages cpupid information as the existing
2253 * information potentially belongs to a now completely
2254 * unrelated process.
2255 */
2256 if (page)
2257 page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
2258
2994302b
JK
2259 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
2260 entry = pte_mkyoung(vmf->orig_pte);
4e047f89 2261 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
82b0f8c3
JK
2262 if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
2263 update_mmu_cache(vma, vmf->address, vmf->pte);
2264 pte_unmap_unlock(vmf->pte, vmf->ptl);
4e047f89
SR
2265}
2266
2f38ab2c
SR
2267/*
2268 * Handle the case of a page which we actually need to copy to a new page.
2269 *
2270 * Called with mmap_sem locked and the old page referenced, but
2271 * without the ptl held.
2272 *
2273 * High level logic flow:
2274 *
2275 * - Allocate a page, copy the content of the old page to the new one.
2276 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
2277 * - Take the PTL. If the pte changed, bail out and release the allocated page
2278 * - If the pte is still the way we remember it, update the page table and all
2279 * relevant references. This includes dropping the reference the page-table
2280 * held to the old page, as well as updating the rmap.
2281 * - In any case, unlock the PTL and drop the reference we took to the old page.
2282 */
a41b70d6 2283static int wp_page_copy(struct vm_fault *vmf)
2f38ab2c 2284{
82b0f8c3 2285 struct vm_area_struct *vma = vmf->vma;
bae473a4 2286 struct mm_struct *mm = vma->vm_mm;
a41b70d6 2287 struct page *old_page = vmf->page;
2f38ab2c 2288 struct page *new_page = NULL;
2f38ab2c
SR
2289 pte_t entry;
2290 int page_copied = 0;
82b0f8c3 2291 const unsigned long mmun_start = vmf->address & PAGE_MASK;
bae473a4 2292 const unsigned long mmun_end = mmun_start + PAGE_SIZE;
2f38ab2c
SR
2293 struct mem_cgroup *memcg;
2294
2295 if (unlikely(anon_vma_prepare(vma)))
2296 goto oom;
2297
2994302b 2298 if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
82b0f8c3
JK
2299 new_page = alloc_zeroed_user_highpage_movable(vma,
2300 vmf->address);
2f38ab2c
SR
2301 if (!new_page)
2302 goto oom;
2303 } else {
bae473a4 2304 new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
82b0f8c3 2305 vmf->address);
2f38ab2c
SR
2306 if (!new_page)
2307 goto oom;
82b0f8c3 2308 cow_user_page(new_page, old_page, vmf->address, vma);
2f38ab2c 2309 }
2f38ab2c 2310
f627c2f5 2311 if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false))
2f38ab2c
SR
2312 goto oom_free_new;
2313
eb3c24f3
MG
2314 __SetPageUptodate(new_page);
2315
2f38ab2c
SR
2316 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2317
2318 /*
2319 * Re-check the pte - we dropped the lock
2320 */
82b0f8c3 2321 vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
2994302b 2322 if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
2f38ab2c
SR
2323 if (old_page) {
2324 if (!PageAnon(old_page)) {
eca56ff9
JM
2325 dec_mm_counter_fast(mm,
2326 mm_counter_file(old_page));
2f38ab2c
SR
2327 inc_mm_counter_fast(mm, MM_ANONPAGES);
2328 }
2329 } else {
2330 inc_mm_counter_fast(mm, MM_ANONPAGES);
2331 }
2994302b 2332 flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
2f38ab2c
SR
2333 entry = mk_pte(new_page, vma->vm_page_prot);
2334 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2335 /*
2336 * Clear the pte entry and flush it first, before updating the
2337 * pte with the new entry. This will avoid a race condition
2338 * seen in the presence of one thread doing SMC and another
2339 * thread doing COW.
2340 */
82b0f8c3
JK
2341 ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
2342 page_add_new_anon_rmap(new_page, vma, vmf->address, false);
f627c2f5 2343 mem_cgroup_commit_charge(new_page, memcg, false, false);
2f38ab2c
SR
2344 lru_cache_add_active_or_unevictable(new_page, vma);
2345 /*
2346 * We call the notify macro here because, when using secondary
2347 * mmu page tables (such as kvm shadow page tables), we want the
2348 * new page to be mapped directly into the secondary page table.
2349 */
82b0f8c3
JK
2350 set_pte_at_notify(mm, vmf->address, vmf->pte, entry);
2351 update_mmu_cache(vma, vmf->address, vmf->pte);
2f38ab2c
SR
2352 if (old_page) {
2353 /*
2354 * Only after switching the pte to the new page may
2355 * we remove the mapcount here. Otherwise another
2356 * process may come and find the rmap count decremented
2357 * before the pte is switched to the new page, and
2358 * "reuse" the old page writing into it while our pte
2359 * here still points into it and can be read by other
2360 * threads.
2361 *
2362 * The critical issue is to order this
2363 * page_remove_rmap with the ptp_clear_flush above.
2364 * Those stores are ordered by (if nothing else,)
2365 * the barrier present in the atomic_add_negative
2366 * in page_remove_rmap.
2367 *
2368 * Then the TLB flush in ptep_clear_flush ensures that
2369 * no process can access the old page before the
2370 * decremented mapcount is visible. And the old page
2371 * cannot be reused until after the decremented
2372 * mapcount is visible. So transitively, TLBs to
2373 * old page will be flushed before it can be reused.
2374 */
d281ee61 2375 page_remove_rmap(old_page, false);
2f38ab2c
SR
2376 }
2377
2378 /* Free the old page.. */
2379 new_page = old_page;
2380 page_copied = 1;
2381 } else {
f627c2f5 2382 mem_cgroup_cancel_charge(new_page, memcg, false);
2f38ab2c
SR
2383 }
2384
2385 if (new_page)
09cbfeaf 2386 put_page(new_page);
2f38ab2c 2387
82b0f8c3 2388 pte_unmap_unlock(vmf->pte, vmf->ptl);
2f38ab2c
SR
2389 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2390 if (old_page) {
2391 /*
2392 * Don't let another task, with possibly unlocked vma,
2393 * keep the mlocked page.
2394 */
2395 if (page_copied && (vma->vm_flags & VM_LOCKED)) {
2396 lock_page(old_page); /* LRU manipulation */
e90309c9
KS
2397 if (PageMlocked(old_page))
2398 munlock_vma_page(old_page);
2f38ab2c
SR
2399 unlock_page(old_page);
2400 }
09cbfeaf 2401 put_page(old_page);
2f38ab2c
SR
2402 }
2403 return page_copied ? VM_FAULT_WRITE : 0;
2404oom_free_new:
09cbfeaf 2405 put_page(new_page);
2f38ab2c
SR
2406oom:
2407 if (old_page)
09cbfeaf 2408 put_page(old_page);
2f38ab2c
SR
2409 return VM_FAULT_OOM;
2410}
2411
66a6197c
JK
2412/**
2413 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
2414 * writeable once the page is prepared
2415 *
2416 * @vmf: structure describing the fault
2417 *
2418 * This function handles all that is needed to finish a write page fault in a
2419 * shared mapping due to PTE being read-only once the mapped page is prepared.
2420 * It handles locking of PTE and modifying it. The function returns
2421 * VM_FAULT_WRITE on success, 0 when PTE got changed before we acquired PTE
2422 * lock.
2423 *
2424 * The function expects the page to be locked or other protection against
2425 * concurrent faults / writeback (such as DAX radix tree locks).
2426 */
2427int finish_mkwrite_fault(struct vm_fault *vmf)
2428{
2429 WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
2430 vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
2431 &vmf->ptl);
2432 /*
2433 * We might have raced with another page fault while we released the
2434 * pte_offset_map_lock.
2435 */
2436 if (!pte_same(*vmf->pte, vmf->orig_pte)) {
2437 pte_unmap_unlock(vmf->pte, vmf->ptl);
a19e2553 2438 return VM_FAULT_NOPAGE;
66a6197c
JK
2439 }
2440 wp_page_reuse(vmf);
a19e2553 2441 return 0;
66a6197c
JK
2442}
2443
dd906184
BH
2444/*
2445 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
2446 * mapping
2447 */
2994302b 2448static int wp_pfn_shared(struct vm_fault *vmf)
dd906184 2449{
82b0f8c3 2450 struct vm_area_struct *vma = vmf->vma;
bae473a4 2451
dd906184 2452 if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
dd906184
BH
2453 int ret;
2454
82b0f8c3 2455 pte_unmap_unlock(vmf->pte, vmf->ptl);
fe82221f 2456 vmf->flags |= FAULT_FLAG_MKWRITE;
11bac800 2457 ret = vma->vm_ops->pfn_mkwrite(vmf);
2f89dc12 2458 if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
dd906184 2459 return ret;
66a6197c 2460 return finish_mkwrite_fault(vmf);
dd906184 2461 }
997dd98d
JK
2462 wp_page_reuse(vmf);
2463 return VM_FAULT_WRITE;
dd906184
BH
2464}
2465
a41b70d6 2466static int wp_page_shared(struct vm_fault *vmf)
82b0f8c3 2467 __releases(vmf->ptl)
93e478d4 2468{
82b0f8c3 2469 struct vm_area_struct *vma = vmf->vma;
93e478d4 2470
a41b70d6 2471 get_page(vmf->page);
93e478d4 2472
93e478d4
SR
2473 if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
2474 int tmp;
2475
82b0f8c3 2476 pte_unmap_unlock(vmf->pte, vmf->ptl);
38b8cb7f 2477 tmp = do_page_mkwrite(vmf);
93e478d4
SR
2478 if (unlikely(!tmp || (tmp &
2479 (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
a41b70d6 2480 put_page(vmf->page);
93e478d4
SR
2481 return tmp;
2482 }
66a6197c 2483 tmp = finish_mkwrite_fault(vmf);
a19e2553 2484 if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
a41b70d6 2485 unlock_page(vmf->page);
a41b70d6 2486 put_page(vmf->page);
66a6197c 2487 return tmp;
93e478d4 2488 }
66a6197c
JK
2489 } else {
2490 wp_page_reuse(vmf);
997dd98d 2491 lock_page(vmf->page);
93e478d4 2492 }
997dd98d
JK
2493 fault_dirty_shared_page(vma, vmf->page);
2494 put_page(vmf->page);
93e478d4 2495
997dd98d 2496 return VM_FAULT_WRITE;
93e478d4
SR
2497}
2498
1da177e4
LT
2499/*
2500 * This routine handles present pages, when users try to write
2501 * to a shared page. It is done by copying the page to a new address
2502 * and decrementing the shared-page counter for the old page.
2503 *
1da177e4
LT
2504 * Note that this routine assumes that the protection checks have been
2505 * done by the caller (the low-level page fault routine in most cases).
2506 * Thus we can safely just mark it writable once we've done any necessary
2507 * COW.
2508 *
2509 * We also mark the page dirty at this point even though the page will
2510 * change only once the write actually happens. This avoids a few races,
2511 * and potentially makes it more efficient.
2512 *
8f4e2101
HD
2513 * We enter with non-exclusive mmap_sem (to exclude vma changes,
2514 * but allow concurrent faults), with pte both mapped and locked.
2515 * We return with mmap_sem still held, but pte unmapped and unlocked.
1da177e4 2516 */
2994302b 2517static int do_wp_page(struct vm_fault *vmf)
82b0f8c3 2518 __releases(vmf->ptl)
1da177e4 2519{
82b0f8c3 2520 struct vm_area_struct *vma = vmf->vma;
1da177e4 2521
a41b70d6
JK
2522 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
2523 if (!vmf->page) {
251b97f5 2524 /*
64e45507
PF
2525 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
2526 * VM_PFNMAP VMA.
251b97f5
PZ
2527 *
2528 * We should not cow pages in a shared writeable mapping.
dd906184 2529 * Just mark the pages writable and/or call ops->pfn_mkwrite.
251b97f5
PZ
2530 */
2531 if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
2532 (VM_WRITE|VM_SHARED))
2994302b 2533 return wp_pfn_shared(vmf);
2f38ab2c 2534
82b0f8c3 2535 pte_unmap_unlock(vmf->pte, vmf->ptl);
a41b70d6 2536 return wp_page_copy(vmf);
251b97f5 2537 }
1da177e4 2538
d08b3851 2539 /*
ee6a6457
PZ
2540 * Take out anonymous pages first, anonymous shared vmas are
2541 * not dirty accountable.
d08b3851 2542 */
a41b70d6 2543 if (PageAnon(vmf->page) && !PageKsm(vmf->page)) {
6d0a07ed 2544 int total_mapcount;
a41b70d6
JK
2545 if (!trylock_page(vmf->page)) {
2546 get_page(vmf->page);
82b0f8c3 2547 pte_unmap_unlock(vmf->pte, vmf->ptl);
a41b70d6 2548 lock_page(vmf->page);
82b0f8c3
JK
2549 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
2550 vmf->address, &vmf->ptl);
2994302b 2551 if (!pte_same(*vmf->pte, vmf->orig_pte)) {
a41b70d6 2552 unlock_page(vmf->page);
82b0f8c3 2553 pte_unmap_unlock(vmf->pte, vmf->ptl);
a41b70d6 2554 put_page(vmf->page);
28766805 2555 return 0;
ab967d86 2556 }
a41b70d6 2557 put_page(vmf->page);
ee6a6457 2558 }
a41b70d6 2559 if (reuse_swap_page(vmf->page, &total_mapcount)) {
6d0a07ed
AA
2560 if (total_mapcount == 1) {
2561 /*
2562 * The page is all ours. Move it to
2563 * our anon_vma so the rmap code will
2564 * not search our parent or siblings.
2565 * Protected against the rmap code by
2566 * the page lock.
2567 */
a41b70d6 2568 page_move_anon_rmap(vmf->page, vma);
6d0a07ed 2569 }
a41b70d6 2570 unlock_page(vmf->page);
997dd98d
JK
2571 wp_page_reuse(vmf);
2572 return VM_FAULT_WRITE;
b009c024 2573 }
a41b70d6 2574 unlock_page(vmf->page);
ee6a6457 2575 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
d08b3851 2576 (VM_WRITE|VM_SHARED))) {
a41b70d6 2577 return wp_page_shared(vmf);
1da177e4 2578 }
1da177e4
LT
2579
2580 /*
2581 * Ok, we need to copy. Oh, well..
2582 */
a41b70d6 2583 get_page(vmf->page);
28766805 2584
82b0f8c3 2585 pte_unmap_unlock(vmf->pte, vmf->ptl);
a41b70d6 2586 return wp_page_copy(vmf);
1da177e4
LT
2587}
2588
97a89413 2589static void unmap_mapping_range_vma(struct vm_area_struct *vma,
1da177e4
LT
2590 unsigned long start_addr, unsigned long end_addr,
2591 struct zap_details *details)
2592{
f5cc4eef 2593 zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
1da177e4
LT
2594}
2595
6b2dbba8 2596static inline void unmap_mapping_range_tree(struct rb_root *root,
1da177e4
LT
2597 struct zap_details *details)
2598{
2599 struct vm_area_struct *vma;
1da177e4
LT
2600 pgoff_t vba, vea, zba, zea;
2601
6b2dbba8 2602 vma_interval_tree_foreach(vma, root,
1da177e4 2603 details->first_index, details->last_index) {
1da177e4
LT
2604
2605 vba = vma->vm_pgoff;
d6e93217 2606 vea = vba + vma_pages(vma) - 1;
1da177e4
LT
2607 zba = details->first_index;
2608 if (zba < vba)
2609 zba = vba;
2610 zea = details->last_index;
2611 if (zea > vea)
2612 zea = vea;
2613
97a89413 2614 unmap_mapping_range_vma(vma,
1da177e4
LT
2615 ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
2616 ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
97a89413 2617 details);
1da177e4
LT
2618 }
2619}
2620
1da177e4 2621/**
8a5f14a2
KS
2622 * unmap_mapping_range - unmap the portion of all mmaps in the specified
2623 * address_space corresponding to the specified page range in the underlying
2624 * file.
2625 *
3d41088f 2626 * @mapping: the address space containing mmaps to be unmapped.
1da177e4
LT
2627 * @holebegin: byte in first page to unmap, relative to the start of
2628 * the underlying file. This will be rounded down to a PAGE_SIZE
25d9e2d1 2629 * boundary. Note that this is different from truncate_pagecache(), which
1da177e4
LT
2630 * must keep the partial page. In contrast, we must get rid of
2631 * partial pages.
2632 * @holelen: size of prospective hole in bytes. This will be rounded
2633 * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
2634 * end of the file.
2635 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
2636 * but 0 when invalidating pagecache, don't throw away private data.
2637 */
2638void unmap_mapping_range(struct address_space *mapping,
2639 loff_t const holebegin, loff_t const holelen, int even_cows)
2640{
aac45363 2641 struct zap_details details = { };
1da177e4
LT
2642 pgoff_t hba = holebegin >> PAGE_SHIFT;
2643 pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
2644
2645 /* Check for overflow. */
2646 if (sizeof(holelen) > sizeof(hlen)) {
2647 long long holeend =
2648 (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
2649 if (holeend & ~(long long)ULONG_MAX)
2650 hlen = ULONG_MAX - hba + 1;
2651 }
2652
166f61b9 2653 details.check_mapping = even_cows ? NULL : mapping;
1da177e4
LT
2654 details.first_index = hba;
2655 details.last_index = hba + hlen - 1;
2656 if (details.last_index < details.first_index)
2657 details.last_index = ULONG_MAX;
1da177e4 2658
46c043ed 2659 i_mmap_lock_write(mapping);
6b2dbba8 2660 if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap)))
1da177e4 2661 unmap_mapping_range_tree(&mapping->i_mmap, &details);
46c043ed 2662 i_mmap_unlock_write(mapping);
1da177e4
LT
2663}
2664EXPORT_SYMBOL(unmap_mapping_range);
2665
1da177e4 2666/*
8f4e2101
HD
2667 * We enter with non-exclusive mmap_sem (to exclude vma changes,
2668 * but allow concurrent faults), and pte mapped but not yet locked.
9a95f3cf
PC
2669 * We return with pte unmapped and unlocked.
2670 *
2671 * We return with the mmap_sem locked or unlocked in the same cases
2672 * as does filemap_fault().
1da177e4 2673 */
2994302b 2674int do_swap_page(struct vm_fault *vmf)
1da177e4 2675{
82b0f8c3 2676 struct vm_area_struct *vma = vmf->vma;
56f31801 2677 struct page *page, *swapcache;
00501b53 2678 struct mem_cgroup *memcg;
65500d23 2679 swp_entry_t entry;
1da177e4 2680 pte_t pte;
d065bd81 2681 int locked;
ad8c2ee8 2682 int exclusive = 0;
83c54070 2683 int ret = 0;
1da177e4 2684
2994302b 2685 if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte))
8f4e2101 2686 goto out;
65500d23 2687
2994302b 2688 entry = pte_to_swp_entry(vmf->orig_pte);
d1737fdb
AK
2689 if (unlikely(non_swap_entry(entry))) {
2690 if (is_migration_entry(entry)) {
82b0f8c3
JK
2691 migration_entry_wait(vma->vm_mm, vmf->pmd,
2692 vmf->address);
d1737fdb
AK
2693 } else if (is_hwpoison_entry(entry)) {
2694 ret = VM_FAULT_HWPOISON;
2695 } else {
2994302b 2696 print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
d99be1a8 2697 ret = VM_FAULT_SIGBUS;
d1737fdb 2698 }
0697212a
CL
2699 goto out;
2700 }
0ff92245 2701 delayacct_set_flag(DELAYACCT_PF_SWAPIN);
1da177e4
LT
2702 page = lookup_swap_cache(entry);
2703 if (!page) {
82b0f8c3
JK
2704 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vma,
2705 vmf->address);
1da177e4
LT
2706 if (!page) {
2707 /*
8f4e2101
HD
2708 * Back out if somebody else faulted in this pte
2709 * while we released the pte lock.
1da177e4 2710 */
82b0f8c3
JK
2711 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
2712 vmf->address, &vmf->ptl);
2994302b 2713 if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
1da177e4 2714 ret = VM_FAULT_OOM;
0ff92245 2715 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
65500d23 2716 goto unlock;
1da177e4
LT
2717 }
2718
2719 /* Had to read the page from swap area: Major fault */
2720 ret = VM_FAULT_MAJOR;
f8891e5e 2721 count_vm_event(PGMAJFAULT);
bae473a4 2722 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
d1737fdb 2723 } else if (PageHWPoison(page)) {
71f72525
WF
2724 /*
2725 * hwpoisoned dirty swapcache pages are kept for killing
2726 * owner processes (which may be unknown at hwpoison time)
2727 */
d1737fdb
AK
2728 ret = VM_FAULT_HWPOISON;
2729 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
56f31801 2730 swapcache = page;
4779cb31 2731 goto out_release;
1da177e4
LT
2732 }
2733
56f31801 2734 swapcache = page;
82b0f8c3 2735 locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags);
e709ffd6 2736
073e587e 2737 delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
d065bd81
ML
2738 if (!locked) {
2739 ret |= VM_FAULT_RETRY;
2740 goto out_release;
2741 }
073e587e 2742
4969c119 2743 /*
31c4a3d3
HD
2744 * Make sure try_to_free_swap or reuse_swap_page or swapoff did not
2745 * release the swapcache from under us. The page pin, and pte_same
2746 * test below, are not enough to exclude that. Even if it is still
2747 * swapcache, we need to check that the page's swap has not changed.
4969c119 2748 */
31c4a3d3 2749 if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val))
4969c119
AA
2750 goto out_page;
2751
82b0f8c3 2752 page = ksm_might_need_to_copy(page, vma, vmf->address);
cbf86cfe
HD
2753 if (unlikely(!page)) {
2754 ret = VM_FAULT_OOM;
2755 page = swapcache;
cbf86cfe 2756 goto out_page;
5ad64688
HD
2757 }
2758
bae473a4
KS
2759 if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL,
2760 &memcg, false)) {
8a9f3ccd 2761 ret = VM_FAULT_OOM;
bc43f75c 2762 goto out_page;
8a9f3ccd
BS
2763 }
2764
1da177e4 2765 /*
8f4e2101 2766 * Back out if somebody else already faulted in this pte.
1da177e4 2767 */
82b0f8c3
JK
2768 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
2769 &vmf->ptl);
2994302b 2770 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
b8107480 2771 goto out_nomap;
b8107480
KK
2772
2773 if (unlikely(!PageUptodate(page))) {
2774 ret = VM_FAULT_SIGBUS;
2775 goto out_nomap;
1da177e4
LT
2776 }
2777
8c7c6e34
KH
2778 /*
2779 * The page isn't present yet, go ahead with the fault.
2780 *
2781 * Be careful about the sequence of operations here.
2782 * To get its accounting right, reuse_swap_page() must be called
2783 * while the page is counted on swap but not yet in mapcount i.e.
2784 * before page_add_anon_rmap() and swap_free(); try_to_free_swap()
2785 * must be called after the swap_free(), or it will never succeed.
8c7c6e34 2786 */
1da177e4 2787
bae473a4
KS
2788 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
2789 dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
1da177e4 2790 pte = mk_pte(page, vma->vm_page_prot);
82b0f8c3 2791 if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
1da177e4 2792 pte = maybe_mkwrite(pte_mkdirty(pte), vma);
82b0f8c3 2793 vmf->flags &= ~FAULT_FLAG_WRITE;
9a5b489b 2794 ret |= VM_FAULT_WRITE;
d281ee61 2795 exclusive = RMAP_EXCLUSIVE;
1da177e4 2796 }
1da177e4 2797 flush_icache_page(vma, page);
2994302b 2798 if (pte_swp_soft_dirty(vmf->orig_pte))
179ef71c 2799 pte = pte_mksoft_dirty(pte);
82b0f8c3 2800 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
2994302b 2801 vmf->orig_pte = pte;
00501b53 2802 if (page == swapcache) {
82b0f8c3 2803 do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
f627c2f5 2804 mem_cgroup_commit_charge(page, memcg, true, false);
1a8018fb 2805 activate_page(page);
00501b53 2806 } else { /* ksm created a completely new copy */
82b0f8c3 2807 page_add_new_anon_rmap(page, vma, vmf->address, false);
f627c2f5 2808 mem_cgroup_commit_charge(page, memcg, false, false);
00501b53
JW
2809 lru_cache_add_active_or_unevictable(page, vma);
2810 }
1da177e4 2811
c475a8ab 2812 swap_free(entry);
5ccc5aba
VD
2813 if (mem_cgroup_swap_full(page) ||
2814 (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
a2c43eed 2815 try_to_free_swap(page);
c475a8ab 2816 unlock_page(page);
56f31801 2817 if (page != swapcache) {
4969c119
AA
2818 /*
2819 * Hold the lock to avoid the swap entry to be reused
2820 * until we take the PT lock for the pte_same() check
2821 * (to avoid false positives from pte_same). For
2822 * further safety release the lock after the swap_free
2823 * so that the swap count won't change under a
2824 * parallel locked swapcache.
2825 */
2826 unlock_page(swapcache);
09cbfeaf 2827 put_page(swapcache);
4969c119 2828 }
c475a8ab 2829
82b0f8c3 2830 if (vmf->flags & FAULT_FLAG_WRITE) {
2994302b 2831 ret |= do_wp_page(vmf);
61469f1d
HD
2832 if (ret & VM_FAULT_ERROR)
2833 ret &= VM_FAULT_ERROR;
1da177e4
LT
2834 goto out;
2835 }
2836
2837 /* No need to invalidate - it was non-present before */
82b0f8c3 2838 update_mmu_cache(vma, vmf->address, vmf->pte);
65500d23 2839unlock:
82b0f8c3 2840 pte_unmap_unlock(vmf->pte, vmf->ptl);
1da177e4
LT
2841out:
2842 return ret;
b8107480 2843out_nomap:
f627c2f5 2844 mem_cgroup_cancel_charge(page, memcg, false);
82b0f8c3 2845 pte_unmap_unlock(vmf->pte, vmf->ptl);
bc43f75c 2846out_page:
b8107480 2847 unlock_page(page);
4779cb31 2848out_release:
09cbfeaf 2849 put_page(page);
56f31801 2850 if (page != swapcache) {
4969c119 2851 unlock_page(swapcache);
09cbfeaf 2852 put_page(swapcache);
4969c119 2853 }
65500d23 2854 return ret;
1da177e4
LT
2855}
2856
320b2b8d 2857/*
8ca3eb08
TL
2858 * This is like a special single-page "expand_{down|up}wards()",
2859 * except we must first make sure that 'address{-|+}PAGE_SIZE'
320b2b8d 2860 * doesn't hit another vma.
320b2b8d
LT
2861 */
2862static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned long address)
2863{
2864 address &= PAGE_MASK;
2865 if ((vma->vm_flags & VM_GROWSDOWN) && address == vma->vm_start) {
0e8e50e2
LT
2866 struct vm_area_struct *prev = vma->vm_prev;
2867
2868 /*
2869 * Is there a mapping abutting this one below?
2870 *
2871 * That's only ok if it's the same stack mapping
2872 * that has gotten split..
2873 */
2874 if (prev && prev->vm_end == address)
2875 return prev->vm_flags & VM_GROWSDOWN ? 0 : -ENOMEM;
320b2b8d 2876
fee7e49d 2877 return expand_downwards(vma, address - PAGE_SIZE);
320b2b8d 2878 }
8ca3eb08
TL
2879 if ((vma->vm_flags & VM_GROWSUP) && address + PAGE_SIZE == vma->vm_end) {
2880 struct vm_area_struct *next = vma->vm_next;
2881
2882 /* As VM_GROWSDOWN but s/below/above/ */
2883 if (next && next->vm_start == address + PAGE_SIZE)
2884 return next->vm_flags & VM_GROWSUP ? 0 : -ENOMEM;
2885
fee7e49d 2886 return expand_upwards(vma, address + PAGE_SIZE);
8ca3eb08 2887 }
320b2b8d
LT
2888 return 0;
2889}
2890
1da177e4 2891/*
8f4e2101
HD
2892 * We enter with non-exclusive mmap_sem (to exclude vma changes,
2893 * but allow concurrent faults), and pte mapped but not yet locked.
2894 * We return with mmap_sem still held, but pte unmapped and unlocked.
1da177e4 2895 */
82b0f8c3 2896static int do_anonymous_page(struct vm_fault *vmf)
1da177e4 2897{
82b0f8c3 2898 struct vm_area_struct *vma = vmf->vma;
00501b53 2899 struct mem_cgroup *memcg;
8f4e2101 2900 struct page *page;
1da177e4 2901 pte_t entry;
1da177e4 2902
6b7339f4
KS
2903 /* File mapping without ->vm_ops ? */
2904 if (vma->vm_flags & VM_SHARED)
2905 return VM_FAULT_SIGBUS;
2906
11ac5524 2907 /* Check if we need to add a guard page to the stack */
82b0f8c3 2908 if (check_stack_guard_page(vma, vmf->address) < 0)
9c145c56 2909 return VM_FAULT_SIGSEGV;
320b2b8d 2910
7267ec00
KS
2911 /*
2912 * Use pte_alloc() instead of pte_alloc_map(). We can't run
2913 * pte_offset_map() on pmds where a huge pmd might be created
2914 * from a different thread.
2915 *
2916 * pte_alloc_map() is safe to use under down_write(mmap_sem) or when
2917 * parallel threads are excluded by other means.
2918 *
2919 * Here we only have down_read(mmap_sem).
2920 */
82b0f8c3 2921 if (pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))
7267ec00
KS
2922 return VM_FAULT_OOM;
2923
2924 /* See the comment in pte_alloc_one_map() */
82b0f8c3 2925 if (unlikely(pmd_trans_unstable(vmf->pmd)))
7267ec00
KS
2926 return 0;
2927
11ac5524 2928 /* Use the zero-page for reads */
82b0f8c3 2929 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
bae473a4 2930 !mm_forbids_zeropage(vma->vm_mm)) {
82b0f8c3 2931 entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
62eede62 2932 vma->vm_page_prot));
82b0f8c3
JK
2933 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
2934 vmf->address, &vmf->ptl);
2935 if (!pte_none(*vmf->pte))
a13ea5b7 2936 goto unlock;
6b251fc9
AA
2937 /* Deliver the page fault to userland, check inside PT lock */
2938 if (userfaultfd_missing(vma)) {
82b0f8c3
JK
2939 pte_unmap_unlock(vmf->pte, vmf->ptl);
2940 return handle_userfault(vmf, VM_UFFD_MISSING);
6b251fc9 2941 }
a13ea5b7
HD
2942 goto setpte;
2943 }
2944
557ed1fa 2945 /* Allocate our own private page. */
557ed1fa
NP
2946 if (unlikely(anon_vma_prepare(vma)))
2947 goto oom;
82b0f8c3 2948 page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
557ed1fa
NP
2949 if (!page)
2950 goto oom;
eb3c24f3 2951
bae473a4 2952 if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false))
eb3c24f3
MG
2953 goto oom_free_page;
2954
52f37629
MK
2955 /*
2956 * The memory barrier inside __SetPageUptodate makes sure that
2957 * preceeding stores to the page contents become visible before
2958 * the set_pte_at() write.
2959 */
0ed361de 2960 __SetPageUptodate(page);
8f4e2101 2961
557ed1fa 2962 entry = mk_pte(page, vma->vm_page_prot);
1ac0cb5d
HD
2963 if (vma->vm_flags & VM_WRITE)
2964 entry = pte_mkwrite(pte_mkdirty(entry));
1da177e4 2965
82b0f8c3
JK
2966 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
2967 &vmf->ptl);
2968 if (!pte_none(*vmf->pte))
557ed1fa 2969 goto release;
9ba69294 2970
6b251fc9
AA
2971 /* Deliver the page fault to userland, check inside PT lock */
2972 if (userfaultfd_missing(vma)) {
82b0f8c3 2973 pte_unmap_unlock(vmf->pte, vmf->ptl);
f627c2f5 2974 mem_cgroup_cancel_charge(page, memcg, false);
09cbfeaf 2975 put_page(page);
82b0f8c3 2976 return handle_userfault(vmf, VM_UFFD_MISSING);
6b251fc9
AA
2977 }
2978
bae473a4 2979 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
82b0f8c3 2980 page_add_new_anon_rmap(page, vma, vmf->address, false);
f627c2f5 2981 mem_cgroup_commit_charge(page, memcg, false, false);
00501b53 2982 lru_cache_add_active_or_unevictable(page, vma);
a13ea5b7 2983setpte:
82b0f8c3 2984 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
1da177e4
LT
2985
2986 /* No need to invalidate - it was non-present before */
82b0f8c3 2987 update_mmu_cache(vma, vmf->address, vmf->pte);
65500d23 2988unlock:
82b0f8c3 2989 pte_unmap_unlock(vmf->pte, vmf->ptl);
83c54070 2990 return 0;
8f4e2101 2991release:
f627c2f5 2992 mem_cgroup_cancel_charge(page, memcg, false);
09cbfeaf 2993 put_page(page);
8f4e2101 2994 goto unlock;
8a9f3ccd 2995oom_free_page:
09cbfeaf 2996 put_page(page);
65500d23 2997oom:
1da177e4
LT
2998 return VM_FAULT_OOM;
2999}
3000
9a95f3cf
PC
3001/*
3002 * The mmap_sem must have been held on entry, and may have been
3003 * released depending on flags and vma->vm_ops->fault() return value.
3004 * See filemap_fault() and __lock_page_retry().
3005 */
936ca80d 3006static int __do_fault(struct vm_fault *vmf)
7eae74af 3007{
82b0f8c3 3008 struct vm_area_struct *vma = vmf->vma;
7eae74af
KS
3009 int ret;
3010
11bac800 3011 ret = vma->vm_ops->fault(vmf);
3917048d 3012 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
b1aa812b 3013 VM_FAULT_DONE_COW)))
bc2466e4 3014 return ret;
7eae74af 3015
667240e0 3016 if (unlikely(PageHWPoison(vmf->page))) {
7eae74af 3017 if (ret & VM_FAULT_LOCKED)
667240e0
JK
3018 unlock_page(vmf->page);
3019 put_page(vmf->page);
936ca80d 3020 vmf->page = NULL;
7eae74af
KS
3021 return VM_FAULT_HWPOISON;
3022 }
3023
3024 if (unlikely(!(ret & VM_FAULT_LOCKED)))
667240e0 3025 lock_page(vmf->page);
7eae74af 3026 else
667240e0 3027 VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
7eae74af 3028
7eae74af
KS
3029 return ret;
3030}
3031
82b0f8c3 3032static int pte_alloc_one_map(struct vm_fault *vmf)
7267ec00 3033{
82b0f8c3 3034 struct vm_area_struct *vma = vmf->vma;
7267ec00 3035
82b0f8c3 3036 if (!pmd_none(*vmf->pmd))
7267ec00 3037 goto map_pte;
82b0f8c3
JK
3038 if (vmf->prealloc_pte) {
3039 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
3040 if (unlikely(!pmd_none(*vmf->pmd))) {
3041 spin_unlock(vmf->ptl);
7267ec00
KS
3042 goto map_pte;
3043 }
3044
3045 atomic_long_inc(&vma->vm_mm->nr_ptes);
82b0f8c3
JK
3046 pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
3047 spin_unlock(vmf->ptl);
7f2b6ce8 3048 vmf->prealloc_pte = NULL;
82b0f8c3 3049 } else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd, vmf->address))) {
7267ec00
KS
3050 return VM_FAULT_OOM;
3051 }
3052map_pte:
3053 /*
3054 * If a huge pmd materialized under us just retry later. Use
3055 * pmd_trans_unstable() instead of pmd_trans_huge() to ensure the pmd
3056 * didn't become pmd_trans_huge under us and then back to pmd_none, as
3057 * a result of MADV_DONTNEED running immediately after a huge pmd fault
3058 * in a different thread of this mm, in turn leading to a misleading
3059 * pmd_trans_huge() retval. All we have to ensure is that it is a
3060 * regular pmd that we can walk with pte_offset_map() and we can do that
3061 * through an atomic read in C, which is what pmd_trans_unstable()
3062 * provides.
3063 */
82b0f8c3 3064 if (pmd_trans_unstable(vmf->pmd) || pmd_devmap(*vmf->pmd))
7267ec00
KS
3065 return VM_FAULT_NOPAGE;
3066
82b0f8c3
JK
3067 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3068 &vmf->ptl);
7267ec00
KS
3069 return 0;
3070}
3071
e496cf3d 3072#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
10102459
KS
3073
3074#define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1)
3075static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
3076 unsigned long haddr)
3077{
3078 if (((vma->vm_start >> PAGE_SHIFT) & HPAGE_CACHE_INDEX_MASK) !=
3079 (vma->vm_pgoff & HPAGE_CACHE_INDEX_MASK))
3080 return false;
3081 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
3082 return false;
3083 return true;
3084}
3085
82b0f8c3 3086static void deposit_prealloc_pte(struct vm_fault *vmf)
953c66c2 3087{
82b0f8c3 3088 struct vm_area_struct *vma = vmf->vma;
953c66c2 3089
82b0f8c3 3090 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
953c66c2
AK
3091 /*
3092 * We are going to consume the prealloc table,
3093 * count that as nr_ptes.
3094 */
3095 atomic_long_inc(&vma->vm_mm->nr_ptes);
7f2b6ce8 3096 vmf->prealloc_pte = NULL;
953c66c2
AK
3097}
3098
82b0f8c3 3099static int do_set_pmd(struct vm_fault *vmf, struct page *page)
10102459 3100{
82b0f8c3
JK
3101 struct vm_area_struct *vma = vmf->vma;
3102 bool write = vmf->flags & FAULT_FLAG_WRITE;
3103 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
10102459
KS
3104 pmd_t entry;
3105 int i, ret;
3106
3107 if (!transhuge_vma_suitable(vma, haddr))
3108 return VM_FAULT_FALLBACK;
3109
3110 ret = VM_FAULT_FALLBACK;
3111 page = compound_head(page);
3112
953c66c2
AK
3113 /*
3114 * Archs like ppc64 need additonal space to store information
3115 * related to pte entry. Use the preallocated table for that.
3116 */
82b0f8c3
JK
3117 if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
3118 vmf->prealloc_pte = pte_alloc_one(vma->vm_mm, vmf->address);
3119 if (!vmf->prealloc_pte)
953c66c2
AK
3120 return VM_FAULT_OOM;
3121 smp_wmb(); /* See comment in __pte_alloc() */
3122 }
3123
82b0f8c3
JK
3124 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
3125 if (unlikely(!pmd_none(*vmf->pmd)))
10102459
KS
3126 goto out;
3127
3128 for (i = 0; i < HPAGE_PMD_NR; i++)
3129 flush_icache_page(vma, page + i);
3130
3131 entry = mk_huge_pmd(page, vma->vm_page_prot);
3132 if (write)
3133 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
3134
3135 add_mm_counter(vma->vm_mm, MM_FILEPAGES, HPAGE_PMD_NR);
3136 page_add_file_rmap(page, true);
953c66c2
AK
3137 /*
3138 * deposit and withdraw with pmd lock held
3139 */
3140 if (arch_needs_pgtable_deposit())
82b0f8c3 3141 deposit_prealloc_pte(vmf);
10102459 3142
82b0f8c3 3143 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
10102459 3144
82b0f8c3 3145 update_mmu_cache_pmd(vma, haddr, vmf->pmd);
10102459
KS
3146
3147 /* fault is handled */
3148 ret = 0;
95ecedcd 3149 count_vm_event(THP_FILE_MAPPED);
10102459 3150out:
82b0f8c3 3151 spin_unlock(vmf->ptl);
10102459
KS
3152 return ret;
3153}
3154#else
82b0f8c3 3155static int do_set_pmd(struct vm_fault *vmf, struct page *page)
10102459
KS
3156{
3157 BUILD_BUG();
3158 return 0;
3159}
3160#endif
3161
8c6e50b0 3162/**
7267ec00
KS
3163 * alloc_set_pte - setup new PTE entry for given page and add reverse page
3164 * mapping. If needed, the fucntion allocates page table or use pre-allocated.
8c6e50b0 3165 *
82b0f8c3 3166 * @vmf: fault environment
7267ec00 3167 * @memcg: memcg to charge page (only for private mappings)
8c6e50b0 3168 * @page: page to map
8c6e50b0 3169 *
82b0f8c3
JK
3170 * Caller must take care of unlocking vmf->ptl, if vmf->pte is non-NULL on
3171 * return.
8c6e50b0
KS
3172 *
3173 * Target users are page handler itself and implementations of
3174 * vm_ops->map_pages.
3175 */
82b0f8c3 3176int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
7267ec00 3177 struct page *page)
3bb97794 3178{
82b0f8c3
JK
3179 struct vm_area_struct *vma = vmf->vma;
3180 bool write = vmf->flags & FAULT_FLAG_WRITE;
3bb97794 3181 pte_t entry;
10102459
KS
3182 int ret;
3183
82b0f8c3 3184 if (pmd_none(*vmf->pmd) && PageTransCompound(page) &&
e496cf3d 3185 IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
10102459
KS
3186 /* THP on COW? */
3187 VM_BUG_ON_PAGE(memcg, page);
3188
82b0f8c3 3189 ret = do_set_pmd(vmf, page);
10102459 3190 if (ret != VM_FAULT_FALLBACK)
b0b9b3df 3191 return ret;
10102459 3192 }
3bb97794 3193
82b0f8c3
JK
3194 if (!vmf->pte) {
3195 ret = pte_alloc_one_map(vmf);
7267ec00 3196 if (ret)
b0b9b3df 3197 return ret;
7267ec00
KS
3198 }
3199
3200 /* Re-check under ptl */
b0b9b3df
HD
3201 if (unlikely(!pte_none(*vmf->pte)))
3202 return VM_FAULT_NOPAGE;
7267ec00 3203
3bb97794
KS
3204 flush_icache_page(vma, page);
3205 entry = mk_pte(page, vma->vm_page_prot);
3206 if (write)
3207 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
bae473a4
KS
3208 /* copy-on-write page */
3209 if (write && !(vma->vm_flags & VM_SHARED)) {
3bb97794 3210 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
82b0f8c3 3211 page_add_new_anon_rmap(page, vma, vmf->address, false);
7267ec00
KS
3212 mem_cgroup_commit_charge(page, memcg, false, false);
3213 lru_cache_add_active_or_unevictable(page, vma);
3bb97794 3214 } else {
eca56ff9 3215 inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
dd78fedd 3216 page_add_file_rmap(page, false);
3bb97794 3217 }
82b0f8c3 3218 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
3bb97794
KS
3219
3220 /* no need to invalidate: a not-present page won't be cached */
82b0f8c3 3221 update_mmu_cache(vma, vmf->address, vmf->pte);
7267ec00 3222
b0b9b3df 3223 return 0;
3bb97794
KS
3224}
3225
9118c0cb
JK
3226
3227/**
3228 * finish_fault - finish page fault once we have prepared the page to fault
3229 *
3230 * @vmf: structure describing the fault
3231 *
3232 * This function handles all that is needed to finish a page fault once the
3233 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
3234 * given page, adds reverse page mapping, handles memcg charges and LRU
3235 * addition. The function returns 0 on success, VM_FAULT_ code in case of
3236 * error.
3237 *
3238 * The function expects the page to be locked and on success it consumes a
3239 * reference of a page being mapped (for the PTE which maps it).
3240 */
3241int finish_fault(struct vm_fault *vmf)
3242{
3243 struct page *page;
3244 int ret;
3245
3246 /* Did we COW the page? */
3247 if ((vmf->flags & FAULT_FLAG_WRITE) &&
3248 !(vmf->vma->vm_flags & VM_SHARED))
3249 page = vmf->cow_page;
3250 else
3251 page = vmf->page;
3252 ret = alloc_set_pte(vmf, vmf->memcg, page);
3253 if (vmf->pte)
3254 pte_unmap_unlock(vmf->pte, vmf->ptl);
3255 return ret;
3256}
3257
3a91053a
KS
3258static unsigned long fault_around_bytes __read_mostly =
3259 rounddown_pow_of_two(65536);
a9b0f861 3260
a9b0f861
KS
3261#ifdef CONFIG_DEBUG_FS
3262static int fault_around_bytes_get(void *data, u64 *val)
1592eef0 3263{
a9b0f861 3264 *val = fault_around_bytes;
1592eef0
KS
3265 return 0;
3266}
3267
b4903d6e
AR
3268/*
3269 * fault_around_pages() and fault_around_mask() expects fault_around_bytes
3270 * rounded down to nearest page order. It's what do_fault_around() expects to
3271 * see.
3272 */
a9b0f861 3273static int fault_around_bytes_set(void *data, u64 val)
1592eef0 3274{
a9b0f861 3275 if (val / PAGE_SIZE > PTRS_PER_PTE)
1592eef0 3276 return -EINVAL;
b4903d6e
AR
3277 if (val > PAGE_SIZE)
3278 fault_around_bytes = rounddown_pow_of_two(val);
3279 else
3280 fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */
1592eef0
KS
3281 return 0;
3282}
a9b0f861
KS
3283DEFINE_SIMPLE_ATTRIBUTE(fault_around_bytes_fops,
3284 fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
1592eef0
KS
3285
3286static int __init fault_around_debugfs(void)
3287{
3288 void *ret;
3289
a9b0f861
KS
3290 ret = debugfs_create_file("fault_around_bytes", 0644, NULL, NULL,
3291 &fault_around_bytes_fops);
1592eef0 3292 if (!ret)
a9b0f861 3293 pr_warn("Failed to create fault_around_bytes in debugfs");
1592eef0
KS
3294 return 0;
3295}
3296late_initcall(fault_around_debugfs);
1592eef0 3297#endif
8c6e50b0 3298
1fdb412b
KS
3299/*
3300 * do_fault_around() tries to map few pages around the fault address. The hope
3301 * is that the pages will be needed soon and this will lower the number of
3302 * faults to handle.
3303 *
3304 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
3305 * not ready to be mapped: not up-to-date, locked, etc.
3306 *
3307 * This function is called with the page table lock taken. In the split ptlock
3308 * case the page table lock only protects only those entries which belong to
3309 * the page table corresponding to the fault address.
3310 *
3311 * This function doesn't cross the VMA boundaries, in order to call map_pages()
3312 * only once.
3313 *
3314 * fault_around_pages() defines how many pages we'll try to map.
3315 * do_fault_around() expects it to return a power of two less than or equal to
3316 * PTRS_PER_PTE.
3317 *
3318 * The virtual address of the area that we map is naturally aligned to the
3319 * fault_around_pages() value (and therefore to page order). This way it's
3320 * easier to guarantee that we don't cross page table boundaries.
3321 */
0721ec8b 3322static int do_fault_around(struct vm_fault *vmf)
8c6e50b0 3323{
82b0f8c3 3324 unsigned long address = vmf->address, nr_pages, mask;
0721ec8b 3325 pgoff_t start_pgoff = vmf->pgoff;
bae473a4 3326 pgoff_t end_pgoff;
7267ec00 3327 int off, ret = 0;
8c6e50b0 3328
4db0c3c2 3329 nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
aecd6f44
KS
3330 mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;
3331
82b0f8c3
JK
3332 vmf->address = max(address & mask, vmf->vma->vm_start);
3333 off = ((address - vmf->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
bae473a4 3334 start_pgoff -= off;
8c6e50b0
KS
3335
3336 /*
bae473a4
KS
3337 * end_pgoff is either end of page table or end of vma
3338 * or fault_around_pages() from start_pgoff, depending what is nearest.
8c6e50b0 3339 */
bae473a4 3340 end_pgoff = start_pgoff -
82b0f8c3 3341 ((vmf->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
8c6e50b0 3342 PTRS_PER_PTE - 1;
82b0f8c3 3343 end_pgoff = min3(end_pgoff, vma_pages(vmf->vma) + vmf->vma->vm_pgoff - 1,
bae473a4 3344 start_pgoff + nr_pages - 1);
8c6e50b0 3345
82b0f8c3
JK
3346 if (pmd_none(*vmf->pmd)) {
3347 vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm,
3348 vmf->address);
3349 if (!vmf->prealloc_pte)
c5f88bd2 3350 goto out;
7267ec00 3351 smp_wmb(); /* See comment in __pte_alloc() */
8c6e50b0
KS
3352 }
3353
82b0f8c3 3354 vmf->vma->vm_ops->map_pages(vmf, start_pgoff, end_pgoff);
7267ec00 3355
7267ec00 3356 /* Huge page is mapped? Page fault is solved */
82b0f8c3 3357 if (pmd_trans_huge(*vmf->pmd)) {
7267ec00
KS
3358 ret = VM_FAULT_NOPAGE;
3359 goto out;
3360 }
3361
3362 /* ->map_pages() haven't done anything useful. Cold page cache? */
82b0f8c3 3363 if (!vmf->pte)
7267ec00
KS
3364 goto out;
3365
3366 /* check if the page fault is solved */
82b0f8c3
JK
3367 vmf->pte -= (vmf->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT);
3368 if (!pte_none(*vmf->pte))
7267ec00 3369 ret = VM_FAULT_NOPAGE;
82b0f8c3 3370 pte_unmap_unlock(vmf->pte, vmf->ptl);
bae473a4 3371out:
82b0f8c3
JK
3372 vmf->address = address;
3373 vmf->pte = NULL;
7267ec00 3374 return ret;
8c6e50b0
KS
3375}
3376
0721ec8b 3377static int do_read_fault(struct vm_fault *vmf)
e655fb29 3378{
82b0f8c3 3379 struct vm_area_struct *vma = vmf->vma;
8c6e50b0
KS
3380 int ret = 0;
3381
3382 /*
3383 * Let's call ->map_pages() first and use ->fault() as fallback
3384 * if page by the offset is not ready to be mapped (cold cache or
3385 * something).
3386 */
9b4bdd2f 3387 if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
0721ec8b 3388 ret = do_fault_around(vmf);
7267ec00
KS
3389 if (ret)
3390 return ret;
8c6e50b0 3391 }
e655fb29 3392
936ca80d 3393 ret = __do_fault(vmf);
e655fb29
KS
3394 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3395 return ret;
3396
9118c0cb 3397 ret |= finish_fault(vmf);
936ca80d 3398 unlock_page(vmf->page);
7267ec00 3399 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
936ca80d 3400 put_page(vmf->page);
e655fb29
KS
3401 return ret;
3402}
3403
0721ec8b 3404static int do_cow_fault(struct vm_fault *vmf)
ec47c3b9 3405{
82b0f8c3 3406 struct vm_area_struct *vma = vmf->vma;
ec47c3b9
KS
3407 int ret;
3408
3409 if (unlikely(anon_vma_prepare(vma)))
3410 return VM_FAULT_OOM;
3411
936ca80d
JK
3412 vmf->cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vmf->address);
3413 if (!vmf->cow_page)
ec47c3b9
KS
3414 return VM_FAULT_OOM;
3415
936ca80d 3416 if (mem_cgroup_try_charge(vmf->cow_page, vma->vm_mm, GFP_KERNEL,
3917048d 3417 &vmf->memcg, false)) {
936ca80d 3418 put_page(vmf->cow_page);
ec47c3b9
KS
3419 return VM_FAULT_OOM;
3420 }
3421
936ca80d 3422 ret = __do_fault(vmf);
ec47c3b9
KS
3423 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3424 goto uncharge_out;
3917048d
JK
3425 if (ret & VM_FAULT_DONE_COW)
3426 return ret;
ec47c3b9 3427
b1aa812b 3428 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);
936ca80d 3429 __SetPageUptodate(vmf->cow_page);
ec47c3b9 3430
9118c0cb 3431 ret |= finish_fault(vmf);
b1aa812b
JK
3432 unlock_page(vmf->page);
3433 put_page(vmf->page);
7267ec00
KS
3434 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3435 goto uncharge_out;
ec47c3b9
KS
3436 return ret;
3437uncharge_out:
3917048d 3438 mem_cgroup_cancel_charge(vmf->cow_page, vmf->memcg, false);
936ca80d 3439 put_page(vmf->cow_page);
ec47c3b9
KS
3440 return ret;
3441}
3442
0721ec8b 3443static int do_shared_fault(struct vm_fault *vmf)
1da177e4 3444{
82b0f8c3 3445 struct vm_area_struct *vma = vmf->vma;
f0c6d4d2 3446 int ret, tmp;
1d65f86d 3447
936ca80d 3448 ret = __do_fault(vmf);
7eae74af 3449 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
f0c6d4d2 3450 return ret;
1da177e4
LT
3451
3452 /*
f0c6d4d2
KS
3453 * Check if the backing address space wants to know that the page is
3454 * about to become writable
1da177e4 3455 */
fb09a464 3456 if (vma->vm_ops->page_mkwrite) {
936ca80d 3457 unlock_page(vmf->page);
38b8cb7f 3458 tmp = do_page_mkwrite(vmf);
fb09a464
KS
3459 if (unlikely(!tmp ||
3460 (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
936ca80d 3461 put_page(vmf->page);
fb09a464 3462 return tmp;
4294621f 3463 }
fb09a464
KS
3464 }
3465
9118c0cb 3466 ret |= finish_fault(vmf);
7267ec00
KS
3467 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
3468 VM_FAULT_RETRY))) {
936ca80d
JK
3469 unlock_page(vmf->page);
3470 put_page(vmf->page);
f0c6d4d2 3471 return ret;
1da177e4 3472 }
b827e496 3473
97ba0c2b 3474 fault_dirty_shared_page(vma, vmf->page);
1d65f86d 3475 return ret;
54cb8821 3476}
d00806b1 3477
9a95f3cf
PC
3478/*
3479 * We enter with non-exclusive mmap_sem (to exclude vma changes,
3480 * but allow concurrent faults).
3481 * The mmap_sem may have been released depending on flags and our
3482 * return value. See filemap_fault() and __lock_page_or_retry().
3483 */
82b0f8c3 3484static int do_fault(struct vm_fault *vmf)
54cb8821 3485{
82b0f8c3 3486 struct vm_area_struct *vma = vmf->vma;
b0b9b3df 3487 int ret;
54cb8821 3488
6b7339f4
KS
3489 /* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */
3490 if (!vma->vm_ops->fault)
b0b9b3df
HD
3491 ret = VM_FAULT_SIGBUS;
3492 else if (!(vmf->flags & FAULT_FLAG_WRITE))
3493 ret = do_read_fault(vmf);
3494 else if (!(vma->vm_flags & VM_SHARED))
3495 ret = do_cow_fault(vmf);
3496 else
3497 ret = do_shared_fault(vmf);
3498
3499 /* preallocated pagetable is unused: free it */
3500 if (vmf->prealloc_pte) {
3501 pte_free(vma->vm_mm, vmf->prealloc_pte);
7f2b6ce8 3502 vmf->prealloc_pte = NULL;
b0b9b3df
HD
3503 }
3504 return ret;
54cb8821
NP
3505}
3506
b19a9939 3507static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
04bb2f94
RR
3508 unsigned long addr, int page_nid,
3509 int *flags)
9532fec1
MG
3510{
3511 get_page(page);
3512
3513 count_vm_numa_event(NUMA_HINT_FAULTS);
04bb2f94 3514 if (page_nid == numa_node_id()) {
9532fec1 3515 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
04bb2f94
RR
3516 *flags |= TNF_FAULT_LOCAL;
3517 }
9532fec1
MG
3518
3519 return mpol_misplaced(page, vma, addr);
3520}
3521
2994302b 3522static int do_numa_page(struct vm_fault *vmf)
d10e63f2 3523{
82b0f8c3 3524 struct vm_area_struct *vma = vmf->vma;
4daae3b4 3525 struct page *page = NULL;
8191acbd 3526 int page_nid = -1;
90572890 3527 int last_cpupid;
cbee9f88 3528 int target_nid;
b8593bfd 3529 bool migrated = false;
cee216a6 3530 pte_t pte;
288bc549 3531 bool was_writable = pte_savedwrite(vmf->orig_pte);
6688cc05 3532 int flags = 0;
d10e63f2
MG
3533
3534 /*
166f61b9
TH
3535 * The "pte" at this point cannot be used safely without
3536 * validation through pte_unmap_same(). It's of NUMA type but
3537 * the pfn may be screwed if the read is non atomic.
166f61b9 3538 */
82b0f8c3
JK
3539 vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
3540 spin_lock(vmf->ptl);
cee216a6 3541 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
82b0f8c3 3542 pte_unmap_unlock(vmf->pte, vmf->ptl);
4daae3b4
MG
3543 goto out;
3544 }
3545
cee216a6
AK
3546 /*
3547 * Make it present again, Depending on how arch implementes non
3548 * accessible ptes, some can allow access by kernel mode.
3549 */
3550 pte = ptep_modify_prot_start(vma->vm_mm, vmf->address, vmf->pte);
4d942466
MG
3551 pte = pte_modify(pte, vma->vm_page_prot);
3552 pte = pte_mkyoung(pte);
b191f9b1
MG
3553 if (was_writable)
3554 pte = pte_mkwrite(pte);
cee216a6 3555 ptep_modify_prot_commit(vma->vm_mm, vmf->address, vmf->pte, pte);
82b0f8c3 3556 update_mmu_cache(vma, vmf->address, vmf->pte);
d10e63f2 3557
82b0f8c3 3558 page = vm_normal_page(vma, vmf->address, pte);
d10e63f2 3559 if (!page) {
82b0f8c3 3560 pte_unmap_unlock(vmf->pte, vmf->ptl);
d10e63f2
MG
3561 return 0;
3562 }
3563
e81c4802
KS
3564 /* TODO: handle PTE-mapped THP */
3565 if (PageCompound(page)) {
82b0f8c3 3566 pte_unmap_unlock(vmf->pte, vmf->ptl);
e81c4802
KS
3567 return 0;
3568 }
3569
6688cc05 3570 /*
bea66fbd
MG
3571 * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
3572 * much anyway since they can be in shared cache state. This misses
3573 * the case where a mapping is writable but the process never writes
3574 * to it but pte_write gets cleared during protection updates and
3575 * pte_dirty has unpredictable behaviour between PTE scan updates,
3576 * background writeback, dirty balancing and application behaviour.
6688cc05 3577 */
d59dc7bc 3578 if (!pte_write(pte))
6688cc05
PZ
3579 flags |= TNF_NO_GROUP;
3580
dabe1d99
RR
3581 /*
3582 * Flag if the page is shared between multiple address spaces. This
3583 * is later used when determining whether to group tasks together
3584 */
3585 if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
3586 flags |= TNF_SHARED;
3587
90572890 3588 last_cpupid = page_cpupid_last(page);
8191acbd 3589 page_nid = page_to_nid(page);
82b0f8c3 3590 target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid,
bae473a4 3591 &flags);
82b0f8c3 3592 pte_unmap_unlock(vmf->pte, vmf->ptl);
4daae3b4 3593 if (target_nid == -1) {
4daae3b4
MG
3594 put_page(page);
3595 goto out;
3596 }
3597
3598 /* Migrate to the requested node */
1bc115d8 3599 migrated = migrate_misplaced_page(page, vma, target_nid);
6688cc05 3600 if (migrated) {
8191acbd 3601 page_nid = target_nid;
6688cc05 3602 flags |= TNF_MIGRATED;
074c2381
MG
3603 } else
3604 flags |= TNF_MIGRATE_FAIL;
4daae3b4
MG
3605
3606out:
8191acbd 3607 if (page_nid != -1)
6688cc05 3608 task_numa_fault(last_cpupid, page_nid, 1, flags);
d10e63f2
MG
3609 return 0;
3610}
3611
82b0f8c3 3612static int create_huge_pmd(struct vm_fault *vmf)
b96375f7 3613{
f4200391 3614 if (vma_is_anonymous(vmf->vma))
82b0f8c3 3615 return do_huge_pmd_anonymous_page(vmf);
a2d58167 3616 if (vmf->vma->vm_ops->huge_fault)
c791ace1 3617 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
b96375f7
MW
3618 return VM_FAULT_FALLBACK;
3619}
3620
82b0f8c3 3621static int wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd)
b96375f7 3622{
82b0f8c3
JK
3623 if (vma_is_anonymous(vmf->vma))
3624 return do_huge_pmd_wp_page(vmf, orig_pmd);
a2d58167 3625 if (vmf->vma->vm_ops->huge_fault)
c791ace1 3626 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
af9e4d5f
KS
3627
3628 /* COW handled on pte level: split pmd */
82b0f8c3
JK
3629 VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);
3630 __split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
af9e4d5f 3631
b96375f7
MW
3632 return VM_FAULT_FALLBACK;
3633}
3634
38e08854
LS
3635static inline bool vma_is_accessible(struct vm_area_struct *vma)
3636{
3637 return vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE);
3638}
3639
a00cc7d9
MW
3640static int create_huge_pud(struct vm_fault *vmf)
3641{
3642#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3643 /* No support for anonymous transparent PUD pages yet */
3644 if (vma_is_anonymous(vmf->vma))
3645 return VM_FAULT_FALLBACK;
3646 if (vmf->vma->vm_ops->huge_fault)
c791ace1 3647 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
a00cc7d9
MW
3648#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3649 return VM_FAULT_FALLBACK;
3650}
3651
3652static int wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
3653{
3654#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3655 /* No support for anonymous transparent PUD pages yet */
3656 if (vma_is_anonymous(vmf->vma))
3657 return VM_FAULT_FALLBACK;
3658 if (vmf->vma->vm_ops->huge_fault)
c791ace1 3659 return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PUD);
a00cc7d9
MW
3660#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3661 return VM_FAULT_FALLBACK;
3662}
3663
1da177e4
LT
3664/*
3665 * These routines also need to handle stuff like marking pages dirty
3666 * and/or accessed for architectures that don't do it in hardware (most
3667 * RISC architectures). The early dirtying is also good on the i386.
3668 *
3669 * There is also a hook called "update_mmu_cache()" that architectures
3670 * with external mmu caches can use to update those (ie the Sparc or
3671 * PowerPC hashed page tables that act as extended TLBs).
3672 *
7267ec00
KS
3673 * We enter with non-exclusive mmap_sem (to exclude vma changes, but allow
3674 * concurrent faults).
9a95f3cf 3675 *
7267ec00
KS
3676 * The mmap_sem may have been released depending on flags and our return value.
3677 * See filemap_fault() and __lock_page_or_retry().
1da177e4 3678 */
82b0f8c3 3679static int handle_pte_fault(struct vm_fault *vmf)
1da177e4
LT
3680{
3681 pte_t entry;
3682
82b0f8c3 3683 if (unlikely(pmd_none(*vmf->pmd))) {
7267ec00
KS
3684 /*
3685 * Leave __pte_alloc() until later: because vm_ops->fault may
3686 * want to allocate huge page, and if we expose page table
3687 * for an instant, it will be difficult to retract from
3688 * concurrent faults and from rmap lookups.
3689 */
82b0f8c3 3690 vmf->pte = NULL;
7267ec00
KS
3691 } else {
3692 /* See comment in pte_alloc_one_map() */
82b0f8c3 3693 if (pmd_trans_unstable(vmf->pmd) || pmd_devmap(*vmf->pmd))
7267ec00
KS
3694 return 0;
3695 /*
3696 * A regular pmd is established and it can't morph into a huge
3697 * pmd from under us anymore at this point because we hold the
3698 * mmap_sem read mode and khugepaged takes it in write mode.
3699 * So now it's safe to run pte_offset_map().
3700 */
82b0f8c3 3701 vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
2994302b 3702 vmf->orig_pte = *vmf->pte;
7267ec00
KS
3703
3704 /*
3705 * some architectures can have larger ptes than wordsize,
3706 * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
3707 * CONFIG_32BIT=y, so READ_ONCE or ACCESS_ONCE cannot guarantee
3708 * atomic accesses. The code below just needs a consistent
3709 * view for the ifs and we later double check anyway with the
3710 * ptl lock held. So here a barrier will do.
3711 */
3712 barrier();
2994302b 3713 if (pte_none(vmf->orig_pte)) {
82b0f8c3
JK
3714 pte_unmap(vmf->pte);
3715 vmf->pte = NULL;
65500d23 3716 }
1da177e4
LT
3717 }
3718
82b0f8c3
JK
3719 if (!vmf->pte) {
3720 if (vma_is_anonymous(vmf->vma))
3721 return do_anonymous_page(vmf);
7267ec00 3722 else
82b0f8c3 3723 return do_fault(vmf);
7267ec00
KS
3724 }
3725
2994302b
JK
3726 if (!pte_present(vmf->orig_pte))
3727 return do_swap_page(vmf);
7267ec00 3728
2994302b
JK
3729 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
3730 return do_numa_page(vmf);
d10e63f2 3731
82b0f8c3
JK
3732 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
3733 spin_lock(vmf->ptl);
2994302b 3734 entry = vmf->orig_pte;
82b0f8c3 3735 if (unlikely(!pte_same(*vmf->pte, entry)))
8f4e2101 3736 goto unlock;
82b0f8c3 3737 if (vmf->flags & FAULT_FLAG_WRITE) {
1da177e4 3738 if (!pte_write(entry))
2994302b 3739 return do_wp_page(vmf);
1da177e4
LT
3740 entry = pte_mkdirty(entry);
3741 }
3742 entry = pte_mkyoung(entry);
82b0f8c3
JK
3743 if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
3744 vmf->flags & FAULT_FLAG_WRITE)) {
3745 update_mmu_cache(vmf->vma, vmf->address, vmf->pte);
1a44e149
AA
3746 } else {
3747 /*
3748 * This is needed only for protection faults but the arch code
3749 * is not yet telling us if this is a protection fault or not.
3750 * This still avoids useless tlb flushes for .text page faults
3751 * with threads.
3752 */
82b0f8c3
JK
3753 if (vmf->flags & FAULT_FLAG_WRITE)
3754 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
1a44e149 3755 }
8f4e2101 3756unlock:
82b0f8c3 3757 pte_unmap_unlock(vmf->pte, vmf->ptl);
83c54070 3758 return 0;
1da177e4
LT
3759}
3760
3761/*
3762 * By the time we get here, we already hold the mm semaphore
9a95f3cf
PC
3763 *
3764 * The mmap_sem may have been released depending on flags and our
3765 * return value. See filemap_fault() and __lock_page_or_retry().
1da177e4 3766 */
dcddffd4
KS
3767static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
3768 unsigned int flags)
1da177e4 3769{
82b0f8c3 3770 struct vm_fault vmf = {
bae473a4 3771 .vma = vma,
1a29d85e 3772 .address = address & PAGE_MASK,
bae473a4 3773 .flags = flags,
0721ec8b 3774 .pgoff = linear_page_index(vma, address),
667240e0 3775 .gfp_mask = __get_fault_gfp_mask(vma),
bae473a4 3776 };
dcddffd4 3777 struct mm_struct *mm = vma->vm_mm;
1da177e4 3778 pgd_t *pgd;
c2febafc 3779 p4d_t *p4d;
a2d58167 3780 int ret;
1da177e4 3781
1da177e4 3782 pgd = pgd_offset(mm, address);
c2febafc
KS
3783 p4d = p4d_alloc(mm, pgd, address);
3784 if (!p4d)
3785 return VM_FAULT_OOM;
a00cc7d9 3786
c2febafc 3787 vmf.pud = pud_alloc(mm, p4d, address);
a00cc7d9 3788 if (!vmf.pud)
c74df32c 3789 return VM_FAULT_OOM;
a00cc7d9 3790 if (pud_none(*vmf.pud) && transparent_hugepage_enabled(vma)) {
a00cc7d9
MW
3791 ret = create_huge_pud(&vmf);
3792 if (!(ret & VM_FAULT_FALLBACK))
3793 return ret;
3794 } else {
3795 pud_t orig_pud = *vmf.pud;
3796
3797 barrier();
3798 if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
3799 unsigned int dirty = flags & FAULT_FLAG_WRITE;
3800
a00cc7d9
MW
3801 /* NUMA case for anonymous PUDs would go here */
3802
3803 if (dirty && !pud_write(orig_pud)) {
3804 ret = wp_huge_pud(&vmf, orig_pud);
3805 if (!(ret & VM_FAULT_FALLBACK))
3806 return ret;
3807 } else {
3808 huge_pud_set_accessed(&vmf, orig_pud);
3809 return 0;
3810 }
3811 }
3812 }
3813
3814 vmf.pmd = pmd_alloc(mm, vmf.pud, address);
82b0f8c3 3815 if (!vmf.pmd)
c74df32c 3816 return VM_FAULT_OOM;
82b0f8c3 3817 if (pmd_none(*vmf.pmd) && transparent_hugepage_enabled(vma)) {
a2d58167 3818 ret = create_huge_pmd(&vmf);
c0292554
KS
3819 if (!(ret & VM_FAULT_FALLBACK))
3820 return ret;
71e3aac0 3821 } else {
82b0f8c3 3822 pmd_t orig_pmd = *vmf.pmd;
1f1d06c3 3823
71e3aac0 3824 barrier();
5c7fb56e 3825 if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) {
38e08854 3826 if (pmd_protnone(orig_pmd) && vma_is_accessible(vma))
82b0f8c3 3827 return do_huge_pmd_numa_page(&vmf, orig_pmd);
d10e63f2 3828
82b0f8c3 3829 if ((vmf.flags & FAULT_FLAG_WRITE) &&
bae473a4 3830 !pmd_write(orig_pmd)) {
82b0f8c3 3831 ret = wp_huge_pmd(&vmf, orig_pmd);
9845cbbd
KS
3832 if (!(ret & VM_FAULT_FALLBACK))
3833 return ret;
a1dd450b 3834 } else {
82b0f8c3 3835 huge_pmd_set_accessed(&vmf, orig_pmd);
9845cbbd 3836 return 0;
1f1d06c3 3837 }
71e3aac0
AA
3838 }
3839 }
3840
82b0f8c3 3841 return handle_pte_fault(&vmf);
1da177e4
LT
3842}
3843
9a95f3cf
PC
3844/*
3845 * By the time we get here, we already hold the mm semaphore
3846 *
3847 * The mmap_sem may have been released depending on flags and our
3848 * return value. See filemap_fault() and __lock_page_or_retry().
3849 */
dcddffd4
KS
3850int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
3851 unsigned int flags)
519e5247
JW
3852{
3853 int ret;
3854
3855 __set_current_state(TASK_RUNNING);
3856
3857 count_vm_event(PGFAULT);
dcddffd4 3858 mem_cgroup_count_vm_event(vma->vm_mm, PGFAULT);
519e5247
JW
3859
3860 /* do counter updates before entering really critical section. */
3861 check_sync_rss_stat(current);
3862
3863 /*
3864 * Enable the memcg OOM handling for faults triggered in user
3865 * space. Kernel faults are handled more gracefully.
3866 */
3867 if (flags & FAULT_FLAG_USER)
49426420 3868 mem_cgroup_oom_enable();
519e5247 3869
bae473a4
KS
3870 if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
3871 flags & FAULT_FLAG_INSTRUCTION,
3872 flags & FAULT_FLAG_REMOTE))
3873 return VM_FAULT_SIGSEGV;
3874
3875 if (unlikely(is_vm_hugetlb_page(vma)))
3876 ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
3877 else
3878 ret = __handle_mm_fault(vma, address, flags);
519e5247 3879
49426420
JW
3880 if (flags & FAULT_FLAG_USER) {
3881 mem_cgroup_oom_disable();
166f61b9
TH
3882 /*
3883 * The task may have entered a memcg OOM situation but
3884 * if the allocation error was handled gracefully (no
3885 * VM_FAULT_OOM), there is no need to kill anything.
3886 * Just clean up the OOM state peacefully.
3887 */
3888 if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
3889 mem_cgroup_oom_synchronize(false);
49426420 3890 }
3812c8c8 3891
3f70dc38
MH
3892 /*
3893 * This mm has been already reaped by the oom reaper and so the
3894 * refault cannot be trusted in general. Anonymous refaults would
3895 * lose data and give a zero page instead e.g. This is especially
3896 * problem for use_mm() because regular tasks will just die and
3897 * the corrupted data will not be visible anywhere while kthread
3898 * will outlive the oom victim and potentially propagate the date
3899 * further.
3900 */
3901 if (unlikely((current->flags & PF_KTHREAD) && !(ret & VM_FAULT_ERROR)
3902 && test_bit(MMF_UNSTABLE, &vma->vm_mm->flags)))
3903 ret = VM_FAULT_SIGBUS;
3904
519e5247
JW
3905 return ret;
3906}
e1d6d01a 3907EXPORT_SYMBOL_GPL(handle_mm_fault);
519e5247 3908
90eceff1
KS
3909#ifndef __PAGETABLE_P4D_FOLDED
3910/*
3911 * Allocate p4d page table.
3912 * We've already handled the fast-path in-line.
3913 */
3914int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
3915{
3916 p4d_t *new = p4d_alloc_one(mm, address);
3917 if (!new)
3918 return -ENOMEM;
3919
3920 smp_wmb(); /* See comment in __pte_alloc */
3921
3922 spin_lock(&mm->page_table_lock);
3923 if (pgd_present(*pgd)) /* Another has populated it */
3924 p4d_free(mm, new);
3925 else
3926 pgd_populate(mm, pgd, new);
3927 spin_unlock(&mm->page_table_lock);
3928 return 0;
3929}
3930#endif /* __PAGETABLE_P4D_FOLDED */
3931
1da177e4
LT
3932#ifndef __PAGETABLE_PUD_FOLDED
3933/*
3934 * Allocate page upper directory.
872fec16 3935 * We've already handled the fast-path in-line.
1da177e4 3936 */
c2febafc 3937int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
1da177e4 3938{
c74df32c
HD
3939 pud_t *new = pud_alloc_one(mm, address);
3940 if (!new)
1bb3630e 3941 return -ENOMEM;
1da177e4 3942
362a61ad
NP
3943 smp_wmb(); /* See comment in __pte_alloc */
3944
872fec16 3945 spin_lock(&mm->page_table_lock);
c2febafc
KS
3946#ifndef __ARCH_HAS_5LEVEL_HACK
3947 if (p4d_present(*p4d)) /* Another has populated it */
5e541973 3948 pud_free(mm, new);
1bb3630e 3949 else
c2febafc
KS
3950 p4d_populate(mm, p4d, new);
3951#else
3952 if (pgd_present(*p4d)) /* Another has populated it */
5e541973 3953 pud_free(mm, new);
1bb3630e 3954 else
c2febafc
KS
3955 pgd_populate(mm, p4d, new);
3956#endif /* __ARCH_HAS_5LEVEL_HACK */
c74df32c 3957 spin_unlock(&mm->page_table_lock);
1bb3630e 3958 return 0;
1da177e4
LT
3959}
3960#endif /* __PAGETABLE_PUD_FOLDED */
3961
3962#ifndef __PAGETABLE_PMD_FOLDED
3963/*
3964 * Allocate page middle directory.
872fec16 3965 * We've already handled the fast-path in-line.
1da177e4 3966 */
1bb3630e 3967int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1da177e4 3968{
a00cc7d9 3969 spinlock_t *ptl;
c74df32c
HD
3970 pmd_t *new = pmd_alloc_one(mm, address);
3971 if (!new)
1bb3630e 3972 return -ENOMEM;
1da177e4 3973
362a61ad
NP
3974 smp_wmb(); /* See comment in __pte_alloc */
3975
a00cc7d9 3976 ptl = pud_lock(mm, pud);
1da177e4 3977#ifndef __ARCH_HAS_4LEVEL_HACK
dc6c9a35
KS
3978 if (!pud_present(*pud)) {
3979 mm_inc_nr_pmds(mm);
1bb3630e 3980 pud_populate(mm, pud, new);
dc6c9a35 3981 } else /* Another has populated it */
5e541973 3982 pmd_free(mm, new);
dc6c9a35
KS
3983#else
3984 if (!pgd_present(*pud)) {
3985 mm_inc_nr_pmds(mm);
1bb3630e 3986 pgd_populate(mm, pud, new);
dc6c9a35
KS
3987 } else /* Another has populated it */
3988 pmd_free(mm, new);
1da177e4 3989#endif /* __ARCH_HAS_4LEVEL_HACK */
a00cc7d9 3990 spin_unlock(ptl);
1bb3630e 3991 return 0;
e0f39591 3992}
1da177e4
LT
3993#endif /* __PAGETABLE_PMD_FOLDED */
3994
09796395
RZ
3995static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
3996 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
f8ad0f49
JW
3997{
3998 pgd_t *pgd;
c2febafc 3999 p4d_t *p4d;
f8ad0f49
JW
4000 pud_t *pud;
4001 pmd_t *pmd;
4002 pte_t *ptep;
4003
4004 pgd = pgd_offset(mm, address);
4005 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
4006 goto out;
4007
c2febafc
KS
4008 p4d = p4d_offset(pgd, address);
4009 if (p4d_none(*p4d) || unlikely(p4d_bad(*p4d)))
4010 goto out;
4011
4012 pud = pud_offset(p4d, address);
f8ad0f49
JW
4013 if (pud_none(*pud) || unlikely(pud_bad(*pud)))
4014 goto out;
4015
4016 pmd = pmd_offset(pud, address);
f66055ab 4017 VM_BUG_ON(pmd_trans_huge(*pmd));
f8ad0f49 4018
09796395
RZ
4019 if (pmd_huge(*pmd)) {
4020 if (!pmdpp)
4021 goto out;
4022
4023 *ptlp = pmd_lock(mm, pmd);
4024 if (pmd_huge(*pmd)) {
4025 *pmdpp = pmd;
4026 return 0;
4027 }
4028 spin_unlock(*ptlp);
4029 }
4030
4031 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
f8ad0f49
JW
4032 goto out;
4033
4034 ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
4035 if (!ptep)
4036 goto out;
4037 if (!pte_present(*ptep))
4038 goto unlock;
4039 *ptepp = ptep;
4040 return 0;
4041unlock:
4042 pte_unmap_unlock(ptep, *ptlp);
4043out:
4044 return -EINVAL;
4045}
4046
f729c8c9
RZ
4047static inline int follow_pte(struct mm_struct *mm, unsigned long address,
4048 pte_t **ptepp, spinlock_t **ptlp)
1b36ba81
NK
4049{
4050 int res;
4051
4052 /* (void) is needed to make gcc happy */
4053 (void) __cond_lock(*ptlp,
09796395
RZ
4054 !(res = __follow_pte_pmd(mm, address, ptepp, NULL,
4055 ptlp)));
4056 return res;
4057}
4058
4059int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
4060 pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
4061{
4062 int res;
4063
4064 /* (void) is needed to make gcc happy */
4065 (void) __cond_lock(*ptlp,
4066 !(res = __follow_pte_pmd(mm, address, ptepp, pmdpp,
4067 ptlp)));
1b36ba81
NK
4068 return res;
4069}
09796395 4070EXPORT_SYMBOL(follow_pte_pmd);
1b36ba81 4071
3b6748e2
JW
4072/**
4073 * follow_pfn - look up PFN at a user virtual address
4074 * @vma: memory mapping
4075 * @address: user virtual address
4076 * @pfn: location to store found PFN
4077 *
4078 * Only IO mappings and raw PFN mappings are allowed.
4079 *
4080 * Returns zero and the pfn at @pfn on success, -ve otherwise.
4081 */
4082int follow_pfn(struct vm_area_struct *vma, unsigned long address,
4083 unsigned long *pfn)
4084{
4085 int ret = -EINVAL;
4086 spinlock_t *ptl;
4087 pte_t *ptep;
4088
4089 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
4090 return ret;
4091
4092 ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
4093 if (ret)
4094 return ret;
4095 *pfn = pte_pfn(*ptep);
4096 pte_unmap_unlock(ptep, ptl);
4097 return 0;
4098}
4099EXPORT_SYMBOL(follow_pfn);
4100
28b2ee20 4101#ifdef CONFIG_HAVE_IOREMAP_PROT
d87fe660 4102int follow_phys(struct vm_area_struct *vma,
4103 unsigned long address, unsigned int flags,
4104 unsigned long *prot, resource_size_t *phys)
28b2ee20 4105{
03668a4d 4106 int ret = -EINVAL;
28b2ee20
RR
4107 pte_t *ptep, pte;
4108 spinlock_t *ptl;
28b2ee20 4109
d87fe660 4110 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
4111 goto out;
28b2ee20 4112
03668a4d 4113 if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
d87fe660 4114 goto out;
28b2ee20 4115 pte = *ptep;
03668a4d 4116
28b2ee20
RR
4117 if ((flags & FOLL_WRITE) && !pte_write(pte))
4118 goto unlock;
28b2ee20
RR
4119
4120 *prot = pgprot_val(pte_pgprot(pte));
03668a4d 4121 *phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
28b2ee20 4122
03668a4d 4123 ret = 0;
28b2ee20
RR
4124unlock:
4125 pte_unmap_unlock(ptep, ptl);
4126out:
d87fe660 4127 return ret;
28b2ee20
RR
4128}
4129
4130int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
4131 void *buf, int len, int write)
4132{
4133 resource_size_t phys_addr;
4134 unsigned long prot = 0;
2bc7273b 4135 void __iomem *maddr;
28b2ee20
RR
4136 int offset = addr & (PAGE_SIZE-1);
4137
d87fe660 4138 if (follow_phys(vma, addr, write, &prot, &phys_addr))
28b2ee20
RR
4139 return -EINVAL;
4140
9cb12d7b 4141 maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
28b2ee20
RR
4142 if (write)
4143 memcpy_toio(maddr + offset, buf, len);
4144 else
4145 memcpy_fromio(buf, maddr + offset, len);
4146 iounmap(maddr);
4147
4148 return len;
4149}
5a73633e 4150EXPORT_SYMBOL_GPL(generic_access_phys);
28b2ee20
RR
4151#endif
4152
0ec76a11 4153/*
206cb636
SW
4154 * Access another process' address space as given in mm. If non-NULL, use the
4155 * given task for page fault accounting.
0ec76a11 4156 */
84d77d3f 4157int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
442486ec 4158 unsigned long addr, void *buf, int len, unsigned int gup_flags)
0ec76a11 4159{
0ec76a11 4160 struct vm_area_struct *vma;
0ec76a11 4161 void *old_buf = buf;
442486ec 4162 int write = gup_flags & FOLL_WRITE;
0ec76a11 4163
0ec76a11 4164 down_read(&mm->mmap_sem);
183ff22b 4165 /* ignore errors, just check how much was successfully transferred */
0ec76a11
DH
4166 while (len) {
4167 int bytes, ret, offset;
4168 void *maddr;
28b2ee20 4169 struct page *page = NULL;
0ec76a11 4170
1e987790 4171 ret = get_user_pages_remote(tsk, mm, addr, 1,
5b56d49f 4172 gup_flags, &page, &vma, NULL);
28b2ee20 4173 if (ret <= 0) {
dbffcd03
RR
4174#ifndef CONFIG_HAVE_IOREMAP_PROT
4175 break;
4176#else
28b2ee20
RR
4177 /*
4178 * Check if this is a VM_IO | VM_PFNMAP VMA, which
4179 * we can access using slightly different code.
4180 */
28b2ee20 4181 vma = find_vma(mm, addr);
fe936dfc 4182 if (!vma || vma->vm_start > addr)
28b2ee20
RR
4183 break;
4184 if (vma->vm_ops && vma->vm_ops->access)
4185 ret = vma->vm_ops->access(vma, addr, buf,
4186 len, write);
4187 if (ret <= 0)
28b2ee20
RR
4188 break;
4189 bytes = ret;
dbffcd03 4190#endif
0ec76a11 4191 } else {
28b2ee20
RR
4192 bytes = len;
4193 offset = addr & (PAGE_SIZE-1);
4194 if (bytes > PAGE_SIZE-offset)
4195 bytes = PAGE_SIZE-offset;
4196
4197 maddr = kmap(page);
4198 if (write) {
4199 copy_to_user_page(vma, page, addr,
4200 maddr + offset, buf, bytes);
4201 set_page_dirty_lock(page);
4202 } else {
4203 copy_from_user_page(vma, page, addr,
4204 buf, maddr + offset, bytes);
4205 }
4206 kunmap(page);
09cbfeaf 4207 put_page(page);
0ec76a11 4208 }
0ec76a11
DH
4209 len -= bytes;
4210 buf += bytes;
4211 addr += bytes;
4212 }
4213 up_read(&mm->mmap_sem);
0ec76a11
DH
4214
4215 return buf - old_buf;
4216}
03252919 4217
5ddd36b9 4218/**
ae91dbfc 4219 * access_remote_vm - access another process' address space
5ddd36b9
SW
4220 * @mm: the mm_struct of the target address space
4221 * @addr: start address to access
4222 * @buf: source or destination buffer
4223 * @len: number of bytes to transfer
6347e8d5 4224 * @gup_flags: flags modifying lookup behaviour
5ddd36b9
SW
4225 *
4226 * The caller must hold a reference on @mm.
4227 */
4228int access_remote_vm(struct mm_struct *mm, unsigned long addr,
6347e8d5 4229 void *buf, int len, unsigned int gup_flags)
5ddd36b9 4230{
6347e8d5 4231 return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
5ddd36b9
SW
4232}
4233
206cb636
SW
4234/*
4235 * Access another process' address space.
4236 * Source/target buffer must be kernel space,
4237 * Do not walk the page table directly, use get_user_pages
4238 */
4239int access_process_vm(struct task_struct *tsk, unsigned long addr,
f307ab6d 4240 void *buf, int len, unsigned int gup_flags)
206cb636
SW
4241{
4242 struct mm_struct *mm;
4243 int ret;
4244
4245 mm = get_task_mm(tsk);
4246 if (!mm)
4247 return 0;
4248
f307ab6d 4249 ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
442486ec 4250
206cb636
SW
4251 mmput(mm);
4252
4253 return ret;
4254}
fcd35857 4255EXPORT_SYMBOL_GPL(access_process_vm);
206cb636 4256
03252919
AK
4257/*
4258 * Print the name of a VMA.
4259 */
4260void print_vma_addr(char *prefix, unsigned long ip)
4261{
4262 struct mm_struct *mm = current->mm;
4263 struct vm_area_struct *vma;
4264
e8bff74a
IM
4265 /*
4266 * Do not print if we are in atomic
4267 * contexts (in exception stacks, etc.):
4268 */
4269 if (preempt_count())
4270 return;
4271
03252919
AK
4272 down_read(&mm->mmap_sem);
4273 vma = find_vma(mm, ip);
4274 if (vma && vma->vm_file) {
4275 struct file *f = vma->vm_file;
4276 char *buf = (char *)__get_free_page(GFP_KERNEL);
4277 if (buf) {
2fbc57c5 4278 char *p;
03252919 4279
9bf39ab2 4280 p = file_path(f, buf, PAGE_SIZE);
03252919
AK
4281 if (IS_ERR(p))
4282 p = "?";
2fbc57c5 4283 printk("%s%s[%lx+%lx]", prefix, kbasename(p),
03252919
AK
4284 vma->vm_start,
4285 vma->vm_end - vma->vm_start);
4286 free_page((unsigned long)buf);
4287 }
4288 }
51a07e50 4289 up_read(&mm->mmap_sem);
03252919 4290}
3ee1afa3 4291
662bbcb2 4292#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
9ec23531 4293void __might_fault(const char *file, int line)
3ee1afa3 4294{
95156f00
PZ
4295 /*
4296 * Some code (nfs/sunrpc) uses socket ops on kernel memory while
4297 * holding the mmap_sem, this is safe because kernel memory doesn't
4298 * get paged out, therefore we'll never actually fault, and the
4299 * below annotations will generate false positives.
4300 */
db68ce10 4301 if (uaccess_kernel())
95156f00 4302 return;
9ec23531 4303 if (pagefault_disabled())
662bbcb2 4304 return;
9ec23531
DH
4305 __might_sleep(file, line, 0);
4306#if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
662bbcb2 4307 if (current->mm)
3ee1afa3 4308 might_lock_read(&current->mm->mmap_sem);
9ec23531 4309#endif
3ee1afa3 4310}
9ec23531 4311EXPORT_SYMBOL(__might_fault);
3ee1afa3 4312#endif
47ad8475
AA
4313
4314#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
4315static void clear_gigantic_page(struct page *page,
4316 unsigned long addr,
4317 unsigned int pages_per_huge_page)
4318{
4319 int i;
4320 struct page *p = page;
4321
4322 might_sleep();
4323 for (i = 0; i < pages_per_huge_page;
4324 i++, p = mem_map_next(p, page, i)) {
4325 cond_resched();
4326 clear_user_highpage(p, addr + i * PAGE_SIZE);
4327 }
4328}
4329void clear_huge_page(struct page *page,
4330 unsigned long addr, unsigned int pages_per_huge_page)
4331{
4332 int i;
4333
4334 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
4335 clear_gigantic_page(page, addr, pages_per_huge_page);
4336 return;
4337 }
4338
4339 might_sleep();
4340 for (i = 0; i < pages_per_huge_page; i++) {
4341 cond_resched();
4342 clear_user_highpage(page + i, addr + i * PAGE_SIZE);
4343 }
4344}
4345
4346static void copy_user_gigantic_page(struct page *dst, struct page *src,
4347 unsigned long addr,
4348 struct vm_area_struct *vma,
4349 unsigned int pages_per_huge_page)
4350{
4351 int i;
4352 struct page *dst_base = dst;
4353 struct page *src_base = src;
4354
4355 for (i = 0; i < pages_per_huge_page; ) {
4356 cond_resched();
4357 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
4358
4359 i++;
4360 dst = mem_map_next(dst, dst_base, i);
4361 src = mem_map_next(src, src_base, i);
4362 }
4363}
4364
4365void copy_user_huge_page(struct page *dst, struct page *src,
4366 unsigned long addr, struct vm_area_struct *vma,
4367 unsigned int pages_per_huge_page)
4368{
4369 int i;
4370
4371 if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
4372 copy_user_gigantic_page(dst, src, addr, vma,
4373 pages_per_huge_page);
4374 return;
4375 }
4376
4377 might_sleep();
4378 for (i = 0; i < pages_per_huge_page; i++) {
4379 cond_resched();
4380 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
4381 }
4382}
fa4d75c1
MK
4383
4384long copy_huge_page_from_user(struct page *dst_page,
4385 const void __user *usr_src,
810a56b9
MK
4386 unsigned int pages_per_huge_page,
4387 bool allow_pagefault)
fa4d75c1
MK
4388{
4389 void *src = (void *)usr_src;
4390 void *page_kaddr;
4391 unsigned long i, rc = 0;
4392 unsigned long ret_val = pages_per_huge_page * PAGE_SIZE;
4393
4394 for (i = 0; i < pages_per_huge_page; i++) {
810a56b9
MK
4395 if (allow_pagefault)
4396 page_kaddr = kmap(dst_page + i);
4397 else
4398 page_kaddr = kmap_atomic(dst_page + i);
fa4d75c1
MK
4399 rc = copy_from_user(page_kaddr,
4400 (const void __user *)(src + i * PAGE_SIZE),
4401 PAGE_SIZE);
810a56b9
MK
4402 if (allow_pagefault)
4403 kunmap(dst_page + i);
4404 else
4405 kunmap_atomic(page_kaddr);
fa4d75c1
MK
4406
4407 ret_val -= (PAGE_SIZE - rc);
4408 if (rc)
4409 break;
4410
4411 cond_resched();
4412 }
4413 return ret_val;
4414}
47ad8475 4415#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
49076ec2 4416
40b64acd 4417#if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
b35f1819
KS
4418
4419static struct kmem_cache *page_ptl_cachep;
4420
4421void __init ptlock_cache_init(void)
4422{
4423 page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
4424 SLAB_PANIC, NULL);
4425}
4426
539edb58 4427bool ptlock_alloc(struct page *page)
49076ec2
KS
4428{
4429 spinlock_t *ptl;
4430
b35f1819 4431 ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
49076ec2
KS
4432 if (!ptl)
4433 return false;
539edb58 4434 page->ptl = ptl;
49076ec2
KS
4435 return true;
4436}
4437
539edb58 4438void ptlock_free(struct page *page)
49076ec2 4439{
b35f1819 4440 kmem_cache_free(page_ptl_cachep, page->ptl);
49076ec2
KS
4441}
4442#endif