]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - mm/madvise.c
hrtimer: Make __hrtimer_get_next_event() static
[mirror_ubuntu-zesty-kernel.git] / mm / madvise.c
CommitLineData
1da177e4
LT
1/*
2 * linux/mm/madvise.c
3 *
4 * Copyright (C) 1999 Linus Torvalds
5 * Copyright (C) 2002 Christoph Hellwig
6 */
7
8#include <linux/mman.h>
9#include <linux/pagemap.h>
10#include <linux/syscalls.h>
05b74384 11#include <linux/mempolicy.h>
afcf938e 12#include <linux/page-isolation.h>
1da177e4 13#include <linux/hugetlb.h>
3f31d075 14#include <linux/falloc.h>
e8edc6e0 15#include <linux/sched.h>
f8af4da3 16#include <linux/ksm.h>
3f31d075 17#include <linux/fs.h>
9ab4233d 18#include <linux/file.h>
1998cc04
SL
19#include <linux/blkdev.h>
20#include <linux/swap.h>
21#include <linux/swapops.h>
1da177e4 22
0a27a14a
NP
23/*
24 * Any behaviour which results in changes to the vma->vm_flags needs to
25 * take mmap_sem for writing. Others, which simply traverse vmas, need
26 * to only take it for reading.
27 */
28static int madvise_need_mmap_write(int behavior)
29{
30 switch (behavior) {
31 case MADV_REMOVE:
32 case MADV_WILLNEED:
33 case MADV_DONTNEED:
34 return 0;
35 default:
36 /* be safe, default to 1. list exceptions explicitly */
37 return 1;
38 }
39}
40
1da177e4
LT
41/*
42 * We can potentially split a vm area into separate
43 * areas, each area with its own behavior.
44 */
ec9bed9d 45static long madvise_behavior(struct vm_area_struct *vma,
05b74384
PM
46 struct vm_area_struct **prev,
47 unsigned long start, unsigned long end, int behavior)
1da177e4 48{
ec9bed9d 49 struct mm_struct *mm = vma->vm_mm;
1da177e4 50 int error = 0;
05b74384 51 pgoff_t pgoff;
3866ea90 52 unsigned long new_flags = vma->vm_flags;
e798c6e8
PM
53
54 switch (behavior) {
f8225661
MT
55 case MADV_NORMAL:
56 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
57 break;
e798c6e8 58 case MADV_SEQUENTIAL:
f8225661 59 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
e798c6e8
PM
60 break;
61 case MADV_RANDOM:
f8225661 62 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
e798c6e8 63 break;
f8225661
MT
64 case MADV_DONTFORK:
65 new_flags |= VM_DONTCOPY;
66 break;
67 case MADV_DOFORK:
3866ea90
HD
68 if (vma->vm_flags & VM_IO) {
69 error = -EINVAL;
70 goto out;
71 }
f8225661 72 new_flags &= ~VM_DONTCOPY;
e798c6e8 73 break;
accb61fe 74 case MADV_DONTDUMP:
0103bd16 75 new_flags |= VM_DONTDUMP;
accb61fe
JB
76 break;
77 case MADV_DODUMP:
0103bd16
KK
78 if (new_flags & VM_SPECIAL) {
79 error = -EINVAL;
80 goto out;
81 }
82 new_flags &= ~VM_DONTDUMP;
accb61fe 83 break;
f8af4da3
HD
84 case MADV_MERGEABLE:
85 case MADV_UNMERGEABLE:
86 error = ksm_madvise(vma, start, end, behavior, &new_flags);
87 if (error)
88 goto out;
89 break;
0af4e98b 90 case MADV_HUGEPAGE:
a664b2d8 91 case MADV_NOHUGEPAGE:
60ab3244 92 error = hugepage_madvise(vma, &new_flags, behavior);
0af4e98b
AA
93 if (error)
94 goto out;
95 break;
e798c6e8
PM
96 }
97
05b74384
PM
98 if (new_flags == vma->vm_flags) {
99 *prev = vma;
836d5ffd 100 goto out;
05b74384
PM
101 }
102
103 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
104 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
105 vma->vm_file, pgoff, vma_policy(vma));
106 if (*prev) {
107 vma = *prev;
108 goto success;
109 }
110
111 *prev = vma;
1da177e4
LT
112
113 if (start != vma->vm_start) {
114 error = split_vma(mm, vma, start, 1);
115 if (error)
116 goto out;
117 }
118
119 if (end != vma->vm_end) {
120 error = split_vma(mm, vma, end, 0);
121 if (error)
122 goto out;
123 }
124
836d5ffd 125success:
1da177e4
LT
126 /*
127 * vm_flags is protected by the mmap_sem held in write mode.
128 */
e798c6e8 129 vma->vm_flags = new_flags;
1da177e4
LT
130
131out:
132 if (error == -ENOMEM)
133 error = -EAGAIN;
134 return error;
135}
136
1998cc04
SL
137#ifdef CONFIG_SWAP
138static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
139 unsigned long end, struct mm_walk *walk)
140{
141 pte_t *orig_pte;
142 struct vm_area_struct *vma = walk->private;
143 unsigned long index;
144
145 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
146 return 0;
147
148 for (index = start; index != end; index += PAGE_SIZE) {
149 pte_t pte;
150 swp_entry_t entry;
151 struct page *page;
152 spinlock_t *ptl;
153
154 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
155 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
156 pte_unmap_unlock(orig_pte, ptl);
157
158 if (pte_present(pte) || pte_none(pte) || pte_file(pte))
159 continue;
160 entry = pte_to_swp_entry(pte);
161 if (unlikely(non_swap_entry(entry)))
162 continue;
163
164 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
165 vma, index);
166 if (page)
167 page_cache_release(page);
168 }
169
170 return 0;
171}
172
173static void force_swapin_readahead(struct vm_area_struct *vma,
174 unsigned long start, unsigned long end)
175{
176 struct mm_walk walk = {
177 .mm = vma->vm_mm,
178 .pmd_entry = swapin_walk_pmd_entry,
179 .private = vma,
180 };
181
182 walk_page_range(start, end, &walk);
183
184 lru_add_drain(); /* Push any new pages onto the LRU now */
185}
186
187static void force_shm_swapin_readahead(struct vm_area_struct *vma,
188 unsigned long start, unsigned long end,
189 struct address_space *mapping)
190{
191 pgoff_t index;
192 struct page *page;
193 swp_entry_t swap;
194
195 for (; start < end; start += PAGE_SIZE) {
196 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
197
55231e5c 198 page = find_get_entry(mapping, index);
1998cc04
SL
199 if (!radix_tree_exceptional_entry(page)) {
200 if (page)
201 page_cache_release(page);
202 continue;
203 }
204 swap = radix_to_swp_entry(page);
205 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
206 NULL, 0);
207 if (page)
208 page_cache_release(page);
209 }
210
211 lru_add_drain(); /* Push any new pages onto the LRU now */
212}
213#endif /* CONFIG_SWAP */
214
1da177e4
LT
215/*
216 * Schedule all required I/O operations. Do not wait for completion.
217 */
ec9bed9d
VC
218static long madvise_willneed(struct vm_area_struct *vma,
219 struct vm_area_struct **prev,
1da177e4
LT
220 unsigned long start, unsigned long end)
221{
222 struct file *file = vma->vm_file;
223
1998cc04
SL
224#ifdef CONFIG_SWAP
225 if (!file || mapping_cap_swap_backed(file->f_mapping)) {
226 *prev = vma;
227 if (!file)
228 force_swapin_readahead(vma, start, end);
229 else
230 force_shm_swapin_readahead(vma, start, end,
231 file->f_mapping);
232 return 0;
233 }
234#endif
235
1bef4003
S
236 if (!file)
237 return -EBADF;
238
70688e4d 239 if (file->f_mapping->a_ops->get_xip_mem) {
fe77ba6f
CO
240 /* no bad return value, but ignore advice */
241 return 0;
242 }
243
05b74384 244 *prev = vma;
1da177e4
LT
245 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
246 if (end > vma->vm_end)
247 end = vma->vm_end;
248 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
249
f7e839dd 250 force_page_cache_readahead(file->f_mapping, file, start, end - start);
1da177e4
LT
251 return 0;
252}
253
254/*
255 * Application no longer needs these pages. If the pages are dirty,
256 * it's OK to just throw them away. The app will be more careful about
257 * data it wants to keep. Be sure to free swap resources too. The
7e6cbea3 258 * zap_page_range call sets things up for shrink_active_list to actually free
1da177e4
LT
259 * these pages later if no one else has touched them in the meantime,
260 * although we could add these pages to a global reuse list for
7e6cbea3 261 * shrink_active_list to pick up before reclaiming other pages.
1da177e4
LT
262 *
263 * NB: This interface discards data rather than pushes it out to swap,
264 * as some implementations do. This has performance implications for
265 * applications like large transactional databases which want to discard
266 * pages in anonymous maps after committing to backing store the data
267 * that was kept in them. There is no reason to write this data out to
268 * the swap area if the application is discarding it.
269 *
270 * An interface that causes the system to free clean pages and flush
271 * dirty pages is already available as msync(MS_INVALIDATE).
272 */
ec9bed9d
VC
273static long madvise_dontneed(struct vm_area_struct *vma,
274 struct vm_area_struct **prev,
1da177e4
LT
275 unsigned long start, unsigned long end)
276{
05b74384 277 *prev = vma;
6aab341e 278 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
1da177e4
LT
279 return -EINVAL;
280
281 if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
282 struct zap_details details = {
283 .nonlinear_vma = vma,
284 .last_index = ULONG_MAX,
285 };
286 zap_page_range(vma, start, end - start, &details);
287 } else
288 zap_page_range(vma, start, end - start, NULL);
289 return 0;
290}
291
f6b3ec23
BP
292/*
293 * Application wants to free up the pages and associated backing store.
294 * This is effectively punching a hole into the middle of a file.
f6b3ec23
BP
295 */
296static long madvise_remove(struct vm_area_struct *vma,
00e9fa2d 297 struct vm_area_struct **prev,
f6b3ec23
BP
298 unsigned long start, unsigned long end)
299{
3f31d075 300 loff_t offset;
90ed52eb 301 int error;
9ab4233d 302 struct file *f;
f6b3ec23 303
90ed52eb 304 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
00e9fa2d 305
f6b3ec23
BP
306 if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB))
307 return -EINVAL;
308
9ab4233d
AL
309 f = vma->vm_file;
310
311 if (!f || !f->f_mapping || !f->f_mapping->host) {
f6b3ec23
BP
312 return -EINVAL;
313 }
314
69cf0fac
HD
315 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
316 return -EACCES;
317
f6b3ec23
BP
318 offset = (loff_t)(start - vma->vm_start)
319 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
90ed52eb 320
9ab4233d
AL
321 /*
322 * Filesystem's fallocate may need to take i_mutex. We need to
323 * explicitly grab a reference because the vma (and hence the
324 * vma's reference to the file) can go away as soon as we drop
325 * mmap_sem.
326 */
327 get_file(f);
0a27a14a 328 up_read(&current->mm->mmap_sem);
72c72bdf 329 error = vfs_fallocate(f,
3f31d075
HD
330 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
331 offset, end - start);
9ab4233d 332 fput(f);
0a27a14a 333 down_read(&current->mm->mmap_sem);
90ed52eb 334 return error;
f6b3ec23
BP
335}
336
9893e49d
AK
337#ifdef CONFIG_MEMORY_FAILURE
338/*
339 * Error injection support for memory error handling.
340 */
afcf938e 341static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
9893e49d 342{
20cb6cab 343 struct page *p;
9893e49d
AK
344 if (!capable(CAP_SYS_ADMIN))
345 return -EPERM;
20cb6cab
WL
346 for (; start < end; start += PAGE_SIZE <<
347 compound_order(compound_head(p))) {
325c4ef5
AM
348 int ret;
349
350 ret = get_user_pages_fast(start, 1, 0, &p);
9893e49d
AK
351 if (ret != 1)
352 return ret;
325c4ef5 353
29b4eede
WL
354 if (PageHWPoison(p)) {
355 put_page(p);
356 continue;
357 }
afcf938e 358 if (bhv == MADV_SOFT_OFFLINE) {
b194b8cd 359 pr_info("Soft offlining page %#lx at %#lx\n",
afcf938e
AK
360 page_to_pfn(p), start);
361 ret = soft_offline_page(p, MF_COUNT_INCREASED);
362 if (ret)
8302423b 363 return ret;
afcf938e
AK
364 continue;
365 }
b194b8cd 366 pr_info("Injecting memory failure for page %#lx at %#lx\n",
9893e49d
AK
367 page_to_pfn(p), start);
368 /* Ignore return value for now */
cd42f4a3 369 memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
9893e49d 370 }
325c4ef5 371 return 0;
9893e49d
AK
372}
373#endif
374
165cd402 375static long
376madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
377 unsigned long start, unsigned long end, int behavior)
1da177e4 378{
1da177e4 379 switch (behavior) {
f6b3ec23 380 case MADV_REMOVE:
3866ea90 381 return madvise_remove(vma, prev, start, end);
1da177e4 382 case MADV_WILLNEED:
3866ea90 383 return madvise_willneed(vma, prev, start, end);
1da177e4 384 case MADV_DONTNEED:
3866ea90 385 return madvise_dontneed(vma, prev, start, end);
1da177e4 386 default:
3866ea90 387 return madvise_behavior(vma, prev, start, end, behavior);
1da177e4 388 }
1da177e4
LT
389}
390
75927af8
NP
391static int
392madvise_behavior_valid(int behavior)
393{
394 switch (behavior) {
395 case MADV_DOFORK:
396 case MADV_DONTFORK:
397 case MADV_NORMAL:
398 case MADV_SEQUENTIAL:
399 case MADV_RANDOM:
400 case MADV_REMOVE:
401 case MADV_WILLNEED:
402 case MADV_DONTNEED:
f8af4da3
HD
403#ifdef CONFIG_KSM
404 case MADV_MERGEABLE:
405 case MADV_UNMERGEABLE:
0af4e98b
AA
406#endif
407#ifdef CONFIG_TRANSPARENT_HUGEPAGE
408 case MADV_HUGEPAGE:
a664b2d8 409 case MADV_NOHUGEPAGE:
f8af4da3 410#endif
accb61fe
JB
411 case MADV_DONTDUMP:
412 case MADV_DODUMP:
75927af8
NP
413 return 1;
414
415 default:
416 return 0;
417 }
418}
3866ea90 419
1da177e4
LT
420/*
421 * The madvise(2) system call.
422 *
423 * Applications can use madvise() to advise the kernel how it should
424 * handle paging I/O in this VM area. The idea is to help the kernel
425 * use appropriate read-ahead and caching techniques. The information
426 * provided is advisory only, and can be safely disregarded by the
427 * kernel without affecting the correct operation of the application.
428 *
429 * behavior values:
430 * MADV_NORMAL - the default behavior is to read clusters. This
431 * results in some read-ahead and read-behind.
432 * MADV_RANDOM - the system should read the minimum amount of data
433 * on any access, since it is unlikely that the appli-
434 * cation will need more than what it asks for.
435 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
436 * once, so they can be aggressively read ahead, and
437 * can be freed soon after they are accessed.
438 * MADV_WILLNEED - the application is notifying the system to read
439 * some pages ahead.
440 * MADV_DONTNEED - the application is finished with the given range,
441 * so the kernel can free resources associated with it.
f6b3ec23
BP
442 * MADV_REMOVE - the application wants to free up the given range of
443 * pages and associated backing store.
3866ea90
HD
444 * MADV_DONTFORK - omit this area from child's address space when forking:
445 * typically, to avoid COWing pages pinned by get_user_pages().
446 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
f8af4da3
HD
447 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
448 * this area with pages of identical content from other such areas.
449 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
1da177e4
LT
450 *
451 * return values:
452 * zero - success
453 * -EINVAL - start + len < 0, start is not page-aligned,
454 * "behavior" is not a valid value, or application
455 * is attempting to release locked or shared pages.
456 * -ENOMEM - addresses in the specified range are not currently
457 * mapped, or are outside the AS of the process.
458 * -EIO - an I/O error occurred while paging in data.
459 * -EBADF - map exists, but area maps something that isn't a file.
460 * -EAGAIN - a kernel resource was temporarily unavailable.
461 */
3480b257 462SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
1da177e4 463{
05b74384 464 unsigned long end, tmp;
ec9bed9d 465 struct vm_area_struct *vma, *prev;
1da177e4
LT
466 int unmapped_error = 0;
467 int error = -EINVAL;
f7977793 468 int write;
1da177e4 469 size_t len;
1998cc04 470 struct blk_plug plug;
1da177e4 471
9893e49d 472#ifdef CONFIG_MEMORY_FAILURE
afcf938e
AK
473 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
474 return madvise_hwpoison(behavior, start, start+len_in);
9893e49d 475#endif
75927af8
NP
476 if (!madvise_behavior_valid(behavior))
477 return error;
478
1da177e4 479 if (start & ~PAGE_MASK)
84d96d89 480 return error;
1da177e4
LT
481 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
482
483 /* Check to see whether len was rounded up from small -ve to zero */
484 if (len_in && !len)
84d96d89 485 return error;
1da177e4
LT
486
487 end = start + len;
488 if (end < start)
84d96d89 489 return error;
1da177e4
LT
490
491 error = 0;
492 if (end == start)
84d96d89
RV
493 return error;
494
495 write = madvise_need_mmap_write(behavior);
496 if (write)
497 down_write(&current->mm->mmap_sem);
498 else
499 down_read(&current->mm->mmap_sem);
1da177e4
LT
500
501 /*
502 * If the interval [start,end) covers some unmapped address
503 * ranges, just ignore them, but return -ENOMEM at the end.
05b74384 504 * - different from the way of handling in mlock etc.
1da177e4 505 */
05b74384 506 vma = find_vma_prev(current->mm, start, &prev);
836d5ffd
HD
507 if (vma && start > vma->vm_start)
508 prev = vma;
509
1998cc04 510 blk_start_plug(&plug);
1da177e4
LT
511 for (;;) {
512 /* Still start < end. */
513 error = -ENOMEM;
514 if (!vma)
84d96d89 515 goto out;
1da177e4 516
05b74384 517 /* Here start < (end|vma->vm_end). */
1da177e4
LT
518 if (start < vma->vm_start) {
519 unmapped_error = -ENOMEM;
520 start = vma->vm_start;
05b74384 521 if (start >= end)
84d96d89 522 goto out;
1da177e4
LT
523 }
524
05b74384
PM
525 /* Here vma->vm_start <= start < (end|vma->vm_end) */
526 tmp = vma->vm_end;
527 if (end < tmp)
528 tmp = end;
1da177e4 529
05b74384
PM
530 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
531 error = madvise_vma(vma, &prev, start, tmp, behavior);
1da177e4 532 if (error)
84d96d89 533 goto out;
05b74384 534 start = tmp;
90ed52eb 535 if (prev && start < prev->vm_end)
05b74384
PM
536 start = prev->vm_end;
537 error = unmapped_error;
538 if (start >= end)
84d96d89 539 goto out;
90ed52eb
HD
540 if (prev)
541 vma = prev->vm_next;
542 else /* madvise_remove dropped mmap_sem */
543 vma = find_vma(current->mm, start);
1da177e4 544 }
1da177e4 545out:
84d96d89 546 blk_finish_plug(&plug);
f7977793 547 if (write)
0a27a14a
NP
548 up_write(&current->mm->mmap_sem);
549 else
550 up_read(&current->mm->mmap_sem);
551
1da177e4
LT
552 return error;
553}