]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - mm/mlock.c
thp: update documentation
[mirror_ubuntu-zesty-kernel.git] / mm / mlock.c
CommitLineData
1da177e4
LT
1/*
2 * linux/mm/mlock.c
3 *
4 * (C) Copyright 1995 Linus Torvalds
5 * (C) Copyright 2002 Christoph Hellwig
6 */
7
c59ede7b 8#include <linux/capability.h>
1da177e4
LT
9#include <linux/mman.h>
10#include <linux/mm.h>
b291f000
NP
11#include <linux/swap.h>
12#include <linux/swapops.h>
13#include <linux/pagemap.h>
7225522b 14#include <linux/pagevec.h>
1da177e4
LT
15#include <linux/mempolicy.h>
16#include <linux/syscalls.h>
e8edc6e0 17#include <linux/sched.h>
b95f1b31 18#include <linux/export.h>
b291f000
NP
19#include <linux/rmap.h>
20#include <linux/mmzone.h>
21#include <linux/hugetlb.h>
7225522b
VB
22#include <linux/memcontrol.h>
23#include <linux/mm_inline.h>
b291f000
NP
24
25#include "internal.h"
1da177e4 26
e8edc6e0
AD
27int can_do_mlock(void)
28{
59e99e5b 29 if (rlimit(RLIMIT_MEMLOCK) != 0)
e8edc6e0 30 return 1;
a5a6579d
JVS
31 if (capable(CAP_IPC_LOCK))
32 return 1;
e8edc6e0
AD
33 return 0;
34}
35EXPORT_SYMBOL(can_do_mlock);
1da177e4 36
b291f000
NP
37/*
38 * Mlocked pages are marked with PageMlocked() flag for efficient testing
39 * in vmscan and, possibly, the fault path; and to support semi-accurate
40 * statistics.
41 *
42 * An mlocked page [PageMlocked(page)] is unevictable. As such, it will
43 * be placed on the LRU "unevictable" list, rather than the [in]active lists.
44 * The unevictable list is an LRU sibling list to the [in]active lists.
45 * PageUnevictable is set to indicate the unevictable state.
46 *
47 * When lazy mlocking via vmscan, it is important to ensure that the
48 * vma's VM_LOCKED status is not concurrently being modified, otherwise we
49 * may have mlocked a page that is being munlocked. So lazy mlock must take
50 * the mmap_sem for read, and verify that the vma really is locked
51 * (see mm/rmap.c).
52 */
53
54/*
55 * LRU accounting for clear_page_mlock()
56 */
e6c509f8 57void clear_page_mlock(struct page *page)
b291f000 58{
e6c509f8 59 if (!TestClearPageMlocked(page))
b291f000 60 return;
b291f000 61
8449d21f
DR
62 mod_zone_page_state(page_zone(page), NR_MLOCK,
63 -hpage_nr_pages(page));
5344b7e6 64 count_vm_event(UNEVICTABLE_PGCLEARED);
b291f000
NP
65 if (!isolate_lru_page(page)) {
66 putback_lru_page(page);
67 } else {
68 /*
8891d6da 69 * We lost the race. the page already moved to evictable list.
b291f000 70 */
8891d6da 71 if (PageUnevictable(page))
5344b7e6 72 count_vm_event(UNEVICTABLE_PGSTRANDED);
b291f000
NP
73 }
74}
75
76/*
77 * Mark page as mlocked if not already.
78 * If page on LRU, isolate and putback to move to unevictable list.
79 */
80void mlock_vma_page(struct page *page)
81{
57e68e9c 82 /* Serialize with page migration */
b291f000
NP
83 BUG_ON(!PageLocked(page));
84
5344b7e6 85 if (!TestSetPageMlocked(page)) {
8449d21f
DR
86 mod_zone_page_state(page_zone(page), NR_MLOCK,
87 hpage_nr_pages(page));
5344b7e6
NP
88 count_vm_event(UNEVICTABLE_PGMLOCKED);
89 if (!isolate_lru_page(page))
90 putback_lru_page(page);
91 }
b291f000
NP
92}
93
01cc2e58
VB
94/*
95 * Isolate a page from LRU with optional get_page() pin.
96 * Assumes lru_lock already held and page already pinned.
97 */
98static bool __munlock_isolate_lru_page(struct page *page, bool getpage)
99{
100 if (PageLRU(page)) {
101 struct lruvec *lruvec;
102
103 lruvec = mem_cgroup_page_lruvec(page, page_zone(page));
104 if (getpage)
105 get_page(page);
106 ClearPageLRU(page);
107 del_page_from_lru_list(page, lruvec, page_lru(page));
108 return true;
109 }
110
111 return false;
112}
113
7225522b
VB
114/*
115 * Finish munlock after successful page isolation
116 *
117 * Page must be locked. This is a wrapper for try_to_munlock()
118 * and putback_lru_page() with munlock accounting.
119 */
120static void __munlock_isolated_page(struct page *page)
121{
122 int ret = SWAP_AGAIN;
123
124 /*
125 * Optimization: if the page was mapped just once, that's our mapping
126 * and we don't need to check all the other vmas.
127 */
128 if (page_mapcount(page) > 1)
129 ret = try_to_munlock(page);
130
131 /* Did try_to_unlock() succeed or punt? */
132 if (ret != SWAP_MLOCK)
133 count_vm_event(UNEVICTABLE_PGMUNLOCKED);
134
135 putback_lru_page(page);
136}
137
138/*
139 * Accounting for page isolation fail during munlock
140 *
141 * Performs accounting when page isolation fails in munlock. There is nothing
142 * else to do because it means some other task has already removed the page
143 * from the LRU. putback_lru_page() will take care of removing the page from
144 * the unevictable list, if necessary. vmscan [page_referenced()] will move
145 * the page back to the unevictable list if some other vma has it mlocked.
146 */
147static void __munlock_isolation_failed(struct page *page)
148{
149 if (PageUnevictable(page))
01cc2e58 150 __count_vm_event(UNEVICTABLE_PGSTRANDED);
7225522b 151 else
01cc2e58 152 __count_vm_event(UNEVICTABLE_PGMUNLOCKED);
7225522b
VB
153}
154
6927c1dd
LS
155/**
156 * munlock_vma_page - munlock a vma page
c424be1c
VB
157 * @page - page to be unlocked, either a normal page or THP page head
158 *
159 * returns the size of the page as a page mask (0 for normal page,
160 * HPAGE_PMD_NR - 1 for THP head page)
b291f000 161 *
6927c1dd
LS
162 * called from munlock()/munmap() path with page supposedly on the LRU.
163 * When we munlock a page, because the vma where we found the page is being
164 * munlock()ed or munmap()ed, we want to check whether other vmas hold the
165 * page locked so that we can leave it on the unevictable lru list and not
166 * bother vmscan with it. However, to walk the page's rmap list in
167 * try_to_munlock() we must isolate the page from the LRU. If some other
168 * task has removed the page from the LRU, we won't be able to do that.
169 * So we clear the PageMlocked as we might not get another chance. If we
170 * can't isolate the page, we leave it for putback_lru_page() and vmscan
171 * [page_referenced()/try_to_unmap()] to deal with.
b291f000 172 */
ff6a6da6 173unsigned int munlock_vma_page(struct page *page)
b291f000 174{
c424be1c 175 unsigned int nr_pages;
01cc2e58 176 struct zone *zone = page_zone(page);
ff6a6da6 177
57e68e9c 178 /* For try_to_munlock() and to serialize with page migration */
b291f000
NP
179 BUG_ON(!PageLocked(page));
180
c424be1c 181 /*
01cc2e58
VB
182 * Serialize with any parallel __split_huge_page_refcount() which
183 * might otherwise copy PageMlocked to part of the tail pages before
184 * we clear it in the head page. It also stabilizes hpage_nr_pages().
c424be1c 185 */
01cc2e58
VB
186 spin_lock_irq(&zone->lru_lock);
187
188 nr_pages = hpage_nr_pages(page);
189 if (!TestClearPageMlocked(page))
190 goto unlock_out;
191
192 __mod_zone_page_state(zone, NR_MLOCK, -nr_pages);
193
194 if (__munlock_isolate_lru_page(page, true)) {
195 spin_unlock_irq(&zone->lru_lock);
196 __munlock_isolated_page(page);
197 goto out;
198 }
199 __munlock_isolation_failed(page);
200
201unlock_out:
202 spin_unlock_irq(&zone->lru_lock);
203
204out:
c424be1c 205 return nr_pages - 1;
b291f000
NP
206}
207
9978ad58
LS
208/*
209 * convert get_user_pages() return value to posix mlock() error
210 */
211static int __mlock_posix_error_return(long retval)
212{
213 if (retval == -EFAULT)
214 retval = -ENOMEM;
215 else if (retval == -ENOMEM)
216 retval = -EAGAIN;
217 return retval;
b291f000
NP
218}
219
56afe477
VB
220/*
221 * Prepare page for fast batched LRU putback via putback_lru_evictable_pagevec()
222 *
223 * The fast path is available only for evictable pages with single mapping.
224 * Then we can bypass the per-cpu pvec and get better performance.
225 * when mapcount > 1 we need try_to_munlock() which can fail.
226 * when !page_evictable(), we need the full redo logic of putback_lru_page to
227 * avoid leaving evictable page in unevictable list.
228 *
229 * In case of success, @page is added to @pvec and @pgrescued is incremented
230 * in case that the page was previously unevictable. @page is also unlocked.
231 */
232static bool __putback_lru_fast_prepare(struct page *page, struct pagevec *pvec,
233 int *pgrescued)
234{
309381fe
SL
235 VM_BUG_ON_PAGE(PageLRU(page), page);
236 VM_BUG_ON_PAGE(!PageLocked(page), page);
56afe477
VB
237
238 if (page_mapcount(page) <= 1 && page_evictable(page)) {
239 pagevec_add(pvec, page);
240 if (TestClearPageUnevictable(page))
241 (*pgrescued)++;
242 unlock_page(page);
243 return true;
244 }
245
246 return false;
247}
248
249/*
250 * Putback multiple evictable pages to the LRU
251 *
252 * Batched putback of evictable pages that bypasses the per-cpu pvec. Some of
253 * the pages might have meanwhile become unevictable but that is OK.
254 */
255static void __putback_lru_fast(struct pagevec *pvec, int pgrescued)
256{
257 count_vm_events(UNEVICTABLE_PGMUNLOCKED, pagevec_count(pvec));
258 /*
259 *__pagevec_lru_add() calls release_pages() so we don't call
260 * put_page() explicitly
261 */
262 __pagevec_lru_add(pvec);
263 count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
264}
265
7225522b
VB
266/*
267 * Munlock a batch of pages from the same zone
268 *
269 * The work is split to two main phases. First phase clears the Mlocked flag
270 * and attempts to isolate the pages, all under a single zone lru lock.
271 * The second phase finishes the munlock only for pages where isolation
272 * succeeded.
273 *
7a8010cd 274 * Note that the pagevec may be modified during the process.
7225522b
VB
275 */
276static void __munlock_pagevec(struct pagevec *pvec, struct zone *zone)
277{
278 int i;
279 int nr = pagevec_count(pvec);
3b25df93 280 int delta_munlocked;
56afe477
VB
281 struct pagevec pvec_putback;
282 int pgrescued = 0;
7225522b 283
3b25df93
VB
284 pagevec_init(&pvec_putback, 0);
285
7225522b
VB
286 /* Phase 1: page isolation */
287 spin_lock_irq(&zone->lru_lock);
288 for (i = 0; i < nr; i++) {
289 struct page *page = pvec->pages[i];
290
291 if (TestClearPageMlocked(page)) {
7225522b 292 /*
01cc2e58
VB
293 * We already have pin from follow_page_mask()
294 * so we can spare the get_page() here.
7225522b 295 */
01cc2e58
VB
296 if (__munlock_isolate_lru_page(page, false))
297 continue;
298 else
299 __munlock_isolation_failed(page);
7225522b 300 }
01cc2e58
VB
301
302 /*
303 * We won't be munlocking this page in the next phase
304 * but we still need to release the follow_page_mask()
305 * pin. We cannot do it under lru_lock however. If it's
306 * the last pin, __page_cache_release() would deadlock.
307 */
308 pagevec_add(&pvec_putback, pvec->pages[i]);
309 pvec->pages[i] = NULL;
7225522b 310 }
3b25df93 311 delta_munlocked = -nr + pagevec_count(&pvec_putback);
1ebb7cc6 312 __mod_zone_page_state(zone, NR_MLOCK, delta_munlocked);
7225522b
VB
313 spin_unlock_irq(&zone->lru_lock);
314
3b25df93
VB
315 /* Now we can release pins of pages that we are not munlocking */
316 pagevec_release(&pvec_putback);
317
56afe477 318 /* Phase 2: page munlock */
7225522b
VB
319 for (i = 0; i < nr; i++) {
320 struct page *page = pvec->pages[i];
321
322 if (page) {
323 lock_page(page);
56afe477
VB
324 if (!__putback_lru_fast_prepare(page, &pvec_putback,
325 &pgrescued)) {
5b40998a
VB
326 /*
327 * Slow path. We don't want to lose the last
328 * pin before unlock_page()
329 */
330 get_page(page); /* for putback_lru_page() */
56afe477
VB
331 __munlock_isolated_page(page);
332 unlock_page(page);
5b40998a 333 put_page(page); /* from follow_page_mask() */
56afe477 334 }
7225522b
VB
335 }
336 }
56afe477 337
5b40998a
VB
338 /*
339 * Phase 3: page putback for pages that qualified for the fast path
340 * This will also call put_page() to return pin from follow_page_mask()
341 */
56afe477
VB
342 if (pagevec_count(&pvec_putback))
343 __putback_lru_fast(&pvec_putback, pgrescued);
7a8010cd
VB
344}
345
346/*
347 * Fill up pagevec for __munlock_pagevec using pte walk
348 *
349 * The function expects that the struct page corresponding to @start address is
350 * a non-TPH page already pinned and in the @pvec, and that it belongs to @zone.
351 *
352 * The rest of @pvec is filled by subsequent pages within the same pmd and same
353 * zone, as long as the pte's are present and vm_normal_page() succeeds. These
354 * pages also get pinned.
355 *
356 * Returns the address of the next page that should be scanned. This equals
357 * @start + PAGE_SIZE when no page could be added by the pte walk.
358 */
359static unsigned long __munlock_pagevec_fill(struct pagevec *pvec,
360 struct vm_area_struct *vma, int zoneid, unsigned long start,
361 unsigned long end)
362{
363 pte_t *pte;
364 spinlock_t *ptl;
365
366 /*
367 * Initialize pte walk starting at the already pinned page where we
eadb41ae
VB
368 * are sure that there is a pte, as it was pinned under the same
369 * mmap_sem write op.
7a8010cd
VB
370 */
371 pte = get_locked_pte(vma->vm_mm, start, &ptl);
eadb41ae
VB
372 /* Make sure we do not cross the page table boundary */
373 end = pgd_addr_end(start, end);
374 end = pud_addr_end(start, end);
375 end = pmd_addr_end(start, end);
7a8010cd
VB
376
377 /* The page next to the pinned page is the first we will try to get */
378 start += PAGE_SIZE;
379 while (start < end) {
380 struct page *page = NULL;
381 pte++;
382 if (pte_present(*pte))
383 page = vm_normal_page(vma, start, *pte);
384 /*
385 * Break if page could not be obtained or the page's node+zone does not
386 * match
387 */
388 if (!page || page_zone_id(page) != zoneid)
389 break;
56afe477 390
7a8010cd
VB
391 get_page(page);
392 /*
393 * Increase the address that will be returned *before* the
394 * eventual break due to pvec becoming full by adding the page
395 */
396 start += PAGE_SIZE;
397 if (pagevec_add(pvec, page) == 0)
398 break;
399 }
400 pte_unmap_unlock(pte, ptl);
401 return start;
7225522b
VB
402}
403
b291f000 404/*
ba470de4
RR
405 * munlock_vma_pages_range() - munlock all pages in the vma range.'
406 * @vma - vma containing range to be munlock()ed.
407 * @start - start address in @vma of the range
408 * @end - end of range in @vma.
409 *
410 * For mremap(), munmap() and exit().
411 *
412 * Called with @vma VM_LOCKED.
413 *
414 * Returns with VM_LOCKED cleared. Callers must be prepared to
415 * deal with this.
416 *
417 * We don't save and restore VM_LOCKED here because pages are
418 * still on lru. In unmap path, pages might be scanned by reclaim
419 * and re-mlocked by try_to_{munlock|unmap} before we unmap and
420 * free them. This will result in freeing mlocked pages.
b291f000 421 */
ba470de4 422void munlock_vma_pages_range(struct vm_area_struct *vma,
408e82b7 423 unsigned long start, unsigned long end)
b291f000 424{
de60f5f1 425 vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
408e82b7 426
ff6a6da6 427 while (start < end) {
ab7a5af7 428 struct page *page;
c424be1c
VB
429 unsigned int page_mask;
430 unsigned long page_increm;
7a8010cd
VB
431 struct pagevec pvec;
432 struct zone *zone;
433 int zoneid;
ff6a6da6 434
7a8010cd 435 pagevec_init(&pvec, 0);
6e919717
HD
436 /*
437 * Although FOLL_DUMP is intended for get_dump_page(),
438 * it just so happens that its special treatment of the
439 * ZERO_PAGE (returning an error instead of doing get_page)
440 * suits munlock very well (and if somehow an abnormal page
441 * has sneaked into the range, we won't oops here: great).
442 */
ff6a6da6 443 page = follow_page_mask(vma, start, FOLL_GET | FOLL_DUMP,
7a8010cd
VB
444 &page_mask);
445
7479df6d
KS
446 if (page && !IS_ERR(page) && !PageTransCompound(page)) {
447 /*
448 * Non-huge pages are handled in batches via
449 * pagevec. The pin from follow_page_mask()
450 * prevents them from collapsing by THP.
451 */
452 pagevec_add(&pvec, page);
453 zone = page_zone(page);
454 zoneid = page_zone_id(page);
7a8010cd 455
7479df6d
KS
456 /*
457 * Try to fill the rest of pagevec using fast
458 * pte walk. This will also update start to
459 * the next page to process. Then munlock the
460 * pagevec.
461 */
462 start = __munlock_pagevec_fill(&pvec, vma,
463 zoneid, start, end);
464 __munlock_pagevec(&pvec, zone);
465 goto next;
408e82b7 466 }
c424be1c
VB
467 /* It's a bug to munlock in the middle of a THP page */
468 VM_BUG_ON((start >> PAGE_SHIFT) & page_mask);
469 page_increm = 1 + page_mask;
ff6a6da6 470 start += page_increm * PAGE_SIZE;
7a8010cd 471next:
408e82b7
HD
472 cond_resched();
473 }
b291f000
NP
474}
475
476/*
477 * mlock_fixup - handle mlock[all]/munlock[all] requests.
478 *
479 * Filters out "special" vmas -- VM_LOCKED never gets set for these, and
480 * munlock is a no-op. However, for some special vmas, we go ahead and
cea10a19 481 * populate the ptes.
b291f000
NP
482 *
483 * For vmas that pass the filters, merge/split as appropriate.
484 */
1da177e4 485static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
ca16d140 486 unsigned long start, unsigned long end, vm_flags_t newflags)
1da177e4 487{
b291f000 488 struct mm_struct *mm = vma->vm_mm;
1da177e4 489 pgoff_t pgoff;
b291f000 490 int nr_pages;
1da177e4 491 int ret = 0;
ca16d140 492 int lock = !!(newflags & VM_LOCKED);
1da177e4 493
fed067da 494 if (newflags == vma->vm_flags || (vma->vm_flags & VM_SPECIAL) ||
31db58b3 495 is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm))
b0f205c2
EM
496 /* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */
497 goto out;
b291f000 498
1da177e4
LT
499 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
500 *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,
19a809af
AA
501 vma->vm_file, pgoff, vma_policy(vma),
502 vma->vm_userfaultfd_ctx);
1da177e4
LT
503 if (*prev) {
504 vma = *prev;
505 goto success;
506 }
507
1da177e4
LT
508 if (start != vma->vm_start) {
509 ret = split_vma(mm, vma, start, 1);
510 if (ret)
511 goto out;
512 }
513
514 if (end != vma->vm_end) {
515 ret = split_vma(mm, vma, end, 0);
516 if (ret)
517 goto out;
518 }
519
520success:
b291f000
NP
521 /*
522 * Keep track of amount of locked VM.
523 */
524 nr_pages = (end - start) >> PAGE_SHIFT;
525 if (!lock)
526 nr_pages = -nr_pages;
527 mm->locked_vm += nr_pages;
528
1da177e4
LT
529 /*
530 * vm_flags is protected by the mmap_sem held in write mode.
531 * It's okay if try_to_unmap_one unmaps a page just after we
fc05f566 532 * set VM_LOCKED, populate_vma_page_range will bring it back.
1da177e4 533 */
1da177e4 534
fed067da 535 if (lock)
408e82b7 536 vma->vm_flags = newflags;
fed067da 537 else
408e82b7 538 munlock_vma_pages_range(vma, start, end);
1da177e4 539
1da177e4 540out:
b291f000 541 *prev = vma;
1da177e4
LT
542 return ret;
543}
544
1aab92ec
EM
545static int apply_vma_lock_flags(unsigned long start, size_t len,
546 vm_flags_t flags)
1da177e4
LT
547{
548 unsigned long nstart, end, tmp;
549 struct vm_area_struct * vma, * prev;
550 int error;
551
8fd9e488 552 VM_BUG_ON(offset_in_page(start));
fed067da 553 VM_BUG_ON(len != PAGE_ALIGN(len));
1da177e4
LT
554 end = start + len;
555 if (end < start)
556 return -EINVAL;
557 if (end == start)
558 return 0;
097d5910 559 vma = find_vma(current->mm, start);
1da177e4
LT
560 if (!vma || vma->vm_start > start)
561 return -ENOMEM;
562
097d5910 563 prev = vma->vm_prev;
1da177e4
LT
564 if (start > vma->vm_start)
565 prev = vma;
566
567 for (nstart = start ; ; ) {
b0f205c2 568 vm_flags_t newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
1da177e4 569
1aab92ec 570 newflags |= flags;
1da177e4 571
1aab92ec 572 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
1da177e4
LT
573 tmp = vma->vm_end;
574 if (tmp > end)
575 tmp = end;
576 error = mlock_fixup(vma, &prev, nstart, tmp, newflags);
577 if (error)
578 break;
579 nstart = tmp;
580 if (nstart < prev->vm_end)
581 nstart = prev->vm_end;
582 if (nstart >= end)
583 break;
584
585 vma = prev->vm_next;
586 if (!vma || vma->vm_start != nstart) {
587 error = -ENOMEM;
588 break;
589 }
590 }
591 return error;
592}
593
1aab92ec 594static int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
1da177e4
LT
595{
596 unsigned long locked;
597 unsigned long lock_limit;
598 int error = -ENOMEM;
599
600 if (!can_do_mlock())
601 return -EPERM;
602
8891d6da
KM
603 lru_add_drain_all(); /* flush pagevec */
604
8fd9e488 605 len = PAGE_ALIGN(len + (offset_in_page(start)));
1da177e4
LT
606 start &= PAGE_MASK;
607
59e99e5b 608 lock_limit = rlimit(RLIMIT_MEMLOCK);
1da177e4 609 lock_limit >>= PAGE_SHIFT;
1f1cd705
DB
610 locked = len >> PAGE_SHIFT;
611
612 down_write(&current->mm->mmap_sem);
613
614 locked += current->mm->locked_vm;
1da177e4
LT
615
616 /* check against resource limits */
617 if ((locked <= lock_limit) || capable(CAP_IPC_LOCK))
1aab92ec 618 error = apply_vma_lock_flags(start, len, flags);
1f1cd705 619
1da177e4 620 up_write(&current->mm->mmap_sem);
c561259c
KS
621 if (error)
622 return error;
623
624 error = __mm_populate(start, len, 0);
625 if (error)
626 return __mlock_posix_error_return(error);
627 return 0;
1da177e4
LT
628}
629
1aab92ec
EM
630SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len)
631{
632 return do_mlock(start, len, VM_LOCKED);
633}
634
a8ca5d0e
EM
635SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags)
636{
b0f205c2
EM
637 vm_flags_t vm_flags = VM_LOCKED;
638
639 if (flags & ~MLOCK_ONFAULT)
a8ca5d0e
EM
640 return -EINVAL;
641
b0f205c2
EM
642 if (flags & MLOCK_ONFAULT)
643 vm_flags |= VM_LOCKONFAULT;
644
645 return do_mlock(start, len, vm_flags);
a8ca5d0e
EM
646}
647
6a6160a7 648SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
1da177e4
LT
649{
650 int ret;
651
8fd9e488 652 len = PAGE_ALIGN(len + (offset_in_page(start)));
1da177e4 653 start &= PAGE_MASK;
1f1cd705
DB
654
655 down_write(&current->mm->mmap_sem);
1aab92ec 656 ret = apply_vma_lock_flags(start, len, 0);
1da177e4 657 up_write(&current->mm->mmap_sem);
1f1cd705 658
1da177e4
LT
659 return ret;
660}
661
b0f205c2
EM
662/*
663 * Take the MCL_* flags passed into mlockall (or 0 if called from munlockall)
664 * and translate into the appropriate modifications to mm->def_flags and/or the
665 * flags for all current VMAs.
666 *
667 * There are a couple of subtleties with this. If mlockall() is called multiple
668 * times with different flags, the values do not necessarily stack. If mlockall
669 * is called once including the MCL_FUTURE flag and then a second time without
670 * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags.
671 */
1aab92ec 672static int apply_mlockall_flags(int flags)
1da177e4
LT
673{
674 struct vm_area_struct * vma, * prev = NULL;
b0f205c2 675 vm_flags_t to_add = 0;
1da177e4 676
b0f205c2
EM
677 current->mm->def_flags &= VM_LOCKED_CLEAR_MASK;
678 if (flags & MCL_FUTURE) {
09a9f1d2 679 current->mm->def_flags |= VM_LOCKED;
1aab92ec 680
b0f205c2
EM
681 if (flags & MCL_ONFAULT)
682 current->mm->def_flags |= VM_LOCKONFAULT;
683
684 if (!(flags & MCL_CURRENT))
685 goto out;
686 }
687
688 if (flags & MCL_CURRENT) {
689 to_add |= VM_LOCKED;
690 if (flags & MCL_ONFAULT)
691 to_add |= VM_LOCKONFAULT;
692 }
1da177e4
LT
693
694 for (vma = current->mm->mmap; vma ; vma = prev->vm_next) {
ca16d140 695 vm_flags_t newflags;
1da177e4 696
b0f205c2
EM
697 newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
698 newflags |= to_add;
1da177e4
LT
699
700 /* Ignore errors */
701 mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
bde6c3aa 702 cond_resched_rcu_qs();
1da177e4
LT
703 }
704out:
705 return 0;
706}
707
3480b257 708SYSCALL_DEFINE1(mlockall, int, flags)
1da177e4
LT
709{
710 unsigned long lock_limit;
86d2adcc 711 int ret;
1da177e4 712
b0f205c2 713 if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT)))
86d2adcc 714 return -EINVAL;
1da177e4 715
1da177e4 716 if (!can_do_mlock())
86d2adcc 717 return -EPERM;
1da177e4 718
df9d6985
CL
719 if (flags & MCL_CURRENT)
720 lru_add_drain_all(); /* flush pagevec */
8891d6da 721
59e99e5b 722 lock_limit = rlimit(RLIMIT_MEMLOCK);
1da177e4
LT
723 lock_limit >>= PAGE_SHIFT;
724
725 ret = -ENOMEM;
1f1cd705
DB
726 down_write(&current->mm->mmap_sem);
727
1da177e4
LT
728 if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) ||
729 capable(CAP_IPC_LOCK))
1aab92ec 730 ret = apply_mlockall_flags(flags);
1da177e4 731 up_write(&current->mm->mmap_sem);
bebeb3d6
ML
732 if (!ret && (flags & MCL_CURRENT))
733 mm_populate(0, TASK_SIZE);
86d2adcc 734
1da177e4
LT
735 return ret;
736}
737
3480b257 738SYSCALL_DEFINE0(munlockall)
1da177e4
LT
739{
740 int ret;
741
742 down_write(&current->mm->mmap_sem);
1aab92ec 743 ret = apply_mlockall_flags(0);
1da177e4
LT
744 up_write(&current->mm->mmap_sem);
745 return ret;
746}
747
748/*
749 * Objects with different lifetime than processes (SHM_LOCK and SHM_HUGETLB
750 * shm segments) get accounted against the user_struct instead.
751 */
752static DEFINE_SPINLOCK(shmlock_user_lock);
753
754int user_shm_lock(size_t size, struct user_struct *user)
755{
756 unsigned long lock_limit, locked;
757 int allowed = 0;
758
759 locked = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
59e99e5b 760 lock_limit = rlimit(RLIMIT_MEMLOCK);
5ed44a40
HB
761 if (lock_limit == RLIM_INFINITY)
762 allowed = 1;
1da177e4
LT
763 lock_limit >>= PAGE_SHIFT;
764 spin_lock(&shmlock_user_lock);
5ed44a40
HB
765 if (!allowed &&
766 locked + user->locked_shm > lock_limit && !capable(CAP_IPC_LOCK))
1da177e4
LT
767 goto out;
768 get_uid(user);
769 user->locked_shm += locked;
770 allowed = 1;
771out:
772 spin_unlock(&shmlock_user_lock);
773 return allowed;
774}
775
776void user_shm_unlock(size_t size, struct user_struct *user)
777{
778 spin_lock(&shmlock_user_lock);
779 user->locked_shm -= (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
780 spin_unlock(&shmlock_user_lock);
781 free_uid(user);
782}