]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - mm/rmap.c
2 * mm/rmap.c - physical to virtual reverse mappings
4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5 * Released under the General Public License (GPL).
7 * Simple, low overhead reverse mapping scheme.
8 * Please try to keep this thing as modular as possible.
10 * Provides methods for unmapping each kind of mapped page:
11 * the anon methods track anonymous pages, and
12 * the file methods track pages belonging to an inode.
14 * Original design by Rik van Riel <riel@conectiva.com.br> 2001
15 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
16 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
17 * Contributions by Hugh Dickins <hugh@veritas.com> 2003, 2004
21 * Lock ordering in mm:
23 * inode->i_mutex (while writing or truncating, not reading or faulting)
24 * inode->i_alloc_sem (vmtruncate_range)
26 * page->flags PG_locked (lock_page)
27 * mapping->i_mmap_lock
29 * mm->page_table_lock or pte_lock
30 * zone->lru_lock (in mark_page_accessed, isolate_lru_page)
31 * swap_lock (in swap_duplicate, swap_info_get)
32 * mmlist_lock (in mmput, drain_mmlist and others)
33 * mapping->private_lock (in __set_page_dirty_buffers)
34 * inode_lock (in set_page_dirty's __mark_inode_dirty)
35 * sb_lock (within inode_lock in fs/fs-writeback.c)
36 * mapping->tree_lock (widely used, in set_page_dirty,
37 * in arch-dependent flush_dcache_mmap_lock,
38 * within inode_lock in __sync_single_inode)
42 #include <linux/pagemap.h>
43 #include <linux/swap.h>
44 #include <linux/swapops.h>
45 #include <linux/slab.h>
46 #include <linux/init.h>
47 #include <linux/rmap.h>
48 #include <linux/rcupdate.h>
49 #include <linux/module.h>
50 #include <linux/kallsyms.h>
52 #include <asm/tlbflush.h>
54 struct kmem_cache
*anon_vma_cachep
;
56 /* This must be called under the mmap_sem. */
57 int anon_vma_prepare(struct vm_area_struct
*vma
)
59 struct anon_vma
*anon_vma
= vma
->anon_vma
;
62 if (unlikely(!anon_vma
)) {
63 struct mm_struct
*mm
= vma
->vm_mm
;
64 struct anon_vma
*allocated
, *locked
;
66 anon_vma
= find_mergeable_anon_vma(vma
);
70 spin_lock(&locked
->lock
);
72 anon_vma
= anon_vma_alloc();
73 if (unlikely(!anon_vma
))
79 /* page_table_lock to protect against threads */
80 spin_lock(&mm
->page_table_lock
);
81 if (likely(!vma
->anon_vma
)) {
82 vma
->anon_vma
= anon_vma
;
83 list_add_tail(&vma
->anon_vma_node
, &anon_vma
->head
);
86 spin_unlock(&mm
->page_table_lock
);
89 spin_unlock(&locked
->lock
);
90 if (unlikely(allocated
))
91 anon_vma_free(allocated
);
96 void __anon_vma_merge(struct vm_area_struct
*vma
, struct vm_area_struct
*next
)
98 BUG_ON(vma
->anon_vma
!= next
->anon_vma
);
99 list_del(&next
->anon_vma_node
);
102 void __anon_vma_link(struct vm_area_struct
*vma
)
104 struct anon_vma
*anon_vma
= vma
->anon_vma
;
107 list_add_tail(&vma
->anon_vma_node
, &anon_vma
->head
);
110 void anon_vma_link(struct vm_area_struct
*vma
)
112 struct anon_vma
*anon_vma
= vma
->anon_vma
;
115 spin_lock(&anon_vma
->lock
);
116 list_add_tail(&vma
->anon_vma_node
, &anon_vma
->head
);
117 spin_unlock(&anon_vma
->lock
);
121 void anon_vma_unlink(struct vm_area_struct
*vma
)
123 struct anon_vma
*anon_vma
= vma
->anon_vma
;
129 spin_lock(&anon_vma
->lock
);
130 list_del(&vma
->anon_vma_node
);
132 /* We must garbage collect the anon_vma if it's empty */
133 empty
= list_empty(&anon_vma
->head
);
134 spin_unlock(&anon_vma
->lock
);
137 anon_vma_free(anon_vma
);
140 static void anon_vma_ctor(struct kmem_cache
*cachep
, void *data
)
142 struct anon_vma
*anon_vma
= data
;
144 spin_lock_init(&anon_vma
->lock
);
145 INIT_LIST_HEAD(&anon_vma
->head
);
148 void __init
anon_vma_init(void)
150 anon_vma_cachep
= kmem_cache_create("anon_vma", sizeof(struct anon_vma
),
151 0, SLAB_DESTROY_BY_RCU
|SLAB_PANIC
, anon_vma_ctor
);
155 * Getting a lock on a stable anon_vma from a page off the LRU is
156 * tricky: page_lock_anon_vma rely on RCU to guard against the races.
158 static struct anon_vma
*page_lock_anon_vma(struct page
*page
)
160 struct anon_vma
*anon_vma
;
161 unsigned long anon_mapping
;
164 anon_mapping
= (unsigned long) page
->mapping
;
165 if (!(anon_mapping
& PAGE_MAPPING_ANON
))
167 if (!page_mapped(page
))
170 anon_vma
= (struct anon_vma
*) (anon_mapping
- PAGE_MAPPING_ANON
);
171 spin_lock(&anon_vma
->lock
);
178 static void page_unlock_anon_vma(struct anon_vma
*anon_vma
)
180 spin_unlock(&anon_vma
->lock
);
185 * At what user virtual address is page expected in vma?
187 static inline unsigned long
188 vma_address(struct page
*page
, struct vm_area_struct
*vma
)
190 pgoff_t pgoff
= page
->index
<< (PAGE_CACHE_SHIFT
- PAGE_SHIFT
);
191 unsigned long address
;
193 address
= vma
->vm_start
+ ((pgoff
- vma
->vm_pgoff
) << PAGE_SHIFT
);
194 if (unlikely(address
< vma
->vm_start
|| address
>= vma
->vm_end
)) {
195 /* page should be within any vma from prio_tree_next */
196 BUG_ON(!PageAnon(page
));
203 * At what user virtual address is page expected in vma? checking that the
204 * page matches the vma: currently only used on anon pages, by unuse_vma;
206 unsigned long page_address_in_vma(struct page
*page
, struct vm_area_struct
*vma
)
208 if (PageAnon(page
)) {
209 if ((void *)vma
->anon_vma
!=
210 (void *)page
->mapping
- PAGE_MAPPING_ANON
)
212 } else if (page
->mapping
&& !(vma
->vm_flags
& VM_NONLINEAR
)) {
214 vma
->vm_file
->f_mapping
!= page
->mapping
)
218 return vma_address(page
, vma
);
222 * Check that @page is mapped at @address into @mm.
224 * On success returns with pte mapped and locked.
226 pte_t
*page_check_address(struct page
*page
, struct mm_struct
*mm
,
227 unsigned long address
, spinlock_t
**ptlp
)
235 pgd
= pgd_offset(mm
, address
);
236 if (!pgd_present(*pgd
))
239 pud
= pud_offset(pgd
, address
);
240 if (!pud_present(*pud
))
243 pmd
= pmd_offset(pud
, address
);
244 if (!pmd_present(*pmd
))
247 pte
= pte_offset_map(pmd
, address
);
248 /* Make a quick check before getting the lock */
249 if (!pte_present(*pte
)) {
254 ptl
= pte_lockptr(mm
, pmd
);
256 if (pte_present(*pte
) && page_to_pfn(page
) == pte_pfn(*pte
)) {
260 pte_unmap_unlock(pte
, ptl
);
265 * Subfunctions of page_referenced: page_referenced_one called
266 * repeatedly from either page_referenced_anon or page_referenced_file.
268 static int page_referenced_one(struct page
*page
,
269 struct vm_area_struct
*vma
, unsigned int *mapcount
)
271 struct mm_struct
*mm
= vma
->vm_mm
;
272 unsigned long address
;
277 address
= vma_address(page
, vma
);
278 if (address
== -EFAULT
)
281 pte
= page_check_address(page
, mm
, address
, &ptl
);
285 if (ptep_clear_flush_young(vma
, address
, pte
))
288 /* Pretend the page is referenced if the task has the
289 swap token and is in the middle of a page fault. */
290 if (mm
!= current
->mm
&& has_swap_token(mm
) &&
291 rwsem_is_locked(&mm
->mmap_sem
))
295 pte_unmap_unlock(pte
, ptl
);
300 static int page_referenced_anon(struct page
*page
)
302 unsigned int mapcount
;
303 struct anon_vma
*anon_vma
;
304 struct vm_area_struct
*vma
;
307 anon_vma
= page_lock_anon_vma(page
);
311 mapcount
= page_mapcount(page
);
312 list_for_each_entry(vma
, &anon_vma
->head
, anon_vma_node
) {
313 referenced
+= page_referenced_one(page
, vma
, &mapcount
);
318 page_unlock_anon_vma(anon_vma
);
323 * page_referenced_file - referenced check for object-based rmap
324 * @page: the page we're checking references on.
326 * For an object-based mapped page, find all the places it is mapped and
327 * check/clear the referenced flag. This is done by following the page->mapping
328 * pointer, then walking the chain of vmas it holds. It returns the number
329 * of references it found.
331 * This function is only called from page_referenced for object-based pages.
333 static int page_referenced_file(struct page
*page
)
335 unsigned int mapcount
;
336 struct address_space
*mapping
= page
->mapping
;
337 pgoff_t pgoff
= page
->index
<< (PAGE_CACHE_SHIFT
- PAGE_SHIFT
);
338 struct vm_area_struct
*vma
;
339 struct prio_tree_iter iter
;
343 * The caller's checks on page->mapping and !PageAnon have made
344 * sure that this is a file page: the check for page->mapping
345 * excludes the case just before it gets set on an anon page.
347 BUG_ON(PageAnon(page
));
350 * The page lock not only makes sure that page->mapping cannot
351 * suddenly be NULLified by truncation, it makes sure that the
352 * structure at mapping cannot be freed and reused yet,
353 * so we can safely take mapping->i_mmap_lock.
355 BUG_ON(!PageLocked(page
));
357 spin_lock(&mapping
->i_mmap_lock
);
360 * i_mmap_lock does not stabilize mapcount at all, but mapcount
361 * is more likely to be accurate if we note it after spinning.
363 mapcount
= page_mapcount(page
);
365 vma_prio_tree_foreach(vma
, &iter
, &mapping
->i_mmap
, pgoff
, pgoff
) {
366 if ((vma
->vm_flags
& (VM_LOCKED
|VM_MAYSHARE
))
367 == (VM_LOCKED
|VM_MAYSHARE
)) {
371 referenced
+= page_referenced_one(page
, vma
, &mapcount
);
376 spin_unlock(&mapping
->i_mmap_lock
);
381 * page_referenced - test if the page was referenced
382 * @page: the page to test
383 * @is_locked: caller holds lock on the page
385 * Quick test_and_clear_referenced for all mappings to a page,
386 * returns the number of ptes which referenced the page.
388 int page_referenced(struct page
*page
, int is_locked
)
392 if (page_test_and_clear_young(page
))
395 if (TestClearPageReferenced(page
))
398 if (page_mapped(page
) && page
->mapping
) {
400 referenced
+= page_referenced_anon(page
);
402 referenced
+= page_referenced_file(page
);
403 else if (TestSetPageLocked(page
))
407 referenced
+= page_referenced_file(page
);
414 static int page_mkclean_one(struct page
*page
, struct vm_area_struct
*vma
)
416 struct mm_struct
*mm
= vma
->vm_mm
;
417 unsigned long address
;
422 address
= vma_address(page
, vma
);
423 if (address
== -EFAULT
)
426 pte
= page_check_address(page
, mm
, address
, &ptl
);
430 if (pte_dirty(*pte
) || pte_write(*pte
)) {
433 flush_cache_page(vma
, address
, pte_pfn(*pte
));
434 entry
= ptep_clear_flush(vma
, address
, pte
);
435 entry
= pte_wrprotect(entry
);
436 entry
= pte_mkclean(entry
);
437 set_pte_at(mm
, address
, pte
, entry
);
441 pte_unmap_unlock(pte
, ptl
);
446 static int page_mkclean_file(struct address_space
*mapping
, struct page
*page
)
448 pgoff_t pgoff
= page
->index
<< (PAGE_CACHE_SHIFT
- PAGE_SHIFT
);
449 struct vm_area_struct
*vma
;
450 struct prio_tree_iter iter
;
453 BUG_ON(PageAnon(page
));
455 spin_lock(&mapping
->i_mmap_lock
);
456 vma_prio_tree_foreach(vma
, &iter
, &mapping
->i_mmap
, pgoff
, pgoff
) {
457 if (vma
->vm_flags
& VM_SHARED
)
458 ret
+= page_mkclean_one(page
, vma
);
460 spin_unlock(&mapping
->i_mmap_lock
);
464 int page_mkclean(struct page
*page
)
468 BUG_ON(!PageLocked(page
));
470 if (page_mapped(page
)) {
471 struct address_space
*mapping
= page_mapping(page
);
473 ret
= page_mkclean_file(mapping
, page
);
474 if (page_test_dirty(page
)) {
475 page_clear_dirty(page
);
482 EXPORT_SYMBOL_GPL(page_mkclean
);
485 * page_set_anon_rmap - setup new anonymous rmap
486 * @page: the page to add the mapping to
487 * @vma: the vm area in which the mapping is added
488 * @address: the user virtual address mapped
490 static void __page_set_anon_rmap(struct page
*page
,
491 struct vm_area_struct
*vma
, unsigned long address
)
493 struct anon_vma
*anon_vma
= vma
->anon_vma
;
496 anon_vma
= (void *) anon_vma
+ PAGE_MAPPING_ANON
;
497 page
->mapping
= (struct address_space
*) anon_vma
;
499 page
->index
= linear_page_index(vma
, address
);
502 * nr_mapped state can be updated without turning off
503 * interrupts because it is not modified via interrupt.
505 __inc_zone_page_state(page
, NR_ANON_PAGES
);
509 * page_set_anon_rmap - sanity check anonymous rmap addition
510 * @page: the page to add the mapping to
511 * @vma: the vm area in which the mapping is added
512 * @address: the user virtual address mapped
514 static void __page_check_anon_rmap(struct page
*page
,
515 struct vm_area_struct
*vma
, unsigned long address
)
517 #ifdef CONFIG_DEBUG_VM
519 * The page's anon-rmap details (mapping and index) are guaranteed to
520 * be set up correctly at this point.
522 * We have exclusion against page_add_anon_rmap because the caller
523 * always holds the page locked, except if called from page_dup_rmap,
524 * in which case the page is already known to be setup.
526 * We have exclusion against page_add_new_anon_rmap because those pages
527 * are initially only visible via the pagetables, and the pte is locked
528 * over the call to page_add_new_anon_rmap.
530 struct anon_vma
*anon_vma
= vma
->anon_vma
;
531 anon_vma
= (void *) anon_vma
+ PAGE_MAPPING_ANON
;
532 BUG_ON(page
->mapping
!= (struct address_space
*)anon_vma
);
533 BUG_ON(page
->index
!= linear_page_index(vma
, address
));
538 * page_add_anon_rmap - add pte mapping to an anonymous page
539 * @page: the page to add the mapping to
540 * @vma: the vm area in which the mapping is added
541 * @address: the user virtual address mapped
543 * The caller needs to hold the pte lock and the page must be locked.
545 void page_add_anon_rmap(struct page
*page
,
546 struct vm_area_struct
*vma
, unsigned long address
)
548 VM_BUG_ON(!PageLocked(page
));
549 VM_BUG_ON(address
< vma
->vm_start
|| address
>= vma
->vm_end
);
550 if (atomic_inc_and_test(&page
->_mapcount
))
551 __page_set_anon_rmap(page
, vma
, address
);
553 __page_check_anon_rmap(page
, vma
, address
);
557 * page_add_new_anon_rmap - add pte mapping to a new anonymous page
558 * @page: the page to add the mapping to
559 * @vma: the vm area in which the mapping is added
560 * @address: the user virtual address mapped
562 * Same as page_add_anon_rmap but must only be called on *new* pages.
563 * This means the inc-and-test can be bypassed.
564 * Page does not have to be locked.
566 void page_add_new_anon_rmap(struct page
*page
,
567 struct vm_area_struct
*vma
, unsigned long address
)
569 BUG_ON(address
< vma
->vm_start
|| address
>= vma
->vm_end
);
570 atomic_set(&page
->_mapcount
, 0); /* elevate count by 1 (starts at -1) */
571 __page_set_anon_rmap(page
, vma
, address
);
575 * page_add_file_rmap - add pte mapping to a file page
576 * @page: the page to add the mapping to
578 * The caller needs to hold the pte lock.
580 void page_add_file_rmap(struct page
*page
)
582 if (atomic_inc_and_test(&page
->_mapcount
))
583 __inc_zone_page_state(page
, NR_FILE_MAPPED
);
586 #ifdef CONFIG_DEBUG_VM
588 * page_dup_rmap - duplicate pte mapping to a page
589 * @page: the page to add the mapping to
591 * For copy_page_range only: minimal extract from page_add_file_rmap /
592 * page_add_anon_rmap, avoiding unnecessary tests (already checked) so it's
595 * The caller needs to hold the pte lock.
597 void page_dup_rmap(struct page
*page
, struct vm_area_struct
*vma
, unsigned long address
)
599 BUG_ON(page_mapcount(page
) == 0);
601 __page_check_anon_rmap(page
, vma
, address
);
602 atomic_inc(&page
->_mapcount
);
607 * page_remove_rmap - take down pte mapping from a page
608 * @page: page to remove mapping from
610 * The caller needs to hold the pte lock.
612 void page_remove_rmap(struct page
*page
, struct vm_area_struct
*vma
)
614 if (atomic_add_negative(-1, &page
->_mapcount
)) {
615 if (unlikely(page_mapcount(page
) < 0)) {
616 printk (KERN_EMERG
"Eeek! page_mapcount(page) went negative! (%d)\n", page_mapcount(page
));
617 printk (KERN_EMERG
" page pfn = %lx\n", page_to_pfn(page
));
618 printk (KERN_EMERG
" page->flags = %lx\n", page
->flags
);
619 printk (KERN_EMERG
" page->count = %x\n", page_count(page
));
620 printk (KERN_EMERG
" page->mapping = %p\n", page
->mapping
);
621 print_symbol (KERN_EMERG
" vma->vm_ops = %s\n", (unsigned long)vma
->vm_ops
);
623 print_symbol (KERN_EMERG
" vma->vm_ops->nopage = %s\n", (unsigned long)vma
->vm_ops
->nopage
);
624 print_symbol (KERN_EMERG
" vma->vm_ops->fault = %s\n", (unsigned long)vma
->vm_ops
->fault
);
626 if (vma
->vm_file
&& vma
->vm_file
->f_op
)
627 print_symbol (KERN_EMERG
" vma->vm_file->f_op->mmap = %s\n", (unsigned long)vma
->vm_file
->f_op
->mmap
);
632 * It would be tidy to reset the PageAnon mapping here,
633 * but that might overwrite a racing page_add_anon_rmap
634 * which increments mapcount after us but sets mapping
635 * before us: so leave the reset to free_hot_cold_page,
636 * and remember that it's only reliable while mapped.
637 * Leaving it set also helps swapoff to reinstate ptes
638 * faster for those pages still in swapcache.
640 if (page_test_dirty(page
)) {
641 page_clear_dirty(page
);
642 set_page_dirty(page
);
644 __dec_zone_page_state(page
,
645 PageAnon(page
) ? NR_ANON_PAGES
: NR_FILE_MAPPED
);
650 * Subfunctions of try_to_unmap: try_to_unmap_one called
651 * repeatedly from either try_to_unmap_anon or try_to_unmap_file.
653 static int try_to_unmap_one(struct page
*page
, struct vm_area_struct
*vma
,
656 struct mm_struct
*mm
= vma
->vm_mm
;
657 unsigned long address
;
661 int ret
= SWAP_AGAIN
;
663 address
= vma_address(page
, vma
);
664 if (address
== -EFAULT
)
667 pte
= page_check_address(page
, mm
, address
, &ptl
);
672 * If the page is mlock()d, we cannot swap it out.
673 * If it's recently referenced (perhaps page_referenced
674 * skipped over this mm) then we should reactivate it.
676 if (!migration
&& ((vma
->vm_flags
& VM_LOCKED
) ||
677 (ptep_clear_flush_young(vma
, address
, pte
)))) {
682 /* Nuke the page table entry. */
683 flush_cache_page(vma
, address
, page_to_pfn(page
));
684 pteval
= ptep_clear_flush(vma
, address
, pte
);
686 /* Move the dirty bit to the physical page now the pte is gone. */
687 if (pte_dirty(pteval
))
688 set_page_dirty(page
);
690 /* Update high watermark before we lower rss */
691 update_hiwater_rss(mm
);
693 if (PageAnon(page
)) {
694 swp_entry_t entry
= { .val
= page_private(page
) };
696 if (PageSwapCache(page
)) {
698 * Store the swap location in the pte.
699 * See handle_pte_fault() ...
701 swap_duplicate(entry
);
702 if (list_empty(&mm
->mmlist
)) {
703 spin_lock(&mmlist_lock
);
704 if (list_empty(&mm
->mmlist
))
705 list_add(&mm
->mmlist
, &init_mm
.mmlist
);
706 spin_unlock(&mmlist_lock
);
708 dec_mm_counter(mm
, anon_rss
);
709 #ifdef CONFIG_MIGRATION
712 * Store the pfn of the page in a special migration
713 * pte. do_swap_page() will wait until the migration
714 * pte is removed and then restart fault handling.
717 entry
= make_migration_entry(page
, pte_write(pteval
));
720 set_pte_at(mm
, address
, pte
, swp_entry_to_pte(entry
));
721 BUG_ON(pte_file(*pte
));
723 #ifdef CONFIG_MIGRATION
725 /* Establish migration entry for a file page */
727 entry
= make_migration_entry(page
, pte_write(pteval
));
728 set_pte_at(mm
, address
, pte
, swp_entry_to_pte(entry
));
731 dec_mm_counter(mm
, file_rss
);
734 page_remove_rmap(page
, vma
);
735 page_cache_release(page
);
738 pte_unmap_unlock(pte
, ptl
);
744 * objrmap doesn't work for nonlinear VMAs because the assumption that
745 * offset-into-file correlates with offset-into-virtual-addresses does not hold.
746 * Consequently, given a particular page and its ->index, we cannot locate the
747 * ptes which are mapping that page without an exhaustive linear search.
749 * So what this code does is a mini "virtual scan" of each nonlinear VMA which
750 * maps the file to which the target page belongs. The ->vm_private_data field
751 * holds the current cursor into that scan. Successive searches will circulate
752 * around the vma's virtual address space.
754 * So as more replacement pressure is applied to the pages in a nonlinear VMA,
755 * more scanning pressure is placed against them as well. Eventually pages
756 * will become fully unmapped and are eligible for eviction.
758 * For very sparsely populated VMAs this is a little inefficient - chances are
759 * there there won't be many ptes located within the scan cluster. In this case
760 * maybe we could scan further - to the end of the pte page, perhaps.
762 #define CLUSTER_SIZE min(32*PAGE_SIZE, PMD_SIZE)
763 #define CLUSTER_MASK (~(CLUSTER_SIZE - 1))
765 static void try_to_unmap_cluster(unsigned long cursor
,
766 unsigned int *mapcount
, struct vm_area_struct
*vma
)
768 struct mm_struct
*mm
= vma
->vm_mm
;
776 unsigned long address
;
779 address
= (vma
->vm_start
+ cursor
) & CLUSTER_MASK
;
780 end
= address
+ CLUSTER_SIZE
;
781 if (address
< vma
->vm_start
)
782 address
= vma
->vm_start
;
783 if (end
> vma
->vm_end
)
786 pgd
= pgd_offset(mm
, address
);
787 if (!pgd_present(*pgd
))
790 pud
= pud_offset(pgd
, address
);
791 if (!pud_present(*pud
))
794 pmd
= pmd_offset(pud
, address
);
795 if (!pmd_present(*pmd
))
798 pte
= pte_offset_map_lock(mm
, pmd
, address
, &ptl
);
800 /* Update high watermark before we lower rss */
801 update_hiwater_rss(mm
);
803 for (; address
< end
; pte
++, address
+= PAGE_SIZE
) {
804 if (!pte_present(*pte
))
806 page
= vm_normal_page(vma
, address
, *pte
);
807 BUG_ON(!page
|| PageAnon(page
));
809 if (ptep_clear_flush_young(vma
, address
, pte
))
812 /* Nuke the page table entry. */
813 flush_cache_page(vma
, address
, pte_pfn(*pte
));
814 pteval
= ptep_clear_flush(vma
, address
, pte
);
816 /* If nonlinear, store the file page offset in the pte. */
817 if (page
->index
!= linear_page_index(vma
, address
))
818 set_pte_at(mm
, address
, pte
, pgoff_to_pte(page
->index
));
820 /* Move the dirty bit to the physical page now the pte is gone. */
821 if (pte_dirty(pteval
))
822 set_page_dirty(page
);
824 page_remove_rmap(page
, vma
);
825 page_cache_release(page
);
826 dec_mm_counter(mm
, file_rss
);
829 pte_unmap_unlock(pte
- 1, ptl
);
832 static int try_to_unmap_anon(struct page
*page
, int migration
)
834 struct anon_vma
*anon_vma
;
835 struct vm_area_struct
*vma
;
836 int ret
= SWAP_AGAIN
;
838 anon_vma
= page_lock_anon_vma(page
);
842 list_for_each_entry(vma
, &anon_vma
->head
, anon_vma_node
) {
843 ret
= try_to_unmap_one(page
, vma
, migration
);
844 if (ret
== SWAP_FAIL
|| !page_mapped(page
))
848 page_unlock_anon_vma(anon_vma
);
853 * try_to_unmap_file - unmap file page using the object-based rmap method
854 * @page: the page to unmap
856 * Find all the mappings of a page using the mapping pointer and the vma chains
857 * contained in the address_space struct it points to.
859 * This function is only called from try_to_unmap for object-based pages.
861 static int try_to_unmap_file(struct page
*page
, int migration
)
863 struct address_space
*mapping
= page
->mapping
;
864 pgoff_t pgoff
= page
->index
<< (PAGE_CACHE_SHIFT
- PAGE_SHIFT
);
865 struct vm_area_struct
*vma
;
866 struct prio_tree_iter iter
;
867 int ret
= SWAP_AGAIN
;
868 unsigned long cursor
;
869 unsigned long max_nl_cursor
= 0;
870 unsigned long max_nl_size
= 0;
871 unsigned int mapcount
;
873 spin_lock(&mapping
->i_mmap_lock
);
874 vma_prio_tree_foreach(vma
, &iter
, &mapping
->i_mmap
, pgoff
, pgoff
) {
875 ret
= try_to_unmap_one(page
, vma
, migration
);
876 if (ret
== SWAP_FAIL
|| !page_mapped(page
))
880 if (list_empty(&mapping
->i_mmap_nonlinear
))
883 list_for_each_entry(vma
, &mapping
->i_mmap_nonlinear
,
884 shared
.vm_set
.list
) {
885 if ((vma
->vm_flags
& VM_LOCKED
) && !migration
)
887 cursor
= (unsigned long) vma
->vm_private_data
;
888 if (cursor
> max_nl_cursor
)
889 max_nl_cursor
= cursor
;
890 cursor
= vma
->vm_end
- vma
->vm_start
;
891 if (cursor
> max_nl_size
)
892 max_nl_size
= cursor
;
895 if (max_nl_size
== 0) { /* any nonlinears locked or reserved */
901 * We don't try to search for this page in the nonlinear vmas,
902 * and page_referenced wouldn't have found it anyway. Instead
903 * just walk the nonlinear vmas trying to age and unmap some.
904 * The mapcount of the page we came in with is irrelevant,
905 * but even so use it as a guide to how hard we should try?
907 mapcount
= page_mapcount(page
);
910 cond_resched_lock(&mapping
->i_mmap_lock
);
912 max_nl_size
= (max_nl_size
+ CLUSTER_SIZE
- 1) & CLUSTER_MASK
;
913 if (max_nl_cursor
== 0)
914 max_nl_cursor
= CLUSTER_SIZE
;
917 list_for_each_entry(vma
, &mapping
->i_mmap_nonlinear
,
918 shared
.vm_set
.list
) {
919 if ((vma
->vm_flags
& VM_LOCKED
) && !migration
)
921 cursor
= (unsigned long) vma
->vm_private_data
;
922 while ( cursor
< max_nl_cursor
&&
923 cursor
< vma
->vm_end
- vma
->vm_start
) {
924 try_to_unmap_cluster(cursor
, &mapcount
, vma
);
925 cursor
+= CLUSTER_SIZE
;
926 vma
->vm_private_data
= (void *) cursor
;
927 if ((int)mapcount
<= 0)
930 vma
->vm_private_data
= (void *) max_nl_cursor
;
932 cond_resched_lock(&mapping
->i_mmap_lock
);
933 max_nl_cursor
+= CLUSTER_SIZE
;
934 } while (max_nl_cursor
<= max_nl_size
);
937 * Don't loop forever (perhaps all the remaining pages are
938 * in locked vmas). Reset cursor on all unreserved nonlinear
939 * vmas, now forgetting on which ones it had fallen behind.
941 list_for_each_entry(vma
, &mapping
->i_mmap_nonlinear
, shared
.vm_set
.list
)
942 vma
->vm_private_data
= NULL
;
944 spin_unlock(&mapping
->i_mmap_lock
);
949 * try_to_unmap - try to remove all page table mappings to a page
950 * @page: the page to get unmapped
952 * Tries to remove all the page table entries which are mapping this
953 * page, used in the pageout path. Caller must hold the page lock.
956 * SWAP_SUCCESS - we succeeded in removing all mappings
957 * SWAP_AGAIN - we missed a mapping, try again later
958 * SWAP_FAIL - the page is unswappable
960 int try_to_unmap(struct page
*page
, int migration
)
964 BUG_ON(!PageLocked(page
));
967 ret
= try_to_unmap_anon(page
, migration
);
969 ret
= try_to_unmap_file(page
, migration
);
971 if (!page_mapped(page
))