]>
git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blob - mm/mempolicy.c
2 * Simple NUMA memory policy for the Linux kernel.
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
6 * Subject to the GNU Public License, version 2.
8 * NUMA policy allows the user to give hints in which node(s) memory should
11 * Support four policies per VMA and per process:
13 * The VMA policy has priority over the process policy for a page fault.
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
22 * bind Only allocate memory on a specific set of nodes,
24 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
28 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
34 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
57 fix mmap readahead to honour policy and enable policy for any page cache
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
62 handle mremap for shared memory (currently ignored for the policy)
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
66 could replace all the switch()es with a mempolicy_ops structure.
69 #include <linux/mempolicy.h>
71 #include <linux/highmem.h>
72 #include <linux/hugetlb.h>
73 #include <linux/kernel.h>
74 #include <linux/sched.h>
76 #include <linux/nodemask.h>
77 #include <linux/cpuset.h>
78 #include <linux/gfp.h>
79 #include <linux/slab.h>
80 #include <linux/string.h>
81 #include <linux/module.h>
82 #include <linux/interrupt.h>
83 #include <linux/init.h>
84 #include <linux/compat.h>
85 #include <linux/mempolicy.h>
86 #include <linux/swap.h>
87 #include <linux/seq_file.h>
88 #include <linux/proc_fs.h>
89 #include <linux/migrate.h>
90 #include <linux/rmap.h>
91 #include <linux/security.h>
93 #include <asm/tlbflush.h>
94 #include <asm/uaccess.h>
97 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
98 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
99 #define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
101 static struct kmem_cache
*policy_cache
;
102 static struct kmem_cache
*sn_cache
;
104 #define PDprintk(fmt...)
106 /* Highest zone. An specific allocation for a zone below that is not
108 enum zone_type policy_zone
= 0;
110 struct mempolicy default_policy
= {
111 .refcnt
= ATOMIC_INIT(1), /* never free it */
112 .policy
= MPOL_DEFAULT
,
115 /* Do sanity checking on a policy */
116 static int mpol_check_policy(int mode
, nodemask_t
*nodes
)
118 int empty
= nodes_empty(*nodes
);
126 case MPOL_INTERLEAVE
:
127 /* Preferred will only use the first bit, but allow
133 return nodes_subset(*nodes
, node_online_map
) ? 0 : -EINVAL
;
136 /* Generate a custom zonelist for the BIND policy. */
137 static struct zonelist
*bind_zonelist(nodemask_t
*nodes
)
143 max
= 1 + MAX_NR_ZONES
* nodes_weight(*nodes
);
144 max
++; /* space for zlcache_ptr (see mmzone.h) */
145 zl
= kmalloc(sizeof(struct zone
*) * max
, GFP_KERNEL
);
148 zl
->zlcache_ptr
= NULL
;
150 /* First put in the highest zones from all nodes, then all the next
151 lower zones etc. Avoid empty zones because the memory allocator
152 doesn't like them. If you implement node hot removal you
156 for_each_node_mask(nd
, *nodes
) {
157 struct zone
*z
= &NODE_DATA(nd
)->node_zones
[k
];
158 if (z
->present_pages
> 0)
159 zl
->zones
[num
++] = z
;
165 zl
->zones
[num
] = NULL
;
169 /* Create a new policy */
170 static struct mempolicy
*mpol_new(int mode
, nodemask_t
*nodes
)
172 struct mempolicy
*policy
;
174 PDprintk("setting mode %d nodes[0] %lx\n", mode
, nodes_addr(*nodes
)[0]);
175 if (mode
== MPOL_DEFAULT
)
177 policy
= kmem_cache_alloc(policy_cache
, GFP_KERNEL
);
179 return ERR_PTR(-ENOMEM
);
180 atomic_set(&policy
->refcnt
, 1);
182 case MPOL_INTERLEAVE
:
183 policy
->v
.nodes
= *nodes
;
184 if (nodes_weight(*nodes
) == 0) {
185 kmem_cache_free(policy_cache
, policy
);
186 return ERR_PTR(-EINVAL
);
190 policy
->v
.preferred_node
= first_node(*nodes
);
191 if (policy
->v
.preferred_node
>= MAX_NUMNODES
)
192 policy
->v
.preferred_node
= -1;
195 policy
->v
.zonelist
= bind_zonelist(nodes
);
196 if (policy
->v
.zonelist
== NULL
) {
197 kmem_cache_free(policy_cache
, policy
);
198 return ERR_PTR(-ENOMEM
);
202 policy
->policy
= mode
;
203 policy
->cpuset_mems_allowed
= cpuset_mems_allowed(current
);
207 static void gather_stats(struct page
*, void *, int pte_dirty
);
208 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
209 unsigned long flags
);
211 /* Scan through pages checking if pages follow certain conditions. */
212 static int check_pte_range(struct vm_area_struct
*vma
, pmd_t
*pmd
,
213 unsigned long addr
, unsigned long end
,
214 const nodemask_t
*nodes
, unsigned long flags
,
221 orig_pte
= pte
= pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, &ptl
);
226 if (!pte_present(*pte
))
228 page
= vm_normal_page(vma
, addr
, *pte
);
232 * The check for PageReserved here is important to avoid
233 * handling zero pages and other pages that may have been
234 * marked special by the system.
236 * If the PageReserved would not be checked here then f.e.
237 * the location of the zero page could have an influence
238 * on MPOL_MF_STRICT, zero pages would be counted for
239 * the per node stats, and there would be useless attempts
240 * to put zero pages on the migration list.
242 if (PageReserved(page
))
244 nid
= page_to_nid(page
);
245 if (node_isset(nid
, *nodes
) == !!(flags
& MPOL_MF_INVERT
))
248 if (flags
& MPOL_MF_STATS
)
249 gather_stats(page
, private, pte_dirty(*pte
));
250 else if (flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
))
251 migrate_page_add(page
, private, flags
);
254 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
255 pte_unmap_unlock(orig_pte
, ptl
);
259 static inline int check_pmd_range(struct vm_area_struct
*vma
, pud_t
*pud
,
260 unsigned long addr
, unsigned long end
,
261 const nodemask_t
*nodes
, unsigned long flags
,
267 pmd
= pmd_offset(pud
, addr
);
269 next
= pmd_addr_end(addr
, end
);
270 if (pmd_none_or_clear_bad(pmd
))
272 if (check_pte_range(vma
, pmd
, addr
, next
, nodes
,
275 } while (pmd
++, addr
= next
, addr
!= end
);
279 static inline int check_pud_range(struct vm_area_struct
*vma
, pgd_t
*pgd
,
280 unsigned long addr
, unsigned long end
,
281 const nodemask_t
*nodes
, unsigned long flags
,
287 pud
= pud_offset(pgd
, addr
);
289 next
= pud_addr_end(addr
, end
);
290 if (pud_none_or_clear_bad(pud
))
292 if (check_pmd_range(vma
, pud
, addr
, next
, nodes
,
295 } while (pud
++, addr
= next
, addr
!= end
);
299 static inline int check_pgd_range(struct vm_area_struct
*vma
,
300 unsigned long addr
, unsigned long end
,
301 const nodemask_t
*nodes
, unsigned long flags
,
307 pgd
= pgd_offset(vma
->vm_mm
, addr
);
309 next
= pgd_addr_end(addr
, end
);
310 if (pgd_none_or_clear_bad(pgd
))
312 if (check_pud_range(vma
, pgd
, addr
, next
, nodes
,
315 } while (pgd
++, addr
= next
, addr
!= end
);
319 /* Check if a vma is migratable */
320 static inline int vma_migratable(struct vm_area_struct
*vma
)
322 if (vma
->vm_flags
& (
323 VM_LOCKED
|VM_IO
|VM_HUGETLB
|VM_PFNMAP
|VM_RESERVED
))
329 * Check if all pages in a range are on a set of nodes.
330 * If pagelist != NULL then isolate pages from the LRU and
331 * put them on the pagelist.
333 static struct vm_area_struct
*
334 check_range(struct mm_struct
*mm
, unsigned long start
, unsigned long end
,
335 const nodemask_t
*nodes
, unsigned long flags
, void *private)
338 struct vm_area_struct
*first
, *vma
, *prev
;
340 if (flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
)) {
342 err
= migrate_prep();
347 first
= find_vma(mm
, start
);
349 return ERR_PTR(-EFAULT
);
351 for (vma
= first
; vma
&& vma
->vm_start
< end
; vma
= vma
->vm_next
) {
352 if (!(flags
& MPOL_MF_DISCONTIG_OK
)) {
353 if (!vma
->vm_next
&& vma
->vm_end
< end
)
354 return ERR_PTR(-EFAULT
);
355 if (prev
&& prev
->vm_end
< vma
->vm_start
)
356 return ERR_PTR(-EFAULT
);
358 if (!is_vm_hugetlb_page(vma
) &&
359 ((flags
& MPOL_MF_STRICT
) ||
360 ((flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
)) &&
361 vma_migratable(vma
)))) {
362 unsigned long endvma
= vma
->vm_end
;
366 if (vma
->vm_start
> start
)
367 start
= vma
->vm_start
;
368 err
= check_pgd_range(vma
, start
, endvma
, nodes
,
371 first
= ERR_PTR(err
);
380 /* Apply policy to a single VMA */
381 static int policy_vma(struct vm_area_struct
*vma
, struct mempolicy
*new)
384 struct mempolicy
*old
= vma
->vm_policy
;
386 PDprintk("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
387 vma
->vm_start
, vma
->vm_end
, vma
->vm_pgoff
,
388 vma
->vm_ops
, vma
->vm_file
,
389 vma
->vm_ops
? vma
->vm_ops
->set_policy
: NULL
);
391 if (vma
->vm_ops
&& vma
->vm_ops
->set_policy
)
392 err
= vma
->vm_ops
->set_policy(vma
, new);
395 vma
->vm_policy
= new;
401 /* Step 2: apply policy to a range and do splits. */
402 static int mbind_range(struct vm_area_struct
*vma
, unsigned long start
,
403 unsigned long end
, struct mempolicy
*new)
405 struct vm_area_struct
*next
;
409 for (; vma
&& vma
->vm_start
< end
; vma
= next
) {
411 if (vma
->vm_start
< start
)
412 err
= split_vma(vma
->vm_mm
, vma
, start
, 1);
413 if (!err
&& vma
->vm_end
> end
)
414 err
= split_vma(vma
->vm_mm
, vma
, end
, 0);
416 err
= policy_vma(vma
, new);
423 static int contextualize_policy(int mode
, nodemask_t
*nodes
)
428 cpuset_update_task_memory_state();
429 if (!cpuset_nodes_subset_current_mems_allowed(*nodes
))
431 return mpol_check_policy(mode
, nodes
);
436 * Update task->flags PF_MEMPOLICY bit: set iff non-default
437 * mempolicy. Allows more rapid checking of this (combined perhaps
438 * with other PF_* flag bits) on memory allocation hot code paths.
440 * If called from outside this file, the task 'p' should -only- be
441 * a newly forked child not yet visible on the task list, because
442 * manipulating the task flags of a visible task is not safe.
444 * The above limitation is why this routine has the funny name
445 * mpol_fix_fork_child_flag().
447 * It is also safe to call this with a task pointer of current,
448 * which the static wrapper mpol_set_task_struct_flag() does,
449 * for use within this file.
452 void mpol_fix_fork_child_flag(struct task_struct
*p
)
455 p
->flags
|= PF_MEMPOLICY
;
457 p
->flags
&= ~PF_MEMPOLICY
;
460 static void mpol_set_task_struct_flag(void)
462 mpol_fix_fork_child_flag(current
);
465 /* Set the process memory policy */
466 long do_set_mempolicy(int mode
, nodemask_t
*nodes
)
468 struct mempolicy
*new;
470 if (contextualize_policy(mode
, nodes
))
472 new = mpol_new(mode
, nodes
);
475 mpol_free(current
->mempolicy
);
476 current
->mempolicy
= new;
477 mpol_set_task_struct_flag();
478 if (new && new->policy
== MPOL_INTERLEAVE
)
479 current
->il_next
= first_node(new->v
.nodes
);
483 /* Fill a zone bitmap for a policy */
484 static void get_zonemask(struct mempolicy
*p
, nodemask_t
*nodes
)
491 for (i
= 0; p
->v
.zonelist
->zones
[i
]; i
++)
492 node_set(zone_to_nid(p
->v
.zonelist
->zones
[i
]),
497 case MPOL_INTERLEAVE
:
501 /* or use current node instead of online map? */
502 if (p
->v
.preferred_node
< 0)
503 *nodes
= node_online_map
;
505 node_set(p
->v
.preferred_node
, *nodes
);
512 static int lookup_node(struct mm_struct
*mm
, unsigned long addr
)
517 err
= get_user_pages(current
, mm
, addr
& PAGE_MASK
, 1, 0, 0, &p
, NULL
);
519 err
= page_to_nid(p
);
525 /* Retrieve NUMA policy */
526 long do_get_mempolicy(int *policy
, nodemask_t
*nmask
,
527 unsigned long addr
, unsigned long flags
)
530 struct mm_struct
*mm
= current
->mm
;
531 struct vm_area_struct
*vma
= NULL
;
532 struct mempolicy
*pol
= current
->mempolicy
;
534 cpuset_update_task_memory_state();
535 if (flags
& ~(unsigned long)(MPOL_F_NODE
|MPOL_F_ADDR
))
537 if (flags
& MPOL_F_ADDR
) {
538 down_read(&mm
->mmap_sem
);
539 vma
= find_vma_intersection(mm
, addr
, addr
+1);
541 up_read(&mm
->mmap_sem
);
544 if (vma
->vm_ops
&& vma
->vm_ops
->get_policy
)
545 pol
= vma
->vm_ops
->get_policy(vma
, addr
);
547 pol
= vma
->vm_policy
;
552 pol
= &default_policy
;
554 if (flags
& MPOL_F_NODE
) {
555 if (flags
& MPOL_F_ADDR
) {
556 err
= lookup_node(mm
, addr
);
560 } else if (pol
== current
->mempolicy
&&
561 pol
->policy
== MPOL_INTERLEAVE
) {
562 *policy
= current
->il_next
;
568 *policy
= pol
->policy
;
571 up_read(¤t
->mm
->mmap_sem
);
577 get_zonemask(pol
, nmask
);
581 up_read(¤t
->mm
->mmap_sem
);
585 #ifdef CONFIG_MIGRATION
589 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
593 * Avoid migrating a page that is shared with others.
595 if ((flags
& MPOL_MF_MOVE_ALL
) || page_mapcount(page
) == 1)
596 isolate_lru_page(page
, pagelist
);
599 static struct page
*new_node_page(struct page
*page
, unsigned long node
, int **x
)
601 return alloc_pages_node(node
, GFP_HIGHUSER
, 0);
605 * Migrate pages from one node to a target node.
606 * Returns error or the number of pages not migrated.
608 int migrate_to_node(struct mm_struct
*mm
, int source
, int dest
, int flags
)
615 node_set(source
, nmask
);
617 check_range(mm
, mm
->mmap
->vm_start
, TASK_SIZE
, &nmask
,
618 flags
| MPOL_MF_DISCONTIG_OK
, &pagelist
);
620 if (!list_empty(&pagelist
))
621 err
= migrate_pages(&pagelist
, new_node_page
, dest
);
627 * Move pages between the two nodesets so as to preserve the physical
628 * layout as much as possible.
630 * Returns the number of page that could not be moved.
632 int do_migrate_pages(struct mm_struct
*mm
,
633 const nodemask_t
*from_nodes
, const nodemask_t
*to_nodes
, int flags
)
640 down_read(&mm
->mmap_sem
);
642 err
= migrate_vmas(mm
, from_nodes
, to_nodes
, flags
);
647 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
648 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
649 * bit in 'tmp', and return that <source, dest> pair for migration.
650 * The pair of nodemasks 'to' and 'from' define the map.
652 * If no pair of bits is found that way, fallback to picking some
653 * pair of 'source' and 'dest' bits that are not the same. If the
654 * 'source' and 'dest' bits are the same, this represents a node
655 * that will be migrating to itself, so no pages need move.
657 * If no bits are left in 'tmp', or if all remaining bits left
658 * in 'tmp' correspond to the same bit in 'to', return false
659 * (nothing left to migrate).
661 * This lets us pick a pair of nodes to migrate between, such that
662 * if possible the dest node is not already occupied by some other
663 * source node, minimizing the risk of overloading the memory on a
664 * node that would happen if we migrated incoming memory to a node
665 * before migrating outgoing memory source that same node.
667 * A single scan of tmp is sufficient. As we go, we remember the
668 * most recent <s, d> pair that moved (s != d). If we find a pair
669 * that not only moved, but what's better, moved to an empty slot
670 * (d is not set in tmp), then we break out then, with that pair.
671 * Otherwise when we finish scannng from_tmp, we at least have the
672 * most recent <s, d> pair that moved. If we get all the way through
673 * the scan of tmp without finding any node that moved, much less
674 * moved to an empty node, then there is nothing left worth migrating.
678 while (!nodes_empty(tmp
)) {
683 for_each_node_mask(s
, tmp
) {
684 d
= node_remap(s
, *from_nodes
, *to_nodes
);
688 source
= s
; /* Node moved. Memorize */
691 /* dest not in remaining from nodes? */
692 if (!node_isset(dest
, tmp
))
698 node_clear(source
, tmp
);
699 err
= migrate_to_node(mm
, source
, dest
, flags
);
706 up_read(&mm
->mmap_sem
);
713 static struct page
*new_vma_page(struct page
*page
, unsigned long private, int **x
)
715 struct vm_area_struct
*vma
= (struct vm_area_struct
*)private;
717 return alloc_page_vma(GFP_HIGHUSER
, vma
, page_address_in_vma(page
, vma
));
721 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
726 int do_migrate_pages(struct mm_struct
*mm
,
727 const nodemask_t
*from_nodes
, const nodemask_t
*to_nodes
, int flags
)
732 static struct page
*new_vma_page(struct page
*page
, unsigned long private, int **x
)
738 long do_mbind(unsigned long start
, unsigned long len
,
739 unsigned long mode
, nodemask_t
*nmask
, unsigned long flags
)
741 struct vm_area_struct
*vma
;
742 struct mm_struct
*mm
= current
->mm
;
743 struct mempolicy
*new;
748 if ((flags
& ~(unsigned long)(MPOL_MF_STRICT
|
749 MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
))
752 if ((flags
& MPOL_MF_MOVE_ALL
) && !capable(CAP_SYS_NICE
))
755 if (start
& ~PAGE_MASK
)
758 if (mode
== MPOL_DEFAULT
)
759 flags
&= ~MPOL_MF_STRICT
;
761 len
= (len
+ PAGE_SIZE
- 1) & PAGE_MASK
;
769 if (mpol_check_policy(mode
, nmask
))
772 new = mpol_new(mode
, nmask
);
777 * If we are using the default policy then operation
778 * on discontinuous address spaces is okay after all
781 flags
|= MPOL_MF_DISCONTIG_OK
;
783 PDprintk("mbind %lx-%lx mode:%ld nodes:%lx\n",start
,start
+len
,
784 mode
,nodes_addr(nodes
)[0]);
786 down_write(&mm
->mmap_sem
);
787 vma
= check_range(mm
, start
, end
, nmask
,
788 flags
| MPOL_MF_INVERT
, &pagelist
);
794 err
= mbind_range(vma
, start
, end
, new);
796 if (!list_empty(&pagelist
))
797 nr_failed
= migrate_pages(&pagelist
, new_vma_page
,
800 if (!err
&& nr_failed
&& (flags
& MPOL_MF_STRICT
))
804 up_write(&mm
->mmap_sem
);
810 * User space interface with variable sized bitmaps for nodelists.
813 /* Copy a node mask from user space. */
814 static int get_nodes(nodemask_t
*nodes
, const unsigned long __user
*nmask
,
815 unsigned long maxnode
)
818 unsigned long nlongs
;
819 unsigned long endmask
;
823 if (maxnode
== 0 || !nmask
)
825 if (maxnode
> PAGE_SIZE
*BITS_PER_BYTE
)
828 nlongs
= BITS_TO_LONGS(maxnode
);
829 if ((maxnode
% BITS_PER_LONG
) == 0)
832 endmask
= (1UL << (maxnode
% BITS_PER_LONG
)) - 1;
834 /* When the user specified more nodes than supported just check
835 if the non supported part is all zero. */
836 if (nlongs
> BITS_TO_LONGS(MAX_NUMNODES
)) {
837 if (nlongs
> PAGE_SIZE
/sizeof(long))
839 for (k
= BITS_TO_LONGS(MAX_NUMNODES
); k
< nlongs
; k
++) {
841 if (get_user(t
, nmask
+ k
))
843 if (k
== nlongs
- 1) {
849 nlongs
= BITS_TO_LONGS(MAX_NUMNODES
);
853 if (copy_from_user(nodes_addr(*nodes
), nmask
, nlongs
*sizeof(unsigned long)))
855 nodes_addr(*nodes
)[nlongs
-1] &= endmask
;
859 /* Copy a kernel node mask to user space */
860 static int copy_nodes_to_user(unsigned long __user
*mask
, unsigned long maxnode
,
863 unsigned long copy
= ALIGN(maxnode
-1, 64) / 8;
864 const int nbytes
= BITS_TO_LONGS(MAX_NUMNODES
) * sizeof(long);
867 if (copy
> PAGE_SIZE
)
869 if (clear_user((char __user
*)mask
+ nbytes
, copy
- nbytes
))
873 return copy_to_user(mask
, nodes_addr(*nodes
), copy
) ? -EFAULT
: 0;
876 asmlinkage
long sys_mbind(unsigned long start
, unsigned long len
,
878 unsigned long __user
*nmask
, unsigned long maxnode
,
884 err
= get_nodes(&nodes
, nmask
, maxnode
);
887 #ifdef CONFIG_CPUSETS
888 /* Restrict the nodes to the allowed nodes in the cpuset */
889 nodes_and(nodes
, nodes
, current
->mems_allowed
);
891 return do_mbind(start
, len
, mode
, &nodes
, flags
);
894 /* Set the process memory policy */
895 asmlinkage
long sys_set_mempolicy(int mode
, unsigned long __user
*nmask
,
896 unsigned long maxnode
)
901 if (mode
< 0 || mode
> MPOL_MAX
)
903 err
= get_nodes(&nodes
, nmask
, maxnode
);
906 return do_set_mempolicy(mode
, &nodes
);
909 asmlinkage
long sys_migrate_pages(pid_t pid
, unsigned long maxnode
,
910 const unsigned long __user
*old_nodes
,
911 const unsigned long __user
*new_nodes
)
913 struct mm_struct
*mm
;
914 struct task_struct
*task
;
917 nodemask_t task_nodes
;
920 err
= get_nodes(&old
, old_nodes
, maxnode
);
924 err
= get_nodes(&new, new_nodes
, maxnode
);
928 /* Find the mm_struct */
929 read_lock(&tasklist_lock
);
930 task
= pid
? find_task_by_pid(pid
) : current
;
932 read_unlock(&tasklist_lock
);
935 mm
= get_task_mm(task
);
936 read_unlock(&tasklist_lock
);
942 * Check if this process has the right to modify the specified
943 * process. The right exists if the process has administrative
944 * capabilities, superuser privileges or the same
945 * userid as the target process.
947 if ((current
->euid
!= task
->suid
) && (current
->euid
!= task
->uid
) &&
948 (current
->uid
!= task
->suid
) && (current
->uid
!= task
->uid
) &&
949 !capable(CAP_SYS_NICE
)) {
954 task_nodes
= cpuset_mems_allowed(task
);
955 /* Is the user allowed to access the target nodes? */
956 if (!nodes_subset(new, task_nodes
) && !capable(CAP_SYS_NICE
)) {
961 err
= security_task_movememory(task
);
965 err
= do_migrate_pages(mm
, &old
, &new,
966 capable(CAP_SYS_NICE
) ? MPOL_MF_MOVE_ALL
: MPOL_MF_MOVE
);
973 /* Retrieve NUMA policy */
974 asmlinkage
long sys_get_mempolicy(int __user
*policy
,
975 unsigned long __user
*nmask
,
976 unsigned long maxnode
,
977 unsigned long addr
, unsigned long flags
)
982 if (nmask
!= NULL
&& maxnode
< MAX_NUMNODES
)
985 err
= do_get_mempolicy(&pval
, &nodes
, addr
, flags
);
990 if (policy
&& put_user(pval
, policy
))
994 err
= copy_nodes_to_user(nmask
, maxnode
, &nodes
);
1001 asmlinkage
long compat_sys_get_mempolicy(int __user
*policy
,
1002 compat_ulong_t __user
*nmask
,
1003 compat_ulong_t maxnode
,
1004 compat_ulong_t addr
, compat_ulong_t flags
)
1007 unsigned long __user
*nm
= NULL
;
1008 unsigned long nr_bits
, alloc_size
;
1009 DECLARE_BITMAP(bm
, MAX_NUMNODES
);
1011 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1012 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1015 nm
= compat_alloc_user_space(alloc_size
);
1017 err
= sys_get_mempolicy(policy
, nm
, nr_bits
+1, addr
, flags
);
1019 if (!err
&& nmask
) {
1020 err
= copy_from_user(bm
, nm
, alloc_size
);
1021 /* ensure entire bitmap is zeroed */
1022 err
|= clear_user(nmask
, ALIGN(maxnode
-1, 8) / 8);
1023 err
|= compat_put_bitmap(nmask
, bm
, nr_bits
);
1029 asmlinkage
long compat_sys_set_mempolicy(int mode
, compat_ulong_t __user
*nmask
,
1030 compat_ulong_t maxnode
)
1033 unsigned long __user
*nm
= NULL
;
1034 unsigned long nr_bits
, alloc_size
;
1035 DECLARE_BITMAP(bm
, MAX_NUMNODES
);
1037 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1038 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1041 err
= compat_get_bitmap(bm
, nmask
, nr_bits
);
1042 nm
= compat_alloc_user_space(alloc_size
);
1043 err
|= copy_to_user(nm
, bm
, alloc_size
);
1049 return sys_set_mempolicy(mode
, nm
, nr_bits
+1);
1052 asmlinkage
long compat_sys_mbind(compat_ulong_t start
, compat_ulong_t len
,
1053 compat_ulong_t mode
, compat_ulong_t __user
*nmask
,
1054 compat_ulong_t maxnode
, compat_ulong_t flags
)
1057 unsigned long __user
*nm
= NULL
;
1058 unsigned long nr_bits
, alloc_size
;
1061 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1062 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1065 err
= compat_get_bitmap(nodes_addr(bm
), nmask
, nr_bits
);
1066 nm
= compat_alloc_user_space(alloc_size
);
1067 err
|= copy_to_user(nm
, nodes_addr(bm
), alloc_size
);
1073 return sys_mbind(start
, len
, mode
, nm
, nr_bits
+1, flags
);
1078 /* Return effective policy for a VMA */
1079 static struct mempolicy
* get_vma_policy(struct task_struct
*task
,
1080 struct vm_area_struct
*vma
, unsigned long addr
)
1082 struct mempolicy
*pol
= task
->mempolicy
;
1085 if (vma
->vm_ops
&& vma
->vm_ops
->get_policy
)
1086 pol
= vma
->vm_ops
->get_policy(vma
, addr
);
1087 else if (vma
->vm_policy
&&
1088 vma
->vm_policy
->policy
!= MPOL_DEFAULT
)
1089 pol
= vma
->vm_policy
;
1092 pol
= &default_policy
;
1096 /* Return a zonelist representing a mempolicy */
1097 static struct zonelist
*zonelist_policy(gfp_t gfp
, struct mempolicy
*policy
)
1101 switch (policy
->policy
) {
1102 case MPOL_PREFERRED
:
1103 nd
= policy
->v
.preferred_node
;
1105 nd
= numa_node_id();
1108 /* Lower zones don't get a policy applied */
1109 /* Careful: current->mems_allowed might have moved */
1110 if (gfp_zone(gfp
) >= policy_zone
)
1111 if (cpuset_zonelist_valid_mems_allowed(policy
->v
.zonelist
))
1112 return policy
->v
.zonelist
;
1114 case MPOL_INTERLEAVE
: /* should not happen */
1116 nd
= numa_node_id();
1122 return NODE_DATA(nd
)->node_zonelists
+ gfp_zone(gfp
);
1125 /* Do dynamic interleaving for a process */
1126 static unsigned interleave_nodes(struct mempolicy
*policy
)
1129 struct task_struct
*me
= current
;
1132 next
= next_node(nid
, policy
->v
.nodes
);
1133 if (next
>= MAX_NUMNODES
)
1134 next
= first_node(policy
->v
.nodes
);
1140 * Depending on the memory policy provide a node from which to allocate the
1143 unsigned slab_node(struct mempolicy
*policy
)
1145 int pol
= policy
? policy
->policy
: MPOL_DEFAULT
;
1148 case MPOL_INTERLEAVE
:
1149 return interleave_nodes(policy
);
1153 * Follow bind policy behavior and start allocation at the
1156 return zone_to_nid(policy
->v
.zonelist
->zones
[0]);
1158 case MPOL_PREFERRED
:
1159 if (policy
->v
.preferred_node
>= 0)
1160 return policy
->v
.preferred_node
;
1164 return numa_node_id();
1168 /* Do static interleaving for a VMA with known offset. */
1169 static unsigned offset_il_node(struct mempolicy
*pol
,
1170 struct vm_area_struct
*vma
, unsigned long off
)
1172 unsigned nnodes
= nodes_weight(pol
->v
.nodes
);
1173 unsigned target
= (unsigned)off
% nnodes
;
1179 nid
= next_node(nid
, pol
->v
.nodes
);
1181 } while (c
<= target
);
1185 /* Determine a node number for interleave */
1186 static inline unsigned interleave_nid(struct mempolicy
*pol
,
1187 struct vm_area_struct
*vma
, unsigned long addr
, int shift
)
1193 * for small pages, there is no difference between
1194 * shift and PAGE_SHIFT, so the bit-shift is safe.
1195 * for huge pages, since vm_pgoff is in units of small
1196 * pages, we need to shift off the always 0 bits to get
1199 BUG_ON(shift
< PAGE_SHIFT
);
1200 off
= vma
->vm_pgoff
>> (shift
- PAGE_SHIFT
);
1201 off
+= (addr
- vma
->vm_start
) >> shift
;
1202 return offset_il_node(pol
, vma
, off
);
1204 return interleave_nodes(pol
);
1207 #ifdef CONFIG_HUGETLBFS
1208 /* Return a zonelist suitable for a huge page allocation. */
1209 struct zonelist
*huge_zonelist(struct vm_area_struct
*vma
, unsigned long addr
)
1211 struct mempolicy
*pol
= get_vma_policy(current
, vma
, addr
);
1213 if (pol
->policy
== MPOL_INTERLEAVE
) {
1216 nid
= interleave_nid(pol
, vma
, addr
, HPAGE_SHIFT
);
1217 return NODE_DATA(nid
)->node_zonelists
+ gfp_zone(GFP_HIGHUSER
);
1219 return zonelist_policy(GFP_HIGHUSER
, pol
);
1223 /* Allocate a page in interleaved policy.
1224 Own path because it needs to do special accounting. */
1225 static struct page
*alloc_page_interleave(gfp_t gfp
, unsigned order
,
1228 struct zonelist
*zl
;
1231 zl
= NODE_DATA(nid
)->node_zonelists
+ gfp_zone(gfp
);
1232 page
= __alloc_pages(gfp
, order
, zl
);
1233 if (page
&& page_zone(page
) == zl
->zones
[0])
1234 inc_zone_page_state(page
, NUMA_INTERLEAVE_HIT
);
1239 * alloc_page_vma - Allocate a page for a VMA.
1242 * %GFP_USER user allocation.
1243 * %GFP_KERNEL kernel allocations,
1244 * %GFP_HIGHMEM highmem/user allocations,
1245 * %GFP_FS allocation should not call back into a file system.
1246 * %GFP_ATOMIC don't sleep.
1248 * @vma: Pointer to VMA or NULL if not available.
1249 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1251 * This function allocates a page from the kernel page pool and applies
1252 * a NUMA policy associated with the VMA or the current process.
1253 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1254 * mm_struct of the VMA to prevent it from going away. Should be used for
1255 * all allocations for pages that will be mapped into
1256 * user space. Returns NULL when no page can be allocated.
1258 * Should be called with the mm_sem of the vma hold.
1261 alloc_page_vma(gfp_t gfp
, struct vm_area_struct
*vma
, unsigned long addr
)
1263 struct mempolicy
*pol
= get_vma_policy(current
, vma
, addr
);
1265 cpuset_update_task_memory_state();
1267 if (unlikely(pol
->policy
== MPOL_INTERLEAVE
)) {
1270 nid
= interleave_nid(pol
, vma
, addr
, PAGE_SHIFT
);
1271 return alloc_page_interleave(gfp
, 0, nid
);
1273 return __alloc_pages(gfp
, 0, zonelist_policy(gfp
, pol
));
1277 * alloc_pages_current - Allocate pages.
1280 * %GFP_USER user allocation,
1281 * %GFP_KERNEL kernel allocation,
1282 * %GFP_HIGHMEM highmem allocation,
1283 * %GFP_FS don't call back into a file system.
1284 * %GFP_ATOMIC don't sleep.
1285 * @order: Power of two of allocation size in pages. 0 is a single page.
1287 * Allocate a page from the kernel page pool. When not in
1288 * interrupt context and apply the current process NUMA policy.
1289 * Returns NULL when no page can be allocated.
1291 * Don't call cpuset_update_task_memory_state() unless
1292 * 1) it's ok to take cpuset_sem (can WAIT), and
1293 * 2) allocating for current task (not interrupt).
1295 struct page
*alloc_pages_current(gfp_t gfp
, unsigned order
)
1297 struct mempolicy
*pol
= current
->mempolicy
;
1299 if ((gfp
& __GFP_WAIT
) && !in_interrupt())
1300 cpuset_update_task_memory_state();
1301 if (!pol
|| in_interrupt() || (gfp
& __GFP_THISNODE
))
1302 pol
= &default_policy
;
1303 if (pol
->policy
== MPOL_INTERLEAVE
)
1304 return alloc_page_interleave(gfp
, order
, interleave_nodes(pol
));
1305 return __alloc_pages(gfp
, order
, zonelist_policy(gfp
, pol
));
1307 EXPORT_SYMBOL(alloc_pages_current
);
1310 * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
1311 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1312 * with the mems_allowed returned by cpuset_mems_allowed(). This
1313 * keeps mempolicies cpuset relative after its cpuset moves. See
1314 * further kernel/cpuset.c update_nodemask().
1316 void *cpuset_being_rebound
;
1318 /* Slow path of a mempolicy copy */
1319 struct mempolicy
*__mpol_copy(struct mempolicy
*old
)
1321 struct mempolicy
*new = kmem_cache_alloc(policy_cache
, GFP_KERNEL
);
1324 return ERR_PTR(-ENOMEM
);
1325 if (current_cpuset_is_being_rebound()) {
1326 nodemask_t mems
= cpuset_mems_allowed(current
);
1327 mpol_rebind_policy(old
, &mems
);
1330 atomic_set(&new->refcnt
, 1);
1331 if (new->policy
== MPOL_BIND
) {
1332 int sz
= ksize(old
->v
.zonelist
);
1333 new->v
.zonelist
= kmemdup(old
->v
.zonelist
, sz
, GFP_KERNEL
);
1334 if (!new->v
.zonelist
) {
1335 kmem_cache_free(policy_cache
, new);
1336 return ERR_PTR(-ENOMEM
);
1342 /* Slow path of a mempolicy comparison */
1343 int __mpol_equal(struct mempolicy
*a
, struct mempolicy
*b
)
1347 if (a
->policy
!= b
->policy
)
1349 switch (a
->policy
) {
1352 case MPOL_INTERLEAVE
:
1353 return nodes_equal(a
->v
.nodes
, b
->v
.nodes
);
1354 case MPOL_PREFERRED
:
1355 return a
->v
.preferred_node
== b
->v
.preferred_node
;
1358 for (i
= 0; a
->v
.zonelist
->zones
[i
]; i
++)
1359 if (a
->v
.zonelist
->zones
[i
] != b
->v
.zonelist
->zones
[i
])
1361 return b
->v
.zonelist
->zones
[i
] == NULL
;
1369 /* Slow path of a mpol destructor. */
1370 void __mpol_free(struct mempolicy
*p
)
1372 if (!atomic_dec_and_test(&p
->refcnt
))
1374 if (p
->policy
== MPOL_BIND
)
1375 kfree(p
->v
.zonelist
);
1376 p
->policy
= MPOL_DEFAULT
;
1377 kmem_cache_free(policy_cache
, p
);
1381 * Shared memory backing store policy support.
1383 * Remember policies even when nobody has shared memory mapped.
1384 * The policies are kept in Red-Black tree linked from the inode.
1385 * They are protected by the sp->lock spinlock, which should be held
1386 * for any accesses to the tree.
1389 /* lookup first element intersecting start-end */
1390 /* Caller holds sp->lock */
1391 static struct sp_node
*
1392 sp_lookup(struct shared_policy
*sp
, unsigned long start
, unsigned long end
)
1394 struct rb_node
*n
= sp
->root
.rb_node
;
1397 struct sp_node
*p
= rb_entry(n
, struct sp_node
, nd
);
1399 if (start
>= p
->end
)
1401 else if (end
<= p
->start
)
1409 struct sp_node
*w
= NULL
;
1410 struct rb_node
*prev
= rb_prev(n
);
1413 w
= rb_entry(prev
, struct sp_node
, nd
);
1414 if (w
->end
<= start
)
1418 return rb_entry(n
, struct sp_node
, nd
);
1421 /* Insert a new shared policy into the list. */
1422 /* Caller holds sp->lock */
1423 static void sp_insert(struct shared_policy
*sp
, struct sp_node
*new)
1425 struct rb_node
**p
= &sp
->root
.rb_node
;
1426 struct rb_node
*parent
= NULL
;
1431 nd
= rb_entry(parent
, struct sp_node
, nd
);
1432 if (new->start
< nd
->start
)
1434 else if (new->end
> nd
->end
)
1435 p
= &(*p
)->rb_right
;
1439 rb_link_node(&new->nd
, parent
, p
);
1440 rb_insert_color(&new->nd
, &sp
->root
);
1441 PDprintk("inserting %lx-%lx: %d\n", new->start
, new->end
,
1442 new->policy
? new->policy
->policy
: 0);
1445 /* Find shared policy intersecting idx */
1447 mpol_shared_policy_lookup(struct shared_policy
*sp
, unsigned long idx
)
1449 struct mempolicy
*pol
= NULL
;
1452 if (!sp
->root
.rb_node
)
1454 spin_lock(&sp
->lock
);
1455 sn
= sp_lookup(sp
, idx
, idx
+1);
1457 mpol_get(sn
->policy
);
1460 spin_unlock(&sp
->lock
);
1464 static void sp_delete(struct shared_policy
*sp
, struct sp_node
*n
)
1466 PDprintk("deleting %lx-l%x\n", n
->start
, n
->end
);
1467 rb_erase(&n
->nd
, &sp
->root
);
1468 mpol_free(n
->policy
);
1469 kmem_cache_free(sn_cache
, n
);
1473 sp_alloc(unsigned long start
, unsigned long end
, struct mempolicy
*pol
)
1475 struct sp_node
*n
= kmem_cache_alloc(sn_cache
, GFP_KERNEL
);
1486 /* Replace a policy range. */
1487 static int shared_policy_replace(struct shared_policy
*sp
, unsigned long start
,
1488 unsigned long end
, struct sp_node
*new)
1490 struct sp_node
*n
, *new2
= NULL
;
1493 spin_lock(&sp
->lock
);
1494 n
= sp_lookup(sp
, start
, end
);
1495 /* Take care of old policies in the same range. */
1496 while (n
&& n
->start
< end
) {
1497 struct rb_node
*next
= rb_next(&n
->nd
);
1498 if (n
->start
>= start
) {
1504 /* Old policy spanning whole new range. */
1507 spin_unlock(&sp
->lock
);
1508 new2
= sp_alloc(end
, n
->end
, n
->policy
);
1514 sp_insert(sp
, new2
);
1522 n
= rb_entry(next
, struct sp_node
, nd
);
1526 spin_unlock(&sp
->lock
);
1528 mpol_free(new2
->policy
);
1529 kmem_cache_free(sn_cache
, new2
);
1534 void mpol_shared_policy_init(struct shared_policy
*info
, int policy
,
1535 nodemask_t
*policy_nodes
)
1537 info
->root
= RB_ROOT
;
1538 spin_lock_init(&info
->lock
);
1540 if (policy
!= MPOL_DEFAULT
) {
1541 struct mempolicy
*newpol
;
1543 /* Falls back to MPOL_DEFAULT on any error */
1544 newpol
= mpol_new(policy
, policy_nodes
);
1545 if (!IS_ERR(newpol
)) {
1546 /* Create pseudo-vma that contains just the policy */
1547 struct vm_area_struct pvma
;
1549 memset(&pvma
, 0, sizeof(struct vm_area_struct
));
1550 /* Policy covers entire file */
1551 pvma
.vm_end
= TASK_SIZE
;
1552 mpol_set_shared_policy(info
, &pvma
, newpol
);
1558 int mpol_set_shared_policy(struct shared_policy
*info
,
1559 struct vm_area_struct
*vma
, struct mempolicy
*npol
)
1562 struct sp_node
*new = NULL
;
1563 unsigned long sz
= vma_pages(vma
);
1565 PDprintk("set_shared_policy %lx sz %lu %d %lx\n",
1567 sz
, npol
? npol
->policy
: -1,
1568 npol
? nodes_addr(npol
->v
.nodes
)[0] : -1);
1571 new = sp_alloc(vma
->vm_pgoff
, vma
->vm_pgoff
+ sz
, npol
);
1575 err
= shared_policy_replace(info
, vma
->vm_pgoff
, vma
->vm_pgoff
+sz
, new);
1577 kmem_cache_free(sn_cache
, new);
1581 /* Free a backing policy store on inode delete. */
1582 void mpol_free_shared_policy(struct shared_policy
*p
)
1585 struct rb_node
*next
;
1587 if (!p
->root
.rb_node
)
1589 spin_lock(&p
->lock
);
1590 next
= rb_first(&p
->root
);
1592 n
= rb_entry(next
, struct sp_node
, nd
);
1593 next
= rb_next(&n
->nd
);
1594 rb_erase(&n
->nd
, &p
->root
);
1595 mpol_free(n
->policy
);
1596 kmem_cache_free(sn_cache
, n
);
1598 spin_unlock(&p
->lock
);
1601 /* assumes fs == KERNEL_DS */
1602 void __init
numa_policy_init(void)
1604 policy_cache
= kmem_cache_create("numa_policy",
1605 sizeof(struct mempolicy
),
1606 0, SLAB_PANIC
, NULL
, NULL
);
1608 sn_cache
= kmem_cache_create("shared_policy_node",
1609 sizeof(struct sp_node
),
1610 0, SLAB_PANIC
, NULL
, NULL
);
1612 /* Set interleaving policy for system init. This way not all
1613 the data structures allocated at system boot end up in node zero. */
1615 if (do_set_mempolicy(MPOL_INTERLEAVE
, &node_online_map
))
1616 printk("numa_policy_init: interleaving failed\n");
1619 /* Reset policy of current process to default */
1620 void numa_default_policy(void)
1622 do_set_mempolicy(MPOL_DEFAULT
, NULL
);
1625 /* Migrate a policy to a different set of nodes */
1626 void mpol_rebind_policy(struct mempolicy
*pol
, const nodemask_t
*newmask
)
1628 nodemask_t
*mpolmask
;
1633 mpolmask
= &pol
->cpuset_mems_allowed
;
1634 if (nodes_equal(*mpolmask
, *newmask
))
1637 switch (pol
->policy
) {
1640 case MPOL_INTERLEAVE
:
1641 nodes_remap(tmp
, pol
->v
.nodes
, *mpolmask
, *newmask
);
1643 *mpolmask
= *newmask
;
1644 current
->il_next
= node_remap(current
->il_next
,
1645 *mpolmask
, *newmask
);
1647 case MPOL_PREFERRED
:
1648 pol
->v
.preferred_node
= node_remap(pol
->v
.preferred_node
,
1649 *mpolmask
, *newmask
);
1650 *mpolmask
= *newmask
;
1655 struct zonelist
*zonelist
;
1658 for (z
= pol
->v
.zonelist
->zones
; *z
; z
++)
1659 node_set(zone_to_nid(*z
), nodes
);
1660 nodes_remap(tmp
, nodes
, *mpolmask
, *newmask
);
1663 zonelist
= bind_zonelist(&nodes
);
1665 /* If no mem, then zonelist is NULL and we keep old zonelist.
1666 * If that old zonelist has no remaining mems_allowed nodes,
1667 * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
1671 /* Good - got mem - substitute new zonelist */
1672 kfree(pol
->v
.zonelist
);
1673 pol
->v
.zonelist
= zonelist
;
1675 *mpolmask
= *newmask
;
1685 * Wrapper for mpol_rebind_policy() that just requires task
1686 * pointer, and updates task mempolicy.
1689 void mpol_rebind_task(struct task_struct
*tsk
, const nodemask_t
*new)
1691 mpol_rebind_policy(tsk
->mempolicy
, new);
1695 * Rebind each vma in mm to new nodemask.
1697 * Call holding a reference to mm. Takes mm->mmap_sem during call.
1700 void mpol_rebind_mm(struct mm_struct
*mm
, nodemask_t
*new)
1702 struct vm_area_struct
*vma
;
1704 down_write(&mm
->mmap_sem
);
1705 for (vma
= mm
->mmap
; vma
; vma
= vma
->vm_next
)
1706 mpol_rebind_policy(vma
->vm_policy
, new);
1707 up_write(&mm
->mmap_sem
);
1711 * Display pages allocated per node and memory policy via /proc.
1714 static const char * const policy_types
[] =
1715 { "default", "prefer", "bind", "interleave" };
1718 * Convert a mempolicy into a string.
1719 * Returns the number of characters in buffer (if positive)
1720 * or an error (negative)
1722 static inline int mpol_to_str(char *buffer
, int maxlen
, struct mempolicy
*pol
)
1727 int mode
= pol
? pol
->policy
: MPOL_DEFAULT
;
1734 case MPOL_PREFERRED
:
1736 node_set(pol
->v
.preferred_node
, nodes
);
1740 get_zonemask(pol
, &nodes
);
1743 case MPOL_INTERLEAVE
:
1744 nodes
= pol
->v
.nodes
;
1752 l
= strlen(policy_types
[mode
]);
1753 if (buffer
+ maxlen
< p
+ l
+ 1)
1756 strcpy(p
, policy_types
[mode
]);
1759 if (!nodes_empty(nodes
)) {
1760 if (buffer
+ maxlen
< p
+ 2)
1763 p
+= nodelist_scnprintf(p
, buffer
+ maxlen
- p
, nodes
);
1769 unsigned long pages
;
1771 unsigned long active
;
1772 unsigned long writeback
;
1773 unsigned long mapcount_max
;
1774 unsigned long dirty
;
1775 unsigned long swapcache
;
1776 unsigned long node
[MAX_NUMNODES
];
1779 static void gather_stats(struct page
*page
, void *private, int pte_dirty
)
1781 struct numa_maps
*md
= private;
1782 int count
= page_mapcount(page
);
1785 if (pte_dirty
|| PageDirty(page
))
1788 if (PageSwapCache(page
))
1791 if (PageActive(page
))
1794 if (PageWriteback(page
))
1800 if (count
> md
->mapcount_max
)
1801 md
->mapcount_max
= count
;
1803 md
->node
[page_to_nid(page
)]++;
1806 #ifdef CONFIG_HUGETLB_PAGE
1807 static void check_huge_range(struct vm_area_struct
*vma
,
1808 unsigned long start
, unsigned long end
,
1809 struct numa_maps
*md
)
1814 for (addr
= start
; addr
< end
; addr
+= HPAGE_SIZE
) {
1815 pte_t
*ptep
= huge_pte_offset(vma
->vm_mm
, addr
& HPAGE_MASK
);
1825 page
= pte_page(pte
);
1829 gather_stats(page
, md
, pte_dirty(*ptep
));
1833 static inline void check_huge_range(struct vm_area_struct
*vma
,
1834 unsigned long start
, unsigned long end
,
1835 struct numa_maps
*md
)
1840 int show_numa_map(struct seq_file
*m
, void *v
)
1842 struct proc_maps_private
*priv
= m
->private;
1843 struct vm_area_struct
*vma
= v
;
1844 struct numa_maps
*md
;
1845 struct file
*file
= vma
->vm_file
;
1846 struct mm_struct
*mm
= vma
->vm_mm
;
1853 md
= kzalloc(sizeof(struct numa_maps
), GFP_KERNEL
);
1857 mpol_to_str(buffer
, sizeof(buffer
),
1858 get_vma_policy(priv
->task
, vma
, vma
->vm_start
));
1860 seq_printf(m
, "%08lx %s", vma
->vm_start
, buffer
);
1863 seq_printf(m
, " file=");
1864 seq_path(m
, file
->f_path
.mnt
, file
->f_path
.dentry
, "\n\t= ");
1865 } else if (vma
->vm_start
<= mm
->brk
&& vma
->vm_end
>= mm
->start_brk
) {
1866 seq_printf(m
, " heap");
1867 } else if (vma
->vm_start
<= mm
->start_stack
&&
1868 vma
->vm_end
>= mm
->start_stack
) {
1869 seq_printf(m
, " stack");
1872 if (is_vm_hugetlb_page(vma
)) {
1873 check_huge_range(vma
, vma
->vm_start
, vma
->vm_end
, md
);
1874 seq_printf(m
, " huge");
1876 check_pgd_range(vma
, vma
->vm_start
, vma
->vm_end
,
1877 &node_online_map
, MPOL_MF_STATS
, md
);
1884 seq_printf(m
," anon=%lu",md
->anon
);
1887 seq_printf(m
," dirty=%lu",md
->dirty
);
1889 if (md
->pages
!= md
->anon
&& md
->pages
!= md
->dirty
)
1890 seq_printf(m
, " mapped=%lu", md
->pages
);
1892 if (md
->mapcount_max
> 1)
1893 seq_printf(m
, " mapmax=%lu", md
->mapcount_max
);
1896 seq_printf(m
," swapcache=%lu", md
->swapcache
);
1898 if (md
->active
< md
->pages
&& !is_vm_hugetlb_page(vma
))
1899 seq_printf(m
," active=%lu", md
->active
);
1902 seq_printf(m
," writeback=%lu", md
->writeback
);
1904 for_each_online_node(n
)
1906 seq_printf(m
, " N%d=%lu", n
, md
->node
[n
]);
1911 if (m
->count
< m
->size
)
1912 m
->version
= (vma
!= priv
->tail_vma
) ? vma
->vm_start
: 0;