2 * Simple NUMA memory policy for the Linux kernel.
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
6 * Subject to the GNU Public License, version 2.
8 * NUMA policy allows the user to give hints in which node(s) memory should
11 * Support four policies per VMA and per process:
13 * The VMA policy has priority over the process policy for a page fault.
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
22 * bind Only allocate memory on a specific set of nodes,
24 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
28 * preferred Try a specific node first before normal fallback.
29 * As a special case node -1 here means do the allocation
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
34 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
57 fix mmap readahead to honour policy and enable policy for any page cache
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
62 handle mremap for shared memory (currently ignored for the policy)
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
66 could replace all the switch()es with a mempolicy_ops structure.
69 #include <linux/mempolicy.h>
71 #include <linux/highmem.h>
72 #include <linux/hugetlb.h>
73 #include <linux/kernel.h>
74 #include <linux/sched.h>
75 #include <linux/nodemask.h>
76 #include <linux/cpuset.h>
77 #include <linux/gfp.h>
78 #include <linux/slab.h>
79 #include <linux/string.h>
80 #include <linux/module.h>
81 #include <linux/nsproxy.h>
82 #include <linux/interrupt.h>
83 #include <linux/init.h>
84 #include <linux/compat.h>
85 #include <linux/swap.h>
86 #include <linux/seq_file.h>
87 #include <linux/proc_fs.h>
88 #include <linux/migrate.h>
89 #include <linux/rmap.h>
90 #include <linux/security.h>
91 #include <linux/syscalls.h>
93 #include <asm/tlbflush.h>
94 #include <asm/uaccess.h>
97 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
98 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
99 #define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2) /* Gather statistics */
101 static struct kmem_cache
*policy_cache
;
102 static struct kmem_cache
*sn_cache
;
104 /* Highest zone. An specific allocation for a zone below that is not
106 enum zone_type policy_zone
= 0;
108 struct mempolicy default_policy
= {
109 .refcnt
= ATOMIC_INIT(1), /* never free it */
110 .policy
= MPOL_DEFAULT
,
113 static void mpol_rebind_policy(struct mempolicy
*pol
,
114 const nodemask_t
*newmask
);
116 /* Do sanity checking on a policy */
117 static int mpol_check_policy(int mode
, nodemask_t
*nodes
)
119 int empty
= nodes_empty(*nodes
);
127 case MPOL_INTERLEAVE
:
128 /* Preferred will only use the first bit, but allow
134 return nodes_subset(*nodes
, node_states
[N_HIGH_MEMORY
]) ? 0 : -EINVAL
;
137 /* Generate a custom zonelist for the BIND policy. */
138 static struct zonelist
*bind_zonelist(nodemask_t
*nodes
)
144 max
= 1 + MAX_NR_ZONES
* nodes_weight(*nodes
);
145 max
++; /* space for zlcache_ptr (see mmzone.h) */
146 zl
= kmalloc(sizeof(struct zone
*) * max
, GFP_KERNEL
);
148 return ERR_PTR(-ENOMEM
);
149 zl
->zlcache_ptr
= NULL
;
151 /* First put in the highest zones from all nodes, then all the next
152 lower zones etc. Avoid empty zones because the memory allocator
153 doesn't like them. If you implement node hot removal you
155 k
= MAX_NR_ZONES
- 1;
157 for_each_node_mask(nd
, *nodes
) {
158 struct zone
*z
= &NODE_DATA(nd
)->node_zones
[k
];
159 if (z
->present_pages
> 0)
160 zl
->zones
[num
++] = z
;
168 return ERR_PTR(-EINVAL
);
170 zl
->zones
[num
] = NULL
;
174 /* Create a new policy */
175 static struct mempolicy
*mpol_new(int mode
, nodemask_t
*nodes
)
177 struct mempolicy
*policy
;
179 pr_debug("setting mode %d nodes[0] %lx\n",
180 mode
, nodes
? nodes_addr(*nodes
)[0] : -1);
182 if (mode
== MPOL_DEFAULT
)
184 policy
= kmem_cache_alloc(policy_cache
, GFP_KERNEL
);
186 return ERR_PTR(-ENOMEM
);
187 atomic_set(&policy
->refcnt
, 1);
189 case MPOL_INTERLEAVE
:
190 policy
->v
.nodes
= *nodes
;
191 nodes_and(policy
->v
.nodes
, policy
->v
.nodes
,
192 node_states
[N_HIGH_MEMORY
]);
193 if (nodes_weight(policy
->v
.nodes
) == 0) {
194 kmem_cache_free(policy_cache
, policy
);
195 return ERR_PTR(-EINVAL
);
199 policy
->v
.preferred_node
= first_node(*nodes
);
200 if (policy
->v
.preferred_node
>= MAX_NUMNODES
)
201 policy
->v
.preferred_node
= -1;
204 policy
->v
.zonelist
= bind_zonelist(nodes
);
205 if (IS_ERR(policy
->v
.zonelist
)) {
206 void *error_code
= policy
->v
.zonelist
;
207 kmem_cache_free(policy_cache
, policy
);
212 policy
->policy
= mode
;
213 policy
->cpuset_mems_allowed
= cpuset_mems_allowed(current
);
217 static void gather_stats(struct page
*, void *, int pte_dirty
);
218 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
219 unsigned long flags
);
221 /* Scan through pages checking if pages follow certain conditions. */
222 static int check_pte_range(struct vm_area_struct
*vma
, pmd_t
*pmd
,
223 unsigned long addr
, unsigned long end
,
224 const nodemask_t
*nodes
, unsigned long flags
,
231 orig_pte
= pte
= pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, &ptl
);
236 if (!pte_present(*pte
))
238 page
= vm_normal_page(vma
, addr
, *pte
);
242 * The check for PageReserved here is important to avoid
243 * handling zero pages and other pages that may have been
244 * marked special by the system.
246 * If the PageReserved would not be checked here then f.e.
247 * the location of the zero page could have an influence
248 * on MPOL_MF_STRICT, zero pages would be counted for
249 * the per node stats, and there would be useless attempts
250 * to put zero pages on the migration list.
252 if (PageReserved(page
))
254 nid
= page_to_nid(page
);
255 if (node_isset(nid
, *nodes
) == !!(flags
& MPOL_MF_INVERT
))
258 if (flags
& MPOL_MF_STATS
)
259 gather_stats(page
, private, pte_dirty(*pte
));
260 else if (flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
))
261 migrate_page_add(page
, private, flags
);
264 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
265 pte_unmap_unlock(orig_pte
, ptl
);
269 static inline int check_pmd_range(struct vm_area_struct
*vma
, pud_t
*pud
,
270 unsigned long addr
, unsigned long end
,
271 const nodemask_t
*nodes
, unsigned long flags
,
277 pmd
= pmd_offset(pud
, addr
);
279 next
= pmd_addr_end(addr
, end
);
280 if (pmd_none_or_clear_bad(pmd
))
282 if (check_pte_range(vma
, pmd
, addr
, next
, nodes
,
285 } while (pmd
++, addr
= next
, addr
!= end
);
289 static inline int check_pud_range(struct vm_area_struct
*vma
, pgd_t
*pgd
,
290 unsigned long addr
, unsigned long end
,
291 const nodemask_t
*nodes
, unsigned long flags
,
297 pud
= pud_offset(pgd
, addr
);
299 next
= pud_addr_end(addr
, end
);
300 if (pud_none_or_clear_bad(pud
))
302 if (check_pmd_range(vma
, pud
, addr
, next
, nodes
,
305 } while (pud
++, addr
= next
, addr
!= end
);
309 static inline int check_pgd_range(struct vm_area_struct
*vma
,
310 unsigned long addr
, unsigned long end
,
311 const nodemask_t
*nodes
, unsigned long flags
,
317 pgd
= pgd_offset(vma
->vm_mm
, addr
);
319 next
= pgd_addr_end(addr
, end
);
320 if (pgd_none_or_clear_bad(pgd
))
322 if (check_pud_range(vma
, pgd
, addr
, next
, nodes
,
325 } while (pgd
++, addr
= next
, addr
!= end
);
330 * Check if all pages in a range are on a set of nodes.
331 * If pagelist != NULL then isolate pages from the LRU and
332 * put them on the pagelist.
334 static struct vm_area_struct
*
335 check_range(struct mm_struct
*mm
, unsigned long start
, unsigned long end
,
336 const nodemask_t
*nodes
, unsigned long flags
, void *private)
339 struct vm_area_struct
*first
, *vma
, *prev
;
341 if (flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
)) {
343 err
= migrate_prep();
348 first
= find_vma(mm
, start
);
350 return ERR_PTR(-EFAULT
);
352 for (vma
= first
; vma
&& vma
->vm_start
< end
; vma
= vma
->vm_next
) {
353 if (!(flags
& MPOL_MF_DISCONTIG_OK
)) {
354 if (!vma
->vm_next
&& vma
->vm_end
< end
)
355 return ERR_PTR(-EFAULT
);
356 if (prev
&& prev
->vm_end
< vma
->vm_start
)
357 return ERR_PTR(-EFAULT
);
359 if (!is_vm_hugetlb_page(vma
) &&
360 ((flags
& MPOL_MF_STRICT
) ||
361 ((flags
& (MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
)) &&
362 vma_migratable(vma
)))) {
363 unsigned long endvma
= vma
->vm_end
;
367 if (vma
->vm_start
> start
)
368 start
= vma
->vm_start
;
369 err
= check_pgd_range(vma
, start
, endvma
, nodes
,
372 first
= ERR_PTR(err
);
381 /* Apply policy to a single VMA */
382 static int policy_vma(struct vm_area_struct
*vma
, struct mempolicy
*new)
385 struct mempolicy
*old
= vma
->vm_policy
;
387 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
388 vma
->vm_start
, vma
->vm_end
, vma
->vm_pgoff
,
389 vma
->vm_ops
, vma
->vm_file
,
390 vma
->vm_ops
? vma
->vm_ops
->set_policy
: NULL
);
392 if (vma
->vm_ops
&& vma
->vm_ops
->set_policy
)
393 err
= vma
->vm_ops
->set_policy(vma
, new);
396 vma
->vm_policy
= new;
402 /* Step 2: apply policy to a range and do splits. */
403 static int mbind_range(struct vm_area_struct
*vma
, unsigned long start
,
404 unsigned long end
, struct mempolicy
*new)
406 struct vm_area_struct
*next
;
410 for (; vma
&& vma
->vm_start
< end
; vma
= next
) {
412 if (vma
->vm_start
< start
)
413 err
= split_vma(vma
->vm_mm
, vma
, start
, 1);
414 if (!err
&& vma
->vm_end
> end
)
415 err
= split_vma(vma
->vm_mm
, vma
, end
, 0);
417 err
= policy_vma(vma
, new);
424 static int contextualize_policy(int mode
, nodemask_t
*nodes
)
429 cpuset_update_task_memory_state();
430 if (!cpuset_nodes_subset_current_mems_allowed(*nodes
))
432 return mpol_check_policy(mode
, nodes
);
437 * Update task->flags PF_MEMPOLICY bit: set iff non-default
438 * mempolicy. Allows more rapid checking of this (combined perhaps
439 * with other PF_* flag bits) on memory allocation hot code paths.
441 * If called from outside this file, the task 'p' should -only- be
442 * a newly forked child not yet visible on the task list, because
443 * manipulating the task flags of a visible task is not safe.
445 * The above limitation is why this routine has the funny name
446 * mpol_fix_fork_child_flag().
448 * It is also safe to call this with a task pointer of current,
449 * which the static wrapper mpol_set_task_struct_flag() does,
450 * for use within this file.
453 void mpol_fix_fork_child_flag(struct task_struct
*p
)
456 p
->flags
|= PF_MEMPOLICY
;
458 p
->flags
&= ~PF_MEMPOLICY
;
461 static void mpol_set_task_struct_flag(void)
463 mpol_fix_fork_child_flag(current
);
466 /* Set the process memory policy */
467 static long do_set_mempolicy(int mode
, nodemask_t
*nodes
)
469 struct mempolicy
*new;
471 if (contextualize_policy(mode
, nodes
))
473 new = mpol_new(mode
, nodes
);
476 mpol_free(current
->mempolicy
);
477 current
->mempolicy
= new;
478 mpol_set_task_struct_flag();
479 if (new && new->policy
== MPOL_INTERLEAVE
)
480 current
->il_next
= first_node(new->v
.nodes
);
484 /* Fill a zone bitmap for a policy */
485 static void get_zonemask(struct mempolicy
*p
, nodemask_t
*nodes
)
492 for (i
= 0; p
->v
.zonelist
->zones
[i
]; i
++)
493 node_set(zone_to_nid(p
->v
.zonelist
->zones
[i
]),
498 case MPOL_INTERLEAVE
:
502 /* or use current node instead of memory_map? */
503 if (p
->v
.preferred_node
< 0)
504 *nodes
= node_states
[N_HIGH_MEMORY
];
506 node_set(p
->v
.preferred_node
, *nodes
);
513 static int lookup_node(struct mm_struct
*mm
, unsigned long addr
)
518 err
= get_user_pages(current
, mm
, addr
& PAGE_MASK
, 1, 0, 0, &p
, NULL
);
520 err
= page_to_nid(p
);
526 /* Retrieve NUMA policy */
527 static long do_get_mempolicy(int *policy
, nodemask_t
*nmask
,
528 unsigned long addr
, unsigned long flags
)
531 struct mm_struct
*mm
= current
->mm
;
532 struct vm_area_struct
*vma
= NULL
;
533 struct mempolicy
*pol
= current
->mempolicy
;
535 cpuset_update_task_memory_state();
537 ~(unsigned long)(MPOL_F_NODE
|MPOL_F_ADDR
|MPOL_F_MEMS_ALLOWED
))
540 if (flags
& MPOL_F_MEMS_ALLOWED
) {
541 if (flags
& (MPOL_F_NODE
|MPOL_F_ADDR
))
543 *policy
= 0; /* just so it's initialized */
544 *nmask
= cpuset_current_mems_allowed
;
548 if (flags
& MPOL_F_ADDR
) {
549 down_read(&mm
->mmap_sem
);
550 vma
= find_vma_intersection(mm
, addr
, addr
+1);
552 up_read(&mm
->mmap_sem
);
555 if (vma
->vm_ops
&& vma
->vm_ops
->get_policy
)
556 pol
= vma
->vm_ops
->get_policy(vma
, addr
);
558 pol
= vma
->vm_policy
;
563 pol
= &default_policy
;
565 if (flags
& MPOL_F_NODE
) {
566 if (flags
& MPOL_F_ADDR
) {
567 err
= lookup_node(mm
, addr
);
571 } else if (pol
== current
->mempolicy
&&
572 pol
->policy
== MPOL_INTERLEAVE
) {
573 *policy
= current
->il_next
;
579 *policy
= pol
->policy
;
582 up_read(¤t
->mm
->mmap_sem
);
588 get_zonemask(pol
, nmask
);
592 up_read(¤t
->mm
->mmap_sem
);
596 #ifdef CONFIG_MIGRATION
600 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
604 * Avoid migrating a page that is shared with others.
606 if ((flags
& MPOL_MF_MOVE_ALL
) || page_mapcount(page
) == 1)
607 isolate_lru_page(page
, pagelist
);
610 static struct page
*new_node_page(struct page
*page
, unsigned long node
, int **x
)
612 return alloc_pages_node(node
, GFP_HIGHUSER_MOVABLE
, 0);
616 * Migrate pages from one node to a target node.
617 * Returns error or the number of pages not migrated.
619 static int migrate_to_node(struct mm_struct
*mm
, int source
, int dest
,
627 node_set(source
, nmask
);
629 check_range(mm
, mm
->mmap
->vm_start
, TASK_SIZE
, &nmask
,
630 flags
| MPOL_MF_DISCONTIG_OK
, &pagelist
);
632 if (!list_empty(&pagelist
))
633 err
= migrate_pages(&pagelist
, new_node_page
, dest
);
639 * Move pages between the two nodesets so as to preserve the physical
640 * layout as much as possible.
642 * Returns the number of page that could not be moved.
644 int do_migrate_pages(struct mm_struct
*mm
,
645 const nodemask_t
*from_nodes
, const nodemask_t
*to_nodes
, int flags
)
652 down_read(&mm
->mmap_sem
);
654 err
= migrate_vmas(mm
, from_nodes
, to_nodes
, flags
);
659 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
660 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
661 * bit in 'tmp', and return that <source, dest> pair for migration.
662 * The pair of nodemasks 'to' and 'from' define the map.
664 * If no pair of bits is found that way, fallback to picking some
665 * pair of 'source' and 'dest' bits that are not the same. If the
666 * 'source' and 'dest' bits are the same, this represents a node
667 * that will be migrating to itself, so no pages need move.
669 * If no bits are left in 'tmp', or if all remaining bits left
670 * in 'tmp' correspond to the same bit in 'to', return false
671 * (nothing left to migrate).
673 * This lets us pick a pair of nodes to migrate between, such that
674 * if possible the dest node is not already occupied by some other
675 * source node, minimizing the risk of overloading the memory on a
676 * node that would happen if we migrated incoming memory to a node
677 * before migrating outgoing memory source that same node.
679 * A single scan of tmp is sufficient. As we go, we remember the
680 * most recent <s, d> pair that moved (s != d). If we find a pair
681 * that not only moved, but what's better, moved to an empty slot
682 * (d is not set in tmp), then we break out then, with that pair.
683 * Otherwise when we finish scannng from_tmp, we at least have the
684 * most recent <s, d> pair that moved. If we get all the way through
685 * the scan of tmp without finding any node that moved, much less
686 * moved to an empty node, then there is nothing left worth migrating.
690 while (!nodes_empty(tmp
)) {
695 for_each_node_mask(s
, tmp
) {
696 d
= node_remap(s
, *from_nodes
, *to_nodes
);
700 source
= s
; /* Node moved. Memorize */
703 /* dest not in remaining from nodes? */
704 if (!node_isset(dest
, tmp
))
710 node_clear(source
, tmp
);
711 err
= migrate_to_node(mm
, source
, dest
, flags
);
718 up_read(&mm
->mmap_sem
);
725 static struct page
*new_vma_page(struct page
*page
, unsigned long private, int **x
)
727 struct vm_area_struct
*vma
= (struct vm_area_struct
*)private;
729 return alloc_page_vma(GFP_HIGHUSER_MOVABLE
, vma
,
730 page_address_in_vma(page
, vma
));
734 static void migrate_page_add(struct page
*page
, struct list_head
*pagelist
,
739 int do_migrate_pages(struct mm_struct
*mm
,
740 const nodemask_t
*from_nodes
, const nodemask_t
*to_nodes
, int flags
)
745 static struct page
*new_vma_page(struct page
*page
, unsigned long private, int **x
)
751 static long do_mbind(unsigned long start
, unsigned long len
,
752 unsigned long mode
, nodemask_t
*nmask
,
755 struct vm_area_struct
*vma
;
756 struct mm_struct
*mm
= current
->mm
;
757 struct mempolicy
*new;
762 if ((flags
& ~(unsigned long)(MPOL_MF_STRICT
|
763 MPOL_MF_MOVE
| MPOL_MF_MOVE_ALL
))
766 if ((flags
& MPOL_MF_MOVE_ALL
) && !capable(CAP_SYS_NICE
))
769 if (start
& ~PAGE_MASK
)
772 if (mode
== MPOL_DEFAULT
)
773 flags
&= ~MPOL_MF_STRICT
;
775 len
= (len
+ PAGE_SIZE
- 1) & PAGE_MASK
;
783 if (mpol_check_policy(mode
, nmask
))
786 new = mpol_new(mode
, nmask
);
791 * If we are using the default policy then operation
792 * on discontinuous address spaces is okay after all
795 flags
|= MPOL_MF_DISCONTIG_OK
;
797 pr_debug("mbind %lx-%lx mode:%ld nodes:%lx\n",start
,start
+len
,
798 mode
, nmask
? nodes_addr(*nmask
)[0] : -1);
800 down_write(&mm
->mmap_sem
);
801 vma
= check_range(mm
, start
, end
, nmask
,
802 flags
| MPOL_MF_INVERT
, &pagelist
);
808 err
= mbind_range(vma
, start
, end
, new);
810 if (!list_empty(&pagelist
))
811 nr_failed
= migrate_pages(&pagelist
, new_vma_page
,
814 if (!err
&& nr_failed
&& (flags
& MPOL_MF_STRICT
))
818 up_write(&mm
->mmap_sem
);
824 * User space interface with variable sized bitmaps for nodelists.
827 /* Copy a node mask from user space. */
828 static int get_nodes(nodemask_t
*nodes
, const unsigned long __user
*nmask
,
829 unsigned long maxnode
)
832 unsigned long nlongs
;
833 unsigned long endmask
;
837 if (maxnode
== 0 || !nmask
)
839 if (maxnode
> PAGE_SIZE
*BITS_PER_BYTE
)
842 nlongs
= BITS_TO_LONGS(maxnode
);
843 if ((maxnode
% BITS_PER_LONG
) == 0)
846 endmask
= (1UL << (maxnode
% BITS_PER_LONG
)) - 1;
848 /* When the user specified more nodes than supported just check
849 if the non supported part is all zero. */
850 if (nlongs
> BITS_TO_LONGS(MAX_NUMNODES
)) {
851 if (nlongs
> PAGE_SIZE
/sizeof(long))
853 for (k
= BITS_TO_LONGS(MAX_NUMNODES
); k
< nlongs
; k
++) {
855 if (get_user(t
, nmask
+ k
))
857 if (k
== nlongs
- 1) {
863 nlongs
= BITS_TO_LONGS(MAX_NUMNODES
);
867 if (copy_from_user(nodes_addr(*nodes
), nmask
, nlongs
*sizeof(unsigned long)))
869 nodes_addr(*nodes
)[nlongs
-1] &= endmask
;
873 /* Copy a kernel node mask to user space */
874 static int copy_nodes_to_user(unsigned long __user
*mask
, unsigned long maxnode
,
877 unsigned long copy
= ALIGN(maxnode
-1, 64) / 8;
878 const int nbytes
= BITS_TO_LONGS(MAX_NUMNODES
) * sizeof(long);
881 if (copy
> PAGE_SIZE
)
883 if (clear_user((char __user
*)mask
+ nbytes
, copy
- nbytes
))
887 return copy_to_user(mask
, nodes_addr(*nodes
), copy
) ? -EFAULT
: 0;
890 asmlinkage
long sys_mbind(unsigned long start
, unsigned long len
,
892 unsigned long __user
*nmask
, unsigned long maxnode
,
898 err
= get_nodes(&nodes
, nmask
, maxnode
);
901 #ifdef CONFIG_CPUSETS
902 /* Restrict the nodes to the allowed nodes in the cpuset */
903 nodes_and(nodes
, nodes
, current
->mems_allowed
);
905 return do_mbind(start
, len
, mode
, &nodes
, flags
);
908 /* Set the process memory policy */
909 asmlinkage
long sys_set_mempolicy(int mode
, unsigned long __user
*nmask
,
910 unsigned long maxnode
)
915 if (mode
< 0 || mode
> MPOL_MAX
)
917 err
= get_nodes(&nodes
, nmask
, maxnode
);
920 return do_set_mempolicy(mode
, &nodes
);
923 asmlinkage
long sys_migrate_pages(pid_t pid
, unsigned long maxnode
,
924 const unsigned long __user
*old_nodes
,
925 const unsigned long __user
*new_nodes
)
927 struct mm_struct
*mm
;
928 struct task_struct
*task
;
931 nodemask_t task_nodes
;
934 err
= get_nodes(&old
, old_nodes
, maxnode
);
938 err
= get_nodes(&new, new_nodes
, maxnode
);
942 /* Find the mm_struct */
943 read_lock(&tasklist_lock
);
945 find_task_by_pid_ns(pid
, current
->nsproxy
->pid_ns
) : current
;
947 read_unlock(&tasklist_lock
);
950 mm
= get_task_mm(task
);
951 read_unlock(&tasklist_lock
);
957 * Check if this process has the right to modify the specified
958 * process. The right exists if the process has administrative
959 * capabilities, superuser privileges or the same
960 * userid as the target process.
962 if ((current
->euid
!= task
->suid
) && (current
->euid
!= task
->uid
) &&
963 (current
->uid
!= task
->suid
) && (current
->uid
!= task
->uid
) &&
964 !capable(CAP_SYS_NICE
)) {
969 task_nodes
= cpuset_mems_allowed(task
);
970 /* Is the user allowed to access the target nodes? */
971 if (!nodes_subset(new, task_nodes
) && !capable(CAP_SYS_NICE
)) {
976 if (!nodes_subset(new, node_states
[N_HIGH_MEMORY
])) {
981 err
= security_task_movememory(task
);
985 err
= do_migrate_pages(mm
, &old
, &new,
986 capable(CAP_SYS_NICE
) ? MPOL_MF_MOVE_ALL
: MPOL_MF_MOVE
);
993 /* Retrieve NUMA policy */
994 asmlinkage
long sys_get_mempolicy(int __user
*policy
,
995 unsigned long __user
*nmask
,
996 unsigned long maxnode
,
997 unsigned long addr
, unsigned long flags
)
1000 int uninitialized_var(pval
);
1003 if (nmask
!= NULL
&& maxnode
< MAX_NUMNODES
)
1006 err
= do_get_mempolicy(&pval
, &nodes
, addr
, flags
);
1011 if (policy
&& put_user(pval
, policy
))
1015 err
= copy_nodes_to_user(nmask
, maxnode
, &nodes
);
1020 #ifdef CONFIG_COMPAT
1022 asmlinkage
long compat_sys_get_mempolicy(int __user
*policy
,
1023 compat_ulong_t __user
*nmask
,
1024 compat_ulong_t maxnode
,
1025 compat_ulong_t addr
, compat_ulong_t flags
)
1028 unsigned long __user
*nm
= NULL
;
1029 unsigned long nr_bits
, alloc_size
;
1030 DECLARE_BITMAP(bm
, MAX_NUMNODES
);
1032 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1033 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1036 nm
= compat_alloc_user_space(alloc_size
);
1038 err
= sys_get_mempolicy(policy
, nm
, nr_bits
+1, addr
, flags
);
1040 if (!err
&& nmask
) {
1041 err
= copy_from_user(bm
, nm
, alloc_size
);
1042 /* ensure entire bitmap is zeroed */
1043 err
|= clear_user(nmask
, ALIGN(maxnode
-1, 8) / 8);
1044 err
|= compat_put_bitmap(nmask
, bm
, nr_bits
);
1050 asmlinkage
long compat_sys_set_mempolicy(int mode
, compat_ulong_t __user
*nmask
,
1051 compat_ulong_t maxnode
)
1054 unsigned long __user
*nm
= NULL
;
1055 unsigned long nr_bits
, alloc_size
;
1056 DECLARE_BITMAP(bm
, MAX_NUMNODES
);
1058 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1059 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1062 err
= compat_get_bitmap(bm
, nmask
, nr_bits
);
1063 nm
= compat_alloc_user_space(alloc_size
);
1064 err
|= copy_to_user(nm
, bm
, alloc_size
);
1070 return sys_set_mempolicy(mode
, nm
, nr_bits
+1);
1073 asmlinkage
long compat_sys_mbind(compat_ulong_t start
, compat_ulong_t len
,
1074 compat_ulong_t mode
, compat_ulong_t __user
*nmask
,
1075 compat_ulong_t maxnode
, compat_ulong_t flags
)
1078 unsigned long __user
*nm
= NULL
;
1079 unsigned long nr_bits
, alloc_size
;
1082 nr_bits
= min_t(unsigned long, maxnode
-1, MAX_NUMNODES
);
1083 alloc_size
= ALIGN(nr_bits
, BITS_PER_LONG
) / 8;
1086 err
= compat_get_bitmap(nodes_addr(bm
), nmask
, nr_bits
);
1087 nm
= compat_alloc_user_space(alloc_size
);
1088 err
|= copy_to_user(nm
, nodes_addr(bm
), alloc_size
);
1094 return sys_mbind(start
, len
, mode
, nm
, nr_bits
+1, flags
);
1100 * get_vma_policy(@task, @vma, @addr)
1101 * @task - task for fallback if vma policy == default
1102 * @vma - virtual memory area whose policy is sought
1103 * @addr - address in @vma for shared policy lookup
1105 * Returns effective policy for a VMA at specified address.
1106 * Falls back to @task or system default policy, as necessary.
1107 * Returned policy has extra reference count if shared, vma,
1108 * or some other task's policy [show_numa_maps() can pass
1109 * @task != current]. It is the caller's responsibility to
1110 * free the reference in these cases.
1112 static struct mempolicy
* get_vma_policy(struct task_struct
*task
,
1113 struct vm_area_struct
*vma
, unsigned long addr
)
1115 struct mempolicy
*pol
= task
->mempolicy
;
1119 if (vma
->vm_ops
&& vma
->vm_ops
->get_policy
) {
1120 pol
= vma
->vm_ops
->get_policy(vma
, addr
);
1121 shared_pol
= 1; /* if pol non-NULL, add ref below */
1122 } else if (vma
->vm_policy
&&
1123 vma
->vm_policy
->policy
!= MPOL_DEFAULT
)
1124 pol
= vma
->vm_policy
;
1127 pol
= &default_policy
;
1128 else if (!shared_pol
&& pol
!= current
->mempolicy
)
1129 mpol_get(pol
); /* vma or other task's policy */
1133 /* Return a zonelist representing a mempolicy */
1134 static struct zonelist
*zonelist_policy(gfp_t gfp
, struct mempolicy
*policy
)
1138 switch (policy
->policy
) {
1139 case MPOL_PREFERRED
:
1140 nd
= policy
->v
.preferred_node
;
1142 nd
= numa_node_id();
1145 /* Lower zones don't get a policy applied */
1146 /* Careful: current->mems_allowed might have moved */
1147 if (gfp_zone(gfp
) >= policy_zone
)
1148 if (cpuset_zonelist_valid_mems_allowed(policy
->v
.zonelist
))
1149 return policy
->v
.zonelist
;
1151 case MPOL_INTERLEAVE
: /* should not happen */
1153 nd
= numa_node_id();
1159 return NODE_DATA(nd
)->node_zonelists
+ gfp_zone(gfp
);
1162 /* Do dynamic interleaving for a process */
1163 static unsigned interleave_nodes(struct mempolicy
*policy
)
1166 struct task_struct
*me
= current
;
1169 next
= next_node(nid
, policy
->v
.nodes
);
1170 if (next
>= MAX_NUMNODES
)
1171 next
= first_node(policy
->v
.nodes
);
1177 * Depending on the memory policy provide a node from which to allocate the
1180 unsigned slab_node(struct mempolicy
*policy
)
1182 int pol
= policy
? policy
->policy
: MPOL_DEFAULT
;
1185 case MPOL_INTERLEAVE
:
1186 return interleave_nodes(policy
);
1190 * Follow bind policy behavior and start allocation at the
1193 return zone_to_nid(policy
->v
.zonelist
->zones
[0]);
1195 case MPOL_PREFERRED
:
1196 if (policy
->v
.preferred_node
>= 0)
1197 return policy
->v
.preferred_node
;
1201 return numa_node_id();
1205 /* Do static interleaving for a VMA with known offset. */
1206 static unsigned offset_il_node(struct mempolicy
*pol
,
1207 struct vm_area_struct
*vma
, unsigned long off
)
1209 unsigned nnodes
= nodes_weight(pol
->v
.nodes
);
1210 unsigned target
= (unsigned)off
% nnodes
;
1216 nid
= next_node(nid
, pol
->v
.nodes
);
1218 } while (c
<= target
);
1222 /* Determine a node number for interleave */
1223 static inline unsigned interleave_nid(struct mempolicy
*pol
,
1224 struct vm_area_struct
*vma
, unsigned long addr
, int shift
)
1230 * for small pages, there is no difference between
1231 * shift and PAGE_SHIFT, so the bit-shift is safe.
1232 * for huge pages, since vm_pgoff is in units of small
1233 * pages, we need to shift off the always 0 bits to get
1236 BUG_ON(shift
< PAGE_SHIFT
);
1237 off
= vma
->vm_pgoff
>> (shift
- PAGE_SHIFT
);
1238 off
+= (addr
- vma
->vm_start
) >> shift
;
1239 return offset_il_node(pol
, vma
, off
);
1241 return interleave_nodes(pol
);
1244 #ifdef CONFIG_HUGETLBFS
1246 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
1247 * @vma = virtual memory area whose policy is sought
1248 * @addr = address in @vma for shared policy lookup and interleave policy
1249 * @gfp_flags = for requested zone
1250 * @mpol = pointer to mempolicy pointer for reference counted 'BIND policy
1252 * Returns a zonelist suitable for a huge page allocation.
1253 * If the effective policy is 'BIND, returns pointer to policy's zonelist.
1254 * If it is also a policy for which get_vma_policy() returns an extra
1255 * reference, we must hold that reference until after allocation.
1256 * In that case, return policy via @mpol so hugetlb allocation can drop
1257 * the reference. For non-'BIND referenced policies, we can/do drop the
1258 * reference here, so the caller doesn't need to know about the special case
1259 * for default and current task policy.
1261 struct zonelist
*huge_zonelist(struct vm_area_struct
*vma
, unsigned long addr
,
1262 gfp_t gfp_flags
, struct mempolicy
**mpol
)
1264 struct mempolicy
*pol
= get_vma_policy(current
, vma
, addr
);
1265 struct zonelist
*zl
;
1267 *mpol
= NULL
; /* probably no unref needed */
1268 if (pol
->policy
== MPOL_INTERLEAVE
) {
1271 nid
= interleave_nid(pol
, vma
, addr
, HPAGE_SHIFT
);
1272 __mpol_free(pol
); /* finished with pol */
1273 return NODE_DATA(nid
)->node_zonelists
+ gfp_zone(gfp_flags
);
1276 zl
= zonelist_policy(GFP_HIGHUSER
, pol
);
1277 if (unlikely(pol
!= &default_policy
&& pol
!= current
->mempolicy
)) {
1278 if (pol
->policy
!= MPOL_BIND
)
1279 __mpol_free(pol
); /* finished with pol */
1281 *mpol
= pol
; /* unref needed after allocation */
1287 /* Allocate a page in interleaved policy.
1288 Own path because it needs to do special accounting. */
1289 static struct page
*alloc_page_interleave(gfp_t gfp
, unsigned order
,
1292 struct zonelist
*zl
;
1295 zl
= NODE_DATA(nid
)->node_zonelists
+ gfp_zone(gfp
);
1296 page
= __alloc_pages(gfp
, order
, zl
);
1297 if (page
&& page_zone(page
) == zl
->zones
[0])
1298 inc_zone_page_state(page
, NUMA_INTERLEAVE_HIT
);
1303 * alloc_page_vma - Allocate a page for a VMA.
1306 * %GFP_USER user allocation.
1307 * %GFP_KERNEL kernel allocations,
1308 * %GFP_HIGHMEM highmem/user allocations,
1309 * %GFP_FS allocation should not call back into a file system.
1310 * %GFP_ATOMIC don't sleep.
1312 * @vma: Pointer to VMA or NULL if not available.
1313 * @addr: Virtual Address of the allocation. Must be inside the VMA.
1315 * This function allocates a page from the kernel page pool and applies
1316 * a NUMA policy associated with the VMA or the current process.
1317 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1318 * mm_struct of the VMA to prevent it from going away. Should be used for
1319 * all allocations for pages that will be mapped into
1320 * user space. Returns NULL when no page can be allocated.
1322 * Should be called with the mm_sem of the vma hold.
1325 alloc_page_vma(gfp_t gfp
, struct vm_area_struct
*vma
, unsigned long addr
)
1327 struct mempolicy
*pol
= get_vma_policy(current
, vma
, addr
);
1328 struct zonelist
*zl
;
1330 cpuset_update_task_memory_state();
1332 if (unlikely(pol
->policy
== MPOL_INTERLEAVE
)) {
1335 nid
= interleave_nid(pol
, vma
, addr
, PAGE_SHIFT
);
1336 return alloc_page_interleave(gfp
, 0, nid
);
1338 zl
= zonelist_policy(gfp
, pol
);
1339 if (pol
!= &default_policy
&& pol
!= current
->mempolicy
) {
1341 * slow path: ref counted policy -- shared or vma
1343 struct page
*page
= __alloc_pages(gfp
, 0, zl
);
1348 * fast path: default or task policy
1350 return __alloc_pages(gfp
, 0, zl
);
1354 * alloc_pages_current - Allocate pages.
1357 * %GFP_USER user allocation,
1358 * %GFP_KERNEL kernel allocation,
1359 * %GFP_HIGHMEM highmem allocation,
1360 * %GFP_FS don't call back into a file system.
1361 * %GFP_ATOMIC don't sleep.
1362 * @order: Power of two of allocation size in pages. 0 is a single page.
1364 * Allocate a page from the kernel page pool. When not in
1365 * interrupt context and apply the current process NUMA policy.
1366 * Returns NULL when no page can be allocated.
1368 * Don't call cpuset_update_task_memory_state() unless
1369 * 1) it's ok to take cpuset_sem (can WAIT), and
1370 * 2) allocating for current task (not interrupt).
1372 struct page
*alloc_pages_current(gfp_t gfp
, unsigned order
)
1374 struct mempolicy
*pol
= current
->mempolicy
;
1376 if ((gfp
& __GFP_WAIT
) && !in_interrupt())
1377 cpuset_update_task_memory_state();
1378 if (!pol
|| in_interrupt() || (gfp
& __GFP_THISNODE
))
1379 pol
= &default_policy
;
1380 if (pol
->policy
== MPOL_INTERLEAVE
)
1381 return alloc_page_interleave(gfp
, order
, interleave_nodes(pol
));
1382 return __alloc_pages(gfp
, order
, zonelist_policy(gfp
, pol
));
1384 EXPORT_SYMBOL(alloc_pages_current
);
1387 * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
1388 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
1389 * with the mems_allowed returned by cpuset_mems_allowed(). This
1390 * keeps mempolicies cpuset relative after its cpuset moves. See
1391 * further kernel/cpuset.c update_nodemask().
1394 /* Slow path of a mempolicy copy */
1395 struct mempolicy
*__mpol_copy(struct mempolicy
*old
)
1397 struct mempolicy
*new = kmem_cache_alloc(policy_cache
, GFP_KERNEL
);
1400 return ERR_PTR(-ENOMEM
);
1401 if (current_cpuset_is_being_rebound()) {
1402 nodemask_t mems
= cpuset_mems_allowed(current
);
1403 mpol_rebind_policy(old
, &mems
);
1406 atomic_set(&new->refcnt
, 1);
1407 if (new->policy
== MPOL_BIND
) {
1408 int sz
= ksize(old
->v
.zonelist
);
1409 new->v
.zonelist
= kmemdup(old
->v
.zonelist
, sz
, GFP_KERNEL
);
1410 if (!new->v
.zonelist
) {
1411 kmem_cache_free(policy_cache
, new);
1412 return ERR_PTR(-ENOMEM
);
1418 /* Slow path of a mempolicy comparison */
1419 int __mpol_equal(struct mempolicy
*a
, struct mempolicy
*b
)
1423 if (a
->policy
!= b
->policy
)
1425 switch (a
->policy
) {
1428 case MPOL_INTERLEAVE
:
1429 return nodes_equal(a
->v
.nodes
, b
->v
.nodes
);
1430 case MPOL_PREFERRED
:
1431 return a
->v
.preferred_node
== b
->v
.preferred_node
;
1434 for (i
= 0; a
->v
.zonelist
->zones
[i
]; i
++)
1435 if (a
->v
.zonelist
->zones
[i
] != b
->v
.zonelist
->zones
[i
])
1437 return b
->v
.zonelist
->zones
[i
] == NULL
;
1445 /* Slow path of a mpol destructor. */
1446 void __mpol_free(struct mempolicy
*p
)
1448 if (!atomic_dec_and_test(&p
->refcnt
))
1450 if (p
->policy
== MPOL_BIND
)
1451 kfree(p
->v
.zonelist
);
1452 p
->policy
= MPOL_DEFAULT
;
1453 kmem_cache_free(policy_cache
, p
);
1457 * Shared memory backing store policy support.
1459 * Remember policies even when nobody has shared memory mapped.
1460 * The policies are kept in Red-Black tree linked from the inode.
1461 * They are protected by the sp->lock spinlock, which should be held
1462 * for any accesses to the tree.
1465 /* lookup first element intersecting start-end */
1466 /* Caller holds sp->lock */
1467 static struct sp_node
*
1468 sp_lookup(struct shared_policy
*sp
, unsigned long start
, unsigned long end
)
1470 struct rb_node
*n
= sp
->root
.rb_node
;
1473 struct sp_node
*p
= rb_entry(n
, struct sp_node
, nd
);
1475 if (start
>= p
->end
)
1477 else if (end
<= p
->start
)
1485 struct sp_node
*w
= NULL
;
1486 struct rb_node
*prev
= rb_prev(n
);
1489 w
= rb_entry(prev
, struct sp_node
, nd
);
1490 if (w
->end
<= start
)
1494 return rb_entry(n
, struct sp_node
, nd
);
1497 /* Insert a new shared policy into the list. */
1498 /* Caller holds sp->lock */
1499 static void sp_insert(struct shared_policy
*sp
, struct sp_node
*new)
1501 struct rb_node
**p
= &sp
->root
.rb_node
;
1502 struct rb_node
*parent
= NULL
;
1507 nd
= rb_entry(parent
, struct sp_node
, nd
);
1508 if (new->start
< nd
->start
)
1510 else if (new->end
> nd
->end
)
1511 p
= &(*p
)->rb_right
;
1515 rb_link_node(&new->nd
, parent
, p
);
1516 rb_insert_color(&new->nd
, &sp
->root
);
1517 pr_debug("inserting %lx-%lx: %d\n", new->start
, new->end
,
1518 new->policy
? new->policy
->policy
: 0);
1521 /* Find shared policy intersecting idx */
1523 mpol_shared_policy_lookup(struct shared_policy
*sp
, unsigned long idx
)
1525 struct mempolicy
*pol
= NULL
;
1528 if (!sp
->root
.rb_node
)
1530 spin_lock(&sp
->lock
);
1531 sn
= sp_lookup(sp
, idx
, idx
+1);
1533 mpol_get(sn
->policy
);
1536 spin_unlock(&sp
->lock
);
1540 static void sp_delete(struct shared_policy
*sp
, struct sp_node
*n
)
1542 pr_debug("deleting %lx-l%lx\n", n
->start
, n
->end
);
1543 rb_erase(&n
->nd
, &sp
->root
);
1544 mpol_free(n
->policy
);
1545 kmem_cache_free(sn_cache
, n
);
1548 static struct sp_node
*sp_alloc(unsigned long start
, unsigned long end
,
1549 struct mempolicy
*pol
)
1551 struct sp_node
*n
= kmem_cache_alloc(sn_cache
, GFP_KERNEL
);
1562 /* Replace a policy range. */
1563 static int shared_policy_replace(struct shared_policy
*sp
, unsigned long start
,
1564 unsigned long end
, struct sp_node
*new)
1566 struct sp_node
*n
, *new2
= NULL
;
1569 spin_lock(&sp
->lock
);
1570 n
= sp_lookup(sp
, start
, end
);
1571 /* Take care of old policies in the same range. */
1572 while (n
&& n
->start
< end
) {
1573 struct rb_node
*next
= rb_next(&n
->nd
);
1574 if (n
->start
>= start
) {
1580 /* Old policy spanning whole new range. */
1583 spin_unlock(&sp
->lock
);
1584 new2
= sp_alloc(end
, n
->end
, n
->policy
);
1590 sp_insert(sp
, new2
);
1598 n
= rb_entry(next
, struct sp_node
, nd
);
1602 spin_unlock(&sp
->lock
);
1604 mpol_free(new2
->policy
);
1605 kmem_cache_free(sn_cache
, new2
);
1610 void mpol_shared_policy_init(struct shared_policy
*info
, int policy
,
1611 nodemask_t
*policy_nodes
)
1613 info
->root
= RB_ROOT
;
1614 spin_lock_init(&info
->lock
);
1616 if (policy
!= MPOL_DEFAULT
) {
1617 struct mempolicy
*newpol
;
1619 /* Falls back to MPOL_DEFAULT on any error */
1620 newpol
= mpol_new(policy
, policy_nodes
);
1621 if (!IS_ERR(newpol
)) {
1622 /* Create pseudo-vma that contains just the policy */
1623 struct vm_area_struct pvma
;
1625 memset(&pvma
, 0, sizeof(struct vm_area_struct
));
1626 /* Policy covers entire file */
1627 pvma
.vm_end
= TASK_SIZE
;
1628 mpol_set_shared_policy(info
, &pvma
, newpol
);
1634 int mpol_set_shared_policy(struct shared_policy
*info
,
1635 struct vm_area_struct
*vma
, struct mempolicy
*npol
)
1638 struct sp_node
*new = NULL
;
1639 unsigned long sz
= vma_pages(vma
);
1641 pr_debug("set_shared_policy %lx sz %lu %d %lx\n",
1643 sz
, npol
? npol
->policy
: -1,
1644 npol
? nodes_addr(npol
->v
.nodes
)[0] : -1);
1647 new = sp_alloc(vma
->vm_pgoff
, vma
->vm_pgoff
+ sz
, npol
);
1651 err
= shared_policy_replace(info
, vma
->vm_pgoff
, vma
->vm_pgoff
+sz
, new);
1653 kmem_cache_free(sn_cache
, new);
1657 /* Free a backing policy store on inode delete. */
1658 void mpol_free_shared_policy(struct shared_policy
*p
)
1661 struct rb_node
*next
;
1663 if (!p
->root
.rb_node
)
1665 spin_lock(&p
->lock
);
1666 next
= rb_first(&p
->root
);
1668 n
= rb_entry(next
, struct sp_node
, nd
);
1669 next
= rb_next(&n
->nd
);
1670 rb_erase(&n
->nd
, &p
->root
);
1671 mpol_free(n
->policy
);
1672 kmem_cache_free(sn_cache
, n
);
1674 spin_unlock(&p
->lock
);
1677 /* assumes fs == KERNEL_DS */
1678 void __init
numa_policy_init(void)
1680 nodemask_t interleave_nodes
;
1681 unsigned long largest
= 0;
1682 int nid
, prefer
= 0;
1684 policy_cache
= kmem_cache_create("numa_policy",
1685 sizeof(struct mempolicy
),
1686 0, SLAB_PANIC
, NULL
);
1688 sn_cache
= kmem_cache_create("shared_policy_node",
1689 sizeof(struct sp_node
),
1690 0, SLAB_PANIC
, NULL
);
1693 * Set interleaving policy for system init. Interleaving is only
1694 * enabled across suitably sized nodes (default is >= 16MB), or
1695 * fall back to the largest node if they're all smaller.
1697 nodes_clear(interleave_nodes
);
1698 for_each_node_state(nid
, N_HIGH_MEMORY
) {
1699 unsigned long total_pages
= node_present_pages(nid
);
1701 /* Preserve the largest node */
1702 if (largest
< total_pages
) {
1703 largest
= total_pages
;
1707 /* Interleave this node? */
1708 if ((total_pages
<< PAGE_SHIFT
) >= (16 << 20))
1709 node_set(nid
, interleave_nodes
);
1712 /* All too small, use the largest */
1713 if (unlikely(nodes_empty(interleave_nodes
)))
1714 node_set(prefer
, interleave_nodes
);
1716 if (do_set_mempolicy(MPOL_INTERLEAVE
, &interleave_nodes
))
1717 printk("numa_policy_init: interleaving failed\n");
1720 /* Reset policy of current process to default */
1721 void numa_default_policy(void)
1723 do_set_mempolicy(MPOL_DEFAULT
, NULL
);
1726 /* Migrate a policy to a different set of nodes */
1727 static void mpol_rebind_policy(struct mempolicy
*pol
,
1728 const nodemask_t
*newmask
)
1730 nodemask_t
*mpolmask
;
1735 mpolmask
= &pol
->cpuset_mems_allowed
;
1736 if (nodes_equal(*mpolmask
, *newmask
))
1739 switch (pol
->policy
) {
1742 case MPOL_INTERLEAVE
:
1743 nodes_remap(tmp
, pol
->v
.nodes
, *mpolmask
, *newmask
);
1745 *mpolmask
= *newmask
;
1746 current
->il_next
= node_remap(current
->il_next
,
1747 *mpolmask
, *newmask
);
1749 case MPOL_PREFERRED
:
1750 pol
->v
.preferred_node
= node_remap(pol
->v
.preferred_node
,
1751 *mpolmask
, *newmask
);
1752 *mpolmask
= *newmask
;
1757 struct zonelist
*zonelist
;
1760 for (z
= pol
->v
.zonelist
->zones
; *z
; z
++)
1761 node_set(zone_to_nid(*z
), nodes
);
1762 nodes_remap(tmp
, nodes
, *mpolmask
, *newmask
);
1765 zonelist
= bind_zonelist(&nodes
);
1767 /* If no mem, then zonelist is NULL and we keep old zonelist.
1768 * If that old zonelist has no remaining mems_allowed nodes,
1769 * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
1772 if (!IS_ERR(zonelist
)) {
1773 /* Good - got mem - substitute new zonelist */
1774 kfree(pol
->v
.zonelist
);
1775 pol
->v
.zonelist
= zonelist
;
1777 *mpolmask
= *newmask
;
1787 * Wrapper for mpol_rebind_policy() that just requires task
1788 * pointer, and updates task mempolicy.
1791 void mpol_rebind_task(struct task_struct
*tsk
, const nodemask_t
*new)
1793 mpol_rebind_policy(tsk
->mempolicy
, new);
1797 * Rebind each vma in mm to new nodemask.
1799 * Call holding a reference to mm. Takes mm->mmap_sem during call.
1802 void mpol_rebind_mm(struct mm_struct
*mm
, nodemask_t
*new)
1804 struct vm_area_struct
*vma
;
1806 down_write(&mm
->mmap_sem
);
1807 for (vma
= mm
->mmap
; vma
; vma
= vma
->vm_next
)
1808 mpol_rebind_policy(vma
->vm_policy
, new);
1809 up_write(&mm
->mmap_sem
);
1813 * Display pages allocated per node and memory policy via /proc.
1816 static const char * const policy_types
[] =
1817 { "default", "prefer", "bind", "interleave" };
1820 * Convert a mempolicy into a string.
1821 * Returns the number of characters in buffer (if positive)
1822 * or an error (negative)
1824 static inline int mpol_to_str(char *buffer
, int maxlen
, struct mempolicy
*pol
)
1829 int mode
= pol
? pol
->policy
: MPOL_DEFAULT
;
1836 case MPOL_PREFERRED
:
1838 node_set(pol
->v
.preferred_node
, nodes
);
1842 get_zonemask(pol
, &nodes
);
1845 case MPOL_INTERLEAVE
:
1846 nodes
= pol
->v
.nodes
;
1854 l
= strlen(policy_types
[mode
]);
1855 if (buffer
+ maxlen
< p
+ l
+ 1)
1858 strcpy(p
, policy_types
[mode
]);
1861 if (!nodes_empty(nodes
)) {
1862 if (buffer
+ maxlen
< p
+ 2)
1865 p
+= nodelist_scnprintf(p
, buffer
+ maxlen
- p
, nodes
);
1871 unsigned long pages
;
1873 unsigned long active
;
1874 unsigned long writeback
;
1875 unsigned long mapcount_max
;
1876 unsigned long dirty
;
1877 unsigned long swapcache
;
1878 unsigned long node
[MAX_NUMNODES
];
1881 static void gather_stats(struct page
*page
, void *private, int pte_dirty
)
1883 struct numa_maps
*md
= private;
1884 int count
= page_mapcount(page
);
1887 if (pte_dirty
|| PageDirty(page
))
1890 if (PageSwapCache(page
))
1893 if (PageActive(page
))
1896 if (PageWriteback(page
))
1902 if (count
> md
->mapcount_max
)
1903 md
->mapcount_max
= count
;
1905 md
->node
[page_to_nid(page
)]++;
1908 #ifdef CONFIG_HUGETLB_PAGE
1909 static void check_huge_range(struct vm_area_struct
*vma
,
1910 unsigned long start
, unsigned long end
,
1911 struct numa_maps
*md
)
1916 for (addr
= start
; addr
< end
; addr
+= HPAGE_SIZE
) {
1917 pte_t
*ptep
= huge_pte_offset(vma
->vm_mm
, addr
& HPAGE_MASK
);
1927 page
= pte_page(pte
);
1931 gather_stats(page
, md
, pte_dirty(*ptep
));
1935 static inline void check_huge_range(struct vm_area_struct
*vma
,
1936 unsigned long start
, unsigned long end
,
1937 struct numa_maps
*md
)
1942 int show_numa_map(struct seq_file
*m
, void *v
)
1944 struct proc_maps_private
*priv
= m
->private;
1945 struct vm_area_struct
*vma
= v
;
1946 struct numa_maps
*md
;
1947 struct file
*file
= vma
->vm_file
;
1948 struct mm_struct
*mm
= vma
->vm_mm
;
1949 struct mempolicy
*pol
;
1956 md
= kzalloc(sizeof(struct numa_maps
), GFP_KERNEL
);
1960 pol
= get_vma_policy(priv
->task
, vma
, vma
->vm_start
);
1961 mpol_to_str(buffer
, sizeof(buffer
), pol
);
1963 * unref shared or other task's mempolicy
1965 if (pol
!= &default_policy
&& pol
!= current
->mempolicy
)
1968 seq_printf(m
, "%08lx %s", vma
->vm_start
, buffer
);
1971 seq_printf(m
, " file=");
1972 seq_path(m
, file
->f_path
.mnt
, file
->f_path
.dentry
, "\n\t= ");
1973 } else if (vma
->vm_start
<= mm
->brk
&& vma
->vm_end
>= mm
->start_brk
) {
1974 seq_printf(m
, " heap");
1975 } else if (vma
->vm_start
<= mm
->start_stack
&&
1976 vma
->vm_end
>= mm
->start_stack
) {
1977 seq_printf(m
, " stack");
1980 if (is_vm_hugetlb_page(vma
)) {
1981 check_huge_range(vma
, vma
->vm_start
, vma
->vm_end
, md
);
1982 seq_printf(m
, " huge");
1984 check_pgd_range(vma
, vma
->vm_start
, vma
->vm_end
,
1985 &node_states
[N_HIGH_MEMORY
], MPOL_MF_STATS
, md
);
1992 seq_printf(m
," anon=%lu",md
->anon
);
1995 seq_printf(m
," dirty=%lu",md
->dirty
);
1997 if (md
->pages
!= md
->anon
&& md
->pages
!= md
->dirty
)
1998 seq_printf(m
, " mapped=%lu", md
->pages
);
2000 if (md
->mapcount_max
> 1)
2001 seq_printf(m
, " mapmax=%lu", md
->mapcount_max
);
2004 seq_printf(m
," swapcache=%lu", md
->swapcache
);
2006 if (md
->active
< md
->pages
&& !is_vm_hugetlb_page(vma
))
2007 seq_printf(m
," active=%lu", md
->active
);
2010 seq_printf(m
," writeback=%lu", md
->writeback
);
2012 for_each_node_state(n
, N_HIGH_MEMORY
)
2014 seq_printf(m
, " N%d=%lu", n
, md
->node
[n
]);
2019 if (m
->count
< m
->size
)
2020 m
->version
= (vma
!= priv
->tail_vma
) ? vma
->vm_start
: 0;