]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - mm/mempolicy.c
mlxsw: spectrum_router: Save requested prefix bitlist when creating tree
[mirror_ubuntu-bionic-kernel.git] / mm / mempolicy.c
CommitLineData
1da177e4
LT
1/*
2 * Simple NUMA memory policy for the Linux kernel.
3 *
4 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
8bccd85f 5 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
1da177e4
LT
6 * Subject to the GNU Public License, version 2.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should
9 * be allocated.
10 *
11 * Support four policies per VMA and per process:
12 *
13 * The VMA policy has priority over the process policy for a page fault.
14 *
15 * interleave Allocate memory interleaved over a set of nodes,
16 * with normal fallback if it fails.
17 * For VMA based allocations this interleaves based on the
18 * offset into the backing object or offset into the mapping
19 * for anonymous memory. For process policy an process counter
20 * is used.
8bccd85f 21 *
1da177e4
LT
22 * bind Only allocate memory on a specific set of nodes,
23 * no fallback.
8bccd85f
CL
24 * FIXME: memory is allocated starting with the first node
25 * to the last. It would be better if bind would truly restrict
26 * the allocation to memory nodes instead
27 *
1da177e4 28 * preferred Try a specific node first before normal fallback.
00ef2d2f 29 * As a special case NUMA_NO_NODE here means do the allocation
1da177e4
LT
30 * on the local CPU. This is normally identical to default,
31 * but useful to set in a VMA when you have a non default
32 * process policy.
8bccd85f 33 *
1da177e4
LT
34 * default Allocate on the local node first, or when on a VMA
35 * use the process policy. This is what Linux always did
36 * in a NUMA aware kernel and still does by, ahem, default.
37 *
38 * The process policy is applied for most non interrupt memory allocations
39 * in that process' context. Interrupts ignore the policies and always
40 * try to allocate on the local CPU. The VMA policy is only applied for memory
41 * allocations for a VMA in the VM.
42 *
43 * Currently there are a few corner cases in swapping where the policy
44 * is not applied, but the majority should be handled. When process policy
45 * is used it is not remembered over swap outs/swap ins.
46 *
47 * Only the highest zone in the zone hierarchy gets policied. Allocations
48 * requesting a lower zone just use default policy. This implies that
49 * on systems with highmem kernel lowmem allocation don't get policied.
50 * Same with GFP_DMA allocations.
51 *
52 * For shmfs/tmpfs/hugetlbfs shared memory the policy is shared between
53 * all users and remembered even when nobody has memory mapped.
54 */
55
56/* Notebook:
57 fix mmap readahead to honour policy and enable policy for any page cache
58 object
59 statistics for bigpages
60 global policy for page cache? currently it uses process policy. Requires
61 first item above.
62 handle mremap for shared memory (currently ignored for the policy)
63 grows down?
64 make bind policy root only? It can trigger oom much faster and the
65 kernel is not always grateful with that.
1da177e4
LT
66*/
67
b1de0d13
MH
68#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
69
1da177e4
LT
70#include <linux/mempolicy.h>
71#include <linux/mm.h>
72#include <linux/highmem.h>
73#include <linux/hugetlb.h>
74#include <linux/kernel.h>
75#include <linux/sched.h>
1da177e4
LT
76#include <linux/nodemask.h>
77#include <linux/cpuset.h>
1da177e4
LT
78#include <linux/slab.h>
79#include <linux/string.h>
b95f1b31 80#include <linux/export.h>
b488893a 81#include <linux/nsproxy.h>
1da177e4
LT
82#include <linux/interrupt.h>
83#include <linux/init.h>
84#include <linux/compat.h>
dc9aa5b9 85#include <linux/swap.h>
1a75a6c8
CL
86#include <linux/seq_file.h>
87#include <linux/proc_fs.h>
b20a3503 88#include <linux/migrate.h>
62b61f61 89#include <linux/ksm.h>
95a402c3 90#include <linux/rmap.h>
86c3a764 91#include <linux/security.h>
dbcb0f19 92#include <linux/syscalls.h>
095f1fc4 93#include <linux/ctype.h>
6d9c285a 94#include <linux/mm_inline.h>
b24f53a0 95#include <linux/mmu_notifier.h>
b1de0d13 96#include <linux/printk.h>
dc9aa5b9 97
1da177e4
LT
98#include <asm/tlbflush.h>
99#include <asm/uaccess.h>
100
62695a84
NP
101#include "internal.h"
102
38e35860 103/* Internal flags */
dc9aa5b9 104#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */
38e35860 105#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */
dc9aa5b9 106
fcc234f8
PE
107static struct kmem_cache *policy_cache;
108static struct kmem_cache *sn_cache;
1da177e4 109
1da177e4
LT
110/* Highest zone. An specific allocation for a zone below that is not
111 policied. */
6267276f 112enum zone_type policy_zone = 0;
1da177e4 113
bea904d5
LS
114/*
115 * run-time system-wide default policy => local allocation
116 */
e754d79d 117static struct mempolicy default_policy = {
1da177e4 118 .refcnt = ATOMIC_INIT(1), /* never free it */
bea904d5 119 .mode = MPOL_PREFERRED,
fc36b8d3 120 .flags = MPOL_F_LOCAL,
1da177e4
LT
121};
122
5606e387
MG
123static struct mempolicy preferred_node_policy[MAX_NUMNODES];
124
74d2c3a0 125struct mempolicy *get_task_policy(struct task_struct *p)
5606e387
MG
126{
127 struct mempolicy *pol = p->mempolicy;
f15ca78e 128 int node;
5606e387 129
f15ca78e
ON
130 if (pol)
131 return pol;
5606e387 132
f15ca78e
ON
133 node = numa_node_id();
134 if (node != NUMA_NO_NODE) {
135 pol = &preferred_node_policy[node];
136 /* preferred_node_policy is not initialised early in boot */
137 if (pol->mode)
138 return pol;
5606e387
MG
139 }
140
f15ca78e 141 return &default_policy;
5606e387
MG
142}
143
37012946
DR
144static const struct mempolicy_operations {
145 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
708c1bbc
MX
146 /*
147 * If read-side task has no lock to protect task->mempolicy, write-side
148 * task will rebind the task->mempolicy by two step. The first step is
149 * setting all the newly nodes, and the second step is cleaning all the
150 * disallowed nodes. In this way, we can avoid finding no node to alloc
151 * page.
152 * If we have a lock to protect task->mempolicy in read-side, we do
153 * rebind directly.
154 *
155 * step:
156 * MPOL_REBIND_ONCE - do rebind work at once
157 * MPOL_REBIND_STEP1 - set all the newly nodes
158 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
159 */
160 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
161 enum mpol_rebind_step step);
37012946
DR
162} mpol_ops[MPOL_MAX];
163
f5b087b5
DR
164static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
165{
6d556294 166 return pol->flags & MPOL_MODE_FLAGS;
4c50bc01
DR
167}
168
169static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
170 const nodemask_t *rel)
171{
172 nodemask_t tmp;
173 nodes_fold(tmp, *orig, nodes_weight(*rel));
174 nodes_onto(*ret, tmp, *rel);
f5b087b5
DR
175}
176
37012946
DR
177static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
178{
179 if (nodes_empty(*nodes))
180 return -EINVAL;
181 pol->v.nodes = *nodes;
182 return 0;
183}
184
185static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
186{
187 if (!nodes)
fc36b8d3 188 pol->flags |= MPOL_F_LOCAL; /* local allocation */
37012946
DR
189 else if (nodes_empty(*nodes))
190 return -EINVAL; /* no allowed nodes */
191 else
192 pol->v.preferred_node = first_node(*nodes);
193 return 0;
194}
195
196static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
197{
859f7ef1 198 if (nodes_empty(*nodes))
37012946
DR
199 return -EINVAL;
200 pol->v.nodes = *nodes;
201 return 0;
202}
203
58568d2a
MX
204/*
205 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
206 * any, for the new policy. mpol_new() has already validated the nodes
207 * parameter with respect to the policy mode and flags. But, we need to
208 * handle an empty nodemask with MPOL_PREFERRED here.
209 *
210 * Must be called holding task's alloc_lock to protect task's mems_allowed
211 * and mempolicy. May also be called holding the mmap_semaphore for write.
212 */
4bfc4495
KH
213static int mpol_set_nodemask(struct mempolicy *pol,
214 const nodemask_t *nodes, struct nodemask_scratch *nsc)
58568d2a 215{
58568d2a
MX
216 int ret;
217
218 /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
219 if (pol == NULL)
220 return 0;
01f13bd6 221 /* Check N_MEMORY */
4bfc4495 222 nodes_and(nsc->mask1,
01f13bd6 223 cpuset_current_mems_allowed, node_states[N_MEMORY]);
58568d2a
MX
224
225 VM_BUG_ON(!nodes);
226 if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
227 nodes = NULL; /* explicit local allocation */
228 else {
229 if (pol->flags & MPOL_F_RELATIVE_NODES)
859f7ef1 230 mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
58568d2a 231 else
4bfc4495
KH
232 nodes_and(nsc->mask2, *nodes, nsc->mask1);
233
58568d2a
MX
234 if (mpol_store_user_nodemask(pol))
235 pol->w.user_nodemask = *nodes;
236 else
237 pol->w.cpuset_mems_allowed =
238 cpuset_current_mems_allowed;
239 }
240
4bfc4495
KH
241 if (nodes)
242 ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
243 else
244 ret = mpol_ops[pol->mode].create(pol, NULL);
58568d2a
MX
245 return ret;
246}
247
248/*
249 * This function just creates a new policy, does some check and simple
250 * initialization. You must invoke mpol_set_nodemask() to set nodes.
251 */
028fec41
DR
252static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
253 nodemask_t *nodes)
1da177e4
LT
254{
255 struct mempolicy *policy;
256
028fec41 257 pr_debug("setting mode %d flags %d nodes[0] %lx\n",
00ef2d2f 258 mode, flags, nodes ? nodes_addr(*nodes)[0] : NUMA_NO_NODE);
140d5a49 259
3e1f0645
DR
260 if (mode == MPOL_DEFAULT) {
261 if (nodes && !nodes_empty(*nodes))
37012946 262 return ERR_PTR(-EINVAL);
d3a71033 263 return NULL;
37012946 264 }
3e1f0645
DR
265 VM_BUG_ON(!nodes);
266
267 /*
268 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
269 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
270 * All other modes require a valid pointer to a non-empty nodemask.
271 */
272 if (mode == MPOL_PREFERRED) {
273 if (nodes_empty(*nodes)) {
274 if (((flags & MPOL_F_STATIC_NODES) ||
275 (flags & MPOL_F_RELATIVE_NODES)))
276 return ERR_PTR(-EINVAL);
3e1f0645 277 }
479e2802
PZ
278 } else if (mode == MPOL_LOCAL) {
279 if (!nodes_empty(*nodes))
280 return ERR_PTR(-EINVAL);
281 mode = MPOL_PREFERRED;
3e1f0645
DR
282 } else if (nodes_empty(*nodes))
283 return ERR_PTR(-EINVAL);
1da177e4
LT
284 policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
285 if (!policy)
286 return ERR_PTR(-ENOMEM);
287 atomic_set(&policy->refcnt, 1);
45c4745a 288 policy->mode = mode;
3e1f0645 289 policy->flags = flags;
37012946 290
1da177e4 291 return policy;
37012946
DR
292}
293
52cd3b07
LS
294/* Slow path of a mpol destructor. */
295void __mpol_put(struct mempolicy *p)
296{
297 if (!atomic_dec_and_test(&p->refcnt))
298 return;
52cd3b07
LS
299 kmem_cache_free(policy_cache, p);
300}
301
708c1bbc
MX
302static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
303 enum mpol_rebind_step step)
37012946
DR
304{
305}
306
708c1bbc
MX
307/*
308 * step:
309 * MPOL_REBIND_ONCE - do rebind work at once
310 * MPOL_REBIND_STEP1 - set all the newly nodes
311 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
312 */
313static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
314 enum mpol_rebind_step step)
37012946
DR
315{
316 nodemask_t tmp;
317
318 if (pol->flags & MPOL_F_STATIC_NODES)
319 nodes_and(tmp, pol->w.user_nodemask, *nodes);
320 else if (pol->flags & MPOL_F_RELATIVE_NODES)
321 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
322 else {
708c1bbc
MX
323 /*
324 * if step == 1, we use ->w.cpuset_mems_allowed to cache the
325 * result
326 */
327 if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
328 nodes_remap(tmp, pol->v.nodes,
329 pol->w.cpuset_mems_allowed, *nodes);
330 pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
331 } else if (step == MPOL_REBIND_STEP2) {
332 tmp = pol->w.cpuset_mems_allowed;
333 pol->w.cpuset_mems_allowed = *nodes;
334 } else
335 BUG();
37012946 336 }
f5b087b5 337
708c1bbc
MX
338 if (nodes_empty(tmp))
339 tmp = *nodes;
340
341 if (step == MPOL_REBIND_STEP1)
342 nodes_or(pol->v.nodes, pol->v.nodes, tmp);
343 else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
344 pol->v.nodes = tmp;
345 else
346 BUG();
347
37012946 348 if (!node_isset(current->il_next, tmp)) {
0edaf86c 349 current->il_next = next_node_in(current->il_next, tmp);
37012946
DR
350 if (current->il_next >= MAX_NUMNODES)
351 current->il_next = numa_node_id();
352 }
353}
354
355static void mpol_rebind_preferred(struct mempolicy *pol,
708c1bbc
MX
356 const nodemask_t *nodes,
357 enum mpol_rebind_step step)
37012946
DR
358{
359 nodemask_t tmp;
360
37012946
DR
361 if (pol->flags & MPOL_F_STATIC_NODES) {
362 int node = first_node(pol->w.user_nodemask);
363
fc36b8d3 364 if (node_isset(node, *nodes)) {
37012946 365 pol->v.preferred_node = node;
fc36b8d3
LS
366 pol->flags &= ~MPOL_F_LOCAL;
367 } else
368 pol->flags |= MPOL_F_LOCAL;
37012946
DR
369 } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
370 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
371 pol->v.preferred_node = first_node(tmp);
fc36b8d3 372 } else if (!(pol->flags & MPOL_F_LOCAL)) {
37012946
DR
373 pol->v.preferred_node = node_remap(pol->v.preferred_node,
374 pol->w.cpuset_mems_allowed,
375 *nodes);
376 pol->w.cpuset_mems_allowed = *nodes;
377 }
1da177e4
LT
378}
379
708c1bbc
MX
380/*
381 * mpol_rebind_policy - Migrate a policy to a different set of nodes
382 *
383 * If read-side task has no lock to protect task->mempolicy, write-side
384 * task will rebind the task->mempolicy by two step. The first step is
385 * setting all the newly nodes, and the second step is cleaning all the
386 * disallowed nodes. In this way, we can avoid finding no node to alloc
387 * page.
388 * If we have a lock to protect task->mempolicy in read-side, we do
389 * rebind directly.
390 *
391 * step:
392 * MPOL_REBIND_ONCE - do rebind work at once
393 * MPOL_REBIND_STEP1 - set all the newly nodes
394 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
395 */
396static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
397 enum mpol_rebind_step step)
1d0d2680 398{
1d0d2680
DR
399 if (!pol)
400 return;
89c522c7 401 if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&
1d0d2680
DR
402 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
403 return;
708c1bbc
MX
404
405 if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
406 return;
407
408 if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
409 BUG();
410
411 if (step == MPOL_REBIND_STEP1)
412 pol->flags |= MPOL_F_REBINDING;
413 else if (step == MPOL_REBIND_STEP2)
414 pol->flags &= ~MPOL_F_REBINDING;
415 else if (step >= MPOL_REBIND_NSTEP)
416 BUG();
417
418 mpol_ops[pol->mode].rebind(pol, newmask, step);
1d0d2680
DR
419}
420
421/*
422 * Wrapper for mpol_rebind_policy() that just requires task
423 * pointer, and updates task mempolicy.
58568d2a
MX
424 *
425 * Called with task's alloc_lock held.
1d0d2680
DR
426 */
427
708c1bbc
MX
428void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
429 enum mpol_rebind_step step)
1d0d2680 430{
708c1bbc 431 mpol_rebind_policy(tsk->mempolicy, new, step);
1d0d2680
DR
432}
433
434/*
435 * Rebind each vma in mm to new nodemask.
436 *
437 * Call holding a reference to mm. Takes mm->mmap_sem during call.
438 */
439
440void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
441{
442 struct vm_area_struct *vma;
443
444 down_write(&mm->mmap_sem);
445 for (vma = mm->mmap; vma; vma = vma->vm_next)
708c1bbc 446 mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
1d0d2680
DR
447 up_write(&mm->mmap_sem);
448}
449
37012946
DR
450static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
451 [MPOL_DEFAULT] = {
452 .rebind = mpol_rebind_default,
453 },
454 [MPOL_INTERLEAVE] = {
455 .create = mpol_new_interleave,
456 .rebind = mpol_rebind_nodemask,
457 },
458 [MPOL_PREFERRED] = {
459 .create = mpol_new_preferred,
460 .rebind = mpol_rebind_preferred,
461 },
462 [MPOL_BIND] = {
463 .create = mpol_new_bind,
464 .rebind = mpol_rebind_nodemask,
465 },
466};
467
fc301289
CL
468static void migrate_page_add(struct page *page, struct list_head *pagelist,
469 unsigned long flags);
1a75a6c8 470
6f4576e3
NH
471struct queue_pages {
472 struct list_head *pagelist;
473 unsigned long flags;
474 nodemask_t *nmask;
475 struct vm_area_struct *prev;
476};
477
98094945
NH
478/*
479 * Scan through pages checking if pages follow certain conditions,
480 * and move them to the pagelist if they do.
481 */
6f4576e3
NH
482static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
483 unsigned long end, struct mm_walk *walk)
1da177e4 484{
6f4576e3
NH
485 struct vm_area_struct *vma = walk->vma;
486 struct page *page;
487 struct queue_pages *qp = walk->private;
488 unsigned long flags = qp->flags;
248db92d 489 int nid, ret;
91612e0d 490 pte_t *pte;
705e87c0 491 spinlock_t *ptl;
941150a3 492
248db92d
KS
493 if (pmd_trans_huge(*pmd)) {
494 ptl = pmd_lock(walk->mm, pmd);
495 if (pmd_trans_huge(*pmd)) {
496 page = pmd_page(*pmd);
497 if (is_huge_zero_page(page)) {
498 spin_unlock(ptl);
499 split_huge_pmd(vma, pmd, addr);
500 } else {
501 get_page(page);
502 spin_unlock(ptl);
503 lock_page(page);
504 ret = split_huge_page(page);
505 unlock_page(page);
506 put_page(page);
507 if (ret)
508 return 0;
509 }
510 } else {
511 spin_unlock(ptl);
512 }
513 }
91612e0d 514
337d9abf
NH
515 if (pmd_trans_unstable(pmd))
516 return 0;
248db92d 517retry:
6f4576e3
NH
518 pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
519 for (; addr != end; pte++, addr += PAGE_SIZE) {
91612e0d 520 if (!pte_present(*pte))
1da177e4 521 continue;
6aab341e
LT
522 page = vm_normal_page(vma, addr, *pte);
523 if (!page)
1da177e4 524 continue;
053837fc 525 /*
62b61f61
HD
526 * vm_normal_page() filters out zero pages, but there might
527 * still be PageReserved pages to skip, perhaps in a VDSO.
053837fc 528 */
b79bc0a0 529 if (PageReserved(page))
f4598c8b 530 continue;
6aab341e 531 nid = page_to_nid(page);
6f4576e3 532 if (node_isset(nid, *qp->nmask) == !!(flags & MPOL_MF_INVERT))
38e35860 533 continue;
800d8c63 534 if (PageTransCompound(page)) {
248db92d
KS
535 get_page(page);
536 pte_unmap_unlock(pte, ptl);
537 lock_page(page);
538 ret = split_huge_page(page);
539 unlock_page(page);
540 put_page(page);
541 /* Failed to split -- skip. */
542 if (ret) {
543 pte = pte_offset_map_lock(walk->mm, pmd,
544 addr, &ptl);
545 continue;
546 }
547 goto retry;
548 }
38e35860 549
77bf45e7 550 migrate_page_add(page, qp->pagelist, flags);
6f4576e3
NH
551 }
552 pte_unmap_unlock(pte - 1, ptl);
553 cond_resched();
554 return 0;
91612e0d
HD
555}
556
6f4576e3
NH
557static int queue_pages_hugetlb(pte_t *pte, unsigned long hmask,
558 unsigned long addr, unsigned long end,
559 struct mm_walk *walk)
e2d8cf40
NH
560{
561#ifdef CONFIG_HUGETLB_PAGE
6f4576e3
NH
562 struct queue_pages *qp = walk->private;
563 unsigned long flags = qp->flags;
e2d8cf40
NH
564 int nid;
565 struct page *page;
cb900f41 566 spinlock_t *ptl;
d4c54919 567 pte_t entry;
e2d8cf40 568
6f4576e3
NH
569 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
570 entry = huge_ptep_get(pte);
d4c54919
NH
571 if (!pte_present(entry))
572 goto unlock;
573 page = pte_page(entry);
e2d8cf40 574 nid = page_to_nid(page);
6f4576e3 575 if (node_isset(nid, *qp->nmask) == !!(flags & MPOL_MF_INVERT))
e2d8cf40
NH
576 goto unlock;
577 /* With MPOL_MF_MOVE, we migrate only unshared hugepage. */
578 if (flags & (MPOL_MF_MOVE_ALL) ||
579 (flags & MPOL_MF_MOVE && page_mapcount(page) == 1))
6f4576e3 580 isolate_huge_page(page, qp->pagelist);
e2d8cf40 581unlock:
cb900f41 582 spin_unlock(ptl);
e2d8cf40
NH
583#else
584 BUG();
585#endif
91612e0d 586 return 0;
1da177e4
LT
587}
588
5877231f 589#ifdef CONFIG_NUMA_BALANCING
b24f53a0 590/*
4b10e7d5
MG
591 * This is used to mark a range of virtual addresses to be inaccessible.
592 * These are later cleared by a NUMA hinting fault. Depending on these
593 * faults, pages may be migrated for better NUMA placement.
594 *
595 * This is assuming that NUMA faults are handled using PROT_NONE. If
596 * an architecture makes a different choice, it will need further
597 * changes to the core.
b24f53a0 598 */
4b10e7d5
MG
599unsigned long change_prot_numa(struct vm_area_struct *vma,
600 unsigned long addr, unsigned long end)
b24f53a0 601{
4b10e7d5 602 int nr_updated;
b24f53a0 603
4d942466 604 nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1);
03c5a6e1
MG
605 if (nr_updated)
606 count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
b24f53a0 607
4b10e7d5 608 return nr_updated;
b24f53a0
LS
609}
610#else
611static unsigned long change_prot_numa(struct vm_area_struct *vma,
612 unsigned long addr, unsigned long end)
613{
614 return 0;
615}
5877231f 616#endif /* CONFIG_NUMA_BALANCING */
b24f53a0 617
6f4576e3
NH
618static int queue_pages_test_walk(unsigned long start, unsigned long end,
619 struct mm_walk *walk)
620{
621 struct vm_area_struct *vma = walk->vma;
622 struct queue_pages *qp = walk->private;
623 unsigned long endvma = vma->vm_end;
624 unsigned long flags = qp->flags;
625
77bf45e7 626 if (!vma_migratable(vma))
48684a65
NH
627 return 1;
628
6f4576e3
NH
629 if (endvma > end)
630 endvma = end;
631 if (vma->vm_start > start)
632 start = vma->vm_start;
633
634 if (!(flags & MPOL_MF_DISCONTIG_OK)) {
635 if (!vma->vm_next && vma->vm_end < end)
636 return -EFAULT;
637 if (qp->prev && qp->prev->vm_end < vma->vm_start)
638 return -EFAULT;
639 }
640
641 qp->prev = vma;
642
6f4576e3
NH
643 if (flags & MPOL_MF_LAZY) {
644 /* Similar to task_numa_work, skip inaccessible VMAs */
4355c018
LC
645 if (!is_vm_hugetlb_page(vma) &&
646 (vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)) &&
647 !(vma->vm_flags & VM_MIXEDMAP))
6f4576e3
NH
648 change_prot_numa(vma, start, endvma);
649 return 1;
650 }
651
77bf45e7
KS
652 /* queue pages from current vma */
653 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
6f4576e3
NH
654 return 0;
655 return 1;
656}
657
dc9aa5b9 658/*
98094945
NH
659 * Walk through page tables and collect pages to be migrated.
660 *
661 * If pages found in a given range are on a set of nodes (determined by
662 * @nodes and @flags,) it's isolated and queued to the pagelist which is
663 * passed via @private.)
dc9aa5b9 664 */
d05f0cdc 665static int
98094945 666queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
6f4576e3
NH
667 nodemask_t *nodes, unsigned long flags,
668 struct list_head *pagelist)
1da177e4 669{
6f4576e3
NH
670 struct queue_pages qp = {
671 .pagelist = pagelist,
672 .flags = flags,
673 .nmask = nodes,
674 .prev = NULL,
675 };
676 struct mm_walk queue_pages_walk = {
677 .hugetlb_entry = queue_pages_hugetlb,
678 .pmd_entry = queue_pages_pte_range,
679 .test_walk = queue_pages_test_walk,
680 .mm = mm,
681 .private = &qp,
682 };
683
684 return walk_page_range(start, end, &queue_pages_walk);
1da177e4
LT
685}
686
869833f2
KM
687/*
688 * Apply policy to a single VMA
689 * This must be called with the mmap_sem held for writing.
690 */
691static int vma_replace_policy(struct vm_area_struct *vma,
692 struct mempolicy *pol)
8d34694c 693{
869833f2
KM
694 int err;
695 struct mempolicy *old;
696 struct mempolicy *new;
8d34694c
KM
697
698 pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
699 vma->vm_start, vma->vm_end, vma->vm_pgoff,
700 vma->vm_ops, vma->vm_file,
701 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
702
869833f2
KM
703 new = mpol_dup(pol);
704 if (IS_ERR(new))
705 return PTR_ERR(new);
706
707 if (vma->vm_ops && vma->vm_ops->set_policy) {
8d34694c 708 err = vma->vm_ops->set_policy(vma, new);
869833f2
KM
709 if (err)
710 goto err_out;
8d34694c 711 }
869833f2
KM
712
713 old = vma->vm_policy;
714 vma->vm_policy = new; /* protected by mmap_sem */
715 mpol_put(old);
716
717 return 0;
718 err_out:
719 mpol_put(new);
8d34694c
KM
720 return err;
721}
722
1da177e4 723/* Step 2: apply policy to a range and do splits. */
9d8cebd4
KM
724static int mbind_range(struct mm_struct *mm, unsigned long start,
725 unsigned long end, struct mempolicy *new_pol)
1da177e4
LT
726{
727 struct vm_area_struct *next;
9d8cebd4
KM
728 struct vm_area_struct *prev;
729 struct vm_area_struct *vma;
730 int err = 0;
e26a5114 731 pgoff_t pgoff;
9d8cebd4
KM
732 unsigned long vmstart;
733 unsigned long vmend;
1da177e4 734
097d5910 735 vma = find_vma(mm, start);
9d8cebd4
KM
736 if (!vma || vma->vm_start > start)
737 return -EFAULT;
738
097d5910 739 prev = vma->vm_prev;
e26a5114
KM
740 if (start > vma->vm_start)
741 prev = vma;
742
9d8cebd4 743 for (; vma && vma->vm_start < end; prev = vma, vma = next) {
1da177e4 744 next = vma->vm_next;
9d8cebd4
KM
745 vmstart = max(start, vma->vm_start);
746 vmend = min(end, vma->vm_end);
747
e26a5114
KM
748 if (mpol_equal(vma_policy(vma), new_pol))
749 continue;
750
751 pgoff = vma->vm_pgoff +
752 ((vmstart - vma->vm_start) >> PAGE_SHIFT);
9d8cebd4 753 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
19a809af
AA
754 vma->anon_vma, vma->vm_file, pgoff,
755 new_pol, vma->vm_userfaultfd_ctx);
9d8cebd4
KM
756 if (prev) {
757 vma = prev;
758 next = vma->vm_next;
3964acd0
ON
759 if (mpol_equal(vma_policy(vma), new_pol))
760 continue;
761 /* vma_merge() joined vma && vma->next, case 8 */
762 goto replace;
9d8cebd4
KM
763 }
764 if (vma->vm_start != vmstart) {
765 err = split_vma(vma->vm_mm, vma, vmstart, 1);
766 if (err)
767 goto out;
768 }
769 if (vma->vm_end != vmend) {
770 err = split_vma(vma->vm_mm, vma, vmend, 0);
771 if (err)
772 goto out;
773 }
3964acd0 774 replace:
869833f2 775 err = vma_replace_policy(vma, new_pol);
8d34694c
KM
776 if (err)
777 goto out;
1da177e4 778 }
9d8cebd4
KM
779
780 out:
1da177e4
LT
781 return err;
782}
783
1da177e4 784/* Set the process memory policy */
028fec41
DR
785static long do_set_mempolicy(unsigned short mode, unsigned short flags,
786 nodemask_t *nodes)
1da177e4 787{
58568d2a 788 struct mempolicy *new, *old;
4bfc4495 789 NODEMASK_SCRATCH(scratch);
58568d2a 790 int ret;
1da177e4 791
4bfc4495
KH
792 if (!scratch)
793 return -ENOMEM;
f4e53d91 794
4bfc4495
KH
795 new = mpol_new(mode, flags, nodes);
796 if (IS_ERR(new)) {
797 ret = PTR_ERR(new);
798 goto out;
799 }
2c7c3a7d 800
58568d2a 801 task_lock(current);
4bfc4495 802 ret = mpol_set_nodemask(new, nodes, scratch);
58568d2a
MX
803 if (ret) {
804 task_unlock(current);
58568d2a 805 mpol_put(new);
4bfc4495 806 goto out;
58568d2a
MX
807 }
808 old = current->mempolicy;
1da177e4 809 current->mempolicy = new;
45c4745a 810 if (new && new->mode == MPOL_INTERLEAVE &&
f5b087b5 811 nodes_weight(new->v.nodes))
dfcd3c0d 812 current->il_next = first_node(new->v.nodes);
58568d2a 813 task_unlock(current);
58568d2a 814 mpol_put(old);
4bfc4495
KH
815 ret = 0;
816out:
817 NODEMASK_SCRATCH_FREE(scratch);
818 return ret;
1da177e4
LT
819}
820
bea904d5
LS
821/*
822 * Return nodemask for policy for get_mempolicy() query
58568d2a
MX
823 *
824 * Called with task's alloc_lock held
bea904d5
LS
825 */
826static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
1da177e4 827{
dfcd3c0d 828 nodes_clear(*nodes);
bea904d5
LS
829 if (p == &default_policy)
830 return;
831
45c4745a 832 switch (p->mode) {
19770b32
MG
833 case MPOL_BIND:
834 /* Fall through */
1da177e4 835 case MPOL_INTERLEAVE:
dfcd3c0d 836 *nodes = p->v.nodes;
1da177e4
LT
837 break;
838 case MPOL_PREFERRED:
fc36b8d3 839 if (!(p->flags & MPOL_F_LOCAL))
dfcd3c0d 840 node_set(p->v.preferred_node, *nodes);
53f2556b 841 /* else return empty node mask for local allocation */
1da177e4
LT
842 break;
843 default:
844 BUG();
845 }
846}
847
d4edcf0d 848static int lookup_node(unsigned long addr)
1da177e4
LT
849{
850 struct page *p;
851 int err;
852
d4edcf0d 853 err = get_user_pages(addr & PAGE_MASK, 1, 0, 0, &p, NULL);
1da177e4
LT
854 if (err >= 0) {
855 err = page_to_nid(p);
856 put_page(p);
857 }
858 return err;
859}
860
1da177e4 861/* Retrieve NUMA policy */
dbcb0f19
AB
862static long do_get_mempolicy(int *policy, nodemask_t *nmask,
863 unsigned long addr, unsigned long flags)
1da177e4 864{
8bccd85f 865 int err;
1da177e4
LT
866 struct mm_struct *mm = current->mm;
867 struct vm_area_struct *vma = NULL;
868 struct mempolicy *pol = current->mempolicy;
869
754af6f5
LS
870 if (flags &
871 ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
1da177e4 872 return -EINVAL;
754af6f5
LS
873
874 if (flags & MPOL_F_MEMS_ALLOWED) {
875 if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
876 return -EINVAL;
877 *policy = 0; /* just so it's initialized */
58568d2a 878 task_lock(current);
754af6f5 879 *nmask = cpuset_current_mems_allowed;
58568d2a 880 task_unlock(current);
754af6f5
LS
881 return 0;
882 }
883
1da177e4 884 if (flags & MPOL_F_ADDR) {
bea904d5
LS
885 /*
886 * Do NOT fall back to task policy if the
887 * vma/shared policy at addr is NULL. We
888 * want to return MPOL_DEFAULT in this case.
889 */
1da177e4
LT
890 down_read(&mm->mmap_sem);
891 vma = find_vma_intersection(mm, addr, addr+1);
892 if (!vma) {
893 up_read(&mm->mmap_sem);
894 return -EFAULT;
895 }
896 if (vma->vm_ops && vma->vm_ops->get_policy)
897 pol = vma->vm_ops->get_policy(vma, addr);
898 else
899 pol = vma->vm_policy;
900 } else if (addr)
901 return -EINVAL;
902
903 if (!pol)
bea904d5 904 pol = &default_policy; /* indicates default behavior */
1da177e4
LT
905
906 if (flags & MPOL_F_NODE) {
907 if (flags & MPOL_F_ADDR) {
d4edcf0d 908 err = lookup_node(addr);
1da177e4
LT
909 if (err < 0)
910 goto out;
8bccd85f 911 *policy = err;
1da177e4 912 } else if (pol == current->mempolicy &&
45c4745a 913 pol->mode == MPOL_INTERLEAVE) {
8bccd85f 914 *policy = current->il_next;
1da177e4
LT
915 } else {
916 err = -EINVAL;
917 goto out;
918 }
bea904d5
LS
919 } else {
920 *policy = pol == &default_policy ? MPOL_DEFAULT :
921 pol->mode;
d79df630
DR
922 /*
923 * Internal mempolicy flags must be masked off before exposing
924 * the policy to userspace.
925 */
926 *policy |= (pol->flags & MPOL_MODE_FLAGS);
bea904d5 927 }
1da177e4
LT
928
929 if (vma) {
930 up_read(&current->mm->mmap_sem);
931 vma = NULL;
932 }
933
1da177e4 934 err = 0;
58568d2a 935 if (nmask) {
c6b6ef8b
LS
936 if (mpol_store_user_nodemask(pol)) {
937 *nmask = pol->w.user_nodemask;
938 } else {
939 task_lock(current);
940 get_policy_nodemask(pol, nmask);
941 task_unlock(current);
942 }
58568d2a 943 }
1da177e4
LT
944
945 out:
52cd3b07 946 mpol_cond_put(pol);
1da177e4
LT
947 if (vma)
948 up_read(&current->mm->mmap_sem);
949 return err;
950}
951
b20a3503 952#ifdef CONFIG_MIGRATION
6ce3c4c0
CL
953/*
954 * page migration
955 */
fc301289
CL
956static void migrate_page_add(struct page *page, struct list_head *pagelist,
957 unsigned long flags)
6ce3c4c0
CL
958{
959 /*
fc301289 960 * Avoid migrating a page that is shared with others.
6ce3c4c0 961 */
62695a84
NP
962 if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
963 if (!isolate_lru_page(page)) {
964 list_add_tail(&page->lru, pagelist);
599d0c95 965 inc_node_page_state(page, NR_ISOLATED_ANON +
6d9c285a 966 page_is_file_cache(page));
62695a84
NP
967 }
968 }
7e2ab150 969}
6ce3c4c0 970
742755a1 971static struct page *new_node_page(struct page *page, unsigned long node, int **x)
95a402c3 972{
e2d8cf40
NH
973 if (PageHuge(page))
974 return alloc_huge_page_node(page_hstate(compound_head(page)),
975 node);
976 else
96db800f 977 return __alloc_pages_node(node, GFP_HIGHUSER_MOVABLE |
b360edb4 978 __GFP_THISNODE, 0);
95a402c3
CL
979}
980
7e2ab150
CL
981/*
982 * Migrate pages from one node to a target node.
983 * Returns error or the number of pages not migrated.
984 */
dbcb0f19
AB
985static int migrate_to_node(struct mm_struct *mm, int source, int dest,
986 int flags)
7e2ab150
CL
987{
988 nodemask_t nmask;
989 LIST_HEAD(pagelist);
990 int err = 0;
991
992 nodes_clear(nmask);
993 node_set(source, nmask);
6ce3c4c0 994
08270807
MK
995 /*
996 * This does not "check" the range but isolates all pages that
997 * need migration. Between passing in the full user address
998 * space range and MPOL_MF_DISCONTIG_OK, this call can not fail.
999 */
1000 VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
98094945 1001 queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
7e2ab150
CL
1002 flags | MPOL_MF_DISCONTIG_OK, &pagelist);
1003
cf608ac1 1004 if (!list_empty(&pagelist)) {
68711a74 1005 err = migrate_pages(&pagelist, new_node_page, NULL, dest,
9c620e2b 1006 MIGRATE_SYNC, MR_SYSCALL);
cf608ac1 1007 if (err)
e2d8cf40 1008 putback_movable_pages(&pagelist);
cf608ac1 1009 }
95a402c3 1010
7e2ab150 1011 return err;
6ce3c4c0
CL
1012}
1013
39743889 1014/*
7e2ab150
CL
1015 * Move pages between the two nodesets so as to preserve the physical
1016 * layout as much as possible.
39743889
CL
1017 *
1018 * Returns the number of page that could not be moved.
1019 */
0ce72d4f
AM
1020int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1021 const nodemask_t *to, int flags)
39743889 1022{
7e2ab150 1023 int busy = 0;
0aedadf9 1024 int err;
7e2ab150 1025 nodemask_t tmp;
39743889 1026
0aedadf9
CL
1027 err = migrate_prep();
1028 if (err)
1029 return err;
1030
53f2556b 1031 down_read(&mm->mmap_sem);
39743889 1032
da0aa138
KM
1033 /*
1034 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1035 * bit in 'to' is not also set in 'tmp'. Clear the found 'source'
1036 * bit in 'tmp', and return that <source, dest> pair for migration.
1037 * The pair of nodemasks 'to' and 'from' define the map.
1038 *
1039 * If no pair of bits is found that way, fallback to picking some
1040 * pair of 'source' and 'dest' bits that are not the same. If the
1041 * 'source' and 'dest' bits are the same, this represents a node
1042 * that will be migrating to itself, so no pages need move.
1043 *
1044 * If no bits are left in 'tmp', or if all remaining bits left
1045 * in 'tmp' correspond to the same bit in 'to', return false
1046 * (nothing left to migrate).
1047 *
1048 * This lets us pick a pair of nodes to migrate between, such that
1049 * if possible the dest node is not already occupied by some other
1050 * source node, minimizing the risk of overloading the memory on a
1051 * node that would happen if we migrated incoming memory to a node
1052 * before migrating outgoing memory source that same node.
1053 *
1054 * A single scan of tmp is sufficient. As we go, we remember the
1055 * most recent <s, d> pair that moved (s != d). If we find a pair
1056 * that not only moved, but what's better, moved to an empty slot
1057 * (d is not set in tmp), then we break out then, with that pair.
ae0e47f0 1058 * Otherwise when we finish scanning from_tmp, we at least have the
da0aa138
KM
1059 * most recent <s, d> pair that moved. If we get all the way through
1060 * the scan of tmp without finding any node that moved, much less
1061 * moved to an empty node, then there is nothing left worth migrating.
1062 */
d4984711 1063
0ce72d4f 1064 tmp = *from;
7e2ab150
CL
1065 while (!nodes_empty(tmp)) {
1066 int s,d;
b76ac7e7 1067 int source = NUMA_NO_NODE;
7e2ab150
CL
1068 int dest = 0;
1069
1070 for_each_node_mask(s, tmp) {
4a5b18cc
LW
1071
1072 /*
1073 * do_migrate_pages() tries to maintain the relative
1074 * node relationship of the pages established between
1075 * threads and memory areas.
1076 *
1077 * However if the number of source nodes is not equal to
1078 * the number of destination nodes we can not preserve
1079 * this node relative relationship. In that case, skip
1080 * copying memory from a node that is in the destination
1081 * mask.
1082 *
1083 * Example: [2,3,4] -> [3,4,5] moves everything.
1084 * [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1085 */
1086
0ce72d4f
AM
1087 if ((nodes_weight(*from) != nodes_weight(*to)) &&
1088 (node_isset(s, *to)))
4a5b18cc
LW
1089 continue;
1090
0ce72d4f 1091 d = node_remap(s, *from, *to);
7e2ab150
CL
1092 if (s == d)
1093 continue;
1094
1095 source = s; /* Node moved. Memorize */
1096 dest = d;
1097
1098 /* dest not in remaining from nodes? */
1099 if (!node_isset(dest, tmp))
1100 break;
1101 }
b76ac7e7 1102 if (source == NUMA_NO_NODE)
7e2ab150
CL
1103 break;
1104
1105 node_clear(source, tmp);
1106 err = migrate_to_node(mm, source, dest, flags);
1107 if (err > 0)
1108 busy += err;
1109 if (err < 0)
1110 break;
39743889
CL
1111 }
1112 up_read(&mm->mmap_sem);
7e2ab150
CL
1113 if (err < 0)
1114 return err;
1115 return busy;
b20a3503
CL
1116
1117}
1118
3ad33b24
LS
1119/*
1120 * Allocate a new page for page migration based on vma policy.
d05f0cdc 1121 * Start by assuming the page is mapped by the same vma as contains @start.
3ad33b24
LS
1122 * Search forward from there, if not. N.B., this assumes that the
1123 * list of pages handed to migrate_pages()--which is how we get here--
1124 * is in virtual address order.
1125 */
d05f0cdc 1126static struct page *new_page(struct page *page, unsigned long start, int **x)
95a402c3 1127{
d05f0cdc 1128 struct vm_area_struct *vma;
3ad33b24 1129 unsigned long uninitialized_var(address);
95a402c3 1130
d05f0cdc 1131 vma = find_vma(current->mm, start);
3ad33b24
LS
1132 while (vma) {
1133 address = page_address_in_vma(page, vma);
1134 if (address != -EFAULT)
1135 break;
1136 vma = vma->vm_next;
1137 }
11c731e8
WL
1138
1139 if (PageHuge(page)) {
cc81717e
MH
1140 BUG_ON(!vma);
1141 return alloc_huge_page_noerr(vma, address, 1);
11c731e8 1142 }
0bf598d8 1143 /*
11c731e8 1144 * if !vma, alloc_page_vma() will use task or system default policy
0bf598d8 1145 */
3ad33b24 1146 return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
95a402c3 1147}
b20a3503
CL
1148#else
1149
1150static void migrate_page_add(struct page *page, struct list_head *pagelist,
1151 unsigned long flags)
1152{
39743889
CL
1153}
1154
0ce72d4f
AM
1155int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1156 const nodemask_t *to, int flags)
b20a3503
CL
1157{
1158 return -ENOSYS;
1159}
95a402c3 1160
d05f0cdc 1161static struct page *new_page(struct page *page, unsigned long start, int **x)
95a402c3
CL
1162{
1163 return NULL;
1164}
b20a3503
CL
1165#endif
1166
dbcb0f19 1167static long do_mbind(unsigned long start, unsigned long len,
028fec41
DR
1168 unsigned short mode, unsigned short mode_flags,
1169 nodemask_t *nmask, unsigned long flags)
6ce3c4c0 1170{
6ce3c4c0
CL
1171 struct mm_struct *mm = current->mm;
1172 struct mempolicy *new;
1173 unsigned long end;
1174 int err;
1175 LIST_HEAD(pagelist);
1176
b24f53a0 1177 if (flags & ~(unsigned long)MPOL_MF_VALID)
6ce3c4c0 1178 return -EINVAL;
74c00241 1179 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
6ce3c4c0
CL
1180 return -EPERM;
1181
1182 if (start & ~PAGE_MASK)
1183 return -EINVAL;
1184
1185 if (mode == MPOL_DEFAULT)
1186 flags &= ~MPOL_MF_STRICT;
1187
1188 len = (len + PAGE_SIZE - 1) & PAGE_MASK;
1189 end = start + len;
1190
1191 if (end < start)
1192 return -EINVAL;
1193 if (end == start)
1194 return 0;
1195
028fec41 1196 new = mpol_new(mode, mode_flags, nmask);
6ce3c4c0
CL
1197 if (IS_ERR(new))
1198 return PTR_ERR(new);
1199
b24f53a0
LS
1200 if (flags & MPOL_MF_LAZY)
1201 new->flags |= MPOL_F_MOF;
1202
6ce3c4c0
CL
1203 /*
1204 * If we are using the default policy then operation
1205 * on discontinuous address spaces is okay after all
1206 */
1207 if (!new)
1208 flags |= MPOL_MF_DISCONTIG_OK;
1209
028fec41
DR
1210 pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
1211 start, start + len, mode, mode_flags,
00ef2d2f 1212 nmask ? nodes_addr(*nmask)[0] : NUMA_NO_NODE);
6ce3c4c0 1213
0aedadf9
CL
1214 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
1215
1216 err = migrate_prep();
1217 if (err)
b05ca738 1218 goto mpol_out;
0aedadf9 1219 }
4bfc4495
KH
1220 {
1221 NODEMASK_SCRATCH(scratch);
1222 if (scratch) {
1223 down_write(&mm->mmap_sem);
1224 task_lock(current);
1225 err = mpol_set_nodemask(new, nmask, scratch);
1226 task_unlock(current);
1227 if (err)
1228 up_write(&mm->mmap_sem);
1229 } else
1230 err = -ENOMEM;
1231 NODEMASK_SCRATCH_FREE(scratch);
1232 }
b05ca738
KM
1233 if (err)
1234 goto mpol_out;
1235
d05f0cdc 1236 err = queue_pages_range(mm, start, end, nmask,
6ce3c4c0 1237 flags | MPOL_MF_INVERT, &pagelist);
d05f0cdc 1238 if (!err)
9d8cebd4 1239 err = mbind_range(mm, start, end, new);
7e2ab150 1240
b24f53a0
LS
1241 if (!err) {
1242 int nr_failed = 0;
1243
cf608ac1 1244 if (!list_empty(&pagelist)) {
b24f53a0 1245 WARN_ON_ONCE(flags & MPOL_MF_LAZY);
d05f0cdc
HD
1246 nr_failed = migrate_pages(&pagelist, new_page, NULL,
1247 start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND);
cf608ac1 1248 if (nr_failed)
74060e4d 1249 putback_movable_pages(&pagelist);
cf608ac1 1250 }
6ce3c4c0 1251
b24f53a0 1252 if (nr_failed && (flags & MPOL_MF_STRICT))
6ce3c4c0 1253 err = -EIO;
ab8a3e14 1254 } else
b0e5fd73 1255 putback_movable_pages(&pagelist);
b20a3503 1256
6ce3c4c0 1257 up_write(&mm->mmap_sem);
b05ca738 1258 mpol_out:
f0be3d32 1259 mpol_put(new);
6ce3c4c0
CL
1260 return err;
1261}
1262
8bccd85f
CL
1263/*
1264 * User space interface with variable sized bitmaps for nodelists.
1265 */
1266
1267/* Copy a node mask from user space. */
39743889 1268static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
8bccd85f
CL
1269 unsigned long maxnode)
1270{
1271 unsigned long k;
1272 unsigned long nlongs;
1273 unsigned long endmask;
1274
1275 --maxnode;
1276 nodes_clear(*nodes);
1277 if (maxnode == 0 || !nmask)
1278 return 0;
a9c930ba 1279 if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
636f13c1 1280 return -EINVAL;
8bccd85f
CL
1281
1282 nlongs = BITS_TO_LONGS(maxnode);
1283 if ((maxnode % BITS_PER_LONG) == 0)
1284 endmask = ~0UL;
1285 else
1286 endmask = (1UL << (maxnode % BITS_PER_LONG)) - 1;
1287
1288 /* When the user specified more nodes than supported just check
1289 if the non supported part is all zero. */
1290 if (nlongs > BITS_TO_LONGS(MAX_NUMNODES)) {
1291 if (nlongs > PAGE_SIZE/sizeof(long))
1292 return -EINVAL;
1293 for (k = BITS_TO_LONGS(MAX_NUMNODES); k < nlongs; k++) {
1294 unsigned long t;
1295 if (get_user(t, nmask + k))
1296 return -EFAULT;
1297 if (k == nlongs - 1) {
1298 if (t & endmask)
1299 return -EINVAL;
1300 } else if (t)
1301 return -EINVAL;
1302 }
1303 nlongs = BITS_TO_LONGS(MAX_NUMNODES);
1304 endmask = ~0UL;
1305 }
1306
1307 if (copy_from_user(nodes_addr(*nodes), nmask, nlongs*sizeof(unsigned long)))
1308 return -EFAULT;
1309 nodes_addr(*nodes)[nlongs-1] &= endmask;
1310 return 0;
1311}
1312
1313/* Copy a kernel node mask to user space */
1314static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1315 nodemask_t *nodes)
1316{
1317 unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1318 const int nbytes = BITS_TO_LONGS(MAX_NUMNODES) * sizeof(long);
1319
1320 if (copy > nbytes) {
1321 if (copy > PAGE_SIZE)
1322 return -EINVAL;
1323 if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1324 return -EFAULT;
1325 copy = nbytes;
1326 }
1327 return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1328}
1329
938bb9f5 1330SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
f7f28ca9 1331 unsigned long, mode, const unsigned long __user *, nmask,
938bb9f5 1332 unsigned long, maxnode, unsigned, flags)
8bccd85f
CL
1333{
1334 nodemask_t nodes;
1335 int err;
028fec41 1336 unsigned short mode_flags;
8bccd85f 1337
028fec41
DR
1338 mode_flags = mode & MPOL_MODE_FLAGS;
1339 mode &= ~MPOL_MODE_FLAGS;
a3b51e01
DR
1340 if (mode >= MPOL_MAX)
1341 return -EINVAL;
4c50bc01
DR
1342 if ((mode_flags & MPOL_F_STATIC_NODES) &&
1343 (mode_flags & MPOL_F_RELATIVE_NODES))
1344 return -EINVAL;
8bccd85f
CL
1345 err = get_nodes(&nodes, nmask, maxnode);
1346 if (err)
1347 return err;
028fec41 1348 return do_mbind(start, len, mode, mode_flags, &nodes, flags);
8bccd85f
CL
1349}
1350
1351/* Set the process memory policy */
23c8902d 1352SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
938bb9f5 1353 unsigned long, maxnode)
8bccd85f
CL
1354{
1355 int err;
1356 nodemask_t nodes;
028fec41 1357 unsigned short flags;
8bccd85f 1358
028fec41
DR
1359 flags = mode & MPOL_MODE_FLAGS;
1360 mode &= ~MPOL_MODE_FLAGS;
1361 if ((unsigned int)mode >= MPOL_MAX)
8bccd85f 1362 return -EINVAL;
4c50bc01
DR
1363 if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))
1364 return -EINVAL;
8bccd85f
CL
1365 err = get_nodes(&nodes, nmask, maxnode);
1366 if (err)
1367 return err;
028fec41 1368 return do_set_mempolicy(mode, flags, &nodes);
8bccd85f
CL
1369}
1370
938bb9f5
HC
1371SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1372 const unsigned long __user *, old_nodes,
1373 const unsigned long __user *, new_nodes)
39743889 1374{
c69e8d9c 1375 const struct cred *cred = current_cred(), *tcred;
596d7cfa 1376 struct mm_struct *mm = NULL;
39743889 1377 struct task_struct *task;
39743889
CL
1378 nodemask_t task_nodes;
1379 int err;
596d7cfa
KM
1380 nodemask_t *old;
1381 nodemask_t *new;
1382 NODEMASK_SCRATCH(scratch);
1383
1384 if (!scratch)
1385 return -ENOMEM;
39743889 1386
596d7cfa
KM
1387 old = &scratch->mask1;
1388 new = &scratch->mask2;
1389
1390 err = get_nodes(old, old_nodes, maxnode);
39743889 1391 if (err)
596d7cfa 1392 goto out;
39743889 1393
596d7cfa 1394 err = get_nodes(new, new_nodes, maxnode);
39743889 1395 if (err)
596d7cfa 1396 goto out;
39743889
CL
1397
1398 /* Find the mm_struct */
55cfaa3c 1399 rcu_read_lock();
228ebcbe 1400 task = pid ? find_task_by_vpid(pid) : current;
39743889 1401 if (!task) {
55cfaa3c 1402 rcu_read_unlock();
596d7cfa
KM
1403 err = -ESRCH;
1404 goto out;
39743889 1405 }
3268c63e 1406 get_task_struct(task);
39743889 1407
596d7cfa 1408 err = -EINVAL;
39743889
CL
1409
1410 /*
1411 * Check if this process has the right to modify the specified
1412 * process. The right exists if the process has administrative
7f927fcc 1413 * capabilities, superuser privileges or the same
39743889
CL
1414 * userid as the target process.
1415 */
c69e8d9c 1416 tcred = __task_cred(task);
b38a86eb
EB
1417 if (!uid_eq(cred->euid, tcred->suid) && !uid_eq(cred->euid, tcred->uid) &&
1418 !uid_eq(cred->uid, tcred->suid) && !uid_eq(cred->uid, tcred->uid) &&
74c00241 1419 !capable(CAP_SYS_NICE)) {
c69e8d9c 1420 rcu_read_unlock();
39743889 1421 err = -EPERM;
3268c63e 1422 goto out_put;
39743889 1423 }
c69e8d9c 1424 rcu_read_unlock();
39743889
CL
1425
1426 task_nodes = cpuset_mems_allowed(task);
1427 /* Is the user allowed to access the target nodes? */
596d7cfa 1428 if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
39743889 1429 err = -EPERM;
3268c63e 1430 goto out_put;
39743889
CL
1431 }
1432
01f13bd6 1433 if (!nodes_subset(*new, node_states[N_MEMORY])) {
3b42d28b 1434 err = -EINVAL;
3268c63e 1435 goto out_put;
3b42d28b
CL
1436 }
1437
86c3a764
DQ
1438 err = security_task_movememory(task);
1439 if (err)
3268c63e 1440 goto out_put;
86c3a764 1441
3268c63e
CL
1442 mm = get_task_mm(task);
1443 put_task_struct(task);
f2a9ef88
SL
1444
1445 if (!mm) {
3268c63e 1446 err = -EINVAL;
f2a9ef88
SL
1447 goto out;
1448 }
1449
1450 err = do_migrate_pages(mm, old, new,
1451 capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
3268c63e
CL
1452
1453 mmput(mm);
1454out:
596d7cfa
KM
1455 NODEMASK_SCRATCH_FREE(scratch);
1456
39743889 1457 return err;
3268c63e
CL
1458
1459out_put:
1460 put_task_struct(task);
1461 goto out;
1462
39743889
CL
1463}
1464
1465
8bccd85f 1466/* Retrieve NUMA policy */
938bb9f5
HC
1467SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1468 unsigned long __user *, nmask, unsigned long, maxnode,
1469 unsigned long, addr, unsigned long, flags)
8bccd85f 1470{
dbcb0f19
AB
1471 int err;
1472 int uninitialized_var(pval);
8bccd85f
CL
1473 nodemask_t nodes;
1474
1475 if (nmask != NULL && maxnode < MAX_NUMNODES)
1476 return -EINVAL;
1477
1478 err = do_get_mempolicy(&pval, &nodes, addr, flags);
1479
1480 if (err)
1481 return err;
1482
1483 if (policy && put_user(pval, policy))
1484 return -EFAULT;
1485
1486 if (nmask)
1487 err = copy_nodes_to_user(nmask, maxnode, &nodes);
1488
1489 return err;
1490}
1491
1da177e4
LT
1492#ifdef CONFIG_COMPAT
1493
c93e0f6c
HC
1494COMPAT_SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1495 compat_ulong_t __user *, nmask,
1496 compat_ulong_t, maxnode,
1497 compat_ulong_t, addr, compat_ulong_t, flags)
1da177e4
LT
1498{
1499 long err;
1500 unsigned long __user *nm = NULL;
1501 unsigned long nr_bits, alloc_size;
1502 DECLARE_BITMAP(bm, MAX_NUMNODES);
1503
1504 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1505 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1506
1507 if (nmask)
1508 nm = compat_alloc_user_space(alloc_size);
1509
1510 err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags);
1511
1512 if (!err && nmask) {
2bbff6c7
KH
1513 unsigned long copy_size;
1514 copy_size = min_t(unsigned long, sizeof(bm), alloc_size);
1515 err = copy_from_user(bm, nm, copy_size);
1da177e4
LT
1516 /* ensure entire bitmap is zeroed */
1517 err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8);
1518 err |= compat_put_bitmap(nmask, bm, nr_bits);
1519 }
1520
1521 return err;
1522}
1523
c93e0f6c
HC
1524COMPAT_SYSCALL_DEFINE3(set_mempolicy, int, mode, compat_ulong_t __user *, nmask,
1525 compat_ulong_t, maxnode)
1da177e4
LT
1526{
1527 long err = 0;
1528 unsigned long __user *nm = NULL;
1529 unsigned long nr_bits, alloc_size;
1530 DECLARE_BITMAP(bm, MAX_NUMNODES);
1531
1532 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1533 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1534
1535 if (nmask) {
1536 err = compat_get_bitmap(bm, nmask, nr_bits);
1537 nm = compat_alloc_user_space(alloc_size);
1538 err |= copy_to_user(nm, bm, alloc_size);
1539 }
1540
1541 if (err)
1542 return -EFAULT;
1543
1544 return sys_set_mempolicy(mode, nm, nr_bits+1);
1545}
1546
c93e0f6c
HC
1547COMPAT_SYSCALL_DEFINE6(mbind, compat_ulong_t, start, compat_ulong_t, len,
1548 compat_ulong_t, mode, compat_ulong_t __user *, nmask,
1549 compat_ulong_t, maxnode, compat_ulong_t, flags)
1da177e4
LT
1550{
1551 long err = 0;
1552 unsigned long __user *nm = NULL;
1553 unsigned long nr_bits, alloc_size;
dfcd3c0d 1554 nodemask_t bm;
1da177e4
LT
1555
1556 nr_bits = min_t(unsigned long, maxnode-1, MAX_NUMNODES);
1557 alloc_size = ALIGN(nr_bits, BITS_PER_LONG) / 8;
1558
1559 if (nmask) {
dfcd3c0d 1560 err = compat_get_bitmap(nodes_addr(bm), nmask, nr_bits);
1da177e4 1561 nm = compat_alloc_user_space(alloc_size);
dfcd3c0d 1562 err |= copy_to_user(nm, nodes_addr(bm), alloc_size);
1da177e4
LT
1563 }
1564
1565 if (err)
1566 return -EFAULT;
1567
1568 return sys_mbind(start, len, mode, nm, nr_bits+1, flags);
1569}
1570
1571#endif
1572
74d2c3a0
ON
1573struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
1574 unsigned long addr)
1da177e4 1575{
8d90274b 1576 struct mempolicy *pol = NULL;
1da177e4
LT
1577
1578 if (vma) {
480eccf9 1579 if (vma->vm_ops && vma->vm_ops->get_policy) {
8d90274b 1580 pol = vma->vm_ops->get_policy(vma, addr);
00442ad0 1581 } else if (vma->vm_policy) {
1da177e4 1582 pol = vma->vm_policy;
00442ad0
MG
1583
1584 /*
1585 * shmem_alloc_page() passes MPOL_F_SHARED policy with
1586 * a pseudo vma whose vma->vm_ops=NULL. Take a reference
1587 * count on these policies which will be dropped by
1588 * mpol_cond_put() later
1589 */
1590 if (mpol_needs_cond_ref(pol))
1591 mpol_get(pol);
1592 }
1da177e4 1593 }
f15ca78e 1594
74d2c3a0
ON
1595 return pol;
1596}
1597
1598/*
dd6eecb9 1599 * get_vma_policy(@vma, @addr)
74d2c3a0
ON
1600 * @vma: virtual memory area whose policy is sought
1601 * @addr: address in @vma for shared policy lookup
1602 *
1603 * Returns effective policy for a VMA at specified address.
dd6eecb9 1604 * Falls back to current->mempolicy or system default policy, as necessary.
74d2c3a0
ON
1605 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
1606 * count--added by the get_policy() vm_op, as appropriate--to protect against
1607 * freeing by another task. It is the caller's responsibility to free the
1608 * extra reference for shared policies.
1609 */
dd6eecb9
ON
1610static struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
1611 unsigned long addr)
74d2c3a0
ON
1612{
1613 struct mempolicy *pol = __get_vma_policy(vma, addr);
1614
8d90274b 1615 if (!pol)
dd6eecb9 1616 pol = get_task_policy(current);
8d90274b 1617
1da177e4
LT
1618 return pol;
1619}
1620
6b6482bb 1621bool vma_policy_mof(struct vm_area_struct *vma)
fc314724 1622{
6b6482bb 1623 struct mempolicy *pol;
fc314724 1624
6b6482bb
ON
1625 if (vma->vm_ops && vma->vm_ops->get_policy) {
1626 bool ret = false;
fc314724 1627
6b6482bb
ON
1628 pol = vma->vm_ops->get_policy(vma, vma->vm_start);
1629 if (pol && (pol->flags & MPOL_F_MOF))
1630 ret = true;
1631 mpol_cond_put(pol);
8d90274b 1632
6b6482bb 1633 return ret;
fc314724
MG
1634 }
1635
6b6482bb 1636 pol = vma->vm_policy;
8d90274b 1637 if (!pol)
6b6482bb 1638 pol = get_task_policy(current);
8d90274b 1639
fc314724
MG
1640 return pol->flags & MPOL_F_MOF;
1641}
1642
d3eb1570
LJ
1643static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
1644{
1645 enum zone_type dynamic_policy_zone = policy_zone;
1646
1647 BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
1648
1649 /*
1650 * if policy->v.nodes has movable memory only,
1651 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
1652 *
1653 * policy->v.nodes is intersect with node_states[N_MEMORY].
1654 * so if the following test faile, it implies
1655 * policy->v.nodes has movable memory only.
1656 */
1657 if (!nodes_intersects(policy->v.nodes, node_states[N_HIGH_MEMORY]))
1658 dynamic_policy_zone = ZONE_MOVABLE;
1659
1660 return zone >= dynamic_policy_zone;
1661}
1662
52cd3b07
LS
1663/*
1664 * Return a nodemask representing a mempolicy for filtering nodes for
1665 * page allocation
1666 */
1667static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
19770b32
MG
1668{
1669 /* Lower zones don't get a nodemask applied for MPOL_BIND */
45c4745a 1670 if (unlikely(policy->mode == MPOL_BIND) &&
d3eb1570 1671 apply_policy_zone(policy, gfp_zone(gfp)) &&
19770b32
MG
1672 cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
1673 return &policy->v.nodes;
1674
1675 return NULL;
1676}
1677
52cd3b07 1678/* Return a zonelist indicated by gfp for node representing a mempolicy */
2f5f9486
AK
1679static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy,
1680 int nd)
1da177e4 1681{
45c4745a 1682 switch (policy->mode) {
1da177e4 1683 case MPOL_PREFERRED:
fc36b8d3
LS
1684 if (!(policy->flags & MPOL_F_LOCAL))
1685 nd = policy->v.preferred_node;
1da177e4
LT
1686 break;
1687 case MPOL_BIND:
19770b32 1688 /*
52cd3b07
LS
1689 * Normally, MPOL_BIND allocations are node-local within the
1690 * allowed nodemask. However, if __GFP_THISNODE is set and the
6eb27e1f 1691 * current node isn't part of the mask, we use the zonelist for
52cd3b07 1692 * the first node in the mask instead.
19770b32 1693 */
19770b32
MG
1694 if (unlikely(gfp & __GFP_THISNODE) &&
1695 unlikely(!node_isset(nd, policy->v.nodes)))
1696 nd = first_node(policy->v.nodes);
1697 break;
1da177e4 1698 default:
1da177e4
LT
1699 BUG();
1700 }
0e88460d 1701 return node_zonelist(nd, gfp);
1da177e4
LT
1702}
1703
1704/* Do dynamic interleaving for a process */
1705static unsigned interleave_nodes(struct mempolicy *policy)
1706{
1707 unsigned nid, next;
1708 struct task_struct *me = current;
1709
1710 nid = me->il_next;
0edaf86c 1711 next = next_node_in(nid, policy->v.nodes);
f5b087b5
DR
1712 if (next < MAX_NUMNODES)
1713 me->il_next = next;
1da177e4
LT
1714 return nid;
1715}
1716
dc85da15
CL
1717/*
1718 * Depending on the memory policy provide a node from which to allocate the
1719 * next slab entry.
1720 */
2a389610 1721unsigned int mempolicy_slab_node(void)
dc85da15 1722{
e7b691b0 1723 struct mempolicy *policy;
2a389610 1724 int node = numa_mem_id();
e7b691b0
AK
1725
1726 if (in_interrupt())
2a389610 1727 return node;
e7b691b0
AK
1728
1729 policy = current->mempolicy;
fc36b8d3 1730 if (!policy || policy->flags & MPOL_F_LOCAL)
2a389610 1731 return node;
bea904d5
LS
1732
1733 switch (policy->mode) {
1734 case MPOL_PREFERRED:
fc36b8d3
LS
1735 /*
1736 * handled MPOL_F_LOCAL above
1737 */
1738 return policy->v.preferred_node;
765c4507 1739
dc85da15
CL
1740 case MPOL_INTERLEAVE:
1741 return interleave_nodes(policy);
1742
dd1a239f 1743 case MPOL_BIND: {
c33d6c06
MG
1744 struct zoneref *z;
1745
dc85da15
CL
1746 /*
1747 * Follow bind policy behavior and start allocation at the
1748 * first node.
1749 */
19770b32 1750 struct zonelist *zonelist;
19770b32 1751 enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
c9634cf0 1752 zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK];
c33d6c06
MG
1753 z = first_zones_zonelist(zonelist, highest_zoneidx,
1754 &policy->v.nodes);
1755 return z->zone ? z->zone->node : node;
dd1a239f 1756 }
dc85da15 1757
dc85da15 1758 default:
bea904d5 1759 BUG();
dc85da15
CL
1760 }
1761}
1762
fee83b3a
AM
1763/*
1764 * Do static interleaving for a VMA with known offset @n. Returns the n'th
1765 * node in pol->v.nodes (starting from n=0), wrapping around if n exceeds the
1766 * number of present nodes.
1767 */
1da177e4 1768static unsigned offset_il_node(struct mempolicy *pol,
fee83b3a 1769 struct vm_area_struct *vma, unsigned long n)
1da177e4 1770{
dfcd3c0d 1771 unsigned nnodes = nodes_weight(pol->v.nodes);
f5b087b5 1772 unsigned target;
fee83b3a
AM
1773 int i;
1774 int nid;
1da177e4 1775
f5b087b5
DR
1776 if (!nnodes)
1777 return numa_node_id();
fee83b3a
AM
1778 target = (unsigned int)n % nnodes;
1779 nid = first_node(pol->v.nodes);
1780 for (i = 0; i < target; i++)
dfcd3c0d 1781 nid = next_node(nid, pol->v.nodes);
1da177e4
LT
1782 return nid;
1783}
1784
5da7ca86
CL
1785/* Determine a node number for interleave */
1786static inline unsigned interleave_nid(struct mempolicy *pol,
1787 struct vm_area_struct *vma, unsigned long addr, int shift)
1788{
1789 if (vma) {
1790 unsigned long off;
1791
3b98b087
NA
1792 /*
1793 * for small pages, there is no difference between
1794 * shift and PAGE_SHIFT, so the bit-shift is safe.
1795 * for huge pages, since vm_pgoff is in units of small
1796 * pages, we need to shift off the always 0 bits to get
1797 * a useful offset.
1798 */
1799 BUG_ON(shift < PAGE_SHIFT);
1800 off = vma->vm_pgoff >> (shift - PAGE_SHIFT);
5da7ca86
CL
1801 off += (addr - vma->vm_start) >> shift;
1802 return offset_il_node(pol, vma, off);
1803 } else
1804 return interleave_nodes(pol);
1805}
1806
00ac59ad 1807#ifdef CONFIG_HUGETLBFS
480eccf9
LS
1808/*
1809 * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
b46e14ac
FF
1810 * @vma: virtual memory area whose policy is sought
1811 * @addr: address in @vma for shared policy lookup and interleave policy
1812 * @gfp_flags: for requested zone
1813 * @mpol: pointer to mempolicy pointer for reference counted mempolicy
1814 * @nodemask: pointer to nodemask pointer for MPOL_BIND nodemask
480eccf9 1815 *
52cd3b07
LS
1816 * Returns a zonelist suitable for a huge page allocation and a pointer
1817 * to the struct mempolicy for conditional unref after allocation.
1818 * If the effective policy is 'BIND, returns a pointer to the mempolicy's
1819 * @nodemask for filtering the zonelist.
c0ff7453 1820 *
d26914d1 1821 * Must be protected by read_mems_allowed_begin()
480eccf9 1822 */
396faf03 1823struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
19770b32
MG
1824 gfp_t gfp_flags, struct mempolicy **mpol,
1825 nodemask_t **nodemask)
5da7ca86 1826{
480eccf9 1827 struct zonelist *zl;
5da7ca86 1828
dd6eecb9 1829 *mpol = get_vma_policy(vma, addr);
19770b32 1830 *nodemask = NULL; /* assume !MPOL_BIND */
5da7ca86 1831
52cd3b07
LS
1832 if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
1833 zl = node_zonelist(interleave_nid(*mpol, vma, addr,
a5516438 1834 huge_page_shift(hstate_vma(vma))), gfp_flags);
52cd3b07 1835 } else {
2f5f9486 1836 zl = policy_zonelist(gfp_flags, *mpol, numa_node_id());
52cd3b07
LS
1837 if ((*mpol)->mode == MPOL_BIND)
1838 *nodemask = &(*mpol)->v.nodes;
480eccf9
LS
1839 }
1840 return zl;
5da7ca86 1841}
06808b08
LS
1842
1843/*
1844 * init_nodemask_of_mempolicy
1845 *
1846 * If the current task's mempolicy is "default" [NULL], return 'false'
1847 * to indicate default policy. Otherwise, extract the policy nodemask
1848 * for 'bind' or 'interleave' policy into the argument nodemask, or
1849 * initialize the argument nodemask to contain the single node for
1850 * 'preferred' or 'local' policy and return 'true' to indicate presence
1851 * of non-default mempolicy.
1852 *
1853 * We don't bother with reference counting the mempolicy [mpol_get/put]
1854 * because the current task is examining it's own mempolicy and a task's
1855 * mempolicy is only ever changed by the task itself.
1856 *
1857 * N.B., it is the caller's responsibility to free a returned nodemask.
1858 */
1859bool init_nodemask_of_mempolicy(nodemask_t *mask)
1860{
1861 struct mempolicy *mempolicy;
1862 int nid;
1863
1864 if (!(mask && current->mempolicy))
1865 return false;
1866
c0ff7453 1867 task_lock(current);
06808b08
LS
1868 mempolicy = current->mempolicy;
1869 switch (mempolicy->mode) {
1870 case MPOL_PREFERRED:
1871 if (mempolicy->flags & MPOL_F_LOCAL)
1872 nid = numa_node_id();
1873 else
1874 nid = mempolicy->v.preferred_node;
1875 init_nodemask_of_node(mask, nid);
1876 break;
1877
1878 case MPOL_BIND:
1879 /* Fall through */
1880 case MPOL_INTERLEAVE:
1881 *mask = mempolicy->v.nodes;
1882 break;
1883
1884 default:
1885 BUG();
1886 }
c0ff7453 1887 task_unlock(current);
06808b08
LS
1888
1889 return true;
1890}
00ac59ad 1891#endif
5da7ca86 1892
6f48d0eb
DR
1893/*
1894 * mempolicy_nodemask_intersects
1895 *
1896 * If tsk's mempolicy is "default" [NULL], return 'true' to indicate default
1897 * policy. Otherwise, check for intersection between mask and the policy
1898 * nodemask for 'bind' or 'interleave' policy. For 'perferred' or 'local'
1899 * policy, always return true since it may allocate elsewhere on fallback.
1900 *
1901 * Takes task_lock(tsk) to prevent freeing of its mempolicy.
1902 */
1903bool mempolicy_nodemask_intersects(struct task_struct *tsk,
1904 const nodemask_t *mask)
1905{
1906 struct mempolicy *mempolicy;
1907 bool ret = true;
1908
1909 if (!mask)
1910 return ret;
1911 task_lock(tsk);
1912 mempolicy = tsk->mempolicy;
1913 if (!mempolicy)
1914 goto out;
1915
1916 switch (mempolicy->mode) {
1917 case MPOL_PREFERRED:
1918 /*
1919 * MPOL_PREFERRED and MPOL_F_LOCAL are only preferred nodes to
1920 * allocate from, they may fallback to other nodes when oom.
1921 * Thus, it's possible for tsk to have allocated memory from
1922 * nodes in mask.
1923 */
1924 break;
1925 case MPOL_BIND:
1926 case MPOL_INTERLEAVE:
1927 ret = nodes_intersects(mempolicy->v.nodes, *mask);
1928 break;
1929 default:
1930 BUG();
1931 }
1932out:
1933 task_unlock(tsk);
1934 return ret;
1935}
1936
1da177e4
LT
1937/* Allocate a page in interleaved policy.
1938 Own path because it needs to do special accounting. */
662f3a0b
AK
1939static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
1940 unsigned nid)
1da177e4
LT
1941{
1942 struct zonelist *zl;
1943 struct page *page;
1944
0e88460d 1945 zl = node_zonelist(nid, gfp);
1da177e4 1946 page = __alloc_pages(gfp, order, zl);
dd1a239f 1947 if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
ca889e6c 1948 inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);
1da177e4
LT
1949 return page;
1950}
1951
1952/**
0bbbc0b3 1953 * alloc_pages_vma - Allocate a page for a VMA.
1da177e4
LT
1954 *
1955 * @gfp:
1956 * %GFP_USER user allocation.
1957 * %GFP_KERNEL kernel allocations,
1958 * %GFP_HIGHMEM highmem/user allocations,
1959 * %GFP_FS allocation should not call back into a file system.
1960 * %GFP_ATOMIC don't sleep.
1961 *
0bbbc0b3 1962 * @order:Order of the GFP allocation.
1da177e4
LT
1963 * @vma: Pointer to VMA or NULL if not available.
1964 * @addr: Virtual Address of the allocation. Must be inside the VMA.
be97a41b
VB
1965 * @node: Which node to prefer for allocation (modulo policy).
1966 * @hugepage: for hugepages try only the preferred node if possible
1da177e4
LT
1967 *
1968 * This function allocates a page from the kernel page pool and applies
1969 * a NUMA policy associated with the VMA or the current process.
1970 * When VMA is not NULL caller must hold down_read on the mmap_sem of the
1971 * mm_struct of the VMA to prevent it from going away. Should be used for
be97a41b
VB
1972 * all allocations for pages that will be mapped into user space. Returns
1973 * NULL when no page can be allocated.
1da177e4
LT
1974 */
1975struct page *
0bbbc0b3 1976alloc_pages_vma(gfp_t gfp, int order, struct vm_area_struct *vma,
be97a41b 1977 unsigned long addr, int node, bool hugepage)
1da177e4 1978{
cc9a6c87 1979 struct mempolicy *pol;
c0ff7453 1980 struct page *page;
cc9a6c87 1981 unsigned int cpuset_mems_cookie;
be97a41b
VB
1982 struct zonelist *zl;
1983 nodemask_t *nmask;
cc9a6c87
MG
1984
1985retry_cpuset:
dd6eecb9 1986 pol = get_vma_policy(vma, addr);
d26914d1 1987 cpuset_mems_cookie = read_mems_allowed_begin();
1da177e4 1988
0867a57c
VB
1989 if (pol->mode == MPOL_INTERLEAVE) {
1990 unsigned nid;
1991
1992 nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
1993 mpol_cond_put(pol);
1994 page = alloc_page_interleave(gfp, order, nid);
1995 goto out;
1996 }
1997
1998 if (unlikely(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && hugepage)) {
1999 int hpage_node = node;
2000
be97a41b
VB
2001 /*
2002 * For hugepage allocation and non-interleave policy which
0867a57c
VB
2003 * allows the current node (or other explicitly preferred
2004 * node) we only try to allocate from the current/preferred
2005 * node and don't fall back to other nodes, as the cost of
2006 * remote accesses would likely offset THP benefits.
be97a41b
VB
2007 *
2008 * If the policy is interleave, or does not allow the current
2009 * node in its nodemask, we allocate the standard way.
2010 */
0867a57c
VB
2011 if (pol->mode == MPOL_PREFERRED &&
2012 !(pol->flags & MPOL_F_LOCAL))
2013 hpage_node = pol->v.preferred_node;
2014
be97a41b 2015 nmask = policy_nodemask(gfp, pol);
0867a57c 2016 if (!nmask || node_isset(hpage_node, *nmask)) {
be97a41b 2017 mpol_cond_put(pol);
96db800f 2018 page = __alloc_pages_node(hpage_node,
5265047a 2019 gfp | __GFP_THISNODE, order);
be97a41b
VB
2020 goto out;
2021 }
2022 }
2023
be97a41b
VB
2024 nmask = policy_nodemask(gfp, pol);
2025 zl = policy_zonelist(gfp, pol, node);
2386740d 2026 mpol_cond_put(pol);
be97a41b
VB
2027 page = __alloc_pages_nodemask(gfp, order, zl, nmask);
2028out:
d26914d1 2029 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
cc9a6c87 2030 goto retry_cpuset;
c0ff7453 2031 return page;
1da177e4
LT
2032}
2033
2034/**
2035 * alloc_pages_current - Allocate pages.
2036 *
2037 * @gfp:
2038 * %GFP_USER user allocation,
2039 * %GFP_KERNEL kernel allocation,
2040 * %GFP_HIGHMEM highmem allocation,
2041 * %GFP_FS don't call back into a file system.
2042 * %GFP_ATOMIC don't sleep.
2043 * @order: Power of two of allocation size in pages. 0 is a single page.
2044 *
2045 * Allocate a page from the kernel page pool. When not in
2046 * interrupt context and apply the current process NUMA policy.
2047 * Returns NULL when no page can be allocated.
2048 *
cf2a473c 2049 * Don't call cpuset_update_task_memory_state() unless
1da177e4
LT
2050 * 1) it's ok to take cpuset_sem (can WAIT), and
2051 * 2) allocating for current task (not interrupt).
2052 */
dd0fc66f 2053struct page *alloc_pages_current(gfp_t gfp, unsigned order)
1da177e4 2054{
8d90274b 2055 struct mempolicy *pol = &default_policy;
c0ff7453 2056 struct page *page;
cc9a6c87 2057 unsigned int cpuset_mems_cookie;
1da177e4 2058
8d90274b
ON
2059 if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2060 pol = get_task_policy(current);
52cd3b07 2061
cc9a6c87 2062retry_cpuset:
d26914d1 2063 cpuset_mems_cookie = read_mems_allowed_begin();
cc9a6c87 2064
52cd3b07
LS
2065 /*
2066 * No reference counting needed for current->mempolicy
2067 * nor system default_policy
2068 */
45c4745a 2069 if (pol->mode == MPOL_INTERLEAVE)
c0ff7453
MX
2070 page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
2071 else
2072 page = __alloc_pages_nodemask(gfp, order,
5c4b4be3
AK
2073 policy_zonelist(gfp, pol, numa_node_id()),
2074 policy_nodemask(gfp, pol));
cc9a6c87 2075
d26914d1 2076 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
cc9a6c87
MG
2077 goto retry_cpuset;
2078
c0ff7453 2079 return page;
1da177e4
LT
2080}
2081EXPORT_SYMBOL(alloc_pages_current);
2082
ef0855d3
ON
2083int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2084{
2085 struct mempolicy *pol = mpol_dup(vma_policy(src));
2086
2087 if (IS_ERR(pol))
2088 return PTR_ERR(pol);
2089 dst->vm_policy = pol;
2090 return 0;
2091}
2092
4225399a 2093/*
846a16bf 2094 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
4225399a
PJ
2095 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2096 * with the mems_allowed returned by cpuset_mems_allowed(). This
2097 * keeps mempolicies cpuset relative after its cpuset moves. See
2098 * further kernel/cpuset.c update_nodemask().
708c1bbc
MX
2099 *
2100 * current's mempolicy may be rebinded by the other task(the task that changes
2101 * cpuset's mems), so we needn't do rebind work for current task.
4225399a 2102 */
4225399a 2103
846a16bf
LS
2104/* Slow path of a mempolicy duplicate */
2105struct mempolicy *__mpol_dup(struct mempolicy *old)
1da177e4
LT
2106{
2107 struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2108
2109 if (!new)
2110 return ERR_PTR(-ENOMEM);
708c1bbc
MX
2111
2112 /* task's mempolicy is protected by alloc_lock */
2113 if (old == current->mempolicy) {
2114 task_lock(current);
2115 *new = *old;
2116 task_unlock(current);
2117 } else
2118 *new = *old;
2119
4225399a
PJ
2120 if (current_cpuset_is_being_rebound()) {
2121 nodemask_t mems = cpuset_mems_allowed(current);
708c1bbc
MX
2122 if (new->flags & MPOL_F_REBINDING)
2123 mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
2124 else
2125 mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);
4225399a 2126 }
1da177e4 2127 atomic_set(&new->refcnt, 1);
1da177e4
LT
2128 return new;
2129}
2130
2131/* Slow path of a mempolicy comparison */
fcfb4dcc 2132bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
1da177e4
LT
2133{
2134 if (!a || !b)
fcfb4dcc 2135 return false;
45c4745a 2136 if (a->mode != b->mode)
fcfb4dcc 2137 return false;
19800502 2138 if (a->flags != b->flags)
fcfb4dcc 2139 return false;
19800502
BL
2140 if (mpol_store_user_nodemask(a))
2141 if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
fcfb4dcc 2142 return false;
19800502 2143
45c4745a 2144 switch (a->mode) {
19770b32
MG
2145 case MPOL_BIND:
2146 /* Fall through */
1da177e4 2147 case MPOL_INTERLEAVE:
fcfb4dcc 2148 return !!nodes_equal(a->v.nodes, b->v.nodes);
1da177e4 2149 case MPOL_PREFERRED:
75719661 2150 return a->v.preferred_node == b->v.preferred_node;
1da177e4
LT
2151 default:
2152 BUG();
fcfb4dcc 2153 return false;
1da177e4
LT
2154 }
2155}
2156
1da177e4
LT
2157/*
2158 * Shared memory backing store policy support.
2159 *
2160 * Remember policies even when nobody has shared memory mapped.
2161 * The policies are kept in Red-Black tree linked from the inode.
4a8c7bb5 2162 * They are protected by the sp->lock rwlock, which should be held
1da177e4
LT
2163 * for any accesses to the tree.
2164 */
2165
4a8c7bb5
NZ
2166/*
2167 * lookup first element intersecting start-end. Caller holds sp->lock for
2168 * reading or for writing
2169 */
1da177e4
LT
2170static struct sp_node *
2171sp_lookup(struct shared_policy *sp, unsigned long start, unsigned long end)
2172{
2173 struct rb_node *n = sp->root.rb_node;
2174
2175 while (n) {
2176 struct sp_node *p = rb_entry(n, struct sp_node, nd);
2177
2178 if (start >= p->end)
2179 n = n->rb_right;
2180 else if (end <= p->start)
2181 n = n->rb_left;
2182 else
2183 break;
2184 }
2185 if (!n)
2186 return NULL;
2187 for (;;) {
2188 struct sp_node *w = NULL;
2189 struct rb_node *prev = rb_prev(n);
2190 if (!prev)
2191 break;
2192 w = rb_entry(prev, struct sp_node, nd);
2193 if (w->end <= start)
2194 break;
2195 n = prev;
2196 }
2197 return rb_entry(n, struct sp_node, nd);
2198}
2199
4a8c7bb5
NZ
2200/*
2201 * Insert a new shared policy into the list. Caller holds sp->lock for
2202 * writing.
2203 */
1da177e4
LT
2204static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2205{
2206 struct rb_node **p = &sp->root.rb_node;
2207 struct rb_node *parent = NULL;
2208 struct sp_node *nd;
2209
2210 while (*p) {
2211 parent = *p;
2212 nd = rb_entry(parent, struct sp_node, nd);
2213 if (new->start < nd->start)
2214 p = &(*p)->rb_left;
2215 else if (new->end > nd->end)
2216 p = &(*p)->rb_right;
2217 else
2218 BUG();
2219 }
2220 rb_link_node(&new->nd, parent, p);
2221 rb_insert_color(&new->nd, &sp->root);
140d5a49 2222 pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
45c4745a 2223 new->policy ? new->policy->mode : 0);
1da177e4
LT
2224}
2225
2226/* Find shared policy intersecting idx */
2227struct mempolicy *
2228mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
2229{
2230 struct mempolicy *pol = NULL;
2231 struct sp_node *sn;
2232
2233 if (!sp->root.rb_node)
2234 return NULL;
4a8c7bb5 2235 read_lock(&sp->lock);
1da177e4
LT
2236 sn = sp_lookup(sp, idx, idx+1);
2237 if (sn) {
2238 mpol_get(sn->policy);
2239 pol = sn->policy;
2240 }
4a8c7bb5 2241 read_unlock(&sp->lock);
1da177e4
LT
2242 return pol;
2243}
2244
63f74ca2
KM
2245static void sp_free(struct sp_node *n)
2246{
2247 mpol_put(n->policy);
2248 kmem_cache_free(sn_cache, n);
2249}
2250
771fb4d8
LS
2251/**
2252 * mpol_misplaced - check whether current page node is valid in policy
2253 *
b46e14ac
FF
2254 * @page: page to be checked
2255 * @vma: vm area where page mapped
2256 * @addr: virtual address where page mapped
771fb4d8
LS
2257 *
2258 * Lookup current policy node id for vma,addr and "compare to" page's
2259 * node id.
2260 *
2261 * Returns:
2262 * -1 - not misplaced, page is in the right node
2263 * node - node id where the page should be
2264 *
2265 * Policy determination "mimics" alloc_page_vma().
2266 * Called from fault path where we know the vma and faulting address.
2267 */
2268int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
2269{
2270 struct mempolicy *pol;
c33d6c06 2271 struct zoneref *z;
771fb4d8
LS
2272 int curnid = page_to_nid(page);
2273 unsigned long pgoff;
90572890
PZ
2274 int thiscpu = raw_smp_processor_id();
2275 int thisnid = cpu_to_node(thiscpu);
771fb4d8
LS
2276 int polnid = -1;
2277 int ret = -1;
2278
2279 BUG_ON(!vma);
2280
dd6eecb9 2281 pol = get_vma_policy(vma, addr);
771fb4d8
LS
2282 if (!(pol->flags & MPOL_F_MOF))
2283 goto out;
2284
2285 switch (pol->mode) {
2286 case MPOL_INTERLEAVE:
2287 BUG_ON(addr >= vma->vm_end);
2288 BUG_ON(addr < vma->vm_start);
2289
2290 pgoff = vma->vm_pgoff;
2291 pgoff += (addr - vma->vm_start) >> PAGE_SHIFT;
2292 polnid = offset_il_node(pol, vma, pgoff);
2293 break;
2294
2295 case MPOL_PREFERRED:
2296 if (pol->flags & MPOL_F_LOCAL)
2297 polnid = numa_node_id();
2298 else
2299 polnid = pol->v.preferred_node;
2300 break;
2301
2302 case MPOL_BIND:
c33d6c06 2303
771fb4d8
LS
2304 /*
2305 * allows binding to multiple nodes.
2306 * use current page if in policy nodemask,
2307 * else select nearest allowed node, if any.
2308 * If no allowed nodes, use current [!misplaced].
2309 */
2310 if (node_isset(curnid, pol->v.nodes))
2311 goto out;
c33d6c06 2312 z = first_zones_zonelist(
771fb4d8
LS
2313 node_zonelist(numa_node_id(), GFP_HIGHUSER),
2314 gfp_zone(GFP_HIGHUSER),
c33d6c06
MG
2315 &pol->v.nodes);
2316 polnid = z->zone->node;
771fb4d8
LS
2317 break;
2318
2319 default:
2320 BUG();
2321 }
5606e387
MG
2322
2323 /* Migrate the page towards the node whose CPU is referencing it */
e42c8ff2 2324 if (pol->flags & MPOL_F_MORON) {
90572890 2325 polnid = thisnid;
5606e387 2326
10f39042 2327 if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
de1c9ce6 2328 goto out;
e42c8ff2
MG
2329 }
2330
771fb4d8
LS
2331 if (curnid != polnid)
2332 ret = polnid;
2333out:
2334 mpol_cond_put(pol);
2335
2336 return ret;
2337}
2338
c11600e4
DR
2339/*
2340 * Drop the (possibly final) reference to task->mempolicy. It needs to be
2341 * dropped after task->mempolicy is set to NULL so that any allocation done as
2342 * part of its kmem_cache_free(), such as by KASAN, doesn't reference a freed
2343 * policy.
2344 */
2345void mpol_put_task_policy(struct task_struct *task)
2346{
2347 struct mempolicy *pol;
2348
2349 task_lock(task);
2350 pol = task->mempolicy;
2351 task->mempolicy = NULL;
2352 task_unlock(task);
2353 mpol_put(pol);
2354}
2355
1da177e4
LT
2356static void sp_delete(struct shared_policy *sp, struct sp_node *n)
2357{
140d5a49 2358 pr_debug("deleting %lx-l%lx\n", n->start, n->end);
1da177e4 2359 rb_erase(&n->nd, &sp->root);
63f74ca2 2360 sp_free(n);
1da177e4
LT
2361}
2362
42288fe3
MG
2363static void sp_node_init(struct sp_node *node, unsigned long start,
2364 unsigned long end, struct mempolicy *pol)
2365{
2366 node->start = start;
2367 node->end = end;
2368 node->policy = pol;
2369}
2370
dbcb0f19
AB
2371static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
2372 struct mempolicy *pol)
1da177e4 2373{
869833f2
KM
2374 struct sp_node *n;
2375 struct mempolicy *newpol;
1da177e4 2376
869833f2 2377 n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
1da177e4
LT
2378 if (!n)
2379 return NULL;
869833f2
KM
2380
2381 newpol = mpol_dup(pol);
2382 if (IS_ERR(newpol)) {
2383 kmem_cache_free(sn_cache, n);
2384 return NULL;
2385 }
2386 newpol->flags |= MPOL_F_SHARED;
42288fe3 2387 sp_node_init(n, start, end, newpol);
869833f2 2388
1da177e4
LT
2389 return n;
2390}
2391
2392/* Replace a policy range. */
2393static int shared_policy_replace(struct shared_policy *sp, unsigned long start,
2394 unsigned long end, struct sp_node *new)
2395{
b22d127a 2396 struct sp_node *n;
42288fe3
MG
2397 struct sp_node *n_new = NULL;
2398 struct mempolicy *mpol_new = NULL;
b22d127a 2399 int ret = 0;
1da177e4 2400
42288fe3 2401restart:
4a8c7bb5 2402 write_lock(&sp->lock);
1da177e4
LT
2403 n = sp_lookup(sp, start, end);
2404 /* Take care of old policies in the same range. */
2405 while (n && n->start < end) {
2406 struct rb_node *next = rb_next(&n->nd);
2407 if (n->start >= start) {
2408 if (n->end <= end)
2409 sp_delete(sp, n);
2410 else
2411 n->start = end;
2412 } else {
2413 /* Old policy spanning whole new range. */
2414 if (n->end > end) {
42288fe3
MG
2415 if (!n_new)
2416 goto alloc_new;
2417
2418 *mpol_new = *n->policy;
2419 atomic_set(&mpol_new->refcnt, 1);
7880639c 2420 sp_node_init(n_new, end, n->end, mpol_new);
1da177e4 2421 n->end = start;
5ca39575 2422 sp_insert(sp, n_new);
42288fe3
MG
2423 n_new = NULL;
2424 mpol_new = NULL;
1da177e4
LT
2425 break;
2426 } else
2427 n->end = start;
2428 }
2429 if (!next)
2430 break;
2431 n = rb_entry(next, struct sp_node, nd);
2432 }
2433 if (new)
2434 sp_insert(sp, new);
4a8c7bb5 2435 write_unlock(&sp->lock);
42288fe3
MG
2436 ret = 0;
2437
2438err_out:
2439 if (mpol_new)
2440 mpol_put(mpol_new);
2441 if (n_new)
2442 kmem_cache_free(sn_cache, n_new);
2443
b22d127a 2444 return ret;
42288fe3
MG
2445
2446alloc_new:
4a8c7bb5 2447 write_unlock(&sp->lock);
42288fe3
MG
2448 ret = -ENOMEM;
2449 n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
2450 if (!n_new)
2451 goto err_out;
2452 mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2453 if (!mpol_new)
2454 goto err_out;
2455 goto restart;
1da177e4
LT
2456}
2457
71fe804b
LS
2458/**
2459 * mpol_shared_policy_init - initialize shared policy for inode
2460 * @sp: pointer to inode shared policy
2461 * @mpol: struct mempolicy to install
2462 *
2463 * Install non-NULL @mpol in inode's shared policy rb-tree.
2464 * On entry, the current task has a reference on a non-NULL @mpol.
2465 * This must be released on exit.
4bfc4495 2466 * This is called at get_inode() calls and we can use GFP_KERNEL.
71fe804b
LS
2467 */
2468void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
2469{
58568d2a
MX
2470 int ret;
2471
71fe804b 2472 sp->root = RB_ROOT; /* empty tree == default mempolicy */
4a8c7bb5 2473 rwlock_init(&sp->lock);
71fe804b
LS
2474
2475 if (mpol) {
2476 struct vm_area_struct pvma;
2477 struct mempolicy *new;
4bfc4495 2478 NODEMASK_SCRATCH(scratch);
71fe804b 2479
4bfc4495 2480 if (!scratch)
5c0c1654 2481 goto put_mpol;
71fe804b
LS
2482 /* contextualize the tmpfs mount point mempolicy */
2483 new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
15d77835 2484 if (IS_ERR(new))
0cae3457 2485 goto free_scratch; /* no valid nodemask intersection */
58568d2a
MX
2486
2487 task_lock(current);
4bfc4495 2488 ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
58568d2a 2489 task_unlock(current);
15d77835 2490 if (ret)
5c0c1654 2491 goto put_new;
71fe804b
LS
2492
2493 /* Create pseudo-vma that contains just the policy */
2494 memset(&pvma, 0, sizeof(struct vm_area_struct));
2495 pvma.vm_end = TASK_SIZE; /* policy covers entire file */
2496 mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
15d77835 2497
5c0c1654 2498put_new:
71fe804b 2499 mpol_put(new); /* drop initial ref */
0cae3457 2500free_scratch:
4bfc4495 2501 NODEMASK_SCRATCH_FREE(scratch);
5c0c1654
LS
2502put_mpol:
2503 mpol_put(mpol); /* drop our incoming ref on sb mpol */
7339ff83
RH
2504 }
2505}
2506
1da177e4
LT
2507int mpol_set_shared_policy(struct shared_policy *info,
2508 struct vm_area_struct *vma, struct mempolicy *npol)
2509{
2510 int err;
2511 struct sp_node *new = NULL;
2512 unsigned long sz = vma_pages(vma);
2513
028fec41 2514 pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",
1da177e4 2515 vma->vm_pgoff,
45c4745a 2516 sz, npol ? npol->mode : -1,
028fec41 2517 npol ? npol->flags : -1,
00ef2d2f 2518 npol ? nodes_addr(npol->v.nodes)[0] : NUMA_NO_NODE);
1da177e4
LT
2519
2520 if (npol) {
2521 new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
2522 if (!new)
2523 return -ENOMEM;
2524 }
2525 err = shared_policy_replace(info, vma->vm_pgoff, vma->vm_pgoff+sz, new);
2526 if (err && new)
63f74ca2 2527 sp_free(new);
1da177e4
LT
2528 return err;
2529}
2530
2531/* Free a backing policy store on inode delete. */
2532void mpol_free_shared_policy(struct shared_policy *p)
2533{
2534 struct sp_node *n;
2535 struct rb_node *next;
2536
2537 if (!p->root.rb_node)
2538 return;
4a8c7bb5 2539 write_lock(&p->lock);
1da177e4
LT
2540 next = rb_first(&p->root);
2541 while (next) {
2542 n = rb_entry(next, struct sp_node, nd);
2543 next = rb_next(&n->nd);
63f74ca2 2544 sp_delete(p, n);
1da177e4 2545 }
4a8c7bb5 2546 write_unlock(&p->lock);
1da177e4
LT
2547}
2548
1a687c2e 2549#ifdef CONFIG_NUMA_BALANCING
c297663c 2550static int __initdata numabalancing_override;
1a687c2e
MG
2551
2552static void __init check_numabalancing_enable(void)
2553{
2554 bool numabalancing_default = false;
2555
2556 if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
2557 numabalancing_default = true;
2558
c297663c
MG
2559 /* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
2560 if (numabalancing_override)
2561 set_numabalancing_state(numabalancing_override == 1);
2562
b0dc2b9b 2563 if (num_online_nodes() > 1 && !numabalancing_override) {
756a025f 2564 pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
c297663c 2565 numabalancing_default ? "Enabling" : "Disabling");
1a687c2e
MG
2566 set_numabalancing_state(numabalancing_default);
2567 }
2568}
2569
2570static int __init setup_numabalancing(char *str)
2571{
2572 int ret = 0;
2573 if (!str)
2574 goto out;
1a687c2e
MG
2575
2576 if (!strcmp(str, "enable")) {
c297663c 2577 numabalancing_override = 1;
1a687c2e
MG
2578 ret = 1;
2579 } else if (!strcmp(str, "disable")) {
c297663c 2580 numabalancing_override = -1;
1a687c2e
MG
2581 ret = 1;
2582 }
2583out:
2584 if (!ret)
4a404bea 2585 pr_warn("Unable to parse numa_balancing=\n");
1a687c2e
MG
2586
2587 return ret;
2588}
2589__setup("numa_balancing=", setup_numabalancing);
2590#else
2591static inline void __init check_numabalancing_enable(void)
2592{
2593}
2594#endif /* CONFIG_NUMA_BALANCING */
2595
1da177e4
LT
2596/* assumes fs == KERNEL_DS */
2597void __init numa_policy_init(void)
2598{
b71636e2
PM
2599 nodemask_t interleave_nodes;
2600 unsigned long largest = 0;
2601 int nid, prefer = 0;
2602
1da177e4
LT
2603 policy_cache = kmem_cache_create("numa_policy",
2604 sizeof(struct mempolicy),
20c2df83 2605 0, SLAB_PANIC, NULL);
1da177e4
LT
2606
2607 sn_cache = kmem_cache_create("shared_policy_node",
2608 sizeof(struct sp_node),
20c2df83 2609 0, SLAB_PANIC, NULL);
1da177e4 2610
5606e387
MG
2611 for_each_node(nid) {
2612 preferred_node_policy[nid] = (struct mempolicy) {
2613 .refcnt = ATOMIC_INIT(1),
2614 .mode = MPOL_PREFERRED,
2615 .flags = MPOL_F_MOF | MPOL_F_MORON,
2616 .v = { .preferred_node = nid, },
2617 };
2618 }
2619
b71636e2
PM
2620 /*
2621 * Set interleaving policy for system init. Interleaving is only
2622 * enabled across suitably sized nodes (default is >= 16MB), or
2623 * fall back to the largest node if they're all smaller.
2624 */
2625 nodes_clear(interleave_nodes);
01f13bd6 2626 for_each_node_state(nid, N_MEMORY) {
b71636e2
PM
2627 unsigned long total_pages = node_present_pages(nid);
2628
2629 /* Preserve the largest node */
2630 if (largest < total_pages) {
2631 largest = total_pages;
2632 prefer = nid;
2633 }
2634
2635 /* Interleave this node? */
2636 if ((total_pages << PAGE_SHIFT) >= (16 << 20))
2637 node_set(nid, interleave_nodes);
2638 }
2639
2640 /* All too small, use the largest */
2641 if (unlikely(nodes_empty(interleave_nodes)))
2642 node_set(prefer, interleave_nodes);
1da177e4 2643
028fec41 2644 if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
b1de0d13 2645 pr_err("%s: interleaving failed\n", __func__);
1a687c2e
MG
2646
2647 check_numabalancing_enable();
1da177e4
LT
2648}
2649
8bccd85f 2650/* Reset policy of current process to default */
1da177e4
LT
2651void numa_default_policy(void)
2652{
028fec41 2653 do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
1da177e4 2654}
68860ec1 2655
095f1fc4
LS
2656/*
2657 * Parse and format mempolicy from/to strings
2658 */
2659
1a75a6c8 2660/*
f2a07f40 2661 * "local" is implemented internally by MPOL_PREFERRED with MPOL_F_LOCAL flag.
1a75a6c8 2662 */
345ace9c
LS
2663static const char * const policy_modes[] =
2664{
2665 [MPOL_DEFAULT] = "default",
2666 [MPOL_PREFERRED] = "prefer",
2667 [MPOL_BIND] = "bind",
2668 [MPOL_INTERLEAVE] = "interleave",
d3a71033 2669 [MPOL_LOCAL] = "local",
345ace9c 2670};
1a75a6c8 2671
095f1fc4
LS
2672
2673#ifdef CONFIG_TMPFS
2674/**
f2a07f40 2675 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
095f1fc4 2676 * @str: string containing mempolicy to parse
71fe804b 2677 * @mpol: pointer to struct mempolicy pointer, returned on success.
095f1fc4
LS
2678 *
2679 * Format of input:
2680 * <mode>[=<flags>][:<nodelist>]
2681 *
71fe804b 2682 * On success, returns 0, else 1
095f1fc4 2683 */
a7a88b23 2684int mpol_parse_str(char *str, struct mempolicy **mpol)
095f1fc4 2685{
71fe804b 2686 struct mempolicy *new = NULL;
b4652e84 2687 unsigned short mode;
f2a07f40 2688 unsigned short mode_flags;
71fe804b 2689 nodemask_t nodes;
095f1fc4
LS
2690 char *nodelist = strchr(str, ':');
2691 char *flags = strchr(str, '=');
095f1fc4
LS
2692 int err = 1;
2693
2694 if (nodelist) {
2695 /* NUL-terminate mode or flags string */
2696 *nodelist++ = '\0';
71fe804b 2697 if (nodelist_parse(nodelist, nodes))
095f1fc4 2698 goto out;
01f13bd6 2699 if (!nodes_subset(nodes, node_states[N_MEMORY]))
095f1fc4 2700 goto out;
71fe804b
LS
2701 } else
2702 nodes_clear(nodes);
2703
095f1fc4
LS
2704 if (flags)
2705 *flags++ = '\0'; /* terminate mode string */
2706
479e2802 2707 for (mode = 0; mode < MPOL_MAX; mode++) {
345ace9c 2708 if (!strcmp(str, policy_modes[mode])) {
095f1fc4
LS
2709 break;
2710 }
2711 }
a720094d 2712 if (mode >= MPOL_MAX)
095f1fc4
LS
2713 goto out;
2714
71fe804b 2715 switch (mode) {
095f1fc4 2716 case MPOL_PREFERRED:
71fe804b
LS
2717 /*
2718 * Insist on a nodelist of one node only
2719 */
095f1fc4
LS
2720 if (nodelist) {
2721 char *rest = nodelist;
2722 while (isdigit(*rest))
2723 rest++;
926f2ae0
KM
2724 if (*rest)
2725 goto out;
095f1fc4
LS
2726 }
2727 break;
095f1fc4
LS
2728 case MPOL_INTERLEAVE:
2729 /*
2730 * Default to online nodes with memory if no nodelist
2731 */
2732 if (!nodelist)
01f13bd6 2733 nodes = node_states[N_MEMORY];
3f226aa1 2734 break;
71fe804b 2735 case MPOL_LOCAL:
3f226aa1 2736 /*
71fe804b 2737 * Don't allow a nodelist; mpol_new() checks flags
3f226aa1 2738 */
71fe804b 2739 if (nodelist)
3f226aa1 2740 goto out;
71fe804b 2741 mode = MPOL_PREFERRED;
3f226aa1 2742 break;
413b43de
RT
2743 case MPOL_DEFAULT:
2744 /*
2745 * Insist on a empty nodelist
2746 */
2747 if (!nodelist)
2748 err = 0;
2749 goto out;
d69b2e63
KM
2750 case MPOL_BIND:
2751 /*
2752 * Insist on a nodelist
2753 */
2754 if (!nodelist)
2755 goto out;
095f1fc4
LS
2756 }
2757
71fe804b 2758 mode_flags = 0;
095f1fc4
LS
2759 if (flags) {
2760 /*
2761 * Currently, we only support two mutually exclusive
2762 * mode flags.
2763 */
2764 if (!strcmp(flags, "static"))
71fe804b 2765 mode_flags |= MPOL_F_STATIC_NODES;
095f1fc4 2766 else if (!strcmp(flags, "relative"))
71fe804b 2767 mode_flags |= MPOL_F_RELATIVE_NODES;
095f1fc4 2768 else
926f2ae0 2769 goto out;
095f1fc4 2770 }
71fe804b
LS
2771
2772 new = mpol_new(mode, mode_flags, &nodes);
2773 if (IS_ERR(new))
926f2ae0
KM
2774 goto out;
2775
f2a07f40
HD
2776 /*
2777 * Save nodes for mpol_to_str() to show the tmpfs mount options
2778 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
2779 */
2780 if (mode != MPOL_PREFERRED)
2781 new->v.nodes = nodes;
2782 else if (nodelist)
2783 new->v.preferred_node = first_node(nodes);
2784 else
2785 new->flags |= MPOL_F_LOCAL;
2786
2787 /*
2788 * Save nodes for contextualization: this will be used to "clone"
2789 * the mempolicy in a specific context [cpuset] at a later time.
2790 */
2791 new->w.user_nodemask = nodes;
2792
926f2ae0 2793 err = 0;
71fe804b 2794
095f1fc4
LS
2795out:
2796 /* Restore string for error message */
2797 if (nodelist)
2798 *--nodelist = ':';
2799 if (flags)
2800 *--flags = '=';
71fe804b
LS
2801 if (!err)
2802 *mpol = new;
095f1fc4
LS
2803 return err;
2804}
2805#endif /* CONFIG_TMPFS */
2806
71fe804b
LS
2807/**
2808 * mpol_to_str - format a mempolicy structure for printing
2809 * @buffer: to contain formatted mempolicy string
2810 * @maxlen: length of @buffer
2811 * @pol: pointer to mempolicy to be formatted
71fe804b 2812 *
948927ee
DR
2813 * Convert @pol into a string. If @buffer is too short, truncate the string.
2814 * Recommend a @maxlen of at least 32 for the longest mode, "interleave", the
2815 * longest flag, "relative", and to display at least a few node ids.
1a75a6c8 2816 */
948927ee 2817void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
1a75a6c8
CL
2818{
2819 char *p = buffer;
948927ee
DR
2820 nodemask_t nodes = NODE_MASK_NONE;
2821 unsigned short mode = MPOL_DEFAULT;
2822 unsigned short flags = 0;
2291990a 2823
8790c71a 2824 if (pol && pol != &default_policy && !(pol->flags & MPOL_F_MORON)) {
bea904d5 2825 mode = pol->mode;
948927ee
DR
2826 flags = pol->flags;
2827 }
bea904d5 2828
1a75a6c8
CL
2829 switch (mode) {
2830 case MPOL_DEFAULT:
1a75a6c8 2831 break;
1a75a6c8 2832 case MPOL_PREFERRED:
fc36b8d3 2833 if (flags & MPOL_F_LOCAL)
f2a07f40 2834 mode = MPOL_LOCAL;
53f2556b 2835 else
fc36b8d3 2836 node_set(pol->v.preferred_node, nodes);
1a75a6c8 2837 break;
1a75a6c8 2838 case MPOL_BIND:
1a75a6c8 2839 case MPOL_INTERLEAVE:
f2a07f40 2840 nodes = pol->v.nodes;
1a75a6c8 2841 break;
1a75a6c8 2842 default:
948927ee
DR
2843 WARN_ON_ONCE(1);
2844 snprintf(p, maxlen, "unknown");
2845 return;
1a75a6c8
CL
2846 }
2847
b7a9f420 2848 p += snprintf(p, maxlen, "%s", policy_modes[mode]);
1a75a6c8 2849
fc36b8d3 2850 if (flags & MPOL_MODE_FLAGS) {
948927ee 2851 p += snprintf(p, buffer + maxlen - p, "=");
f5b087b5 2852
2291990a
LS
2853 /*
2854 * Currently, the only defined flags are mutually exclusive
2855 */
f5b087b5 2856 if (flags & MPOL_F_STATIC_NODES)
2291990a
LS
2857 p += snprintf(p, buffer + maxlen - p, "static");
2858 else if (flags & MPOL_F_RELATIVE_NODES)
2859 p += snprintf(p, buffer + maxlen - p, "relative");
f5b087b5
DR
2860 }
2861
9e763e0f
TH
2862 if (!nodes_empty(nodes))
2863 p += scnprintf(p, buffer + maxlen - p, ":%*pbl",
2864 nodemask_pr_args(&nodes));
1a75a6c8 2865}