4 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
11 #include <linux/threads.h>
12 #include <linux/bootmem.h>
13 #include <linux/init.h>
15 #include <linux/mmzone.h>
16 #include <linux/module.h>
17 #include <linux/nodemask.h>
18 #include <linux/cpu.h>
19 #include <linux/notifier.h>
20 #include <asm/sparsemem.h>
22 #include <asm/system.h>
25 static int numa_enabled
= 1;
27 static int numa_debug
;
28 #define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
30 int numa_cpu_lookup_table
[NR_CPUS
];
31 cpumask_t numa_cpumask_lookup_table
[MAX_NUMNODES
];
32 struct pglist_data
*node_data
[MAX_NUMNODES
];
34 EXPORT_SYMBOL(numa_cpu_lookup_table
);
35 EXPORT_SYMBOL(numa_cpumask_lookup_table
);
36 EXPORT_SYMBOL(node_data
);
38 static bootmem_data_t __initdata plat_node_bdata
[MAX_NUMNODES
];
39 static int min_common_depth
;
40 static int n_mem_addr_cells
, n_mem_size_cells
;
42 static void __cpuinit
map_cpu_to_node(int cpu
, int node
)
44 numa_cpu_lookup_table
[cpu
] = node
;
46 dbg("adding cpu %d to node %d\n", cpu
, node
);
48 if (!(cpu_isset(cpu
, numa_cpumask_lookup_table
[node
])))
49 cpu_set(cpu
, numa_cpumask_lookup_table
[node
]);
52 #ifdef CONFIG_HOTPLUG_CPU
53 static void unmap_cpu_from_node(unsigned long cpu
)
55 int node
= numa_cpu_lookup_table
[cpu
];
57 dbg("removing cpu %lu from node %d\n", cpu
, node
);
59 if (cpu_isset(cpu
, numa_cpumask_lookup_table
[node
])) {
60 cpu_clear(cpu
, numa_cpumask_lookup_table
[node
]);
62 printk(KERN_ERR
"WARNING: cpu %lu not found in node %d\n",
66 #endif /* CONFIG_HOTPLUG_CPU */
68 static struct device_node
* __cpuinit
find_cpu_node(unsigned int cpu
)
70 unsigned int hw_cpuid
= get_hard_smp_processor_id(cpu
);
71 struct device_node
*cpu_node
= NULL
;
72 const unsigned int *interrupt_server
, *reg
;
75 while ((cpu_node
= of_find_node_by_type(cpu_node
, "cpu")) != NULL
) {
76 /* Try interrupt server first */
77 interrupt_server
= get_property(cpu_node
,
78 "ibm,ppc-interrupt-server#s", &len
);
80 len
= len
/ sizeof(u32
);
82 if (interrupt_server
&& (len
> 0)) {
84 if (interrupt_server
[len
] == hw_cpuid
)
88 reg
= get_property(cpu_node
, "reg", &len
);
89 if (reg
&& (len
> 0) && (reg
[0] == hw_cpuid
))
97 /* must hold reference to node during call */
98 static const int *of_get_associativity(struct device_node
*dev
)
100 return get_property(dev
, "ibm,associativity", NULL
);
103 /* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa
106 static int of_node_to_nid_single(struct device_node
*device
)
109 const unsigned int *tmp
;
111 if (min_common_depth
== -1)
114 tmp
= of_get_associativity(device
);
118 if (tmp
[0] >= min_common_depth
)
119 nid
= tmp
[min_common_depth
];
121 /* POWER4 LPAR uses 0xffff as invalid node */
122 if (nid
== 0xffff || nid
>= MAX_NUMNODES
)
128 /* Walk the device tree upwards, looking for an associativity id */
129 int of_node_to_nid(struct device_node
*device
)
131 struct device_node
*tmp
;
136 nid
= of_node_to_nid_single(device
);
141 device
= of_get_parent(tmp
);
148 EXPORT_SYMBOL_GPL(of_node_to_nid
);
151 * In theory, the "ibm,associativity" property may contain multiple
152 * associativity lists because a resource may be multiply connected
153 * into the machine. This resource then has different associativity
154 * characteristics relative to its multiple connections. We ignore
155 * this for now. We also assume that all cpu and memory sets have
156 * their distances represented at a common level. This won't be
157 * true for heirarchical NUMA.
159 * In any case the ibm,associativity-reference-points should give
160 * the correct depth for a normal NUMA system.
162 * - Dave Hansen <haveblue@us.ibm.com>
164 static int __init
find_min_common_depth(void)
167 const unsigned int *ref_points
;
168 struct device_node
*rtas_root
;
171 rtas_root
= of_find_node_by_path("/rtas");
177 * this property is 2 32-bit integers, each representing a level of
178 * depth in the associativity nodes. The first is for an SMP
179 * configuration (should be all 0's) and the second is for a normal
180 * NUMA configuration.
182 ref_points
= get_property(rtas_root
,
183 "ibm,associativity-reference-points", &len
);
185 if ((len
>= 1) && ref_points
) {
186 depth
= ref_points
[1];
188 dbg("NUMA: ibm,associativity-reference-points not found.\n");
191 of_node_put(rtas_root
);
196 static void __init
get_n_mem_cells(int *n_addr_cells
, int *n_size_cells
)
198 struct device_node
*memory
= NULL
;
200 memory
= of_find_node_by_type(memory
, "memory");
202 panic("numa.c: No memory nodes found!");
204 *n_addr_cells
= prom_n_addr_cells(memory
);
205 *n_size_cells
= prom_n_size_cells(memory
);
209 static unsigned long __devinit
read_n_cells(int n
, const unsigned int **buf
)
211 unsigned long result
= 0;
214 result
= (result
<< 32) | **buf
;
221 * Figure out to which domain a cpu belongs and stick it there.
222 * Return the id of the domain used.
224 static int __cpuinit
numa_setup_cpu(unsigned long lcpu
)
227 struct device_node
*cpu
= find_cpu_node(lcpu
);
234 nid
= of_node_to_nid_single(cpu
);
236 if (nid
< 0 || !node_online(nid
))
237 nid
= any_online_node(NODE_MASK_ALL
);
239 map_cpu_to_node(lcpu
, nid
);
246 static int __cpuinit
cpu_numa_callback(struct notifier_block
*nfb
,
247 unsigned long action
,
250 unsigned long lcpu
= (unsigned long)hcpu
;
251 int ret
= NOTIFY_DONE
;
255 numa_setup_cpu(lcpu
);
258 #ifdef CONFIG_HOTPLUG_CPU
260 case CPU_UP_CANCELED
:
261 unmap_cpu_from_node(lcpu
);
270 * Check and possibly modify a memory region to enforce the memory limit.
272 * Returns the size the region should have to enforce the memory limit.
273 * This will either be the original value of size, a truncated value,
274 * or zero. If the returned value of size is 0 the region should be
275 * discarded as it lies wholy above the memory limit.
277 static unsigned long __init
numa_enforce_memory_limit(unsigned long start
,
281 * We use lmb_end_of_DRAM() in here instead of memory_limit because
282 * we've already adjusted it for the limit and it takes care of
283 * having memory holes below the limit.
289 if (start
+ size
<= lmb_end_of_DRAM())
292 if (start
>= lmb_end_of_DRAM())
295 return lmb_end_of_DRAM() - start
;
298 static int __init
parse_numa_properties(void)
300 struct device_node
*cpu
= NULL
;
301 struct device_node
*memory
= NULL
;
305 if (numa_enabled
== 0) {
306 printk(KERN_WARNING
"NUMA disabled by user\n");
310 min_common_depth
= find_min_common_depth();
312 if (min_common_depth
< 0)
313 return min_common_depth
;
315 dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth
);
318 * Even though we connect cpus to numa domains later in SMP
319 * init, we need to know the node ids now. This is because
320 * each node to be onlined must have NODE_DATA etc backing it.
322 for_each_present_cpu(i
) {
325 cpu
= find_cpu_node(i
);
327 nid
= of_node_to_nid_single(cpu
);
331 * Don't fall back to default_nid yet -- we will plug
332 * cpus into nodes once the memory scan has discovered
337 node_set_online(nid
);
340 get_n_mem_cells(&n_mem_addr_cells
, &n_mem_size_cells
);
342 while ((memory
= of_find_node_by_type(memory
, "memory")) != NULL
) {
347 const unsigned int *memcell_buf
;
350 memcell_buf
= get_property(memory
,
351 "linux,usable-memory", &len
);
352 if (!memcell_buf
|| len
<= 0)
353 memcell_buf
= get_property(memory
, "reg", &len
);
354 if (!memcell_buf
|| len
<= 0)
358 ranges
= (len
>> 2) / (n_mem_addr_cells
+ n_mem_size_cells
);
360 /* these are order-sensitive, and modify the buffer pointer */
361 start
= read_n_cells(n_mem_addr_cells
, &memcell_buf
);
362 size
= read_n_cells(n_mem_size_cells
, &memcell_buf
);
365 * Assumption: either all memory nodes or none will
366 * have associativity properties. If none, then
367 * everything goes to default_nid.
369 nid
= of_node_to_nid_single(memory
);
372 node_set_online(nid
);
374 if (!(size
= numa_enforce_memory_limit(start
, size
))) {
381 add_active_range(nid
, start
>> PAGE_SHIFT
,
382 (start
>> PAGE_SHIFT
) + (size
>> PAGE_SHIFT
));
391 static void __init
setup_nonnuma(void)
393 unsigned long top_of_ram
= lmb_end_of_DRAM();
394 unsigned long total_ram
= lmb_phys_mem_size();
395 unsigned long start_pfn
, end_pfn
;
398 printk(KERN_DEBUG
"Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
399 top_of_ram
, total_ram
);
400 printk(KERN_DEBUG
"Memory hole size: %ldMB\n",
401 (top_of_ram
- total_ram
) >> 20);
403 for (i
= 0; i
< lmb
.memory
.cnt
; ++i
) {
404 start_pfn
= lmb
.memory
.region
[i
].base
>> PAGE_SHIFT
;
405 end_pfn
= start_pfn
+ lmb_size_pages(&lmb
.memory
, i
);
406 add_active_range(0, start_pfn
, end_pfn
);
411 void __init
dump_numa_cpu_topology(void)
414 unsigned int cpu
, count
;
416 if (min_common_depth
== -1 || !numa_enabled
)
419 for_each_online_node(node
) {
420 printk(KERN_DEBUG
"Node %d CPUs:", node
);
424 * If we used a CPU iterator here we would miss printing
425 * the holes in the cpumap.
427 for (cpu
= 0; cpu
< NR_CPUS
; cpu
++) {
428 if (cpu_isset(cpu
, numa_cpumask_lookup_table
[node
])) {
434 printk("-%u", cpu
- 1);
440 printk("-%u", NR_CPUS
- 1);
445 static void __init
dump_numa_memory_topology(void)
450 if (min_common_depth
== -1 || !numa_enabled
)
453 for_each_online_node(node
) {
456 printk(KERN_DEBUG
"Node %d Memory:", node
);
460 for (i
= 0; i
< lmb_end_of_DRAM();
461 i
+= (1 << SECTION_SIZE_BITS
)) {
462 if (early_pfn_to_nid(i
>> PAGE_SHIFT
) == node
) {
480 * Allocate some memory, satisfying the lmb or bootmem allocator where
481 * required. nid is the preferred node and end is the physical address of
482 * the highest address in the node.
484 * Returns the physical address of the memory.
486 static void __init
*careful_allocation(int nid
, unsigned long size
,
488 unsigned long end_pfn
)
491 unsigned long ret
= __lmb_alloc_base(size
, align
, end_pfn
<< PAGE_SHIFT
);
493 /* retry over all memory */
495 ret
= __lmb_alloc_base(size
, align
, lmb_end_of_DRAM());
498 panic("numa.c: cannot allocate %lu bytes on node %d",
502 * If the memory came from a previously allocated node, we must
503 * retry with the bootmem allocator.
505 new_nid
= early_pfn_to_nid(ret
>> PAGE_SHIFT
);
507 ret
= (unsigned long)__alloc_bootmem_node(NODE_DATA(new_nid
),
511 panic("numa.c: cannot allocate %lu bytes on node %d",
516 dbg("alloc_bootmem %lx %lx\n", ret
, size
);
522 static struct notifier_block __cpuinitdata ppc64_numa_nb
= {
523 .notifier_call
= cpu_numa_callback
,
524 .priority
= 1 /* Must run before sched domains notifier. */
527 void __init
do_init_bootmem(void)
533 max_low_pfn
= lmb_end_of_DRAM() >> PAGE_SHIFT
;
534 max_pfn
= max_low_pfn
;
536 if (parse_numa_properties())
539 dump_numa_memory_topology();
541 register_cpu_notifier(&ppc64_numa_nb
);
542 cpu_numa_callback(&ppc64_numa_nb
, CPU_UP_PREPARE
,
543 (void *)(unsigned long)boot_cpuid
);
545 for_each_online_node(nid
) {
546 unsigned long start_pfn
, end_pfn
;
547 unsigned long bootmem_paddr
;
548 unsigned long bootmap_pages
;
550 get_pfn_range_for_nid(nid
, &start_pfn
, &end_pfn
);
552 /* Allocate the node structure node local if possible */
553 NODE_DATA(nid
) = careful_allocation(nid
,
554 sizeof(struct pglist_data
),
555 SMP_CACHE_BYTES
, end_pfn
);
556 NODE_DATA(nid
) = __va(NODE_DATA(nid
));
557 memset(NODE_DATA(nid
), 0, sizeof(struct pglist_data
));
559 dbg("node %d\n", nid
);
560 dbg("NODE_DATA() = %p\n", NODE_DATA(nid
));
562 NODE_DATA(nid
)->bdata
= &plat_node_bdata
[nid
];
563 NODE_DATA(nid
)->node_start_pfn
= start_pfn
;
564 NODE_DATA(nid
)->node_spanned_pages
= end_pfn
- start_pfn
;
566 if (NODE_DATA(nid
)->node_spanned_pages
== 0)
569 dbg("start_paddr = %lx\n", start_pfn
<< PAGE_SHIFT
);
570 dbg("end_paddr = %lx\n", end_pfn
<< PAGE_SHIFT
);
572 bootmap_pages
= bootmem_bootmap_pages(end_pfn
- start_pfn
);
573 bootmem_paddr
= (unsigned long)careful_allocation(nid
,
574 bootmap_pages
<< PAGE_SHIFT
,
576 memset(__va(bootmem_paddr
), 0, bootmap_pages
<< PAGE_SHIFT
);
578 dbg("bootmap_paddr = %lx\n", bootmem_paddr
);
580 init_bootmem_node(NODE_DATA(nid
), bootmem_paddr
>> PAGE_SHIFT
,
583 free_bootmem_with_active_regions(nid
, end_pfn
);
585 /* Mark reserved regions on this node */
586 for (i
= 0; i
< lmb
.reserved
.cnt
; i
++) {
587 unsigned long physbase
= lmb
.reserved
.region
[i
].base
;
588 unsigned long size
= lmb
.reserved
.region
[i
].size
;
589 unsigned long start_paddr
= start_pfn
<< PAGE_SHIFT
;
590 unsigned long end_paddr
= end_pfn
<< PAGE_SHIFT
;
592 if (early_pfn_to_nid(physbase
>> PAGE_SHIFT
) != nid
&&
593 early_pfn_to_nid((physbase
+size
-1) >> PAGE_SHIFT
) != nid
)
596 if (physbase
< end_paddr
&&
597 (physbase
+size
) > start_paddr
) {
599 if (physbase
< start_paddr
) {
600 size
-= start_paddr
- physbase
;
601 physbase
= start_paddr
;
604 if (size
> end_paddr
- physbase
)
605 size
= end_paddr
- physbase
;
607 dbg("reserve_bootmem %lx %lx\n", physbase
,
609 reserve_bootmem_node(NODE_DATA(nid
), physbase
,
614 sparse_memory_present_with_active_regions(nid
);
618 void __init
paging_init(void)
620 unsigned long max_zone_pfns
[MAX_NR_ZONES
] = {
621 lmb_end_of_DRAM() >> PAGE_SHIFT
623 free_area_init_nodes(max_zone_pfns
);
626 static int __init
early_numa(char *p
)
631 if (strstr(p
, "off"))
634 if (strstr(p
, "debug"))
639 early_param("numa", early_numa
);
641 #ifdef CONFIG_MEMORY_HOTPLUG
643 * Find the node associated with a hot added memory section. Section
644 * corresponds to a SPARSEMEM section, not an LMB. It is assumed that
645 * sections are fully contained within a single LMB.
647 int hot_add_scn_to_nid(unsigned long scn_addr
)
649 struct device_node
*memory
= NULL
;
651 int default_nid
= any_online_node(NODE_MASK_ALL
);
654 if (!numa_enabled
|| (min_common_depth
< 0))
657 while ((memory
= of_find_node_by_type(memory
, "memory")) != NULL
) {
658 unsigned long start
, size
;
660 const unsigned int *memcell_buf
;
663 memcell_buf
= get_property(memory
, "reg", &len
);
664 if (!memcell_buf
|| len
<= 0)
668 ranges
= (len
>> 2) / (n_mem_addr_cells
+ n_mem_size_cells
);
670 start
= read_n_cells(n_mem_addr_cells
, &memcell_buf
);
671 size
= read_n_cells(n_mem_size_cells
, &memcell_buf
);
672 nid
= of_node_to_nid_single(memory
);
674 /* Domains not present at boot default to 0 */
675 if (nid
< 0 || !node_online(nid
))
678 if ((scn_addr
>= start
) && (scn_addr
< (start
+ size
))) {
683 if (--ranges
) /* process all ranges in cell */
686 BUG(); /* section address should be found above */
689 /* Temporary code to ensure that returned node is not empty */
692 while (NODE_DATA(nid
)->node_spanned_pages
== 0) {
693 node_clear(nid
, nodes
);
694 nid
= any_online_node(nodes
);
698 #endif /* CONFIG_MEMORY_HOTPLUG */