1 // SPDX-License-Identifier: GPL-2.0-or-later
5 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
7 #define pr_fmt(fmt) "numa: " fmt
9 #include <linux/threads.h>
10 #include <linux/memblock.h>
11 #include <linux/init.h>
13 #include <linux/mmzone.h>
14 #include <linux/export.h>
15 #include <linux/nodemask.h>
16 #include <linux/cpu.h>
17 #include <linux/notifier.h>
19 #include <linux/pfn.h>
20 #include <linux/cpuset.h>
21 #include <linux/node.h>
22 #include <linux/stop_machine.h>
23 #include <linux/proc_fs.h>
24 #include <linux/seq_file.h>
25 #include <linux/uaccess.h>
26 #include <linux/slab.h>
27 #include <asm/cputhreads.h>
28 #include <asm/sparsemem.h>
31 #include <asm/topology.h>
32 #include <asm/firmware.h>
34 #include <asm/hvcall.h>
35 #include <asm/setup.h>
37 #include <asm/drmem.h>
39 static int numa_enabled
= 1;
41 static char *cmdline __initdata
;
43 static int numa_debug
;
44 #define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
46 int numa_cpu_lookup_table
[NR_CPUS
];
47 cpumask_var_t node_to_cpumask_map
[MAX_NUMNODES
];
48 struct pglist_data
*node_data
[MAX_NUMNODES
];
50 EXPORT_SYMBOL(numa_cpu_lookup_table
);
51 EXPORT_SYMBOL(node_to_cpumask_map
);
52 EXPORT_SYMBOL(node_data
);
54 static int primary_domain_index
;
55 static int n_mem_addr_cells
, n_mem_size_cells
;
57 #define FORM0_AFFINITY 0
58 #define FORM1_AFFINITY 1
59 static int affinity_form
;
61 #define MAX_DISTANCE_REF_POINTS 4
62 static int distance_ref_points_depth
;
63 static const __be32
*distance_ref_points
;
64 static int distance_lookup_table
[MAX_NUMNODES
][MAX_DISTANCE_REF_POINTS
];
67 * Allocate node_to_cpumask_map based on number of available nodes
68 * Requires node_possible_map to be valid.
70 * Note: cpumask_of_node() is not valid until after this is done.
72 static void __init
setup_node_to_cpumask_map(void)
76 /* setup nr_node_ids if not done yet */
77 if (nr_node_ids
== MAX_NUMNODES
)
80 /* allocate the map */
82 alloc_bootmem_cpumask_var(&node_to_cpumask_map
[node
]);
84 /* cpumask_of_node() will now work */
85 dbg("Node to cpumask map for %u nodes\n", nr_node_ids
);
88 static int __init
fake_numa_create_new_node(unsigned long end_pfn
,
91 unsigned long long mem
;
93 static unsigned int fake_nid
;
94 static unsigned long long curr_boundary
;
97 * Modify node id, iff we started creating NUMA nodes
98 * We want to continue from where we left of the last time
103 * In case there are no more arguments to parse, the
104 * node_id should be the same as the last fake node id
105 * (we've handled this above).
110 mem
= memparse(p
, &p
);
114 if (mem
< curr_boundary
)
119 if ((end_pfn
<< PAGE_SHIFT
) > mem
) {
121 * Skip commas and spaces
123 while (*p
== ',' || *p
== ' ' || *p
== '\t')
129 dbg("created new fake_node with id %d\n", fake_nid
);
135 static void reset_numa_cpu_lookup_table(void)
139 for_each_possible_cpu(cpu
)
140 numa_cpu_lookup_table
[cpu
] = -1;
143 static void map_cpu_to_node(int cpu
, int node
)
145 update_numa_cpu_lookup_table(cpu
, node
);
147 dbg("adding cpu %d to node %d\n", cpu
, node
);
149 if (!(cpumask_test_cpu(cpu
, node_to_cpumask_map
[node
])))
150 cpumask_set_cpu(cpu
, node_to_cpumask_map
[node
]);
153 #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR)
154 static void unmap_cpu_from_node(unsigned long cpu
)
156 int node
= numa_cpu_lookup_table
[cpu
];
158 dbg("removing cpu %lu from node %d\n", cpu
, node
);
160 if (cpumask_test_cpu(cpu
, node_to_cpumask_map
[node
])) {
161 cpumask_clear_cpu(cpu
, node_to_cpumask_map
[node
]);
163 printk(KERN_ERR
"WARNING: cpu %lu not found in node %d\n",
167 #endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */
169 static int __cpu_form1_relative_distance(__be32
*cpu1_assoc
, __be32
*cpu2_assoc
)
175 for (i
= 0; i
< distance_ref_points_depth
; i
++) {
176 index
= be32_to_cpu(distance_ref_points
[i
]);
177 if (cpu1_assoc
[index
] == cpu2_assoc
[index
])
185 int cpu_relative_distance(__be32
*cpu1_assoc
, __be32
*cpu2_assoc
)
187 /* We should not get called with FORM0 */
188 VM_WARN_ON(affinity_form
== FORM0_AFFINITY
);
190 return __cpu_form1_relative_distance(cpu1_assoc
, cpu2_assoc
);
193 /* must hold reference to node during call */
194 static const __be32
*of_get_associativity(struct device_node
*dev
)
196 return of_get_property(dev
, "ibm,associativity", NULL
);
199 int __node_distance(int a
, int b
)
202 int distance
= LOCAL_DISTANCE
;
204 if (affinity_form
== FORM0_AFFINITY
)
205 return ((a
== b
) ? LOCAL_DISTANCE
: REMOTE_DISTANCE
);
207 for (i
= 0; i
< distance_ref_points_depth
; i
++) {
208 if (distance_lookup_table
[a
][i
] == distance_lookup_table
[b
][i
])
211 /* Double the distance for each NUMA level */
217 EXPORT_SYMBOL(__node_distance
);
219 static int __associativity_to_nid(const __be32
*associativity
,
224 * primary_domain_index is 1 based array index.
226 int index
= primary_domain_index
- 1;
228 if (!numa_enabled
|| index
>= max_array_sz
)
231 nid
= of_read_number(&associativity
[index
], 1);
233 /* POWER4 LPAR uses 0xffff as invalid node */
234 if (nid
== 0xffff || nid
>= nr_node_ids
)
239 * Returns nid in the range [0..nr_node_ids], or -1 if no useful NUMA
242 static int associativity_to_nid(const __be32
*associativity
)
244 int array_sz
= of_read_number(associativity
, 1);
246 /* Skip the first element in the associativity array */
247 return __associativity_to_nid((associativity
+ 1), array_sz
);
250 /* Returns the nid associated with the given device tree node,
251 * or -1 if not found.
253 static int of_node_to_nid_single(struct device_node
*device
)
255 int nid
= NUMA_NO_NODE
;
258 tmp
= of_get_associativity(device
);
260 nid
= associativity_to_nid(tmp
);
264 /* Walk the device tree upwards, looking for an associativity id */
265 int of_node_to_nid(struct device_node
*device
)
267 int nid
= NUMA_NO_NODE
;
271 nid
= of_node_to_nid_single(device
);
275 device
= of_get_next_parent(device
);
281 EXPORT_SYMBOL(of_node_to_nid
);
283 static void __initialize_form1_numa_distance(const __be32
*associativity
,
288 if (affinity_form
!= FORM1_AFFINITY
)
291 nid
= __associativity_to_nid(associativity
, max_array_sz
);
292 if (nid
!= NUMA_NO_NODE
) {
293 for (i
= 0; i
< distance_ref_points_depth
; i
++) {
295 int index
= be32_to_cpu(distance_ref_points
[i
]) - 1;
298 * broken hierarchy, return with broken distance table
300 if (WARN(index
>= max_array_sz
, "Broken ibm,associativity property"))
303 entry
= &associativity
[index
];
304 distance_lookup_table
[nid
][i
] = of_read_number(entry
, 1);
309 static void initialize_form1_numa_distance(const __be32
*associativity
)
313 array_sz
= of_read_number(associativity
, 1);
314 /* Skip the first element in the associativity array */
315 __initialize_form1_numa_distance(associativity
+ 1, array_sz
);
319 * Used to update distance information w.r.t newly added node.
321 void update_numa_distance(struct device_node
*node
)
323 if (affinity_form
== FORM0_AFFINITY
)
325 else if (affinity_form
== FORM1_AFFINITY
) {
326 const __be32
*associativity
;
328 associativity
= of_get_associativity(node
);
332 initialize_form1_numa_distance(associativity
);
337 static int __init
find_primary_domain_index(void)
340 struct device_node
*root
;
343 * Check for which form of affinity.
345 if (firmware_has_feature(FW_FEATURE_OPAL
)) {
346 affinity_form
= FORM1_AFFINITY
;
347 } else if (firmware_has_feature(FW_FEATURE_FORM1_AFFINITY
)) {
348 dbg("Using form 1 affinity\n");
349 affinity_form
= FORM1_AFFINITY
;
351 affinity_form
= FORM0_AFFINITY
;
353 if (firmware_has_feature(FW_FEATURE_OPAL
))
354 root
= of_find_node_by_path("/ibm,opal");
356 root
= of_find_node_by_path("/rtas");
358 root
= of_find_node_by_path("/");
361 * This property is a set of 32-bit integers, each representing
362 * an index into the ibm,associativity nodes.
364 * With form 0 affinity the first integer is for an SMP configuration
365 * (should be all 0's) and the second is for a normal NUMA
366 * configuration. We have only one level of NUMA.
368 * With form 1 affinity the first integer is the most significant
369 * NUMA boundary and the following are progressively less significant
370 * boundaries. There can be more than one level of NUMA.
372 distance_ref_points
= of_get_property(root
,
373 "ibm,associativity-reference-points",
374 &distance_ref_points_depth
);
376 if (!distance_ref_points
) {
377 dbg("NUMA: ibm,associativity-reference-points not found.\n");
381 distance_ref_points_depth
/= sizeof(int);
382 if (affinity_form
== FORM0_AFFINITY
) {
383 if (distance_ref_points_depth
< 2) {
384 printk(KERN_WARNING
"NUMA: "
385 "short ibm,associativity-reference-points\n");
389 index
= of_read_number(&distance_ref_points
[1], 1);
391 index
= of_read_number(distance_ref_points
, 1);
395 * Warn and cap if the hardware supports more than
396 * MAX_DISTANCE_REF_POINTS domains.
398 if (distance_ref_points_depth
> MAX_DISTANCE_REF_POINTS
) {
399 printk(KERN_WARNING
"NUMA: distance array capped at "
400 "%d entries\n", MAX_DISTANCE_REF_POINTS
);
401 distance_ref_points_depth
= MAX_DISTANCE_REF_POINTS
;
412 static void __init
get_n_mem_cells(int *n_addr_cells
, int *n_size_cells
)
414 struct device_node
*memory
= NULL
;
416 memory
= of_find_node_by_type(memory
, "memory");
418 panic("numa.c: No memory nodes found!");
420 *n_addr_cells
= of_n_addr_cells(memory
);
421 *n_size_cells
= of_n_size_cells(memory
);
425 static unsigned long read_n_cells(int n
, const __be32
**buf
)
427 unsigned long result
= 0;
430 result
= (result
<< 32) | of_read_number(*buf
, 1);
436 struct assoc_arrays
{
439 const __be32
*arrays
;
443 * Retrieve and validate the list of associativity arrays for drconf
444 * memory from the ibm,associativity-lookup-arrays property of the
447 * The layout of the ibm,associativity-lookup-arrays property is a number N
448 * indicating the number of associativity arrays, followed by a number M
449 * indicating the size of each associativity array, followed by a list
450 * of N associativity arrays.
452 static int of_get_assoc_arrays(struct assoc_arrays
*aa
)
454 struct device_node
*memory
;
458 memory
= of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
462 prop
= of_get_property(memory
, "ibm,associativity-lookup-arrays", &len
);
463 if (!prop
|| len
< 2 * sizeof(unsigned int)) {
468 aa
->n_arrays
= of_read_number(prop
++, 1);
469 aa
->array_sz
= of_read_number(prop
++, 1);
473 /* Now that we know the number of arrays and size of each array,
474 * revalidate the size of the property read in.
476 if (len
< (aa
->n_arrays
* aa
->array_sz
+ 2) * sizeof(unsigned int))
483 static int get_nid_and_numa_distance(struct drmem_lmb
*lmb
)
485 struct assoc_arrays aa
= { .arrays
= NULL
};
486 int default_nid
= NUMA_NO_NODE
;
487 int nid
= default_nid
;
490 if ((primary_domain_index
< 0) || !numa_enabled
)
493 rc
= of_get_assoc_arrays(&aa
);
497 if (primary_domain_index
<= aa
.array_sz
&&
498 !(lmb
->flags
& DRCONF_MEM_AI_INVALID
) && lmb
->aa_index
< aa
.n_arrays
) {
499 const __be32
*associativity
;
501 index
= lmb
->aa_index
* aa
.array_sz
;
502 associativity
= &aa
.arrays
[index
];
503 nid
= __associativity_to_nid(associativity
, aa
.array_sz
);
504 if (nid
> 0 && affinity_form
== FORM1_AFFINITY
) {
506 * lookup array associativity entries have
507 * no length of the array as the first element.
509 __initialize_form1_numa_distance(associativity
, aa
.array_sz
);
516 * This is like of_node_to_nid_single() for memory represented in the
517 * ibm,dynamic-reconfiguration-memory node.
519 int of_drconf_to_nid_single(struct drmem_lmb
*lmb
)
521 struct assoc_arrays aa
= { .arrays
= NULL
};
522 int default_nid
= NUMA_NO_NODE
;
523 int nid
= default_nid
;
526 if ((primary_domain_index
< 0) || !numa_enabled
)
529 rc
= of_get_assoc_arrays(&aa
);
533 if (primary_domain_index
<= aa
.array_sz
&&
534 !(lmb
->flags
& DRCONF_MEM_AI_INVALID
) && lmb
->aa_index
< aa
.n_arrays
) {
535 const __be32
*associativity
;
537 index
= lmb
->aa_index
* aa
.array_sz
;
538 associativity
= &aa
.arrays
[index
];
539 nid
= __associativity_to_nid(associativity
, aa
.array_sz
);
544 #ifdef CONFIG_PPC_SPLPAR
546 static int __vphn_get_associativity(long lcpu
, __be32
*associativity
)
551 * On a shared lpar, device tree will not have node associativity.
552 * At this time lppaca, or its __old_status field may not be
553 * updated. Hence kernel cannot detect if its on a shared lpar. So
554 * request an explicit associativity irrespective of whether the
555 * lpar is shared or dedicated. Use the device tree property as a
556 * fallback. cpu_to_phys_id is only valid between
557 * smp_setup_cpu_maps() and smp_setup_pacas().
559 if (firmware_has_feature(FW_FEATURE_VPHN
)) {
561 hwid
= cpu_to_phys_id
[lcpu
];
563 hwid
= get_hard_smp_processor_id(lcpu
);
565 rc
= hcall_vphn(hwid
, VPHN_FLAG_VCPU
, associativity
);
573 static int vphn_get_nid(long lcpu
)
575 __be32 associativity
[VPHN_ASSOC_BUFSIZE
] = {0};
578 if (!__vphn_get_associativity(lcpu
, associativity
))
579 return associativity_to_nid(associativity
);
586 static int __vphn_get_associativity(long lcpu
, __be32
*associativity
)
591 static int vphn_get_nid(long unused
)
595 #endif /* CONFIG_PPC_SPLPAR */
598 * Figure out to which domain a cpu belongs and stick it there.
599 * Return the id of the domain used.
601 static int numa_setup_cpu(unsigned long lcpu
)
603 struct device_node
*cpu
;
604 int fcpu
= cpu_first_thread_sibling(lcpu
);
605 int nid
= NUMA_NO_NODE
;
607 if (!cpu_present(lcpu
)) {
608 set_cpu_numa_node(lcpu
, first_online_node
);
609 return first_online_node
;
613 * If a valid cpu-to-node mapping is already available, use it
614 * directly instead of querying the firmware, since it represents
615 * the most recent mapping notified to us by the platform (eg: VPHN).
616 * Since cpu_to_node binding remains the same for all threads in the
617 * core. If a valid cpu-to-node mapping is already available, for
618 * the first thread in the core, use it.
620 nid
= numa_cpu_lookup_table
[fcpu
];
622 map_cpu_to_node(lcpu
, nid
);
626 nid
= vphn_get_nid(lcpu
);
627 if (nid
!= NUMA_NO_NODE
)
630 cpu
= of_get_cpu_node(lcpu
, NULL
);
634 if (cpu_present(lcpu
))
640 nid
= of_node_to_nid_single(cpu
);
644 if (nid
< 0 || !node_possible(nid
))
645 nid
= first_online_node
;
648 * Update for the first thread of the core. All threads of a core
649 * have to be part of the same node. This not only avoids querying
650 * for every other thread in the core, but always avoids a case
651 * where virtual node associativity change causes subsequent threads
652 * of a core to be associated with different nid. However if first
653 * thread is already online, expect it to have a valid mapping.
656 WARN_ON(cpu_online(fcpu
));
657 map_cpu_to_node(fcpu
, nid
);
660 map_cpu_to_node(lcpu
, nid
);
665 static void verify_cpu_node_mapping(int cpu
, int node
)
667 int base
, sibling
, i
;
669 /* Verify that all the threads in the core belong to the same node */
670 base
= cpu_first_thread_sibling(cpu
);
672 for (i
= 0; i
< threads_per_core
; i
++) {
675 if (sibling
== cpu
|| cpu_is_offline(sibling
))
678 if (cpu_to_node(sibling
) != node
) {
679 WARN(1, "CPU thread siblings %d and %d don't belong"
680 " to the same node!\n", cpu
, sibling
);
686 /* Must run before sched domains notifier. */
687 static int ppc_numa_cpu_prepare(unsigned int cpu
)
691 nid
= numa_setup_cpu(cpu
);
692 verify_cpu_node_mapping(cpu
, nid
);
696 static int ppc_numa_cpu_dead(unsigned int cpu
)
698 #ifdef CONFIG_HOTPLUG_CPU
699 unmap_cpu_from_node(cpu
);
705 * Check and possibly modify a memory region to enforce the memory limit.
707 * Returns the size the region should have to enforce the memory limit.
708 * This will either be the original value of size, a truncated value,
709 * or zero. If the returned value of size is 0 the region should be
710 * discarded as it lies wholly above the memory limit.
712 static unsigned long __init
numa_enforce_memory_limit(unsigned long start
,
716 * We use memblock_end_of_DRAM() in here instead of memory_limit because
717 * we've already adjusted it for the limit and it takes care of
718 * having memory holes below the limit. Also, in the case of
719 * iommu_is_off, memory_limit is not set but is implicitly enforced.
722 if (start
+ size
<= memblock_end_of_DRAM())
725 if (start
>= memblock_end_of_DRAM())
728 return memblock_end_of_DRAM() - start
;
732 * Reads the counter for a given entry in
733 * linux,drconf-usable-memory property
735 static inline int __init
read_usm_ranges(const __be32
**usm
)
738 * For each lmb in ibm,dynamic-memory a corresponding
739 * entry in linux,drconf-usable-memory property contains
740 * a counter followed by that many (base, size) duple.
741 * read the counter from linux,drconf-usable-memory
743 return read_n_cells(n_mem_size_cells
, usm
);
747 * Extract NUMA information from the ibm,dynamic-reconfiguration-memory
748 * node. This assumes n_mem_{addr,size}_cells have been set.
750 static int __init
numa_setup_drmem_lmb(struct drmem_lmb
*lmb
,
754 unsigned int ranges
, is_kexec_kdump
= 0;
755 unsigned long base
, size
, sz
;
759 * Skip this block if the reserved bit is set in flags (0x80)
760 * or if the block is not assigned to this partition (0x8)
762 if ((lmb
->flags
& DRCONF_MEM_RESERVED
)
763 || !(lmb
->flags
& DRCONF_MEM_ASSIGNED
))
769 base
= lmb
->base_addr
;
770 size
= drmem_lmb_size();
773 if (is_kexec_kdump
) {
774 ranges
= read_usm_ranges(usm
);
775 if (!ranges
) /* there are no (base, size) duple */
780 if (is_kexec_kdump
) {
781 base
= read_n_cells(n_mem_addr_cells
, usm
);
782 size
= read_n_cells(n_mem_size_cells
, usm
);
785 nid
= get_nid_and_numa_distance(lmb
);
786 fake_numa_create_new_node(((base
+ size
) >> PAGE_SHIFT
),
788 node_set_online(nid
);
789 sz
= numa_enforce_memory_limit(base
, size
);
791 memblock_set_node(base
, sz
, &memblock
.memory
, nid
);
797 static int __init
parse_numa_properties(void)
799 struct device_node
*memory
;
802 const __be32
*associativity
;
804 if (numa_enabled
== 0) {
805 printk(KERN_WARNING
"NUMA disabled by user\n");
809 primary_domain_index
= find_primary_domain_index();
811 if (primary_domain_index
< 0) {
813 * if we fail to parse primary_domain_index from device tree
814 * mark the numa disabled, boot with numa disabled.
816 numa_enabled
= false;
817 return primary_domain_index
;
820 dbg("NUMA associativity depth for CPU/Memory: %d\n", primary_domain_index
);
823 * Even though we connect cpus to numa domains later in SMP
824 * init, we need to know the node ids now. This is because
825 * each node to be onlined must have NODE_DATA etc backing it.
827 for_each_present_cpu(i
) {
828 __be32 vphn_assoc
[VPHN_ASSOC_BUFSIZE
];
829 struct device_node
*cpu
;
830 int nid
= NUMA_NO_NODE
;
832 memset(vphn_assoc
, 0, VPHN_ASSOC_BUFSIZE
* sizeof(__be32
));
834 if (__vphn_get_associativity(i
, vphn_assoc
) == 0) {
835 nid
= associativity_to_nid(vphn_assoc
);
836 initialize_form1_numa_distance(vphn_assoc
);
840 * Don't fall back to default_nid yet -- we will plug
841 * cpus into nodes once the memory scan has discovered
844 cpu
= of_get_cpu_node(i
, NULL
);
847 associativity
= of_get_associativity(cpu
);
849 nid
= associativity_to_nid(associativity
);
850 initialize_form1_numa_distance(associativity
);
855 node_set_online(nid
);
858 get_n_mem_cells(&n_mem_addr_cells
, &n_mem_size_cells
);
860 for_each_node_by_type(memory
, "memory") {
865 const __be32
*memcell_buf
;
868 memcell_buf
= of_get_property(memory
,
869 "linux,usable-memory", &len
);
870 if (!memcell_buf
|| len
<= 0)
871 memcell_buf
= of_get_property(memory
, "reg", &len
);
872 if (!memcell_buf
|| len
<= 0)
876 ranges
= (len
>> 2) / (n_mem_addr_cells
+ n_mem_size_cells
);
878 /* these are order-sensitive, and modify the buffer pointer */
879 start
= read_n_cells(n_mem_addr_cells
, &memcell_buf
);
880 size
= read_n_cells(n_mem_size_cells
, &memcell_buf
);
883 * Assumption: either all memory nodes or none will
884 * have associativity properties. If none, then
885 * everything goes to default_nid.
887 associativity
= of_get_associativity(memory
);
889 nid
= associativity_to_nid(associativity
);
890 initialize_form1_numa_distance(associativity
);
894 fake_numa_create_new_node(((start
+ size
) >> PAGE_SHIFT
), &nid
);
895 node_set_online(nid
);
897 size
= numa_enforce_memory_limit(start
, size
);
899 memblock_set_node(start
, size
, &memblock
.memory
, nid
);
906 * Now do the same thing for each MEMBLOCK listed in the
907 * ibm,dynamic-memory property in the
908 * ibm,dynamic-reconfiguration-memory node.
910 memory
= of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
912 walk_drmem_lmbs(memory
, NULL
, numa_setup_drmem_lmb
);
919 static void __init
setup_nonnuma(void)
921 unsigned long top_of_ram
= memblock_end_of_DRAM();
922 unsigned long total_ram
= memblock_phys_mem_size();
923 unsigned long start_pfn
, end_pfn
;
924 unsigned int nid
= 0;
927 printk(KERN_DEBUG
"Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
928 top_of_ram
, total_ram
);
929 printk(KERN_DEBUG
"Memory hole size: %ldMB\n",
930 (top_of_ram
- total_ram
) >> 20);
932 for_each_mem_pfn_range(i
, MAX_NUMNODES
, &start_pfn
, &end_pfn
, NULL
) {
933 fake_numa_create_new_node(end_pfn
, &nid
);
934 memblock_set_node(PFN_PHYS(start_pfn
),
935 PFN_PHYS(end_pfn
- start_pfn
),
936 &memblock
.memory
, nid
);
937 node_set_online(nid
);
941 void __init
dump_numa_cpu_topology(void)
944 unsigned int cpu
, count
;
949 for_each_online_node(node
) {
950 pr_info("Node %d CPUs:", node
);
954 * If we used a CPU iterator here we would miss printing
955 * the holes in the cpumap.
957 for (cpu
= 0; cpu
< nr_cpu_ids
; cpu
++) {
958 if (cpumask_test_cpu(cpu
,
959 node_to_cpumask_map
[node
])) {
965 pr_cont("-%u", cpu
- 1);
971 pr_cont("-%u", nr_cpu_ids
- 1);
976 /* Initialize NODE_DATA for a node on the local memory */
977 static void __init
setup_node_data(int nid
, u64 start_pfn
, u64 end_pfn
)
979 u64 spanned_pages
= end_pfn
- start_pfn
;
980 const size_t nd_size
= roundup(sizeof(pg_data_t
), SMP_CACHE_BYTES
);
985 nd_pa
= memblock_phys_alloc_try_nid(nd_size
, SMP_CACHE_BYTES
, nid
);
987 panic("Cannot allocate %zu bytes for node %d data\n",
992 /* report and initialize */
993 pr_info(" NODE_DATA [mem %#010Lx-%#010Lx]\n",
994 nd_pa
, nd_pa
+ nd_size
- 1);
995 tnid
= early_pfn_to_nid(nd_pa
>> PAGE_SHIFT
);
997 pr_info(" NODE_DATA(%d) on node %d\n", nid
, tnid
);
1000 memset(NODE_DATA(nid
), 0, sizeof(pg_data_t
));
1001 NODE_DATA(nid
)->node_id
= nid
;
1002 NODE_DATA(nid
)->node_start_pfn
= start_pfn
;
1003 NODE_DATA(nid
)->node_spanned_pages
= spanned_pages
;
1006 static void __init
find_possible_nodes(void)
1008 struct device_node
*rtas
;
1009 const __be32
*domains
= NULL
;
1010 int prop_length
, max_nodes
;
1016 rtas
= of_find_node_by_path("/rtas");
1021 * ibm,current-associativity-domains is a fairly recent property. If
1022 * it doesn't exist, then fallback on ibm,max-associativity-domains.
1023 * Current denotes what the platform can support compared to max
1024 * which denotes what the Hypervisor can support.
1026 * If the LPAR is migratable, new nodes might be activated after a LPM,
1027 * so we should consider the max number in that case.
1029 if (!of_get_property(of_root
, "ibm,migratable-partition", NULL
))
1030 domains
= of_get_property(rtas
,
1031 "ibm,current-associativity-domains",
1034 domains
= of_get_property(rtas
, "ibm,max-associativity-domains",
1040 max_nodes
= of_read_number(&domains
[primary_domain_index
], 1);
1041 pr_info("Partition configured for %d NUMA nodes.\n", max_nodes
);
1043 for (i
= 0; i
< max_nodes
; i
++) {
1044 if (!node_possible(i
))
1045 node_set(i
, node_possible_map
);
1048 prop_length
/= sizeof(int);
1049 if (prop_length
> primary_domain_index
+ 2)
1050 coregroup_enabled
= 1;
1056 void __init
mem_topology_setup(void)
1061 * Linux/mm assumes node 0 to be online at boot. However this is not
1062 * true on PowerPC, where node 0 is similar to any other node, it
1063 * could be cpuless, memoryless node. So force node 0 to be offline
1064 * for now. This will prevent cpuless, memoryless node 0 showing up
1065 * unnecessarily as online. If a node has cpus or memory that need
1066 * to be online, then node will anyway be marked online.
1068 node_set_offline(0);
1070 if (parse_numa_properties())
1074 * Modify the set of possible NUMA nodes to reflect information
1075 * available about the set of online nodes, and the set of nodes
1076 * that we expect to make use of for this platform's affinity
1079 nodes_and(node_possible_map
, node_possible_map
, node_online_map
);
1081 find_possible_nodes();
1083 setup_node_to_cpumask_map();
1085 reset_numa_cpu_lookup_table();
1087 for_each_possible_cpu(cpu
) {
1089 * Powerpc with CONFIG_NUMA always used to have a node 0,
1090 * even if it was memoryless or cpuless. For all cpus that
1091 * are possible but not present, cpu_to_node() would point
1092 * to node 0. To remove a cpuless, memoryless dummy node,
1093 * powerpc need to make sure all possible but not present
1094 * cpu_to_node are set to a proper node.
1096 numa_setup_cpu(cpu
);
1100 void __init
initmem_init(void)
1104 max_low_pfn
= memblock_end_of_DRAM() >> PAGE_SHIFT
;
1105 max_pfn
= max_low_pfn
;
1107 memblock_dump_all();
1109 for_each_online_node(nid
) {
1110 unsigned long start_pfn
, end_pfn
;
1112 get_pfn_range_for_nid(nid
, &start_pfn
, &end_pfn
);
1113 setup_node_data(nid
, start_pfn
, end_pfn
);
1119 * We need the numa_cpu_lookup_table to be accurate for all CPUs,
1120 * even before we online them, so that we can use cpu_to_{node,mem}
1121 * early in boot, cf. smp_prepare_cpus().
1122 * _nocalls() + manual invocation is used because cpuhp is not yet
1123 * initialized for the boot CPU.
1125 cpuhp_setup_state_nocalls(CPUHP_POWER_NUMA_PREPARE
, "powerpc/numa:prepare",
1126 ppc_numa_cpu_prepare
, ppc_numa_cpu_dead
);
1129 static int __init
early_numa(char *p
)
1134 if (strstr(p
, "off"))
1137 if (strstr(p
, "debug"))
1140 p
= strstr(p
, "fake=");
1142 cmdline
= p
+ strlen("fake=");
1146 early_param("numa", early_numa
);
1148 #ifdef CONFIG_MEMORY_HOTPLUG
1150 * Find the node associated with a hot added memory section for
1151 * memory represented in the device tree by the property
1152 * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory.
1154 static int hot_add_drconf_scn_to_nid(unsigned long scn_addr
)
1156 struct drmem_lmb
*lmb
;
1157 unsigned long lmb_size
;
1158 int nid
= NUMA_NO_NODE
;
1160 lmb_size
= drmem_lmb_size();
1162 for_each_drmem_lmb(lmb
) {
1163 /* skip this block if it is reserved or not assigned to
1165 if ((lmb
->flags
& DRCONF_MEM_RESERVED
)
1166 || !(lmb
->flags
& DRCONF_MEM_ASSIGNED
))
1169 if ((scn_addr
< lmb
->base_addr
)
1170 || (scn_addr
>= (lmb
->base_addr
+ lmb_size
)))
1173 nid
= of_drconf_to_nid_single(lmb
);
1181 * Find the node associated with a hot added memory section for memory
1182 * represented in the device tree as a node (i.e. memory@XXXX) for
1185 static int hot_add_node_scn_to_nid(unsigned long scn_addr
)
1187 struct device_node
*memory
;
1188 int nid
= NUMA_NO_NODE
;
1190 for_each_node_by_type(memory
, "memory") {
1191 unsigned long start
, size
;
1193 const __be32
*memcell_buf
;
1196 memcell_buf
= of_get_property(memory
, "reg", &len
);
1197 if (!memcell_buf
|| len
<= 0)
1200 /* ranges in cell */
1201 ranges
= (len
>> 2) / (n_mem_addr_cells
+ n_mem_size_cells
);
1204 start
= read_n_cells(n_mem_addr_cells
, &memcell_buf
);
1205 size
= read_n_cells(n_mem_size_cells
, &memcell_buf
);
1207 if ((scn_addr
< start
) || (scn_addr
>= (start
+ size
)))
1210 nid
= of_node_to_nid_single(memory
);
1218 of_node_put(memory
);
1224 * Find the node associated with a hot added memory section. Section
1225 * corresponds to a SPARSEMEM section, not an MEMBLOCK. It is assumed that
1226 * sections are fully contained within a single MEMBLOCK.
1228 int hot_add_scn_to_nid(unsigned long scn_addr
)
1230 struct device_node
*memory
= NULL
;
1234 return first_online_node
;
1236 memory
= of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1238 nid
= hot_add_drconf_scn_to_nid(scn_addr
);
1239 of_node_put(memory
);
1241 nid
= hot_add_node_scn_to_nid(scn_addr
);
1244 if (nid
< 0 || !node_possible(nid
))
1245 nid
= first_online_node
;
1250 static u64
hot_add_drconf_memory_max(void)
1252 struct device_node
*memory
= NULL
;
1253 struct device_node
*dn
= NULL
;
1254 const __be64
*lrdr
= NULL
;
1256 dn
= of_find_node_by_path("/rtas");
1258 lrdr
= of_get_property(dn
, "ibm,lrdr-capacity", NULL
);
1261 return be64_to_cpup(lrdr
);
1264 memory
= of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
1266 of_node_put(memory
);
1267 return drmem_lmb_memory_max();
1273 * memory_hotplug_max - return max address of memory that may be added
1275 * This is currently only used on systems that support drconfig memory
1278 u64
memory_hotplug_max(void)
1280 return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM());
1282 #endif /* CONFIG_MEMORY_HOTPLUG */
1284 /* Virtual Processor Home Node (VPHN) support */
1285 #ifdef CONFIG_PPC_SPLPAR
1286 static int topology_inited
;
1289 * Retrieve the new associativity information for a virtual processor's
1292 static long vphn_get_associativity(unsigned long cpu
,
1293 __be32
*associativity
)
1297 rc
= hcall_vphn(get_hard_smp_processor_id(cpu
),
1298 VPHN_FLAG_VCPU
, associativity
);
1302 dbg("VPHN hcall succeeded. Reset polling...\n");
1306 pr_err_ratelimited("VPHN unsupported. Disabling polling...\n");
1309 pr_err_ratelimited("hcall_vphn() experienced a hardware fault "
1310 "preventing VPHN. Disabling polling...\n");
1313 pr_err_ratelimited("hcall_vphn() was passed an invalid parameter. "
1314 "Disabling polling...\n");
1317 pr_err_ratelimited("hcall_vphn() returned %ld. Disabling polling...\n"
1325 int find_and_online_cpu_nid(int cpu
)
1327 __be32 associativity
[VPHN_ASSOC_BUFSIZE
] = {0};
1330 /* Use associativity from first thread for all siblings */
1331 if (vphn_get_associativity(cpu
, associativity
))
1332 return cpu_to_node(cpu
);
1334 new_nid
= associativity_to_nid(associativity
);
1335 if (new_nid
< 0 || !node_possible(new_nid
))
1336 new_nid
= first_online_node
;
1338 if (NODE_DATA(new_nid
) == NULL
) {
1339 #ifdef CONFIG_MEMORY_HOTPLUG
1341 * Need to ensure that NODE_DATA is initialized for a node from
1342 * available memory (see memblock_alloc_try_nid). If unable to
1343 * init the node, then default to nearest node that has memory
1344 * installed. Skip onlining a node if the subsystems are not
1347 if (!topology_inited
|| try_online_node(new_nid
))
1348 new_nid
= first_online_node
;
1351 * Default to using the nearest node that has memory installed.
1352 * Otherwise, it would be necessary to patch the kernel MM code
1353 * to deal with more memoryless-node error conditions.
1355 new_nid
= first_online_node
;
1359 pr_debug("%s:%d cpu %d nid %d\n", __FUNCTION__
, __LINE__
,
1364 int cpu_to_coregroup_id(int cpu
)
1366 __be32 associativity
[VPHN_ASSOC_BUFSIZE
] = {0};
1369 if (cpu
< 0 || cpu
> nr_cpu_ids
)
1372 if (!coregroup_enabled
)
1375 if (!firmware_has_feature(FW_FEATURE_VPHN
))
1378 if (vphn_get_associativity(cpu
, associativity
))
1381 index
= of_read_number(associativity
, 1);
1382 if (index
> primary_domain_index
+ 1)
1383 return of_read_number(&associativity
[index
- 1], 1);
1386 return cpu_to_core_id(cpu
);
1389 static int topology_update_init(void)
1391 topology_inited
= 1;
1394 device_initcall(topology_update_init
);
1395 #endif /* CONFIG_PPC_SPLPAR */