[mirror_ubuntu-jammy-kernel.git] / arch / x86 / mm / srat_64.c

/*
 * ACPI 3.0 based NUMA setup
 * Copyright 2004 Andi Kleen, SuSE Labs.
 *
 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
 *
 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
 * Assumes all memory regions belonging to a single proximity domain
 * are in one chunk. Holes between them will be included in the node.
 */

#include <linux/kernel.h>
#include <linux/acpi.h>
#include <linux/mmzone.h>
#include <linux/bitmap.h>
#include <linux/module.h>
#include <linux/topology.h>
#include <linux/bootmem.h>
#include <linux/mm.h>
#include <asm/proto.h>
#include <asm/numa.h>
#include <asm/e820.h>
#include <asm/apic.h>
#include <asm/uv/uv.h>

int acpi_numa __initdata;

static struct acpi_table_slit *acpi_slit;

static nodemask_t nodes_parsed __initdata;
static nodemask_t cpu_nodes_parsed __initdata;
static struct bootnode nodes[MAX_NUMNODES] __initdata;
static struct bootnode nodes_add[MAX_NUMNODES];

static int num_node_memblks __initdata;
static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;

static __init int setup_node(int pxm)
{
	return acpi_map_pxm_to_node(pxm);
}

static __init int conflicting_memblks(unsigned long start, unsigned long end)
{
	int i;
	for (i = 0; i < num_node_memblks; i++) {
		struct bootnode *nd = &node_memblk_range[i];
		if (nd->start == nd->end)
			continue;
		if (nd->end > start && nd->start < end)
			return memblk_nodeid[i];
		if (nd->end == end && nd->start == start)
			return memblk_nodeid[i];
	}
	return -1;
}

static __init void cutoff_node(int i, unsigned long start, unsigned long end)
{
	struct bootnode *nd = &nodes[i];

	if (nd->start < start) {
		nd->start = start;
		if (nd->end < nd->start)
			nd->start = nd->end;
	}
	if (nd->end > end) {
		nd->end = end;
		if (nd->start > nd->end)
			nd->start = nd->end;
	}
}

static __init void bad_srat(void)
{
	int i;
	printk(KERN_ERR "SRAT: SRAT not used.\n");
	acpi_numa = -1;
	for (i = 0; i < MAX_LOCAL_APIC; i++)
		apicid_to_node[i] = NUMA_NO_NODE;
	for (i = 0; i < MAX_NUMNODES; i++)
		nodes_add[i].start = nodes[i].end = 0;
	remove_all_active_ranges();
}

static __init inline int srat_disabled(void)
{
	return numa_off || acpi_numa < 0;
}

/* Callback for SLIT parsing */
void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
{
	unsigned length;
	unsigned long phys;

	length = slit->header.length;
	phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length,
		 PAGE_SIZE);

	if (phys == -1L)
		panic(" Can not save slit!\n");

	acpi_slit = __va(phys);
	memcpy(acpi_slit, slit, length);
	reserve_early(phys, phys + length, "ACPI SLIT");
}

/* Callback for Proximity Domain -> x2APIC mapping */
void __init
acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
{
	int pxm, node;
	int apic_id;

	if (srat_disabled())
		return;
	if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
		bad_srat();
		return;
	}
	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
		return;
	pxm = pa->proximity_domain;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
		bad_srat();
		return;
	}

	apic_id = pa->apic_id;
	apicid_to_node[apic_id] = node;
	node_set(node, cpu_nodes_parsed);
	acpi_numa = 1;
	printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n",
	       pxm, apic_id, node);
}

/* Callback for Proximity Domain -> LAPIC mapping */
void __init
acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
{
	int pxm, node;
	int apic_id;

	if (srat_disabled())
		return;
	if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
		bad_srat();
		return;
	}
	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
		return;
	pxm = pa->proximity_domain_lo;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
		bad_srat();
		return;
	}

	if (get_uv_system_type() >= UV_X2APIC)
		apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
	else
		apic_id = pa->apic_id;
	apicid_to_node[apic_id] = node;
	node_set(node, cpu_nodes_parsed);
	acpi_numa = 1;
	printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n",
	       pxm, apic_id, node);
}

#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
static inline int save_add_info(void) {return 1;}
#else
static inline int save_add_info(void) {return 0;}
#endif
/*
 * Update nodes_add[]
 * This code supports one contiguous hot add area per node
 */
static void __init
update_nodes_add(int node, unsigned long start, unsigned long end)
{
	unsigned long s_pfn = start >> PAGE_SHIFT;
	unsigned long e_pfn = end >> PAGE_SHIFT;
	int changed = 0;
	struct bootnode *nd = &nodes_add[node];

	/* I had some trouble with strange memory hotadd regions breaking
	   the boot. Be very strict here and reject anything unexpected.
	   If you want working memory hotadd write correct SRATs.

	   The node size check is a basic sanity check to guard against
	   mistakes */
	if ((signed long)(end - start) < NODE_MIN_SIZE) {
		printk(KERN_ERR "SRAT: Hotplug area too small\n");
		return;
	}

	/* This check might be a bit too strict, but I'm keeping it for now. */
	if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
		printk(KERN_ERR
			"SRAT: Hotplug area %lu -> %lu has existing memory\n",
			s_pfn, e_pfn);
		return;
	}

	/* Looks good */

	if (nd->start == nd->end) {
		nd->start = start;
		nd->end = end;
		changed = 1;
	} else {
		if (nd->start == end) {
			nd->start = start;
			changed = 1;
		}
		if (nd->end == start) {
			nd->end = end;
			changed = 1;
		}
		if (!changed)
			printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
	}

	if (changed)
		printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
				 nd->start, nd->end);
}

/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
void __init
acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
{
	struct bootnode *nd, oldnode;
	unsigned long start, end;
	int node, pxm;
	int i;

	if (srat_disabled())
		return;
	if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
		bad_srat();
		return;
	}
	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
		return;

	if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
		return;
	start = ma->base_address;
	end = start + ma->length;
	pxm = ma->proximity_domain;
	node = setup_node(pxm);
	if (node < 0) {
		printk(KERN_ERR "SRAT: Too many proximity domains.\n");
		bad_srat();
		return;
	}
	i = conflicting_memblks(start, end);
	if (i == node) {
		printk(KERN_WARNING
		"SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
			pxm, start, end, nodes[i].start, nodes[i].end);
	} else if (i >= 0) {
		printk(KERN_ERR
		       "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
		       pxm, start, end, node_to_pxm(i),
			nodes[i].start, nodes[i].end);
		bad_srat();
		return;
	}
	nd = &nodes[node];
	oldnode = *nd;
	if (!node_test_and_set(node, nodes_parsed)) {
		nd->start = start;
		nd->end = end;
	} else {
		if (start < nd->start)
			nd->start = start;
		if (nd->end < end)
			nd->end = end;
	}

	printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
	       start, end);
	e820_register_active_regions(node, start >> PAGE_SHIFT,
				     end >> PAGE_SHIFT);

	if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
		update_nodes_add(node, start, end);
		/* restore nodes[node] */
		*nd = oldnode;
		if ((nd->start | nd->end) == 0)
			node_clear(node, nodes_parsed);
	}

	node_memblk_range[num_node_memblks].start = start;
	node_memblk_range[num_node_memblks].end = end;
	memblk_nodeid[num_node_memblks] = node;
	num_node_memblks++;
}

/* Sanity check to catch more bad SRATs (they are amazingly common).
   Make sure the PXMs cover all memory. */
static int __init nodes_cover_memory(const struct bootnode *nodes)
{
	int i;
	unsigned long pxmram, e820ram;

	pxmram = 0;
	for_each_node_mask(i, nodes_parsed) {
		unsigned long s = nodes[i].start >> PAGE_SHIFT;
		unsigned long e = nodes[i].end >> PAGE_SHIFT;
		pxmram += e - s;
		pxmram -= absent_pages_in_range(s, e);
		if ((long)pxmram < 0)
			pxmram = 0;
	}

	e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
	/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
	if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
		printk(KERN_ERR
	"SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
			(pxmram << PAGE_SHIFT) >> 20,
			(e820ram << PAGE_SHIFT) >> 20);
		return 0;
	}
	return 1;
}

void __init acpi_numa_arch_fixup(void) {}

/* Use the information discovered above to actually set up the nodes. */
int __init acpi_scan_nodes(unsigned long start, unsigned long end)
{
	int i;

	if (acpi_numa <= 0)
		return -1;

	/* First clean up the node list */
	for (i = 0; i < MAX_NUMNODES; i++)
		cutoff_node(i, start, end);

	if (!nodes_cover_memory(nodes)) {
		bad_srat();
		return -1;
	}

	memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks,
					   memblk_nodeid);
	if (memnode_shift < 0) {
		printk(KERN_ERR
		     "SRAT: No NUMA node hash function found. Contact maintainer\n");
		bad_srat();
		return -1;
	}

	/* Account for nodes with cpus and no memory */
	nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed);

	/* Finally register nodes */
	for_each_node_mask(i, node_possible_map)
		setup_node_bootmem(i, nodes[i].start, nodes[i].end);
	/* Try again in case setup_node_bootmem missed one due
	   to missing bootmem */
	for_each_node_mask(i, node_possible_map)
		if (!node_online(i))
			setup_node_bootmem(i, nodes[i].start, nodes[i].end);

	for (i = 0; i < nr_cpu_ids; i++) {
		int node = early_cpu_to_node(i);

		if (node == NUMA_NO_NODE)
			continue;
		if (!node_online(node))
			numa_clear_node(i);
	}
	numa_init_array();
	return 0;
}

#ifdef CONFIG_NUMA_EMU
static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
	[0 ... MAX_NUMNODES-1] = PXM_INVAL
};
static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
};
static int __init find_node_by_addr(unsigned long addr)
{
	int ret = NUMA_NO_NODE;
	int i;

	for_each_node_mask(i, nodes_parsed) {
		/*
		 * Find the real node that this emulated node appears on.  For
		 * the sake of simplicity, we only use a real node's starting
		 * address to determine which emulated node it appears on.
		 */
		if (addr >= nodes[i].start && addr < nodes[i].end) {
			ret = i;
			break;
		}
	}
	return ret;
}

/*
 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
 * mappings that respect the real ACPI topology but reflect our emulated
 * environment.  For each emulated node, we find which real node it appears on
 * and create PXM to NID mappings for those fake nodes which mirror that
 * locality.  SLIT will now represent the correct distances between emulated
 * nodes as a result of the real topology.
 */
void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
{
	int i, j;

	printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
			 "topology.\n");
	for (i = 0; i < num_nodes; i++) {
		int nid, pxm;

		nid = find_node_by_addr(fake_nodes[i].start);
		if (nid == NUMA_NO_NODE)
			continue;
		pxm = node_to_pxm(nid);
		if (pxm == PXM_INVAL)
			continue;
		fake_node_to_pxm_map[i] = pxm;
		/*
		 * For each apicid_to_node mapping that exists for this real
		 * node, it must now point to the fake node ID.
		 */
		for (j = 0; j < MAX_LOCAL_APIC; j++)
			if (apicid_to_node[j] == nid)
				fake_apicid_to_node[j] = i;
	}
	for (i = 0; i < num_nodes; i++)
		__acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
	memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));

	nodes_clear(nodes_parsed);
	for (i = 0; i < num_nodes; i++)
		if (fake_nodes[i].start != fake_nodes[i].end)
			node_set(i, nodes_parsed);
	WARN_ON(!nodes_cover_memory(fake_nodes));
}

static int null_slit_node_compare(int a, int b)
{
	return node_to_pxm(a) == node_to_pxm(b);
}
#else
static int null_slit_node_compare(int a, int b)
{
	return a == b;
}
#endif /* CONFIG_NUMA_EMU */

int __node_distance(int a, int b)
{
	int index;

	if (!acpi_slit)
		return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
						      REMOTE_DISTANCE;
	index = acpi_slit->locality_count * node_to_pxm(a);
	return acpi_slit->entry[index + node_to_pxm(b)];
}

EXPORT_SYMBOL(__node_distance);

#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
int memory_add_physaddr_to_nid(u64 start)
{
	int i, ret = 0;

	for_each_node(i)
		if (nodes_add[i].start <= start && nodes_add[i].end > start)
			ret = i;

	return ret;
}
EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* ACPI 3.0 based NUMA setup
	3	* Copyright 2004 Andi Kleen, SuSE Labs.
	4	*
	5	* Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
	6	*
	7	* Called from acpi_numa_init while reading the SRAT and SLIT tables.
	8	* Assumes all memory regions belonging to a single proximity domain
	9	* are in one chunk. Holes between them will be included in the node.
	10	*/
	11
	12	#include <linux/kernel.h>
	13	#include <linux/acpi.h>
	14	#include <linux/mmzone.h>
	15	#include <linux/bitmap.h>
	16	#include <linux/module.h>
	17	#include <linux/topology.h>
68a3a7fe AK	18	#include <linux/bootmem.h>
68a3a7fe AK	19	#include <linux/mm.h>
1da177e4 LT	20	#include <asm/proto.h>
1da177e4 LT	21	#include <asm/numa.h>
8a6fdd3e	22	#include <asm/e820.h>
7b6aa335	23	#include <asm/apic.h>
4ec71fa2	24	#include <asm/uv/uv.h>
1da177e4	25
c31fbb1a AK	26	int acpi_numa __initdata;
c31fbb1a AK	27
1da177e4 LT	28	static struct acpi_table_slit *acpi_slit;
	29
	30	static nodemask_t nodes_parsed __initdata;
dc098551	31	static nodemask_t cpu_nodes_parsed __initdata;
abe059e7	32	static struct bootnode nodes[MAX_NUMNODES] __initdata;
4942e998	33	static struct bootnode nodes_add[MAX_NUMNODES];
1da177e4	34
6ec6e0d9 SS	35	static int num_node_memblks __initdata;
	36	static struct bootnode node_memblk_range[NR_NODE_MEMBLKS] __initdata;
	37	static int memblk_nodeid[NR_NODE_MEMBLKS] __initdata;
	38
1da177e4 LT	39	static __init int setup_node(int pxm)
1da177e4 LT	40	{
762834e8	41	return acpi_map_pxm_to_node(pxm);
1da177e4 LT	42	}
1da177e4 LT	43
6ec6e0d9	44	static __init int conflicting_memblks(unsigned long start, unsigned long end)
1da177e4 LT	45	{
1da177e4 LT	46	int i;
6ec6e0d9 SS	47	for (i = 0; i < num_node_memblks; i++) {
6ec6e0d9 SS	48	struct bootnode *nd = &node_memblk_range[i];
1da177e4 LT	49	if (nd->start == nd->end)
	50	continue;
	51	if (nd->end > start && nd->start < end)
6ec6e0d9	52	return memblk_nodeid[i];
1da177e4	53	if (nd->end == end && nd->start == start)
6ec6e0d9	54	return memblk_nodeid[i];
1da177e4 LT	55	}
	56	return -1;
	57	}
	58
	59	static __init void cutoff_node(int i, unsigned long start, unsigned long end)
	60	{
abe059e7	61	struct bootnode *nd = &nodes[i];
68a3a7fe	62
1da177e4 LT	63	if (nd->start < start) {
	64	nd->start = start;
	65	if (nd->end < nd->start)
	66	nd->start = nd->end;
	67	}
	68	if (nd->end > end) {
1da177e4 LT	69	nd->end = end;
	70	if (nd->start > nd->end)
	71	nd->start = nd->end;
	72	}
	73	}
	74
	75	static __init void bad_srat(void)
	76	{
2bce2b54	77	int i;
1da177e4 LT	78	printk(KERN_ERR "SRAT: SRAT not used.\n");
1da177e4 LT	79	acpi_numa = -1;
2bce2b54 AK	80	for (i = 0; i < MAX_LOCAL_APIC; i++)
2bce2b54 AK	81	apicid_to_node[i] = NUMA_NO_NODE;
68a3a7fe AK	82	for (i = 0; i < MAX_NUMNODES; i++)
68a3a7fe AK	83	nodes_add[i].start = nodes[i].end = 0;
5cb248ab	84	remove_all_active_ranges();
1da177e4 LT	85	}
	86
	87	static __init inline int srat_disabled(void)
	88	{
	89	return numa_off \|\| acpi_numa < 0;
	90	}
	91
	92	/* Callback for SLIT parsing */
	93	void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
	94	{
f302a5bb YL	95	unsigned length;
	96	unsigned long phys;
	97
	98	length = slit->header.length;
	99	phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length,
	100	PAGE_SIZE);
	101
	102	if (phys == -1L)
	103	panic(" Can not save slit!\n");
	104
	105	acpi_slit = __va(phys);
	106	memcpy(acpi_slit, slit, length);
	107	reserve_early(phys, phys + length, "ACPI SLIT");
1da177e4 LT	108	}
1da177e4 LT	109
7237d3de SS	110	/* Callback for Proximity Domain -> x2APIC mapping */
	111	void __init
	112	acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
	113	{
	114	int pxm, node;
	115	int apic_id;
	116
	117	if (srat_disabled())
	118	return;
	119	if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
	120	bad_srat();
	121	return;
	122	}
	123	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
	124	return;
	125	pxm = pa->proximity_domain;
	126	node = setup_node(pxm);
	127	if (node < 0) {
	128	printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
	129	bad_srat();
	130	return;
	131	}
	132
	133	apic_id = pa->apic_id;
	134	apicid_to_node[apic_id] = node;
dc098551	135	node_set(node, cpu_nodes_parsed);
7237d3de SS	136	acpi_numa = 1;
	137	printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n",
	138	pxm, apic_id, node);
	139	}
	140
1da177e4 LT	141	/* Callback for Proximity Domain -> LAPIC mapping */
1da177e4 LT	142	void __init
15a58ed1	143	acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
1da177e4 LT	144	{
1da177e4 LT	145	int pxm, node;
ef97001f	146	int apic_id;
ef97001f	147
d22fe808 AK	148	if (srat_disabled())
d22fe808 AK	149	return;
15a58ed1	150	if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
fad7906d	151	bad_srat();
d22fe808 AK	152	return;
d22fe808 AK	153	}
15a58ed1	154	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
1da177e4	155	return;
15a58ed1	156	pxm = pa->proximity_domain_lo;
1da177e4 LT	157	node = setup_node(pxm);
	158	if (node < 0) {
	159	printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
	160	bad_srat();
	161	return;
	162	}
beafe91f	163
2e42060c	164	if (get_uv_system_type() >= UV_X2APIC)
a65d1d64 JS	165	apic_id = (pa->apic_id << 8) \| pa->local_sapic_eid;
	166	else
	167	apic_id = pa->apic_id;
ef97001f	168	apicid_to_node[apic_id] = node;
dc098551	169	node_set(node, cpu_nodes_parsed);
1da177e4	170	acpi_numa = 1;
0b07e984	171	printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n",
ef97001f	172	pxm, apic_id, node);
1da177e4 LT	173	}
1da177e4 LT	174
71efa8fd KM	175	#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
	176	static inline int save_add_info(void) {return 1;}
	177	#else
	178	static inline int save_add_info(void) {return 0;}
	179	#endif
68a3a7fe	180	/*
888a589f YL	181	* Update nodes_add[]
888a589f YL	182	* This code supports one contiguous hot add area per node
68a3a7fe	183	*/
888a589f YL	184	static void __init
888a589f YL	185	update_nodes_add(int node, unsigned long start, unsigned long end)
68a3a7fe AK	186	{
	187	unsigned long s_pfn = start >> PAGE_SHIFT;
	188	unsigned long e_pfn = end >> PAGE_SHIFT;
888a589f	189	int changed = 0;
68a3a7fe AK	190	struct bootnode *nd = &nodes_add[node];
	191
	192	/* I had some trouble with strange memory hotadd regions breaking
	193	the boot. Be very strict here and reject anything unexpected.
	194	If you want working memory hotadd write correct SRATs.
	195
	196	The node size check is a basic sanity check to guard against
	197	mistakes */
	198	if ((signed long)(end - start) < NODE_MIN_SIZE) {
	199	printk(KERN_ERR "SRAT: Hotplug area too small\n");
888a589f	200	return;
68a3a7fe AK	201	}
	202
	203	/* This check might be a bit too strict, but I'm keeping it for now. */
5cb248ab	204	if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
9c7cd687 MG	205	printk(KERN_ERR
	206	"SRAT: Hotplug area %lu -> %lu has existing memory\n",
	207	s_pfn, e_pfn);
888a589f	208	return;
68a3a7fe AK	209	}
	210
	211	/* Looks good */
	212
68a3a7fe	213	if (nd->start == nd->end) {
15a58ed1 AS	214	nd->start = start;
15a58ed1 AS	215	nd->end = end;
68a3a7fe	216	changed = 1;
15a58ed1 AS	217	} else {
	218	if (nd->start == end) {
	219	nd->start = start;
68a3a7fe AK	220	changed = 1;
68a3a7fe AK	221	}
15a58ed1 AS	222	if (nd->end == start) {
15a58ed1 AS	223	nd->end = end;
68a3a7fe AK	224	changed = 1;
	225	}
	226	if (!changed)
	227	printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
15a58ed1	228	}
68a3a7fe	229
68a3a7fe	230	if (changed)
888a589f YL	231	printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
888a589f YL	232	nd->start, nd->end);
68a3a7fe	233	}
68a3a7fe	234
1da177e4 LT	235	/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
1da177e4 LT	236	void __init
15a58ed1	237	acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
1da177e4	238	{
68a3a7fe	239	struct bootnode *nd, oldnode;
1da177e4 LT	240	unsigned long start, end;
	241	int node, pxm;
	242	int i;
	243
d22fe808	244	if (srat_disabled())
1da177e4	245	return;
15a58ed1	246	if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
d22fe808 AK	247	bad_srat();
	248	return;
	249	}
15a58ed1	250	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
d22fe808	251	return;
15a58ed1 AS	252
15a58ed1 AS	253	if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
68a3a7fe	254	return;
15a58ed1 AS	255	start = ma->base_address;
15a58ed1 AS	256	end = start + ma->length;
1da177e4 LT	257	pxm = ma->proximity_domain;
	258	node = setup_node(pxm);
	259	if (node < 0) {
	260	printk(KERN_ERR "SRAT: Too many proximity domains.\n");
	261	bad_srat();
	262	return;
	263	}
6ec6e0d9	264	i = conflicting_memblks(start, end);
05d1fa4b AK	265	if (i == node) {
	266	printk(KERN_WARNING
	267	"SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
	268	pxm, start, end, nodes[i].start, nodes[i].end);
	269	} else if (i >= 0) {
1da177e4	270	printk(KERN_ERR
05d1fa4b AK	271	"SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
	272	pxm, start, end, node_to_pxm(i),
	273	nodes[i].start, nodes[i].end);
1da177e4 LT	274	bad_srat();
	275	return;
	276	}
	277	nd = &nodes[node];
68a3a7fe	278	oldnode = *nd;
1da177e4 LT	279	if (!node_test_and_set(node, nodes_parsed)) {
	280	nd->start = start;
	281	nd->end = end;
	282	} else {
	283	if (start < nd->start)
	284	nd->start = start;
	285	if (nd->end < end)
	286	nd->end = end;
	287	}
68a3a7fe	288
6ec6e0d9 SS	289	printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
	290	start, end);
	291	e820_register_active_regions(node, start >> PAGE_SHIFT,
	292	end >> PAGE_SHIFT);
68a3a7fe	293
888a589f YL	294	if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) {
	295	update_nodes_add(node, start, end);
	296	/* restore nodes[node] */
68a3a7fe AK	297	*nd = oldnode;
	298	if ((nd->start \| nd->end) == 0)
	299	node_clear(node, nodes_parsed);
	300	}
6ec6e0d9 SS	301
	302	node_memblk_range[num_node_memblks].start = start;
	303	node_memblk_range[num_node_memblks].end = end;
	304	memblk_nodeid[num_node_memblks] = node;
	305	num_node_memblks++;
1da177e4 LT	306	}
1da177e4 LT	307
8a6fdd3e AK	308	/* Sanity check to catch more bad SRATs (they are amazingly common).
8a6fdd3e AK	309	Make sure the PXMs cover all memory. */
3484d798	310	static int __init nodes_cover_memory(const struct bootnode *nodes)
8a6fdd3e AK	311	{
	312	int i;
	313	unsigned long pxmram, e820ram;
	314
	315	pxmram = 0;
	316	for_each_node_mask(i, nodes_parsed) {
	317	unsigned long s = nodes[i].start >> PAGE_SHIFT;
	318	unsigned long e = nodes[i].end >> PAGE_SHIFT;
	319	pxmram += e - s;
5cb248ab	320	pxmram -= absent_pages_in_range(s, e);
68a3a7fe AK	321	if ((long)pxmram < 0)
68a3a7fe AK	322	pxmram = 0;
8a6fdd3e AK	323	}
8a6fdd3e AK	324
b37ab919	325	e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
0964b056 YL	326	/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
0964b056 YL	327	if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
8a6fdd3e AK	328	printk(KERN_ERR
	329	"SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
	330	(pxmram << PAGE_SHIFT) >> 20,
	331	(e820ram << PAGE_SHIFT) >> 20);
	332	return 0;
	333	}
	334	return 1;
	335	}
	336
1da177e4 LT	337	void __init acpi_numa_arch_fixup(void) {}
	338
	339	/* Use the information discovered above to actually set up the nodes. */
	340	int __init acpi_scan_nodes(unsigned long start, unsigned long end)
	341	{
	342	int i;
8a6fdd3e	343
ae2c6dcf DR	344	if (acpi_numa <= 0)
	345	return -1;
	346
e58e0d03	347	/* First clean up the node list */
7c43769a	348	for (i = 0; i < MAX_NUMNODES; i++)
15a58ed1	349	cutoff_node(i, start, end);
e58e0d03	350
3484d798	351	if (!nodes_cover_memory(nodes)) {
8a6fdd3e AK	352	bad_srat();
	353	return -1;
	354	}
	355
6ec6e0d9 SS	356	memnode_shift = compute_hash_shift(node_memblk_range, num_node_memblks,
6ec6e0d9 SS	357	memblk_nodeid);
1da177e4 LT	358	if (memnode_shift < 0) {
	359	printk(KERN_ERR
	360	"SRAT: No NUMA node hash function found. Contact maintainer\n");
	361	bad_srat();
	362	return -1;
	363	}
e58e0d03	364
dc098551 JS	365	/* Account for nodes with cpus and no memory */
dc098551 JS	366	nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed);
e3f1caee	367
e58e0d03	368	/* Finally register nodes */
e3f1caee	369	for_each_node_mask(i, node_possible_map)
1da177e4	370	setup_node_bootmem(i, nodes[i].start, nodes[i].end);
a8062231 AK	371	/* Try again in case setup_node_bootmem missed one due
a8062231 AK	372	to missing bootmem */
e3f1caee	373	for_each_node_mask(i, node_possible_map)
a8062231 AK	374	if (!node_online(i))
	375	setup_node_bootmem(i, nodes[i].start, nodes[i].end);
	376
168ef543	377	for (i = 0; i < nr_cpu_ids; i++) {
0164fe16 MT	378	int node = early_cpu_to_node(i);
0164fe16 MT	379
834beda1	380	if (node == NUMA_NO_NODE)
1da177e4	381	continue;
7c43769a	382	if (!node_online(node))
23ca4bba	383	numa_clear_node(i);
1da177e4 LT	384	}
	385	numa_init_array();
	386	return 0;
	387	}
	388
3484d798	389	#ifdef CONFIG_NUMA_EMU
ef97001f	390	static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
	391	[0 ... MAX_NUMNODES-1] = PXM_INVAL
	392	};
602a54a8	393	static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
ef97001f	394	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
ef97001f	395	};
3484d798 DR	396	static int __init find_node_by_addr(unsigned long addr)
	397	{
	398	int ret = NUMA_NO_NODE;
	399	int i;
	400
	401	for_each_node_mask(i, nodes_parsed) {
	402	/*
	403	* Find the real node that this emulated node appears on. For
	404	* the sake of simplicity, we only use a real node's starting
	405	* address to determine which emulated node it appears on.
	406	*/
	407	if (addr >= nodes[i].start && addr < nodes[i].end) {
	408	ret = i;
	409	break;
	410	}
	411	}
9a1b62fe	412	return ret;
3484d798 DR	413	}
	414
	415	/*
	416	* In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
	417	* mappings that respect the real ACPI topology but reflect our emulated
	418	* environment. For each emulated node, we find which real node it appears on
	419	* and create PXM to NID mappings for those fake nodes which mirror that
	420	* locality. SLIT will now represent the correct distances between emulated
	421	* nodes as a result of the real topology.
	422	*/
	423	void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
	424	{
08705b89	425	int i, j;
3484d798 DR	426
	427	printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
	428	"topology.\n");
	429	for (i = 0; i < num_nodes; i++) {
	430	int nid, pxm;
	431
	432	nid = find_node_by_addr(fake_nodes[i].start);
	433	if (nid == NUMA_NO_NODE)
	434	continue;
	435	pxm = node_to_pxm(nid);
	436	if (pxm == PXM_INVAL)
	437	continue;
	438	fake_node_to_pxm_map[i] = pxm;
08705b89 DR	439	/*
	440	* For each apicid_to_node mapping that exists for this real
	441	* node, it must now point to the fake node ID.
	442	*/
	443	for (j = 0; j < MAX_LOCAL_APIC; j++)
	444	if (apicid_to_node[j] == nid)
	445	fake_apicid_to_node[j] = i;
3484d798 DR	446	}
	447	for (i = 0; i < num_nodes; i++)
	448	__acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
08705b89	449	memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));
3484d798 DR	450
	451	nodes_clear(nodes_parsed);
	452	for (i = 0; i < num_nodes; i++)
	453	if (fake_nodes[i].start != fake_nodes[i].end)
	454	node_set(i, nodes_parsed);
	455	WARN_ON(!nodes_cover_memory(fake_nodes));
	456	}
	457
	458	static int null_slit_node_compare(int a, int b)
	459	{
	460	return node_to_pxm(a) == node_to_pxm(b);
	461	}
	462	#else
	463	static int null_slit_node_compare(int a, int b)
	464	{
	465	return a == b;
	466	}
	467	#endif /* CONFIG_NUMA_EMU */
	468
1da177e4 LT	469	int __node_distance(int a, int b)
	470	{
	471	int index;
	472
	473	if (!acpi_slit)
3484d798 DR	474	return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
3484d798 DR	475	REMOTE_DISTANCE;
15a58ed1	476	index = acpi_slit->locality_count * node_to_pxm(a);
1da177e4 LT	477	return acpi_slit->entry[index + node_to_pxm(b)];
	478	}
	479
	480	EXPORT_SYMBOL(__node_distance);
4942e998	481
6a1673ae	482	#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) \|\| defined(CONFIG_ACPI_HOTPLUG_MEMORY)
4942e998 KM	483	int memory_add_physaddr_to_nid(u64 start)
	484	{
	485	int i, ret = 0;
	486
	487	for_each_node(i)
	488	if (nodes_add[i].start <= start && nodes_add[i].end > start)
	489	ret = i;
	490
	491	return ret;
	492	}
8c2676a5	493	EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
6a1673ae	494	#endif