[mirror_ubuntu-bionic-kernel.git] / arch / x86 / kernel / setup_percpu.c

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/percpu.h>
#include <linux/kexec.h>
#include <linux/crash_dump.h>
#include <linux/smp.h>
#include <linux/topology.h>
#include <linux/pfn.h>
#include <asm/sections.h>
#include <asm/processor.h>
#include <asm/setup.h>
#include <asm/mpspec.h>
#include <asm/apicdef.h>
#include <asm/highmem.h>
#include <asm/proto.h>
#include <asm/cpumask.h>
#include <asm/cpu.h>
#include <asm/stackprotector.h>

#ifdef CONFIG_DEBUG_PER_CPU_MAPS
# define DBG(x...) printk(KERN_DEBUG x)
#else
# define DBG(x...)
#endif

DEFINE_PER_CPU(int, cpu_number);
EXPORT_PER_CPU_SYMBOL(cpu_number);

#ifdef CONFIG_X86_64
#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
#else
#define BOOT_PERCPU_OFFSET 0
#endif

DEFINE_PER_CPU(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
EXPORT_PER_CPU_SYMBOL(this_cpu_off);

unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
	[0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
};
EXPORT_SYMBOL(__per_cpu_offset);

/*
 * On x86_64 symbols referenced from code should be reachable using
 * 32bit relocations.  Reserve space for static percpu variables in
 * modules so that they are always served from the first chunk which
 * is located at the percpu segment base.  On x86_32, anything can
 * address anywhere.  No need to reserve space in the first chunk.
 */
#ifdef CONFIG_X86_64
#define PERCPU_FIRST_CHUNK_RESERVE	PERCPU_MODULE_RESERVE
#else
#define PERCPU_FIRST_CHUNK_RESERVE	0
#endif

/**
 * pcpu_need_numa - determine percpu allocation needs to consider NUMA
 *
 * If NUMA is not configured or there is only one NUMA node available,
 * there is no reason to consider NUMA.  This function determines
 * whether percpu allocation should consider NUMA or not.
 *
 * RETURNS:
 * true if NUMA should be considered; otherwise, false.
 */
static bool __init pcpu_need_numa(void)
{
#ifdef CONFIG_NEED_MULTIPLE_NODES
	pg_data_t *last = NULL;
	unsigned int cpu;

	for_each_possible_cpu(cpu) {
		int node = early_cpu_to_node(cpu);

		if (node_online(node) && NODE_DATA(node) &&
		    last && last != NODE_DATA(node))
			return true;

		last = NODE_DATA(node);
	}
#endif
	return false;
}

/**
 * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu
 * @cpu: cpu to allocate for
 * @size: size allocation in bytes
 * @align: alignment
 *
 * Allocate @size bytes aligned at @align for cpu @cpu.  This wrapper
 * does the right thing for NUMA regardless of the current
 * configuration.
 *
 * RETURNS:
 * Pointer to the allocated area on success, NULL on failure.
 */
static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size,
					unsigned long align)
{
	const unsigned long goal = __pa(MAX_DMA_ADDRESS);
#ifdef CONFIG_NEED_MULTIPLE_NODES
	int node = early_cpu_to_node(cpu);
	void *ptr;

	if (!node_online(node) || !NODE_DATA(node)) {
		ptr = __alloc_bootmem_nopanic(size, align, goal);
		pr_info("cpu %d has no node %d or node-local memory\n",
			cpu, node);
		pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",
			 cpu, size, __pa(ptr));
	} else {
		ptr = __alloc_bootmem_node_nopanic(NODE_DATA(node),
						   size, align, goal);
		pr_debug("per cpu data for cpu%d %lu bytes on node%d at "
			 "%016lx\n", cpu, size, node, __pa(ptr));
	}
	return ptr;
#else
	return __alloc_bootmem_nopanic(size, align, goal);
#endif
}

/*
 * Remap allocator
 *
 * This allocator uses PMD page as unit.  A PMD page is allocated for
 * each cpu and each is remapped into vmalloc area using PMD mapping.
 * As PMD page is quite large, only part of it is used for the first
 * chunk.  Unused part is returned to the bootmem allocator.
 *
 * So, the PMD pages are mapped twice - once to the physical mapping
 * and to the vmalloc area for the first percpu chunk.  The double
 * mapping does add one more PMD TLB entry pressure but still is much
 * better than only using 4k mappings while still being NUMA friendly.
 */
#ifdef CONFIG_NEED_MULTIPLE_NODES
static size_t pcpur_size __initdata;
static void **pcpur_ptrs __initdata;

static struct page * __init pcpur_get_page(unsigned int cpu, int pageno)
{
	size_t off = (size_t)pageno << PAGE_SHIFT;

	if (off >= pcpur_size)
		return NULL;

	return virt_to_page(pcpur_ptrs[cpu] + off);
}

static ssize_t __init setup_pcpu_remap(size_t static_size)
{
	static struct vm_struct vm;
	size_t ptrs_size, dyn_size;
	unsigned int cpu;
	ssize_t ret;

	/*
	 * If large page isn't supported, there's no benefit in doing
	 * this.  Also, on non-NUMA, embedding is better.
	 *
	 * NOTE: disabled for now.
	 */
	if (true || !cpu_has_pse || !pcpu_need_numa())
		return -EINVAL;

	/*
	 * Currently supports only single page.  Supporting multiple
	 * pages won't be too difficult if it ever becomes necessary.
	 */
	pcpur_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +
			       PERCPU_DYNAMIC_RESERVE);
	if (pcpur_size > PMD_SIZE) {
		pr_warning("PERCPU: static data is larger than large page, "
			   "can't use large page\n");
		return -EINVAL;
	}
	dyn_size = pcpur_size - static_size - PERCPU_FIRST_CHUNK_RESERVE;

	/* allocate pointer array and alloc large pages */
	ptrs_size = PFN_ALIGN(num_possible_cpus() * sizeof(pcpur_ptrs[0]));
	pcpur_ptrs = alloc_bootmem(ptrs_size);

	for_each_possible_cpu(cpu) {
		pcpur_ptrs[cpu] = pcpu_alloc_bootmem(cpu, PMD_SIZE, PMD_SIZE);
		if (!pcpur_ptrs[cpu])
			goto enomem;

		/*
		 * Only use pcpur_size bytes and give back the rest.
		 *
		 * Ingo: The 2MB up-rounding bootmem is needed to make
		 * sure the partial 2MB page is still fully RAM - it's
		 * not well-specified to have a PAT-incompatible area
		 * (unmapped RAM, device memory, etc.) in that hole.
		 */
		free_bootmem(__pa(pcpur_ptrs[cpu] + pcpur_size),
			     PMD_SIZE - pcpur_size);

		memcpy(pcpur_ptrs[cpu], __per_cpu_load, static_size);
	}

	/* allocate address and map */
	vm.flags = VM_ALLOC;
	vm.size = num_possible_cpus() * PMD_SIZE;
	vm_area_register_early(&vm, PMD_SIZE);

	for_each_possible_cpu(cpu) {
		pmd_t *pmd;

		pmd = populate_extra_pmd((unsigned long)vm.addr
					 + cpu * PMD_SIZE);
		set_pmd(pmd, pfn_pmd(page_to_pfn(virt_to_page(pcpur_ptrs[cpu])),
				     PAGE_KERNEL_LARGE));
	}

	/* we're ready, commit */
	pr_info("PERCPU: Remapped at %p with large pages, static data "
		"%zu bytes\n", vm.addr, static_size);

	ret = pcpu_setup_first_chunk(pcpur_get_page, static_size,
				     PERCPU_FIRST_CHUNK_RESERVE, dyn_size,
				     PMD_SIZE, vm.addr, NULL);
	goto out_free_ar;

enomem:
	for_each_possible_cpu(cpu)
		if (pcpur_ptrs[cpu])
			free_bootmem(__pa(pcpur_ptrs[cpu]), pcpur_size);
	ret = -ENOMEM;
out_free_ar:
	free_bootmem(__pa(pcpur_ptrs), ptrs_size);
	return ret;
}
#else
static ssize_t __init setup_pcpu_remap(size_t static_size)
{
	return -EINVAL;
}
#endif

/*
 * Embedding allocator
 *
 * The first chunk is sized to just contain the static area plus
 * module and dynamic reserves and embedded into linear physical
 * mapping so that it can use PMD mapping without additional TLB
 * pressure.
 */
static ssize_t __init setup_pcpu_embed(size_t static_size)
{
	size_t reserve = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;

	/*
	 * If large page isn't supported, there's no benefit in doing
	 * this.  Also, embedding allocation doesn't play well with
	 * NUMA.
	 */
	if (!cpu_has_pse || pcpu_need_numa())
		return -EINVAL;

	return pcpu_embed_first_chunk(static_size, PERCPU_FIRST_CHUNK_RESERVE,
				      reserve - PERCPU_FIRST_CHUNK_RESERVE, -1);
}

/*
 * 4k page allocator
 *
 * This is the basic allocator.  Static percpu area is allocated
 * page-by-page and most of initialization is done by the generic
 * setup function.
 */
static struct page **pcpu4k_pages __initdata;
static int pcpu4k_nr_static_pages __initdata;

static struct page * __init pcpu4k_get_page(unsigned int cpu, int pageno)
{
	if (pageno < pcpu4k_nr_static_pages)
		return pcpu4k_pages[cpu * pcpu4k_nr_static_pages + pageno];
	return NULL;
}

static void __init pcpu4k_populate_pte(unsigned long addr)
{
	populate_extra_pte(addr);
}

static ssize_t __init setup_pcpu_4k(size_t static_size)
{
	size_t pages_size;
	unsigned int cpu;
	int i, j;
	ssize_t ret;

	pcpu4k_nr_static_pages = PFN_UP(static_size);

	/* unaligned allocations can't be freed, round up to page size */
	pages_size = PFN_ALIGN(pcpu4k_nr_static_pages * num_possible_cpus()
			       * sizeof(pcpu4k_pages[0]));
	pcpu4k_pages = alloc_bootmem(pages_size);

	/* allocate and copy */
	j = 0;
	for_each_possible_cpu(cpu)
		for (i = 0; i < pcpu4k_nr_static_pages; i++) {
			void *ptr;

			ptr = pcpu_alloc_bootmem(cpu, PAGE_SIZE, PAGE_SIZE);
			if (!ptr)
				goto enomem;

			memcpy(ptr, __per_cpu_load + i * PAGE_SIZE, PAGE_SIZE);
			pcpu4k_pages[j++] = virt_to_page(ptr);
		}

	/* we're ready, commit */
	pr_info("PERCPU: Allocated %d 4k pages, static data %zu bytes\n",
		pcpu4k_nr_static_pages, static_size);

	ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size,
				     PERCPU_FIRST_CHUNK_RESERVE, -1,
				     -1, NULL, pcpu4k_populate_pte);
	goto out_free_ar;

enomem:
	while (--j >= 0)
		free_bootmem(__pa(page_address(pcpu4k_pages[j])), PAGE_SIZE);
	ret = -ENOMEM;
out_free_ar:
	free_bootmem(__pa(pcpu4k_pages), pages_size);
	return ret;
}

static inline void setup_percpu_segment(int cpu)
{
#ifdef CONFIG_X86_32
	struct desc_struct gdt;

	pack_descriptor(&gdt, per_cpu_offset(cpu), 0xFFFFF,
			0x2 | DESCTYPE_S, 0x8);
	gdt.s = 1;
	write_gdt_entry(get_cpu_gdt_table(cpu),
			GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
#endif
}

/*
 * Great future plan:
 * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
 * Always point %gs to its beginning
 */
void __init setup_per_cpu_areas(void)
{
	size_t static_size = __per_cpu_end - __per_cpu_start;
	unsigned int cpu;
	unsigned long delta;
	size_t pcpu_unit_size;
	ssize_t ret;

	pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
		NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);

	/*
	 * Allocate percpu area.  If PSE is supported, try to make use
	 * of large page mappings.  Please read comments on top of
	 * each allocator for details.
	 */
	ret = setup_pcpu_remap(static_size);
	if (ret < 0)
		ret = setup_pcpu_embed(static_size);
	if (ret < 0)
		ret = setup_pcpu_4k(static_size);
	if (ret < 0)
		panic("cannot allocate static percpu area (%zu bytes, err=%zd)",
		      static_size, ret);

	pcpu_unit_size = ret;

	/* alrighty, percpu areas up and running */
	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
	for_each_possible_cpu(cpu) {
		per_cpu_offset(cpu) = delta + cpu * pcpu_unit_size;
		per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
		per_cpu(cpu_number, cpu) = cpu;
		setup_percpu_segment(cpu);
		setup_stack_canary_segment(cpu);
		/*
		 * Copy data used in early init routines from the
		 * initial arrays to the per cpu data areas.  These
		 * arrays then become expendable and the *_early_ptr's
		 * are zeroed indicating that the static arrays are
		 * gone.
		 */
#ifdef CONFIG_X86_LOCAL_APIC
		per_cpu(x86_cpu_to_apicid, cpu) =
			early_per_cpu_map(x86_cpu_to_apicid, cpu);
		per_cpu(x86_bios_cpu_apicid, cpu) =
			early_per_cpu_map(x86_bios_cpu_apicid, cpu);
#endif
#ifdef CONFIG_X86_64
		per_cpu(irq_stack_ptr, cpu) =
			per_cpu(irq_stack_union.irq_stack, cpu) +
			IRQ_STACK_SIZE - 64;
#ifdef CONFIG_NUMA
		per_cpu(x86_cpu_to_node_map, cpu) =
			early_per_cpu_map(x86_cpu_to_node_map, cpu);
#endif
#endif
		/*
		 * Up to this point, the boot CPU has been using .data.init
		 * area.  Reload any changed state for the boot CPU.
		 */
		if (cpu == boot_cpu_id)
			switch_to_new_gdt(cpu);
	}

	/* indicate the early static arrays will soon be gone */
#ifdef CONFIG_X86_LOCAL_APIC
	early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
	early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
#endif
#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
	early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
#endif

#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
	/*
	 * make sure boot cpu node_number is right, when boot cpu is on the
	 * node that doesn't have mem installed
	 */
	per_cpu(node_number, boot_cpu_id) = cpu_to_node(boot_cpu_id);
#endif

	/* Setup node to cpumask map */
	setup_node_to_cpumask_map();

	/* Setup cpu initialized, callin, callout masks */
	setup_cpu_local_masks();
}
Commit	Line	Data
4fe29a85 GOC	1	#include <linux/kernel.h>
	2	#include <linux/module.h>
	3	#include <linux/init.h>
	4	#include <linux/bootmem.h>
	5	#include <linux/percpu.h>
1ecd2765	6	#include <linux/kexec.h>
17b4cceb	7	#include <linux/crash_dump.h>
8a87dd9a JSR	8	#include <linux/smp.h>
8a87dd9a JSR	9	#include <linux/topology.h>
5f5d8405	10	#include <linux/pfn.h>
4fe29a85 GOC	11	#include <asm/sections.h>
	12	#include <asm/processor.h>
	13	#include <asm/setup.h>
0fc0906e	14	#include <asm/mpspec.h>
76eb4131	15	#include <asm/apicdef.h>
1ecd2765	16	#include <asm/highmem.h>
1a51e3a0	17	#include <asm/proto.h>
06879033	18	#include <asm/cpumask.h>
34019be1	19	#include <asm/cpu.h>
60a5317f	20	#include <asm/stackprotector.h>
76eb4131	21
c90aa894 MT	22	#ifdef CONFIG_DEBUG_PER_CPU_MAPS
	23	# define DBG(x...) printk(KERN_DEBUG x)
	24	#else
	25	# define DBG(x...)
	26	#endif
	27
ea927906 BG	28	DEFINE_PER_CPU(int, cpu_number);
ea927906 BG	29	EXPORT_PER_CPU_SYMBOL(cpu_number);
ea927906	30
1688401a BG	31	#ifdef CONFIG_X86_64
	32	#define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
	33	#else
	34	#define BOOT_PERCPU_OFFSET 0
	35	#endif
	36
	37	DEFINE_PER_CPU(unsigned long, this_cpu_off) = BOOT_PERCPU_OFFSET;
	38	EXPORT_PER_CPU_SYMBOL(this_cpu_off);
	39
9939ddaf	40	unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
34019be1	41	[0 ... NR_CPUS-1] = BOOT_PERCPU_OFFSET,
9939ddaf	42	};
9939ddaf	43	EXPORT_SYMBOL(__per_cpu_offset);
4fe29a85	44
6b19b0c2 TH	45	/*
	46	* On x86_64 symbols referenced from code should be reachable using
	47	* 32bit relocations. Reserve space for static percpu variables in
	48	* modules so that they are always served from the first chunk which
	49	* is located at the percpu segment base. On x86_32, anything can
	50	* address anywhere. No need to reserve space in the first chunk.
	51	*/
	52	#ifdef CONFIG_X86_64
	53	#define PERCPU_FIRST_CHUNK_RESERVE PERCPU_MODULE_RESERVE
	54	#else
	55	#define PERCPU_FIRST_CHUNK_RESERVE 0
	56	#endif
	57
89c92151 TH	58	/**
	59	* pcpu_need_numa - determine percpu allocation needs to consider NUMA
	60	*
	61	* If NUMA is not configured or there is only one NUMA node available,
	62	* there is no reason to consider NUMA. This function determines
	63	* whether percpu allocation should consider NUMA or not.
	64	*
	65	* RETURNS:
	66	* true if NUMA should be considered; otherwise, false.
	67	*/
	68	static bool __init pcpu_need_numa(void)
	69	{
	70	#ifdef CONFIG_NEED_MULTIPLE_NODES
	71	pg_data_t *last = NULL;
	72	unsigned int cpu;
	73
	74	for_each_possible_cpu(cpu) {
	75	int node = early_cpu_to_node(cpu);
	76
	77	if (node_online(node) && NODE_DATA(node) &&
	78	last && last != NODE_DATA(node))
	79	return true;
	80
	81	last = NODE_DATA(node);
	82	}
	83	#endif
	84	return false;
	85	}
	86
5f5d8405 TH	87	/**
	88	* pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu
	89	* @cpu: cpu to allocate for
	90	* @size: size allocation in bytes
	91	* @align: alignment
	92	*
	93	* Allocate @size bytes aligned at @align for cpu @cpu. This wrapper
	94	* does the right thing for NUMA regardless of the current
	95	* configuration.
	96	*
	97	* RETURNS:
	98	* Pointer to the allocated area on success, NULL on failure.
	99	*/
	100	static void * __init pcpu_alloc_bootmem(unsigned int cpu, unsigned long size,
	101	unsigned long align)
	102	{
	103	const unsigned long goal = __pa(MAX_DMA_ADDRESS);
	104	#ifdef CONFIG_NEED_MULTIPLE_NODES
	105	int node = early_cpu_to_node(cpu);
	106	void *ptr;
	107
	108	if (!node_online(node) \|\| !NODE_DATA(node)) {
	109	ptr = __alloc_bootmem_nopanic(size, align, goal);
	110	pr_info("cpu %d has no node %d or node-local memory\n",
	111	cpu, node);
	112	pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",
	113	cpu, size, __pa(ptr));
	114	} else {
	115	ptr = __alloc_bootmem_node_nopanic(NODE_DATA(node),
	116	size, align, goal);
	117	pr_debug("per cpu data for cpu%d %lu bytes on node%d at "
	118	"%016lx\n", cpu, size, node, __pa(ptr));
	119	}
	120	return ptr;
	121	#else
	122	return __alloc_bootmem_nopanic(size, align, goal);
	123	#endif
	124	}
	125
8ac83757 TH	126	/*
	127	* Remap allocator
	128	*
	129	* This allocator uses PMD page as unit. A PMD page is allocated for
	130	* each cpu and each is remapped into vmalloc area using PMD mapping.
	131	* As PMD page is quite large, only part of it is used for the first
	132	* chunk. Unused part is returned to the bootmem allocator.
	133	*
	134	* So, the PMD pages are mapped twice - once to the physical mapping
	135	* and to the vmalloc area for the first percpu chunk. The double
	136	* mapping does add one more PMD TLB entry pressure but still is much
	137	* better than only using 4k mappings while still being NUMA friendly.
	138	*/
	139	#ifdef CONFIG_NEED_MULTIPLE_NODES
	140	static size_t pcpur_size __initdata;
	141	static void **pcpur_ptrs __initdata;
	142
	143	static struct page * __init pcpur_get_page(unsigned int cpu, int pageno)
	144	{
	145	size_t off = (size_t)pageno << PAGE_SHIFT;
	146
	147	if (off >= pcpur_size)
	148	return NULL;
	149
	150	return virt_to_page(pcpur_ptrs[cpu] + off);
	151	}
	152
	153	static ssize_t __init setup_pcpu_remap(size_t static_size)
	154	{
	155	static struct vm_struct vm;
6b19b0c2	156	size_t ptrs_size, dyn_size;
8ac83757 TH	157	unsigned int cpu;
	158	ssize_t ret;
	159
	160	/*
	161	* If large page isn't supported, there's no benefit in doing
	162	* this. Also, on non-NUMA, embedding is better.
71c9d8b6 TH	163	*
71c9d8b6 TH	164	* NOTE: disabled for now.
8ac83757	165	*/
71c9d8b6	166	if (true \|\| !cpu_has_pse \|\| !pcpu_need_numa())
8ac83757 TH	167	return -EINVAL;
8ac83757 TH	168
8ac83757 TH	169	/*
	170	* Currently supports only single page. Supporting multiple
	171	* pages won't be too difficult if it ever becomes necessary.
	172	*/
6b19b0c2 TH	173	pcpur_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +
6b19b0c2 TH	174	PERCPU_DYNAMIC_RESERVE);
8ac83757 TH	175	if (pcpur_size > PMD_SIZE) {
	176	pr_warning("PERCPU: static data is larger than large page, "
	177	"can't use large page\n");
	178	return -EINVAL;
	179	}
6b19b0c2	180	dyn_size = pcpur_size - static_size - PERCPU_FIRST_CHUNK_RESERVE;
8ac83757 TH	181
	182	/* allocate pointer array and alloc large pages */
	183	ptrs_size = PFN_ALIGN(num_possible_cpus() * sizeof(pcpur_ptrs[0]));
	184	pcpur_ptrs = alloc_bootmem(ptrs_size);
	185
	186	for_each_possible_cpu(cpu) {
	187	pcpur_ptrs[cpu] = pcpu_alloc_bootmem(cpu, PMD_SIZE, PMD_SIZE);
	188	if (!pcpur_ptrs[cpu])
	189	goto enomem;
	190
	191	/*
	192	* Only use pcpur_size bytes and give back the rest.
	193	*
	194	* Ingo: The 2MB up-rounding bootmem is needed to make
	195	* sure the partial 2MB page is still fully RAM - it's
	196	* not well-specified to have a PAT-incompatible area
	197	* (unmapped RAM, device memory, etc.) in that hole.
	198	*/
	199	free_bootmem(__pa(pcpur_ptrs[cpu] + pcpur_size),
	200	PMD_SIZE - pcpur_size);
	201
	202	memcpy(pcpur_ptrs[cpu], __per_cpu_load, static_size);
	203	}
	204
	205	/* allocate address and map */
	206	vm.flags = VM_ALLOC;
	207	vm.size = num_possible_cpus() * PMD_SIZE;
	208	vm_area_register_early(&vm, PMD_SIZE);
	209
	210	for_each_possible_cpu(cpu) {
	211	pmd_t *pmd;
	212
	213	pmd = populate_extra_pmd((unsigned long)vm.addr
	214	+ cpu * PMD_SIZE);
	215	set_pmd(pmd, pfn_pmd(page_to_pfn(virt_to_page(pcpur_ptrs[cpu])),
	216	PAGE_KERNEL_LARGE));
	217	}
	218
	219	/* we're ready, commit */
	220	pr_info("PERCPU: Remapped at %p with large pages, static data "
	221	"%zu bytes\n", vm.addr, static_size);
	222
6b19b0c2	223	ret = pcpu_setup_first_chunk(pcpur_get_page, static_size,
6074d5b0 TH	224	PERCPU_FIRST_CHUNK_RESERVE, dyn_size,
6074d5b0 TH	225	PMD_SIZE, vm.addr, NULL);
8ac83757 TH	226	goto out_free_ar;
	227
	228	enomem:
	229	for_each_possible_cpu(cpu)
	230	if (pcpur_ptrs[cpu])
c5806df9	231	free_bootmem(__pa(pcpur_ptrs[cpu]), pcpur_size);
8ac83757 TH	232	ret = -ENOMEM;
	233	out_free_ar:
	234	free_bootmem(__pa(pcpur_ptrs), ptrs_size);
	235	return ret;
	236	}
	237	#else
	238	static ssize_t __init setup_pcpu_remap(size_t static_size)
	239	{
	240	return -EINVAL;
	241	}
	242	#endif
	243
89c92151 TH	244	/*
	245	* Embedding allocator
	246	*
	247	* The first chunk is sized to just contain the static area plus
66c3a757 TH	248	* module and dynamic reserves and embedded into linear physical
	249	* mapping so that it can use PMD mapping without additional TLB
	250	* pressure.
89c92151	251	*/
89c92151 TH	252	static ssize_t __init setup_pcpu_embed(size_t static_size)
89c92151 TH	253	{
66c3a757	254	size_t reserve = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;
89c92151 TH	255
	256	/*
	257	* If large page isn't supported, there's no benefit in doing
	258	* this. Also, embedding allocation doesn't play well with
	259	* NUMA.
	260	*/
	261	if (!cpu_has_pse \|\| pcpu_need_numa())
	262	return -EINVAL;
	263
66c3a757 TH	264	return pcpu_embed_first_chunk(static_size, PERCPU_FIRST_CHUNK_RESERVE,
66c3a757 TH	265	reserve - PERCPU_FIRST_CHUNK_RESERVE, -1);
89c92151 TH	266	}
89c92151 TH	267
5f5d8405 TH	268	/*
	269	* 4k page allocator
	270	*
	271	* This is the basic allocator. Static percpu area is allocated
	272	* page-by-page and most of initialization is done by the generic
	273	* setup function.
	274	*/
8d408b4b TH	275	static struct page **pcpu4k_pages __initdata;
	276	static int pcpu4k_nr_static_pages __initdata;
	277
	278	static struct page * __init pcpu4k_get_page(unsigned int cpu, int pageno)
	279	{
	280	if (pageno < pcpu4k_nr_static_pages)
	281	return pcpu4k_pages[cpu * pcpu4k_nr_static_pages + pageno];
	282	return NULL;
	283	}
	284
458a3e64 TH	285	static void __init pcpu4k_populate_pte(unsigned long addr)
	286	{
	287	populate_extra_pte(addr);
	288	}
	289
5f5d8405 TH	290	static ssize_t __init setup_pcpu_4k(size_t static_size)
	291	{
	292	size_t pages_size;
	293	unsigned int cpu;
	294	int i, j;
	295	ssize_t ret;
	296
	297	pcpu4k_nr_static_pages = PFN_UP(static_size);
	298
	299	/* unaligned allocations can't be freed, round up to page size */
	300	pages_size = PFN_ALIGN(pcpu4k_nr_static_pages * num_possible_cpus()
	301	* sizeof(pcpu4k_pages[0]));
	302	pcpu4k_pages = alloc_bootmem(pages_size);
	303
	304	/* allocate and copy */
	305	j = 0;
	306	for_each_possible_cpu(cpu)
	307	for (i = 0; i < pcpu4k_nr_static_pages; i++) {
	308	void *ptr;
	309
	310	ptr = pcpu_alloc_bootmem(cpu, PAGE_SIZE, PAGE_SIZE);
	311	if (!ptr)
	312	goto enomem;
	313
	314	memcpy(ptr, __per_cpu_load + i * PAGE_SIZE, PAGE_SIZE);
	315	pcpu4k_pages[j++] = virt_to_page(ptr);
	316	}
	317
	318	/* we're ready, commit */
	319	pr_info("PERCPU: Allocated %d 4k pages, static data %zu bytes\n",
	320	pcpu4k_nr_static_pages, static_size);
	321
6b19b0c2	322	ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size,
6074d5b0 TH	323	PERCPU_FIRST_CHUNK_RESERVE, -1,
6074d5b0 TH	324	-1, NULL, pcpu4k_populate_pte);
5f5d8405 TH	325	goto out_free_ar;
	326
	327	enomem:
	328	while (--j >= 0)
	329	free_bootmem(__pa(page_address(pcpu4k_pages[j])), PAGE_SIZE);
	330	ret = -ENOMEM;
	331	out_free_ar:
	332	free_bootmem(__pa(pcpu4k_pages), pages_size);
	333	return ret;
	334	}
	335
b2d2f431 BG	336	static inline void setup_percpu_segment(int cpu)
	337	{
	338	#ifdef CONFIG_X86_32
	339	struct desc_struct gdt;
	340
	341	pack_descriptor(&gdt, per_cpu_offset(cpu), 0xFFFFF,
	342	0x2 \| DESCTYPE_S, 0x8);
	343	gdt.s = 1;
	344	write_gdt_entry(get_cpu_gdt_table(cpu),
	345	GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
	346	#endif
	347	}
	348
4fe29a85 GOC	349	/*
	350	* Great future plan:
	351	* Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
	352	* Always point %gs to its beginning
	353	*/
	354	void __init setup_per_cpu_areas(void)
	355	{
5f5d8405 TH	356	size_t static_size = __per_cpu_end - __per_cpu_start;
5f5d8405 TH	357	unsigned int cpu;
11124411 TH	358	unsigned long delta;
11124411 TH	359	size_t pcpu_unit_size;
5f5d8405	360	ssize_t ret;
a1681965	361
ab14398a	362	pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
a1681965	363	NR_CPUS, nr_cpumask_bits, nr_cpu_ids, nr_node_ids);
11124411	364
8ac83757 TH	365	/*
	366	* Allocate percpu area. If PSE is supported, try to make use
	367	* of large page mappings. Please read comments on top of
	368	* each allocator for details.
	369	*/
	370	ret = setup_pcpu_remap(static_size);
	371	if (ret < 0)
	372	ret = setup_pcpu_embed(static_size);
89c92151 TH	373	if (ret < 0)
89c92151 TH	374	ret = setup_pcpu_4k(static_size);
5f5d8405 TH	375	if (ret < 0)
	376	panic("cannot allocate static percpu area (%zu bytes, err=%zd)",
	377	static_size, ret);
1a51e3a0	378
5f5d8405	379	pcpu_unit_size = ret;
11124411	380
5f5d8405	381	/* alrighty, percpu areas up and running */
11124411 TH	382	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
	383	for_each_possible_cpu(cpu) {
	384	per_cpu_offset(cpu) = delta + cpu * pcpu_unit_size;
26f80bd6	385	per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
ea927906	386	per_cpu(cpu_number, cpu) = cpu;
b2d2f431	387	setup_percpu_segment(cpu);
60a5317f	388	setup_stack_canary_segment(cpu);
0d77e7f0	389	/*
cf3997f5 TH	390	* Copy data used in early init routines from the
	391	* initial arrays to the per cpu data areas. These
	392	* arrays then become expendable and the *_early_ptr's
	393	* are zeroed indicating that the static arrays are
	394	* gone.
0d77e7f0	395	*/
ec70de8b	396	#ifdef CONFIG_X86_LOCAL_APIC
0d77e7f0	397	per_cpu(x86_cpu_to_apicid, cpu) =
cf3997f5	398	early_per_cpu_map(x86_cpu_to_apicid, cpu);
0d77e7f0	399	per_cpu(x86_bios_cpu_apicid, cpu) =
cf3997f5	400	early_per_cpu_map(x86_bios_cpu_apicid, cpu);
ec70de8b	401	#endif
1a51e3a0	402	#ifdef CONFIG_X86_64
26f80bd6	403	per_cpu(irq_stack_ptr, cpu) =
cf3997f5 TH	404	per_cpu(irq_stack_union.irq_stack, cpu) +
cf3997f5 TH	405	IRQ_STACK_SIZE - 64;
6470aff6 BG	406	#ifdef CONFIG_NUMA
6470aff6 BG	407	per_cpu(x86_cpu_to_node_map, cpu) =
cf3997f5	408	early_per_cpu_map(x86_cpu_to_node_map, cpu);
2697fbd5	409	#endif
6470aff6	410	#endif
1a51e3a0	411	/*
34019be1	412	* Up to this point, the boot CPU has been using .data.init
2697fbd5	413	* area. Reload any changed state for the boot CPU.
1a51e3a0	414	*/
34019be1	415	if (cpu == boot_cpu_id)
552be871	416	switch_to_new_gdt(cpu);
4fe29a85 GOC	417	}
4fe29a85 GOC	418
0d77e7f0	419	/* indicate the early static arrays will soon be gone */
22f25138	420	#ifdef CONFIG_X86_LOCAL_APIC
0d77e7f0 BG	421	early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
0d77e7f0 BG	422	early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
22f25138	423	#endif
6470aff6	424	#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
0d77e7f0 BG	425	early_per_cpu_ptr(x86_cpu_to_node_map) = NULL;
0d77e7f0 BG	426	#endif
9f0e8d04	427
35d5a9a6 YL	428	#if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
	429	/*
	430	* make sure boot cpu node_number is right, when boot cpu is on the
	431	* node that doesn't have mem installed
	432	*/
	433	per_cpu(node_number, boot_cpu_id) = cpu_to_node(boot_cpu_id);
	434	#endif
	435
9f248bde MT	436	/* Setup node to cpumask map */
9f248bde MT	437	setup_node_to_cpumask_map();
c2d1cec1 MT	438
	439	/* Setup cpu initialized, callin, callout masks */
	440	setup_cpu_local_masks();
4fe29a85	441	}