]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - arch/x86/kernel/setup_percpu.c
1 #include <linux/kernel.h>
2 #include <linux/module.h>
3 #include <linux/init.h>
4 #include <linux/bootmem.h>
5 #include <linux/percpu.h>
6 #include <linux/kexec.h>
7 #include <linux/crash_dump.h>
9 #include <linux/topology.h>
10 #include <asm/sections.h>
11 #include <asm/processor.h>
12 #include <asm/setup.h>
13 #include <asm/mpspec.h>
14 #include <asm/apicdef.h>
15 #include <asm/highmem.h>
16 #include <asm/proto.h>
17 #include <asm/cpumask.h>
19 #include <asm/stackprotector.h>
21 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
22 # define DBG(x...) printk(KERN_DEBUG x)
27 DEFINE_PER_CPU(int, cpu_number
);
28 EXPORT_PER_CPU_SYMBOL(cpu_number
);
31 #define BOOT_PERCPU_OFFSET ((unsigned long)__per_cpu_load)
33 #define BOOT_PERCPU_OFFSET 0
36 DEFINE_PER_CPU(unsigned long, this_cpu_off
) = BOOT_PERCPU_OFFSET
;
37 EXPORT_PER_CPU_SYMBOL(this_cpu_off
);
39 unsigned long __per_cpu_offset
[NR_CPUS
] __read_mostly
= {
40 [0 ... NR_CPUS
-1] = BOOT_PERCPU_OFFSET
,
42 EXPORT_SYMBOL(__per_cpu_offset
);
44 static void __init
pcpu4k_populate_pte(unsigned long addr
)
46 populate_extra_pte(addr
);
49 static inline void setup_percpu_segment(int cpu
)
52 struct desc_struct gdt
;
54 pack_descriptor(&gdt
, per_cpu_offset(cpu
), 0xFFFFF,
55 0x2 | DESCTYPE_S
, 0x8);
57 write_gdt_entry(get_cpu_gdt_table(cpu
),
58 GDT_ENTRY_PERCPU
, &gdt
, DESCTYPE_S
);
64 * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
65 * Always point %gs to its beginning
67 void __init
setup_per_cpu_areas(void)
69 ssize_t size
= __per_cpu_end
- __per_cpu_start
;
70 unsigned int nr_cpu_pages
= DIV_ROUND_UP(size
, PAGE_SIZE
);
71 static struct page
**pages
;
73 unsigned int cpu
, i
, j
;
75 size_t pcpu_unit_size
;
77 pr_info("NR_CPUS:%d nr_cpumask_bits:%d nr_cpu_ids:%d nr_node_ids:%d\n",
78 NR_CPUS
, nr_cpumask_bits
, nr_cpu_ids
, nr_node_ids
);
79 pr_info("PERCPU: Allocating %zd bytes for static per cpu data\n", size
);
81 pages_size
= nr_cpu_pages
* num_possible_cpus() * sizeof(pages
[0]);
82 pages
= alloc_bootmem(pages_size
);
85 for_each_possible_cpu(cpu
) {
88 for (i
= 0; i
< nr_cpu_pages
; i
++) {
89 #ifndef CONFIG_NEED_MULTIPLE_NODES
90 ptr
= alloc_bootmem_pages(PAGE_SIZE
);
92 int node
= early_cpu_to_node(cpu
);
94 if (!node_online(node
) || !NODE_DATA(node
)) {
95 ptr
= alloc_bootmem_pages(PAGE_SIZE
);
96 pr_info("cpu %d has no node %d or node-local "
97 "memory\n", cpu
, node
);
98 pr_debug("per cpu data for cpu%d at %016lx\n",
101 ptr
= alloc_bootmem_pages_node(NODE_DATA(node
),
103 pr_debug("per cpu data for cpu%d on node%d "
104 "at %016lx\n", cpu
, node
, __pa(ptr
));
107 memcpy(ptr
, __per_cpu_load
+ i
* PAGE_SIZE
, PAGE_SIZE
);
108 pages
[j
++] = virt_to_page(ptr
);
112 pcpu_unit_size
= pcpu_setup_static(pcpu4k_populate_pte
, pages
, size
);
114 free_bootmem(__pa(pages
), pages_size
);
116 delta
= (unsigned long)pcpu_base_addr
- (unsigned long)__per_cpu_start
;
117 for_each_possible_cpu(cpu
) {
118 per_cpu_offset(cpu
) = delta
+ cpu
* pcpu_unit_size
;
119 per_cpu(this_cpu_off
, cpu
) = per_cpu_offset(cpu
);
120 per_cpu(cpu_number
, cpu
) = cpu
;
121 setup_percpu_segment(cpu
);
122 setup_stack_canary_segment(cpu
);
124 * Copy data used in early init routines from the
125 * initial arrays to the per cpu data areas. These
126 * arrays then become expendable and the *_early_ptr's
127 * are zeroed indicating that the static arrays are
130 #ifdef CONFIG_X86_LOCAL_APIC
131 per_cpu(x86_cpu_to_apicid
, cpu
) =
132 early_per_cpu_map(x86_cpu_to_apicid
, cpu
);
133 per_cpu(x86_bios_cpu_apicid
, cpu
) =
134 early_per_cpu_map(x86_bios_cpu_apicid
, cpu
);
137 per_cpu(irq_stack_ptr
, cpu
) =
138 per_cpu(irq_stack_union
.irq_stack
, cpu
) +
141 per_cpu(x86_cpu_to_node_map
, cpu
) =
142 early_per_cpu_map(x86_cpu_to_node_map
, cpu
);
146 * Up to this point, the boot CPU has been using .data.init
147 * area. Reload any changed state for the boot CPU.
149 if (cpu
== boot_cpu_id
)
150 switch_to_new_gdt(cpu
);
152 DBG("PERCPU: cpu %4d %p\n", cpu
, ptr
);
155 /* indicate the early static arrays will soon be gone */
156 #ifdef CONFIG_X86_LOCAL_APIC
157 early_per_cpu_ptr(x86_cpu_to_apicid
) = NULL
;
158 early_per_cpu_ptr(x86_bios_cpu_apicid
) = NULL
;
160 #if defined(CONFIG_X86_64) && defined(CONFIG_NUMA)
161 early_per_cpu_ptr(x86_cpu_to_node_map
) = NULL
;
164 /* Setup node to cpumask map */
165 setup_node_to_cpumask_map();
167 /* Setup cpu initialized, callin, callout masks */
168 setup_cpu_local_masks();