]>
Commit | Line | Data |
---|---|---|
0c867537 | 1 | /* |
ae02679c | 2 | * x86 FPU boot time init code: |
0c867537 | 3 | */ |
78f7f1e5 | 4 | #include <asm/fpu/internal.h> |
0c867537 | 5 | #include <asm/tlbflush.h> |
4f81cbaf | 6 | #include <asm/setup.h> |
7 | #include <asm/cmdline.h> | |
0c867537 | 8 | |
5aaeb5c0 | 9 | #include <linux/sched.h> |
29930025 | 10 | #include <linux/sched/task.h> |
4f81cbaf | 11 | #include <linux/init.h> |
5aaeb5c0 | 12 | |
41e78410 IM |
13 | /* |
14 | * Initialize the registers found in all CPUs, CR0 and CR4: | |
15 | */ | |
16 | static void fpu__init_cpu_generic(void) | |
17 | { | |
18 | unsigned long cr0; | |
19 | unsigned long cr4_mask = 0; | |
20 | ||
01f8fd73 | 21 | if (boot_cpu_has(X86_FEATURE_FXSR)) |
41e78410 | 22 | cr4_mask |= X86_CR4_OSFXSR; |
dda9edf7 | 23 | if (boot_cpu_has(X86_FEATURE_XMM)) |
41e78410 IM |
24 | cr4_mask |= X86_CR4_OSXMMEXCPT; |
25 | if (cr4_mask) | |
26 | cr4_set_bits(cr4_mask); | |
27 | ||
28 | cr0 = read_cr0(); | |
29 | cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */ | |
a402a8df | 30 | if (!boot_cpu_has(X86_FEATURE_FPU)) |
41e78410 IM |
31 | cr0 |= X86_CR0_EM; |
32 | write_cr0(cr0); | |
b1276c48 IM |
33 | |
34 | /* Flush out any pending x87 state: */ | |
5fc96038 | 35 | #ifdef CONFIG_MATH_EMULATION |
a402a8df | 36 | if (!boot_cpu_has(X86_FEATURE_FPU)) |
5fc96038 IM |
37 | fpstate_init_soft(¤t->thread.fpu.state.soft); |
38 | else | |
39 | #endif | |
40 | asm volatile ("fninit"); | |
41e78410 IM |
41 | } |
42 | ||
43 | /* | |
ae02679c | 44 | * Enable all supported FPU features. Called when a CPU is brought online: |
41e78410 IM |
45 | */ |
46 | void fpu__init_cpu(void) | |
47 | { | |
48 | fpu__init_cpu_generic(); | |
49 | fpu__init_cpu_xstate(); | |
41e78410 IM |
50 | } |
51 | ||
37ac78b6 | 52 | static bool fpu__probe_without_cpuid(void) |
2e2f3da7 IM |
53 | { |
54 | unsigned long cr0; | |
55 | u16 fsw, fcw; | |
56 | ||
57 | fsw = fcw = 0xffff; | |
58 | ||
59 | cr0 = read_cr0(); | |
60 | cr0 &= ~(X86_CR0_TS | X86_CR0_EM); | |
61 | write_cr0(cr0); | |
62 | ||
37ac78b6 AL |
63 | asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw)); |
64 | ||
65 | pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw); | |
f363938c | 66 | |
37ac78b6 AL |
67 | return fsw == 0 && (fcw & 0x103f) == 0x003f; |
68 | } | |
69 | ||
70 | static void fpu__init_system_early_generic(struct cpuinfo_x86 *c) | |
71 | { | |
72 | if (!boot_cpu_has(X86_FEATURE_CPUID) && | |
73 | !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) { | |
74 | if (fpu__probe_without_cpuid()) | |
75 | setup_force_cpu_cap(X86_FEATURE_FPU); | |
f363938c | 76 | else |
37ac78b6 | 77 | setup_clear_cpu_cap(X86_FEATURE_FPU); |
f363938c | 78 | } |
e83ab9ad IM |
79 | |
80 | #ifndef CONFIG_MATH_EMULATION | |
9729017f | 81 | if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) { |
ae02679c | 82 | pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n"); |
e83ab9ad IM |
83 | for (;;) |
84 | asm volatile("hlt"); | |
85 | } | |
86 | #endif | |
2e2f3da7 IM |
87 | } |
88 | ||
4d164092 IM |
89 | /* |
90 | * Boot time FPU feature detection code: | |
91 | */ | |
0c867537 | 92 | unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu; |
a575813b | 93 | EXPORT_SYMBOL_GPL(mxcsr_feature_mask); |
91a8c2a5 | 94 | |
32231879 | 95 | static void __init fpu__init_system_mxcsr(void) |
0c867537 | 96 | { |
91a8c2a5 | 97 | unsigned int mask = 0; |
0c867537 | 98 | |
01f8fd73 | 99 | if (boot_cpu_has(X86_FEATURE_FXSR)) { |
b96fecbf IM |
100 | /* Static because GCC does not get 16-byte stack alignment right: */ |
101 | static struct fxregs_state fxregs __initdata; | |
91a8c2a5 | 102 | |
b96fecbf | 103 | asm volatile("fxsave %0" : "+m" (fxregs)); |
91a8c2a5 | 104 | |
b96fecbf | 105 | mask = fxregs.mxcsr_mask; |
91a8c2a5 IM |
106 | |
107 | /* | |
108 | * If zero then use the default features mask, | |
109 | * which has all features set, except the | |
110 | * denormals-are-zero feature bit: | |
111 | */ | |
0c867537 IM |
112 | if (mask == 0) |
113 | mask = 0x0000ffbf; | |
114 | } | |
115 | mxcsr_feature_mask &= mask; | |
116 | } | |
117 | ||
7218e8b7 IM |
118 | /* |
119 | * Once per bootup FPU initialization sequences that will run on most x86 CPUs: | |
120 | */ | |
32231879 | 121 | static void __init fpu__init_system_generic(void) |
7218e8b7 IM |
122 | { |
123 | /* | |
124 | * Set up the legacy init FPU context. (xstate init might overwrite this | |
125 | * with a more modern format, if the CPU supports it.) | |
126 | */ | |
6e686709 | 127 | fpstate_init(&init_fpstate); |
7218e8b7 IM |
128 | |
129 | fpu__init_system_mxcsr(); | |
130 | } | |
131 | ||
ae02679c IM |
132 | /* |
133 | * Size of the FPU context state. All tasks in the system use the | |
134 | * same context size, regardless of what portion they use. | |
135 | * This is inherent to the XSAVE architecture which puts all state | |
136 | * components into a single, continuous memory block: | |
137 | */ | |
bf15a8cf FY |
138 | unsigned int fpu_kernel_xstate_size; |
139 | EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size); | |
41e78410 | 140 | |
25ec02f2 JO |
141 | /* Get alignment of the TYPE. */ |
142 | #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test) | |
143 | ||
144 | /* | |
145 | * Enforce that 'MEMBER' is the last field of 'TYPE'. | |
146 | * | |
147 | * Align the computed size with alignment of the TYPE, | |
148 | * because that's how C aligns structs. | |
149 | */ | |
5aaeb5c0 | 150 | #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \ |
25ec02f2 JO |
151 | BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \ |
152 | TYPE_ALIGN(TYPE))) | |
0c8c0f03 DH |
153 | |
154 | /* | |
5aaeb5c0 | 155 | * We append the 'struct fpu' to the task_struct: |
0c8c0f03 | 156 | */ |
5aaeb5c0 | 157 | static void __init fpu__init_task_struct_size(void) |
0c8c0f03 DH |
158 | { |
159 | int task_size = sizeof(struct task_struct); | |
160 | ||
161 | /* | |
162 | * Subtract off the static size of the register state. | |
163 | * It potentially has a bunch of padding. | |
164 | */ | |
165 | task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state); | |
166 | ||
167 | /* | |
168 | * Add back the dynamically-calculated register state | |
169 | * size. | |
170 | */ | |
bf15a8cf | 171 | task_size += fpu_kernel_xstate_size; |
0c8c0f03 DH |
172 | |
173 | /* | |
174 | * We dynamically size 'struct fpu', so we require that | |
175 | * it be at the end of 'thread_struct' and that | |
176 | * 'thread_struct' be at the end of 'task_struct'. If | |
177 | * you hit a compile error here, check the structure to | |
178 | * see if something got added to the end. | |
179 | */ | |
180 | CHECK_MEMBER_AT_END_OF(struct fpu, state); | |
181 | CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu); | |
182 | CHECK_MEMBER_AT_END_OF(struct task_struct, thread); | |
183 | ||
5aaeb5c0 | 184 | arch_task_struct_size = task_size; |
0c8c0f03 DH |
185 | } |
186 | ||
41e78410 | 187 | /* |
bf15a8cf | 188 | * Set up the user and kernel xstate sizes based on the legacy FPU context size. |
41e78410 IM |
189 | * |
190 | * We set this up first, and later it will be overwritten by | |
191 | * fpu__init_system_xstate() if the CPU knows about xstates. | |
192 | */ | |
32231879 | 193 | static void __init fpu__init_system_xstate_size_legacy(void) |
0c867537 | 194 | { |
e49a449b | 195 | static int on_boot_cpu __initdata = 1; |
e97131a8 IM |
196 | |
197 | WARN_ON_FPU(!on_boot_cpu); | |
198 | on_boot_cpu = 0; | |
199 | ||
0c867537 | 200 | /* |
bf15a8cf | 201 | * Note that xstate sizes might be overwritten later during |
c42103b2 | 202 | * fpu__init_system_xstate(). |
0c867537 IM |
203 | */ |
204 | ||
a402a8df | 205 | if (!boot_cpu_has(X86_FEATURE_FPU)) { |
0c867537 IM |
206 | /* |
207 | * Disable xsave as we do not support it if i387 | |
208 | * emulation is enabled. | |
209 | */ | |
210 | setup_clear_cpu_cap(X86_FEATURE_XSAVE); | |
211 | setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT); | |
bf15a8cf | 212 | fpu_kernel_xstate_size = sizeof(struct swregs_state); |
6a133207 | 213 | } else { |
01f8fd73 | 214 | if (boot_cpu_has(X86_FEATURE_FXSR)) |
bf15a8cf FY |
215 | fpu_kernel_xstate_size = |
216 | sizeof(struct fxregs_state); | |
6a133207 | 217 | else |
bf15a8cf FY |
218 | fpu_kernel_xstate_size = |
219 | sizeof(struct fregs_state); | |
0c867537 | 220 | } |
a1141e0b | 221 | |
bf15a8cf | 222 | fpu_user_xstate_size = fpu_kernel_xstate_size; |
0c867537 IM |
223 | } |
224 | ||
a5fe93a5 | 225 | /* |
226 | * Find supported xfeatures based on cpu features and command-line input. | |
227 | * This must be called after fpu__init_parse_early_param() is called and | |
228 | * xfeatures_mask is enumerated. | |
229 | */ | |
230 | u64 __init fpu__get_supported_xfeatures_mask(void) | |
231 | { | |
ca6938a1 | 232 | return XCNTXT_MASK; |
a5fe93a5 | 233 | } |
234 | ||
ca6938a1 | 235 | /* Legacy code to initialize eager fpu mode. */ |
32231879 | 236 | static void __init fpu__init_system_ctx_switch(void) |
6f5d265a | 237 | { |
e49a449b | 238 | static bool on_boot_cpu __initdata = 1; |
e97131a8 IM |
239 | |
240 | WARN_ON_FPU(!on_boot_cpu); | |
241 | on_boot_cpu = 0; | |
242 | ||
e4a81bfc | 243 | WARN_ON_FPU(current->thread.fpu.initialized); |
6f5d265a IM |
244 | } |
245 | ||
4f81cbaf | 246 | /* |
247 | * We parse fpu parameters early because fpu__init_system() is executed | |
248 | * before parse_early_param(). | |
249 | */ | |
250 | static void __init fpu__init_parse_early_param(void) | |
251 | { | |
4f81cbaf | 252 | if (cmdline_find_option_bool(boot_command_line, "no387")) |
253 | setup_clear_cpu_cap(X86_FEATURE_FPU); | |
254 | ||
255 | if (cmdline_find_option_bool(boot_command_line, "nofxsr")) { | |
256 | setup_clear_cpu_cap(X86_FEATURE_FXSR); | |
257 | setup_clear_cpu_cap(X86_FEATURE_FXSR_OPT); | |
258 | setup_clear_cpu_cap(X86_FEATURE_XMM); | |
259 | } | |
260 | ||
261 | if (cmdline_find_option_bool(boot_command_line, "noxsave")) | |
262 | fpu__xstate_clear_all_cpu_caps(); | |
263 | ||
264 | if (cmdline_find_option_bool(boot_command_line, "noxsaveopt")) | |
265 | setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT); | |
266 | ||
267 | if (cmdline_find_option_bool(boot_command_line, "noxsaves")) | |
268 | setup_clear_cpu_cap(X86_FEATURE_XSAVES); | |
269 | } | |
270 | ||
e35f6f14 | 271 | /* |
ae02679c IM |
272 | * Called on the boot CPU once per system bootup, to set up the initial |
273 | * FPU state that is later cloned into all processes: | |
e35f6f14 | 274 | */ |
32231879 | 275 | void __init fpu__init_system(struct cpuinfo_x86 *c) |
e35f6f14 | 276 | { |
4f81cbaf | 277 | fpu__init_parse_early_param(); |
dd863880 IM |
278 | fpu__init_system_early_generic(c); |
279 | ||
ae02679c IM |
280 | /* |
281 | * The FPU has to be operational for some of the | |
282 | * later FPU init activities: | |
283 | */ | |
e35f6f14 | 284 | fpu__init_cpu(); |
0c867537 | 285 | |
7218e8b7 | 286 | fpu__init_system_generic(); |
7638b74b | 287 | fpu__init_system_xstate_size_legacy(); |
c42103b2 | 288 | fpu__init_system_xstate(); |
5aaeb5c0 | 289 | fpu__init_task_struct_size(); |
997578b1 | 290 | |
011545b5 | 291 | fpu__init_system_ctx_switch(); |
0c867537 | 292 | } |