2 * x86 FPU boot time init code:
4 #include <asm/fpu/internal.h>
5 #include <asm/tlbflush.h>
7 #include <asm/cmdline.h>
9 #include <linux/sched.h>
10 #include <linux/sched/task.h>
11 #include <linux/init.h>
14 * Initialize the registers found in all CPUs, CR0 and CR4:
16 static void fpu__init_cpu_generic(void)
19 unsigned long cr4_mask
= 0;
21 if (boot_cpu_has(X86_FEATURE_FXSR
))
22 cr4_mask
|= X86_CR4_OSFXSR
;
23 if (boot_cpu_has(X86_FEATURE_XMM
))
24 cr4_mask
|= X86_CR4_OSXMMEXCPT
;
26 cr4_set_bits(cr4_mask
);
29 cr0
&= ~(X86_CR0_TS
|X86_CR0_EM
); /* clear TS and EM */
30 if (!boot_cpu_has(X86_FEATURE_FPU
))
34 /* Flush out any pending x87 state: */
35 #ifdef CONFIG_MATH_EMULATION
36 if (!boot_cpu_has(X86_FEATURE_FPU
))
37 fpstate_init_soft(¤t
->thread
.fpu
.state
.soft
);
40 asm volatile ("fninit");
44 * Enable all supported FPU features. Called when a CPU is brought online:
46 void fpu__init_cpu(void)
48 fpu__init_cpu_generic();
49 fpu__init_cpu_xstate();
52 static bool fpu__probe_without_cpuid(void)
60 cr0
&= ~(X86_CR0_TS
| X86_CR0_EM
);
63 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw
), "+m" (fcw
));
65 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw
, fcw
);
67 return fsw
== 0 && (fcw
& 0x103f) == 0x003f;
70 static void fpu__init_system_early_generic(struct cpuinfo_x86
*c
)
72 if (!boot_cpu_has(X86_FEATURE_CPUID
) &&
73 !test_bit(X86_FEATURE_FPU
, (unsigned long *)cpu_caps_cleared
)) {
74 if (fpu__probe_without_cpuid())
75 setup_force_cpu_cap(X86_FEATURE_FPU
);
77 setup_clear_cpu_cap(X86_FEATURE_FPU
);
80 #ifndef CONFIG_MATH_EMULATION
81 if (!test_cpu_cap(&boot_cpu_data
, X86_FEATURE_FPU
)) {
82 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
90 * Boot time FPU feature detection code:
92 unsigned int mxcsr_feature_mask __read_mostly
= 0xffffffffu
;
94 static void __init
fpu__init_system_mxcsr(void)
96 unsigned int mask
= 0;
98 if (boot_cpu_has(X86_FEATURE_FXSR
)) {
99 /* Static because GCC does not get 16-byte stack alignment right: */
100 static struct fxregs_state fxregs __initdata
;
102 asm volatile("fxsave %0" : "+m" (fxregs
));
104 mask
= fxregs
.mxcsr_mask
;
107 * If zero then use the default features mask,
108 * which has all features set, except the
109 * denormals-are-zero feature bit:
114 mxcsr_feature_mask
&= mask
;
118 * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
120 static void __init
fpu__init_system_generic(void)
123 * Set up the legacy init FPU context. (xstate init might overwrite this
124 * with a more modern format, if the CPU supports it.)
126 fpstate_init(&init_fpstate
);
128 fpu__init_system_mxcsr();
132 * Size of the FPU context state. All tasks in the system use the
133 * same context size, regardless of what portion they use.
134 * This is inherent to the XSAVE architecture which puts all state
135 * components into a single, continuous memory block:
137 unsigned int fpu_kernel_xstate_size
;
138 EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size
);
140 /* Get alignment of the TYPE. */
141 #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
144 * Enforce that 'MEMBER' is the last field of 'TYPE'.
146 * Align the computed size with alignment of the TYPE,
147 * because that's how C aligns structs.
149 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
150 BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
154 * We append the 'struct fpu' to the task_struct:
156 static void __init
fpu__init_task_struct_size(void)
158 int task_size
= sizeof(struct task_struct
);
161 * Subtract off the static size of the register state.
162 * It potentially has a bunch of padding.
164 task_size
-= sizeof(((struct task_struct
*)0)->thread
.fpu
.state
);
167 * Add back the dynamically-calculated register state
170 task_size
+= fpu_kernel_xstate_size
;
173 * We dynamically size 'struct fpu', so we require that
174 * it be at the end of 'thread_struct' and that
175 * 'thread_struct' be at the end of 'task_struct'. If
176 * you hit a compile error here, check the structure to
177 * see if something got added to the end.
179 CHECK_MEMBER_AT_END_OF(struct fpu
, state
);
180 CHECK_MEMBER_AT_END_OF(struct thread_struct
, fpu
);
181 CHECK_MEMBER_AT_END_OF(struct task_struct
, thread
);
183 arch_task_struct_size
= task_size
;
187 * Set up the user and kernel xstate sizes based on the legacy FPU context size.
189 * We set this up first, and later it will be overwritten by
190 * fpu__init_system_xstate() if the CPU knows about xstates.
192 static void __init
fpu__init_system_xstate_size_legacy(void)
194 static int on_boot_cpu __initdata
= 1;
196 WARN_ON_FPU(!on_boot_cpu
);
200 * Note that xstate sizes might be overwritten later during
201 * fpu__init_system_xstate().
204 if (!boot_cpu_has(X86_FEATURE_FPU
)) {
206 * Disable xsave as we do not support it if i387
207 * emulation is enabled.
209 setup_clear_cpu_cap(X86_FEATURE_XSAVE
);
210 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT
);
211 fpu_kernel_xstate_size
= sizeof(struct swregs_state
);
213 if (boot_cpu_has(X86_FEATURE_FXSR
))
214 fpu_kernel_xstate_size
=
215 sizeof(struct fxregs_state
);
217 fpu_kernel_xstate_size
=
218 sizeof(struct fregs_state
);
221 fpu_user_xstate_size
= fpu_kernel_xstate_size
;
225 * Find supported xfeatures based on cpu features and command-line input.
226 * This must be called after fpu__init_parse_early_param() is called and
227 * xfeatures_mask is enumerated.
229 u64 __init
fpu__get_supported_xfeatures_mask(void)
234 /* Legacy code to initialize eager fpu mode. */
235 static void __init
fpu__init_system_ctx_switch(void)
237 static bool on_boot_cpu __initdata
= 1;
239 WARN_ON_FPU(!on_boot_cpu
);
242 WARN_ON_FPU(current
->thread
.fpu
.fpstate_active
);
246 * We parse fpu parameters early because fpu__init_system() is executed
247 * before parse_early_param().
249 static void __init
fpu__init_parse_early_param(void)
251 if (cmdline_find_option_bool(boot_command_line
, "no387"))
252 setup_clear_cpu_cap(X86_FEATURE_FPU
);
254 if (cmdline_find_option_bool(boot_command_line
, "nofxsr")) {
255 setup_clear_cpu_cap(X86_FEATURE_FXSR
);
256 setup_clear_cpu_cap(X86_FEATURE_FXSR_OPT
);
257 setup_clear_cpu_cap(X86_FEATURE_XMM
);
260 if (cmdline_find_option_bool(boot_command_line
, "noxsave"))
261 fpu__xstate_clear_all_cpu_caps();
263 if (cmdline_find_option_bool(boot_command_line
, "noxsaveopt"))
264 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT
);
266 if (cmdline_find_option_bool(boot_command_line
, "noxsaves"))
267 setup_clear_cpu_cap(X86_FEATURE_XSAVES
);
271 * Called on the boot CPU once per system bootup, to set up the initial
272 * FPU state that is later cloned into all processes:
274 void __init
fpu__init_system(struct cpuinfo_x86
*c
)
276 fpu__init_parse_early_param();
277 fpu__init_system_early_generic(c
);
280 * The FPU has to be operational for some of the
281 * later FPU init activities:
285 fpu__init_system_generic();
286 fpu__init_system_xstate_size_legacy();
287 fpu__init_system_xstate();
288 fpu__init_task_struct_size();
290 fpu__init_system_ctx_switch();