]>
Commit | Line | Data |
---|---|---|
1eeaed76 RM |
1 | /* |
2 | * Copyright (C) 1994 Linus Torvalds | |
3 | * | |
4 | * Pentium III FXSR, SSE support | |
5 | * General FPU state handling cleanups | |
6 | * Gareth Hughes <gareth@valinux.com>, May 2000 | |
7 | * x86-64 work by Andi Kleen 2002 | |
8 | */ | |
9 | ||
1965aae3 PA |
10 | #ifndef _ASM_X86_I387_H |
11 | #define _ASM_X86_I387_H | |
1eeaed76 RM |
12 | |
13 | #include <linux/sched.h> | |
14 | #include <linux/kernel_stat.h> | |
15 | #include <linux/regset.h> | |
e4914012 | 16 | #include <linux/hardirq.h> |
92c37fa3 | 17 | #include <asm/asm.h> |
1eeaed76 RM |
18 | #include <asm/processor.h> |
19 | #include <asm/sigcontext.h> | |
20 | #include <asm/user.h> | |
21 | #include <asm/uaccess.h> | |
dc1e35c6 | 22 | #include <asm/xsave.h> |
1eeaed76 | 23 | |
3c1c7f10 | 24 | extern unsigned int sig_xstate_size; |
1eeaed76 | 25 | extern void fpu_init(void); |
1eeaed76 | 26 | extern void mxcsr_feature_mask_init(void); |
aa283f49 | 27 | extern int init_fpu(struct task_struct *child); |
1eeaed76 | 28 | extern asmlinkage void math_state_restore(void); |
61c4628b | 29 | extern void init_thread_xstate(void); |
36454936 | 30 | extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); |
1eeaed76 RM |
31 | |
32 | extern user_regset_active_fn fpregs_active, xfpregs_active; | |
33 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get; | |
34 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set; | |
35 | ||
c37b5efe | 36 | extern struct _fpx_sw_bytes fx_sw_reserved; |
1eeaed76 | 37 | #ifdef CONFIG_IA32_EMULATION |
3c1c7f10 | 38 | extern unsigned int sig_xstate_ia32_size; |
c37b5efe | 39 | extern struct _fpx_sw_bytes fx_sw_reserved_ia32; |
1eeaed76 | 40 | struct _fpstate_ia32; |
ab513701 SS |
41 | struct _xstate_ia32; |
42 | extern int save_i387_xstate_ia32(void __user *buf); | |
43 | extern int restore_i387_xstate_ia32(void __user *buf); | |
1eeaed76 RM |
44 | #endif |
45 | ||
b359e8a4 SS |
46 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ |
47 | ||
1eeaed76 RM |
48 | #ifdef CONFIG_X86_64 |
49 | ||
50 | /* Ignore delayed exceptions from user space */ | |
51 | static inline void tolerant_fwait(void) | |
52 | { | |
53 | asm volatile("1: fwait\n" | |
54 | "2:\n" | |
affe6637 | 55 | _ASM_EXTABLE(1b, 2b)); |
1eeaed76 RM |
56 | } |
57 | ||
b359e8a4 | 58 | static inline int fxrstor_checking(struct i387_fxsave_struct *fx) |
1eeaed76 RM |
59 | { |
60 | int err; | |
61 | ||
62 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" | |
63 | "2:\n" | |
64 | ".section .fixup,\"ax\"\n" | |
65 | "3: movl $-1,%[err]\n" | |
66 | " jmp 2b\n" | |
67 | ".previous\n" | |
affe6637 | 68 | _ASM_EXTABLE(1b, 3b) |
1eeaed76 RM |
69 | : [err] "=r" (err) |
70 | #if 0 /* See comment in __save_init_fpu() below. */ | |
71 | : [fx] "r" (fx), "m" (*fx), "0" (0)); | |
72 | #else | |
73 | : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0)); | |
74 | #endif | |
1eeaed76 RM |
75 | return err; |
76 | } | |
77 | ||
b359e8a4 SS |
78 | static inline int restore_fpu_checking(struct task_struct *tsk) |
79 | { | |
80 | if (task_thread_info(tsk)->status & TS_XSAVE) | |
81 | return xrstor_checking(&tsk->thread.xstate->xsave); | |
82 | else | |
83 | return fxrstor_checking(&tsk->thread.xstate->fxsave); | |
84 | } | |
1eeaed76 RM |
85 | |
86 | /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception | |
87 | is pending. Clear the x87 state here by setting it to fixed | |
88 | values. The kernel data segment can be sometimes 0 and sometimes | |
89 | new user value. Both should be ok. | |
90 | Use the PDA as safe address because it should be already in L1. */ | |
b359e8a4 | 91 | static inline void clear_fpu_state(struct task_struct *tsk) |
1eeaed76 | 92 | { |
b359e8a4 SS |
93 | struct xsave_struct *xstate = &tsk->thread.xstate->xsave; |
94 | struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave; | |
95 | ||
96 | /* | |
97 | * xsave header may indicate the init state of the FP. | |
98 | */ | |
99 | if ((task_thread_info(tsk)->status & TS_XSAVE) && | |
100 | !(xstate->xsave_hdr.xstate_bv & XSTATE_FP)) | |
101 | return; | |
102 | ||
1eeaed76 | 103 | if (unlikely(fx->swd & X87_FSW_ES)) |
affe6637 | 104 | asm volatile("fnclex"); |
1eeaed76 | 105 | alternative_input(ASM_NOP8 ASM_NOP2, |
affe6637 JP |
106 | " emms\n" /* clear stack tags */ |
107 | " fildl %%gs:0", /* load to clear state */ | |
108 | X86_FEATURE_FXSAVE_LEAK); | |
1eeaed76 RM |
109 | } |
110 | ||
c37b5efe | 111 | static inline int fxsave_user(struct i387_fxsave_struct __user *fx) |
1eeaed76 RM |
112 | { |
113 | int err; | |
114 | ||
115 | asm volatile("1: rex64/fxsave (%[fx])\n\t" | |
116 | "2:\n" | |
117 | ".section .fixup,\"ax\"\n" | |
118 | "3: movl $-1,%[err]\n" | |
119 | " jmp 2b\n" | |
120 | ".previous\n" | |
affe6637 | 121 | _ASM_EXTABLE(1b, 3b) |
1eeaed76 RM |
122 | : [err] "=r" (err), "=m" (*fx) |
123 | #if 0 /* See comment in __fxsave_clear() below. */ | |
124 | : [fx] "r" (fx), "0" (0)); | |
125 | #else | |
126 | : [fx] "cdaSDb" (fx), "0" (0)); | |
127 | #endif | |
affe6637 JP |
128 | if (unlikely(err) && |
129 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) | |
1eeaed76 RM |
130 | err = -EFAULT; |
131 | /* No need to clear here because the caller clears USED_MATH */ | |
132 | return err; | |
133 | } | |
134 | ||
b359e8a4 | 135 | static inline void fxsave(struct task_struct *tsk) |
1eeaed76 RM |
136 | { |
137 | /* Using "rex64; fxsave %0" is broken because, if the memory operand | |
138 | uses any extended registers for addressing, a second REX prefix | |
139 | will be generated (to the assembler, rex64 followed by semicolon | |
140 | is a separate instruction), and hence the 64-bitness is lost. */ | |
141 | #if 0 | |
142 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported | |
143 | starting with gas 2.16. */ | |
144 | __asm__ __volatile__("fxsaveq %0" | |
61c4628b | 145 | : "=m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
146 | #elif 0 |
147 | /* Using, as a workaround, the properly prefixed form below isn't | |
148 | accepted by any binutils version so far released, complaining that | |
149 | the same type of prefix is used twice if an extended register is | |
150 | needed for addressing (fix submitted to mainline 2005-11-21). */ | |
151 | __asm__ __volatile__("rex64/fxsave %0" | |
61c4628b | 152 | : "=m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
153 | #else |
154 | /* This, however, we can work around by forcing the compiler to select | |
155 | an addressing mode that doesn't require extended registers. */ | |
61c4628b SS |
156 | __asm__ __volatile__("rex64/fxsave (%1)" |
157 | : "=m" (tsk->thread.xstate->fxsave) | |
158 | : "cdaSDb" (&tsk->thread.xstate->fxsave)); | |
1eeaed76 | 159 | #endif |
b359e8a4 SS |
160 | } |
161 | ||
162 | static inline void __save_init_fpu(struct task_struct *tsk) | |
163 | { | |
164 | if (task_thread_info(tsk)->status & TS_XSAVE) | |
165 | xsave(tsk); | |
166 | else | |
167 | fxsave(tsk); | |
168 | ||
169 | clear_fpu_state(tsk); | |
1eeaed76 RM |
170 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
171 | } | |
172 | ||
1eeaed76 RM |
173 | #else /* CONFIG_X86_32 */ |
174 | ||
ab9e1858 DG |
175 | #ifdef CONFIG_MATH_EMULATION |
176 | extern void finit_task(struct task_struct *tsk); | |
177 | #else | |
178 | static inline void finit_task(struct task_struct *tsk) | |
179 | { | |
180 | } | |
181 | #endif | |
e8a496ac | 182 | |
1eeaed76 RM |
183 | static inline void tolerant_fwait(void) |
184 | { | |
185 | asm volatile("fnclex ; fwait"); | |
186 | } | |
187 | ||
188 | static inline void restore_fpu(struct task_struct *tsk) | |
189 | { | |
b359e8a4 SS |
190 | if (task_thread_info(tsk)->status & TS_XSAVE) { |
191 | xrstor_checking(&tsk->thread.xstate->xsave); | |
192 | return; | |
193 | } | |
1eeaed76 RM |
194 | /* |
195 | * The "nop" is needed to make the instructions the same | |
196 | * length. | |
197 | */ | |
198 | alternative_input( | |
199 | "nop ; frstor %1", | |
200 | "fxrstor %1", | |
201 | X86_FEATURE_FXSR, | |
61c4628b | 202 | "m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
203 | } |
204 | ||
205 | /* We need a safe address that is cheap to find and that is already | |
206 | in L1 during context switch. The best choices are unfortunately | |
207 | different for UP and SMP */ | |
208 | #ifdef CONFIG_SMP | |
209 | #define safe_address (__per_cpu_offset[0]) | |
210 | #else | |
211 | #define safe_address (kstat_cpu(0).cpustat.user) | |
212 | #endif | |
213 | ||
214 | /* | |
215 | * These must be called with preempt disabled | |
216 | */ | |
217 | static inline void __save_init_fpu(struct task_struct *tsk) | |
218 | { | |
b359e8a4 SS |
219 | if (task_thread_info(tsk)->status & TS_XSAVE) { |
220 | struct xsave_struct *xstate = &tsk->thread.xstate->xsave; | |
221 | struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave; | |
222 | ||
223 | xsave(tsk); | |
224 | ||
225 | /* | |
226 | * xsave header may indicate the init state of the FP. | |
227 | */ | |
228 | if (!(xstate->xsave_hdr.xstate_bv & XSTATE_FP)) | |
229 | goto end; | |
230 | ||
231 | if (unlikely(fx->swd & X87_FSW_ES)) | |
232 | asm volatile("fnclex"); | |
233 | ||
234 | /* | |
235 | * we can do a simple return here or be paranoid :) | |
236 | */ | |
237 | goto clear_state; | |
238 | } | |
239 | ||
1eeaed76 RM |
240 | /* Use more nops than strictly needed in case the compiler |
241 | varies code */ | |
242 | alternative_input( | |
243 | "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4, | |
244 | "fxsave %[fx]\n" | |
245 | "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", | |
246 | X86_FEATURE_FXSR, | |
61c4628b SS |
247 | [fx] "m" (tsk->thread.xstate->fxsave), |
248 | [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); | |
b359e8a4 | 249 | clear_state: |
1eeaed76 RM |
250 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
251 | is pending. Clear the x87 state here by setting it to fixed | |
252 | values. safe_address is a random variable that should be in L1 */ | |
253 | alternative_input( | |
254 | GENERIC_NOP8 GENERIC_NOP2, | |
255 | "emms\n\t" /* clear stack tags */ | |
256 | "fildl %[addr]", /* set F?P to defined value */ | |
257 | X86_FEATURE_FXSAVE_LEAK, | |
258 | [addr] "m" (safe_address)); | |
b359e8a4 | 259 | end: |
1eeaed76 RM |
260 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
261 | } | |
262 | ||
ab513701 SS |
263 | #endif /* CONFIG_X86_64 */ |
264 | ||
1eeaed76 RM |
265 | /* |
266 | * Signal frame handlers... | |
267 | */ | |
ab513701 SS |
268 | extern int save_i387_xstate(void __user *buf); |
269 | extern int restore_i387_xstate(void __user *buf); | |
1eeaed76 RM |
270 | |
271 | static inline void __unlazy_fpu(struct task_struct *tsk) | |
272 | { | |
273 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
274 | __save_init_fpu(tsk); | |
275 | stts(); | |
276 | } else | |
277 | tsk->fpu_counter = 0; | |
278 | } | |
279 | ||
280 | static inline void __clear_fpu(struct task_struct *tsk) | |
281 | { | |
282 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
283 | tolerant_fwait(); | |
284 | task_thread_info(tsk)->status &= ~TS_USEDFPU; | |
285 | stts(); | |
286 | } | |
287 | } | |
288 | ||
289 | static inline void kernel_fpu_begin(void) | |
290 | { | |
291 | struct thread_info *me = current_thread_info(); | |
292 | preempt_disable(); | |
293 | if (me->status & TS_USEDFPU) | |
294 | __save_init_fpu(me->task); | |
295 | else | |
296 | clts(); | |
297 | } | |
298 | ||
299 | static inline void kernel_fpu_end(void) | |
300 | { | |
301 | stts(); | |
302 | preempt_enable(); | |
303 | } | |
304 | ||
e4914012 SS |
305 | /* |
306 | * Some instructions like VIA's padlock instructions generate a spurious | |
307 | * DNA fault but don't modify SSE registers. And these instructions | |
308 | * get used from interrupt context aswell. To prevent these kernel instructions | |
309 | * in interrupt context interact wrongly with other user/kernel fpu usage, we | |
310 | * should use them only in the context of irq_ts_save/restore() | |
311 | */ | |
312 | static inline int irq_ts_save(void) | |
313 | { | |
314 | /* | |
315 | * If we are in process context, we are ok to take a spurious DNA fault. | |
316 | * Otherwise, doing clts() in process context require pre-emption to | |
317 | * be disabled or some heavy lifting like kernel_fpu_begin() | |
318 | */ | |
319 | if (!in_interrupt()) | |
320 | return 0; | |
321 | ||
322 | if (read_cr0() & X86_CR0_TS) { | |
323 | clts(); | |
324 | return 1; | |
325 | } | |
326 | ||
327 | return 0; | |
328 | } | |
329 | ||
330 | static inline void irq_ts_restore(int TS_state) | |
331 | { | |
332 | if (TS_state) | |
333 | stts(); | |
334 | } | |
335 | ||
1eeaed76 RM |
336 | #ifdef CONFIG_X86_64 |
337 | ||
338 | static inline void save_init_fpu(struct task_struct *tsk) | |
339 | { | |
340 | __save_init_fpu(tsk); | |
341 | stts(); | |
342 | } | |
343 | ||
344 | #define unlazy_fpu __unlazy_fpu | |
345 | #define clear_fpu __clear_fpu | |
346 | ||
347 | #else /* CONFIG_X86_32 */ | |
348 | ||
349 | /* | |
350 | * These disable preemption on their own and are safe | |
351 | */ | |
352 | static inline void save_init_fpu(struct task_struct *tsk) | |
353 | { | |
354 | preempt_disable(); | |
355 | __save_init_fpu(tsk); | |
356 | stts(); | |
357 | preempt_enable(); | |
358 | } | |
359 | ||
360 | static inline void unlazy_fpu(struct task_struct *tsk) | |
361 | { | |
362 | preempt_disable(); | |
363 | __unlazy_fpu(tsk); | |
364 | preempt_enable(); | |
365 | } | |
366 | ||
367 | static inline void clear_fpu(struct task_struct *tsk) | |
368 | { | |
369 | preempt_disable(); | |
370 | __clear_fpu(tsk); | |
371 | preempt_enable(); | |
372 | } | |
373 | ||
374 | #endif /* CONFIG_X86_64 */ | |
375 | ||
1eeaed76 RM |
376 | /* |
377 | * i387 state interaction | |
378 | */ | |
379 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) | |
380 | { | |
381 | if (cpu_has_fxsr) { | |
61c4628b | 382 | return tsk->thread.xstate->fxsave.cwd; |
1eeaed76 | 383 | } else { |
1679f271 | 384 | return (unsigned short)tsk->thread.xstate->fsave.cwd; |
1eeaed76 RM |
385 | } |
386 | } | |
387 | ||
388 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) | |
389 | { | |
390 | if (cpu_has_fxsr) { | |
61c4628b | 391 | return tsk->thread.xstate->fxsave.swd; |
1eeaed76 | 392 | } else { |
1679f271 | 393 | return (unsigned short)tsk->thread.xstate->fsave.swd; |
1eeaed76 RM |
394 | } |
395 | } | |
396 | ||
397 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) | |
398 | { | |
399 | if (cpu_has_xmm) { | |
61c4628b | 400 | return tsk->thread.xstate->fxsave.mxcsr; |
1eeaed76 RM |
401 | } else { |
402 | return MXCSR_DEFAULT; | |
403 | } | |
404 | } | |
405 | ||
1965aae3 | 406 | #endif /* _ASM_X86_I387_H */ |