2 * Copyright (C) 1994 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * General FPU state handling cleanups
6 * Gareth Hughes <gareth@valinux.com>, May 2000
7 * x86-64 work by Andi Kleen 2002
10 #ifndef _FPU_INTERNAL_H
11 #define _FPU_INTERNAL_H
13 #include <linux/regset.h>
14 #include <linux/compat.h>
15 #include <linux/slab.h>
19 #include <asm/xsave.h>
22 # include <asm/sigcontext32.h>
23 # include <asm/user32.h>
25 int ia32_setup_rt_frame(int sig
, struct ksignal
*ksig
,
26 compat_sigset_t
*set
, struct pt_regs
*regs
);
27 int ia32_setup_frame(int sig
, struct ksignal
*ksig
,
28 compat_sigset_t
*set
, struct pt_regs
*regs
);
30 # define user_i387_ia32_struct user_i387_struct
31 # define user32_fxsr_struct user_fxsr_struct
32 # define ia32_setup_frame __setup_frame
33 # define ia32_setup_rt_frame __setup_rt_frame
36 extern unsigned int mxcsr_feature_mask
;
37 extern void fpu__cpu_init(void);
38 extern void eager_fpu_init(void);
40 DECLARE_PER_CPU(struct fpu
*, fpu_fpregs_owner_ctx
);
42 extern void convert_from_fxsr(struct user_i387_ia32_struct
*env
,
43 struct task_struct
*tsk
);
44 extern void convert_to_fxsr(struct task_struct
*tsk
,
45 const struct user_i387_ia32_struct
*env
);
47 extern user_regset_active_fn fpregs_active
, xfpregs_active
;
48 extern user_regset_get_fn fpregs_get
, xfpregs_get
, fpregs_soft_get
,
50 extern user_regset_set_fn fpregs_set
, xfpregs_set
, fpregs_soft_set
,
54 * xstateregs_active == fpregs_active. Please refer to the comment
55 * at the definition of fpregs_active.
57 #define xstateregs_active fpregs_active
59 #ifdef CONFIG_MATH_EMULATION
60 extern void finit_soft_fpu(struct i387_soft_struct
*soft
);
62 static inline void finit_soft_fpu(struct i387_soft_struct
*soft
) {}
66 * Must be run with preemption disabled: this clears the fpu_fpregs_owner_ctx,
69 * This will disable any lazy FPU state restore of the current FPU state,
70 * but if the current thread owns the FPU, it will still be saved by.
72 static inline void __cpu_disable_lazy_restore(unsigned int cpu
)
74 per_cpu(fpu_fpregs_owner_ctx
, cpu
) = NULL
;
78 * Used to indicate that the FPU state in memory is newer than the FPU
79 * state in registers, and the FPU state should be reloaded next time the
80 * task is run. Only safe on the current task, or non-running tasks.
82 static inline void task_disable_lazy_fpu_restore(struct task_struct
*tsk
)
84 tsk
->thread
.fpu
.last_cpu
= ~0;
87 static inline int fpu_lazy_restore(struct task_struct
*new, unsigned int cpu
)
89 return &new->thread
.fpu
== this_cpu_read_stable(fpu_fpregs_owner_ctx
) &&
90 cpu
== new->thread
.fpu
.last_cpu
;
93 static inline int is_ia32_compat_frame(void)
95 return config_enabled(CONFIG_IA32_EMULATION
) &&
96 test_thread_flag(TIF_IA32
);
99 static inline int is_ia32_frame(void)
101 return config_enabled(CONFIG_X86_32
) || is_ia32_compat_frame();
104 static inline int is_x32_frame(void)
106 return config_enabled(CONFIG_X86_X32_ABI
) && test_thread_flag(TIF_X32
);
109 #define X87_FSW_ES (1 << 7) /* Exception Summary */
111 static __always_inline __pure
bool use_eager_fpu(void)
113 return static_cpu_has_safe(X86_FEATURE_EAGER_FPU
);
116 static __always_inline __pure
bool use_xsaveopt(void)
118 return static_cpu_has_safe(X86_FEATURE_XSAVEOPT
);
121 static __always_inline __pure
bool use_xsave(void)
123 return static_cpu_has_safe(X86_FEATURE_XSAVE
);
126 static __always_inline __pure
bool use_fxsr(void)
128 return static_cpu_has_safe(X86_FEATURE_FXSR
);
131 static inline void fx_finit(struct i387_fxsave_struct
*fx
)
134 fx
->mxcsr
= MXCSR_DEFAULT
;
137 extern void __sanitize_i387_state(struct task_struct
*);
139 static inline void sanitize_i387_state(struct task_struct
*tsk
)
143 __sanitize_i387_state(tsk
);
146 #define user_insn(insn, output, input...) \
149 asm volatile(ASM_STAC "\n" \
151 "2: " ASM_CLAC "\n" \
152 ".section .fixup,\"ax\"\n" \
153 "3: movl $-1,%[err]\n" \
156 _ASM_EXTABLE(1b, 3b) \
157 : [err] "=r" (err), output \
162 #define check_insn(insn, output, input...) \
165 asm volatile("1:" #insn "\n\t" \
167 ".section .fixup,\"ax\"\n" \
168 "3: movl $-1,%[err]\n" \
171 _ASM_EXTABLE(1b, 3b) \
172 : [err] "=r" (err), output \
177 static inline int fsave_user(struct i387_fsave_struct __user
*fx
)
179 return user_insn(fnsave
%[fx
]; fwait
, [fx
] "=m" (*fx
), "m" (*fx
));
182 static inline int fxsave_user(struct i387_fxsave_struct __user
*fx
)
184 if (config_enabled(CONFIG_X86_32
))
185 return user_insn(fxsave
%[fx
], [fx
] "=m" (*fx
), "m" (*fx
));
186 else if (config_enabled(CONFIG_AS_FXSAVEQ
))
187 return user_insn(fxsaveq
%[fx
], [fx
] "=m" (*fx
), "m" (*fx
));
189 /* See comment in fpu_fxsave() below. */
190 return user_insn(rex64
/fxsave (%[fx
]), "=m" (*fx
), [fx
] "R" (fx
));
193 static inline int fxrstor_checking(struct i387_fxsave_struct
*fx
)
195 if (config_enabled(CONFIG_X86_32
))
196 return check_insn(fxrstor
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
197 else if (config_enabled(CONFIG_AS_FXSAVEQ
))
198 return check_insn(fxrstorq
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
200 /* See comment in fpu_fxsave() below. */
201 return check_insn(rex64
/fxrstor (%[fx
]), "=m" (*fx
), [fx
] "R" (fx
),
205 static inline int fxrstor_user(struct i387_fxsave_struct __user
*fx
)
207 if (config_enabled(CONFIG_X86_32
))
208 return user_insn(fxrstor
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
209 else if (config_enabled(CONFIG_AS_FXSAVEQ
))
210 return user_insn(fxrstorq
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
212 /* See comment in fpu_fxsave() below. */
213 return user_insn(rex64
/fxrstor (%[fx
]), "=m" (*fx
), [fx
] "R" (fx
),
217 static inline int frstor_checking(struct i387_fsave_struct
*fx
)
219 return check_insn(frstor
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
222 static inline int frstor_user(struct i387_fsave_struct __user
*fx
)
224 return user_insn(frstor
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
227 static inline void fpu_fxsave(struct fpu
*fpu
)
229 if (config_enabled(CONFIG_X86_32
))
230 asm volatile( "fxsave %[fx]" : [fx
] "=m" (fpu
->state
->fxsave
));
231 else if (config_enabled(CONFIG_AS_FXSAVEQ
))
232 asm volatile("fxsaveq %[fx]" : [fx
] "=m" (fpu
->state
->fxsave
));
234 /* Using "rex64; fxsave %0" is broken because, if the memory
235 * operand uses any extended registers for addressing, a second
236 * REX prefix will be generated (to the assembler, rex64
237 * followed by semicolon is a separate instruction), and hence
238 * the 64-bitness is lost.
240 * Using "fxsaveq %0" would be the ideal choice, but is only
241 * supported starting with gas 2.16.
243 * Using, as a workaround, the properly prefixed form below
244 * isn't accepted by any binutils version so far released,
245 * complaining that the same type of prefix is used twice if
246 * an extended register is needed for addressing (fix submitted
247 * to mainline 2005-11-21).
249 * asm volatile("rex64/fxsave %0" : "=m" (fpu->state->fxsave));
251 * This, however, we can work around by forcing the compiler to
252 * select an addressing mode that doesn't require extended
255 asm volatile( "rex64/fxsave (%[fx])"
256 : "=m" (fpu
->state
->fxsave
)
257 : [fx
] "R" (&fpu
->state
->fxsave
));
262 * These must be called with preempt disabled. Returns
263 * 'true' if the FPU state is still intact.
265 static inline int fpu_save_init(struct fpu
*fpu
)
268 xsave_state(&fpu
->state
->xsave
);
271 * xsave header may indicate the init state of the FP.
273 if (!(fpu
->state
->xsave
.xsave_hdr
.xstate_bv
& XSTATE_FP
))
275 } else if (use_fxsr()) {
278 asm volatile("fnsave %[fx]; fwait"
279 : [fx
] "=m" (fpu
->state
->fsave
));
284 * If exceptions are pending, we need to clear them so
285 * that we don't randomly get exceptions later.
287 * FIXME! Is this perhaps only true for the old-style
288 * irq13 case? Maybe we could leave the x87 state
291 if (unlikely(fpu
->state
->fxsave
.swd
& X87_FSW_ES
)) {
292 asm volatile("fnclex");
298 static inline int fpu_restore_checking(struct fpu
*fpu
)
301 return fpu_xrstor_checking(&fpu
->state
->xsave
);
303 return fxrstor_checking(&fpu
->state
->fxsave
);
305 return frstor_checking(&fpu
->state
->fsave
);
308 static inline int restore_fpu_checking(struct task_struct
*tsk
)
311 * AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is
312 * pending. Clear the x87 state here by setting it to fixed values.
313 * "m" is a random variable that should be in L1.
315 if (unlikely(static_cpu_has_bug_safe(X86_BUG_FXSAVE_LEAK
))) {
319 "fildl %P[addr]" /* set F?P to defined value */
320 : : [addr
] "m" (tsk
->thread
.fpu
.has_fpu
));
323 return fpu_restore_checking(&tsk
->thread
.fpu
);
326 /* Must be paired with an 'stts' after! */
327 static inline void __thread_clear_has_fpu(struct fpu
*fpu
)
330 this_cpu_write(fpu_fpregs_owner_ctx
, NULL
);
333 /* Must be paired with a 'clts' before! */
334 static inline void __thread_set_has_fpu(struct task_struct
*tsk
)
336 tsk
->thread
.fpu
.has_fpu
= 1;
337 this_cpu_write(fpu_fpregs_owner_ctx
, &tsk
->thread
.fpu
);
341 * Encapsulate the CR0.TS handling together with the
344 * These generally need preemption protection to work,
345 * do try to avoid using these on their own.
347 static inline void __thread_fpu_end(struct task_struct
*tsk
)
349 __thread_clear_has_fpu(&tsk
->thread
.fpu
);
350 if (!use_eager_fpu())
354 static inline void __thread_fpu_begin(struct task_struct
*tsk
)
356 if (!use_eager_fpu())
358 __thread_set_has_fpu(tsk
);
361 static inline void drop_fpu(struct task_struct
*tsk
)
363 struct fpu
*fpu
= &tsk
->thread
.fpu
;
365 * Forget coprocessor state..
368 tsk
->thread
.fpu
.counter
= 0;
371 /* Ignore delayed exceptions from user space */
372 asm volatile("1: fwait\n"
374 _ASM_EXTABLE(1b
, 2b
));
375 __thread_fpu_end(tsk
);
378 clear_stopped_child_used_math(tsk
);
382 static inline void restore_init_xstate(void)
385 xrstor_state(init_xstate_buf
, -1);
387 fxrstor_checking(&init_xstate_buf
->i387
);
391 * Reset the FPU state in the eager case and drop it in the lazy case (later use
394 static inline void fpu_reset_state(struct task_struct
*tsk
)
396 if (!use_eager_fpu())
399 restore_init_xstate();
403 * FPU state switching for scheduling.
405 * This is a two-stage process:
407 * - switch_fpu_prepare() saves the old state and
408 * sets the new state of the CR0.TS bit. This is
409 * done within the context of the old process.
411 * - switch_fpu_finish() restores the new state as
414 typedef struct { int preload
; } fpu_switch_t
;
416 static inline fpu_switch_t
switch_fpu_prepare(struct task_struct
*old
, struct task_struct
*new, int cpu
)
418 struct fpu
*old_fpu
= &old
->thread
.fpu
;
422 * If the task has used the math, pre-load the FPU on xsave processors
423 * or if the past 5 consecutive context-switches used math.
425 fpu
.preload
= tsk_used_math(new) &&
426 (use_eager_fpu() || new->thread
.fpu
.counter
> 5);
428 if (old_fpu
->has_fpu
) {
429 if (!fpu_save_init(&old
->thread
.fpu
))
430 task_disable_lazy_fpu_restore(old
);
432 old
->thread
.fpu
.last_cpu
= cpu
;
434 /* But leave fpu_fpregs_owner_ctx! */
435 old
->thread
.fpu
.has_fpu
= 0;
437 /* Don't change CR0.TS if we just switch! */
439 new->thread
.fpu
.counter
++;
440 __thread_set_has_fpu(new);
441 prefetch(new->thread
.fpu
.state
);
442 } else if (!use_eager_fpu())
445 old
->thread
.fpu
.counter
= 0;
446 task_disable_lazy_fpu_restore(old
);
448 new->thread
.fpu
.counter
++;
449 if (fpu_lazy_restore(new, cpu
))
452 prefetch(new->thread
.fpu
.state
);
453 __thread_fpu_begin(new);
460 * By the time this gets called, we've already cleared CR0.TS and
461 * given the process the FPU if we are going to preload the FPU
462 * state - all we need to do is to conditionally restore the register
465 static inline void switch_fpu_finish(struct task_struct
*new, fpu_switch_t fpu
)
468 if (unlikely(restore_fpu_checking(new)))
469 fpu_reset_state(new);
474 * Signal frame handlers...
476 extern int save_xstate_sig(void __user
*buf
, void __user
*fx
, int size
);
477 extern int __restore_xstate_sig(void __user
*buf
, void __user
*fx
, int size
);
479 static inline int xstate_sigframe_size(void)
481 return use_xsave() ? xstate_size
+ FP_XSTATE_MAGIC2_SIZE
: xstate_size
;
484 static inline int restore_xstate_sig(void __user
*buf
, int ia32_frame
)
486 void __user
*buf_fx
= buf
;
487 int size
= xstate_sigframe_size();
489 if (ia32_frame
&& use_fxsr()) {
490 buf_fx
= buf
+ sizeof(struct i387_fsave_struct
);
491 size
+= sizeof(struct i387_fsave_struct
);
494 return __restore_xstate_sig(buf
, buf_fx
, size
);
498 * Needs to be preemption-safe.
500 * NOTE! user_fpu_begin() must be used only immediately before restoring
501 * the save state. It does not do any saving/restoring on its own. In
502 * lazy FPU mode, it is just an optimization to avoid a #NM exception,
503 * the task can lose the FPU right after preempt_enable().
505 static inline void user_fpu_begin(void)
509 __thread_fpu_begin(current
);
513 static inline void __save_fpu(struct task_struct
*tsk
)
516 if (unlikely(system_state
== SYSTEM_BOOTING
))
517 xsave_state_booting(&tsk
->thread
.fpu
.state
->xsave
);
519 xsave_state(&tsk
->thread
.fpu
.state
->xsave
);
521 fpu_fxsave(&tsk
->thread
.fpu
);
525 * i387 state interaction
527 static inline unsigned short get_fpu_cwd(struct task_struct
*tsk
)
530 return tsk
->thread
.fpu
.state
->fxsave
.cwd
;
532 return (unsigned short)tsk
->thread
.fpu
.state
->fsave
.cwd
;
536 static inline unsigned short get_fpu_swd(struct task_struct
*tsk
)
539 return tsk
->thread
.fpu
.state
->fxsave
.swd
;
541 return (unsigned short)tsk
->thread
.fpu
.state
->fsave
.swd
;
545 static inline unsigned short get_fpu_mxcsr(struct task_struct
*tsk
)
548 return tsk
->thread
.fpu
.state
->fxsave
.mxcsr
;
550 return MXCSR_DEFAULT
;
554 extern void fpstate_cache_init(void);
556 extern int fpstate_alloc(struct fpu
*fpu
);
557 extern void fpstate_free(struct fpu
*fpu
);
558 extern int fpu__copy(struct task_struct
*dst
, struct task_struct
*src
);
560 static inline unsigned long
561 alloc_mathframe(unsigned long sp
, int ia32_frame
, unsigned long *buf_fx
,
564 unsigned long frame_size
= xstate_sigframe_size();
566 *buf_fx
= sp
= round_down(sp
- frame_size
, 64);
567 if (ia32_frame
&& use_fxsr()) {
568 frame_size
+= sizeof(struct i387_fsave_struct
);
569 sp
-= sizeof(struct i387_fsave_struct
);