2 * Copyright (C) 1994 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * General FPU state handling cleanups
6 * Gareth Hughes <gareth@valinux.com>, May 2000
7 * x86-64 work by Andi Kleen 2002
10 #ifndef _FPU_INTERNAL_H
11 #define _FPU_INTERNAL_H
13 #include <linux/regset.h>
14 #include <linux/compat.h>
15 #include <linux/slab.h>
18 #include <asm/fpu/api.h>
19 #include <asm/fpu/xsave.h>
22 # include <asm/sigcontext32.h>
23 # include <asm/user32.h>
25 int ia32_setup_rt_frame(int sig
, struct ksignal
*ksig
,
26 compat_sigset_t
*set
, struct pt_regs
*regs
);
27 int ia32_setup_frame(int sig
, struct ksignal
*ksig
,
28 compat_sigset_t
*set
, struct pt_regs
*regs
);
30 # define user_i387_ia32_struct user_i387_struct
31 # define user32_fxsr_struct user_fxsr_struct
32 # define ia32_setup_frame __setup_frame
33 # define ia32_setup_rt_frame __setup_rt_frame
36 extern unsigned int mxcsr_feature_mask
;
37 extern void fpu__cpu_init(void);
38 extern void eager_fpu_init(void);
40 DECLARE_PER_CPU(struct fpu
*, fpu_fpregs_owner_ctx
);
42 extern void convert_from_fxsr(struct user_i387_ia32_struct
*env
,
43 struct task_struct
*tsk
);
44 extern void convert_to_fxsr(struct task_struct
*tsk
,
45 const struct user_i387_ia32_struct
*env
);
47 extern user_regset_active_fn fpregs_active
, xfpregs_active
;
48 extern user_regset_get_fn fpregs_get
, xfpregs_get
, fpregs_soft_get
,
50 extern user_regset_set_fn fpregs_set
, xfpregs_set
, fpregs_soft_set
,
54 * xstateregs_active == fpregs_active. Please refer to the comment
55 * at the definition of fpregs_active.
57 #define xstateregs_active fpregs_active
59 #ifdef CONFIG_MATH_EMULATION
60 extern void finit_soft_fpu(struct i387_soft_struct
*soft
);
62 static inline void finit_soft_fpu(struct i387_soft_struct
*soft
) {}
66 * Must be run with preemption disabled: this clears the fpu_fpregs_owner_ctx,
69 * This will disable any lazy FPU state restore of the current FPU state,
70 * but if the current thread owns the FPU, it will still be saved by.
72 static inline void __cpu_disable_lazy_restore(unsigned int cpu
)
74 per_cpu(fpu_fpregs_owner_ctx
, cpu
) = NULL
;
77 static inline int fpu_want_lazy_restore(struct fpu
*fpu
, unsigned int cpu
)
79 return fpu
== this_cpu_read_stable(fpu_fpregs_owner_ctx
) && cpu
== fpu
->last_cpu
;
82 static inline int is_ia32_compat_frame(void)
84 return config_enabled(CONFIG_IA32_EMULATION
) &&
85 test_thread_flag(TIF_IA32
);
88 static inline int is_ia32_frame(void)
90 return config_enabled(CONFIG_X86_32
) || is_ia32_compat_frame();
93 static inline int is_x32_frame(void)
95 return config_enabled(CONFIG_X86_X32_ABI
) && test_thread_flag(TIF_X32
);
98 #define X87_FSW_ES (1 << 7) /* Exception Summary */
100 static __always_inline __pure
bool use_eager_fpu(void)
102 return static_cpu_has_safe(X86_FEATURE_EAGER_FPU
);
105 static __always_inline __pure
bool use_xsaveopt(void)
107 return static_cpu_has_safe(X86_FEATURE_XSAVEOPT
);
110 static __always_inline __pure
bool use_xsave(void)
112 return static_cpu_has_safe(X86_FEATURE_XSAVE
);
115 static __always_inline __pure
bool use_fxsr(void)
117 return static_cpu_has_safe(X86_FEATURE_FXSR
);
120 static inline void fx_finit(struct i387_fxsave_struct
*fx
)
123 fx
->mxcsr
= MXCSR_DEFAULT
;
126 extern void __sanitize_i387_state(struct task_struct
*);
128 static inline void sanitize_i387_state(struct task_struct
*tsk
)
132 __sanitize_i387_state(tsk
);
135 #define user_insn(insn, output, input...) \
138 asm volatile(ASM_STAC "\n" \
140 "2: " ASM_CLAC "\n" \
141 ".section .fixup,\"ax\"\n" \
142 "3: movl $-1,%[err]\n" \
145 _ASM_EXTABLE(1b, 3b) \
146 : [err] "=r" (err), output \
151 #define check_insn(insn, output, input...) \
154 asm volatile("1:" #insn "\n\t" \
156 ".section .fixup,\"ax\"\n" \
157 "3: movl $-1,%[err]\n" \
160 _ASM_EXTABLE(1b, 3b) \
161 : [err] "=r" (err), output \
166 static inline int fsave_user(struct i387_fsave_struct __user
*fx
)
168 return user_insn(fnsave
%[fx
]; fwait
, [fx
] "=m" (*fx
), "m" (*fx
));
171 static inline int fxsave_user(struct i387_fxsave_struct __user
*fx
)
173 if (config_enabled(CONFIG_X86_32
))
174 return user_insn(fxsave
%[fx
], [fx
] "=m" (*fx
), "m" (*fx
));
175 else if (config_enabled(CONFIG_AS_FXSAVEQ
))
176 return user_insn(fxsaveq
%[fx
], [fx
] "=m" (*fx
), "m" (*fx
));
178 /* See comment in fpu_fxsave() below. */
179 return user_insn(rex64
/fxsave (%[fx
]), "=m" (*fx
), [fx
] "R" (fx
));
182 static inline int fxrstor_checking(struct i387_fxsave_struct
*fx
)
184 if (config_enabled(CONFIG_X86_32
))
185 return check_insn(fxrstor
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
186 else if (config_enabled(CONFIG_AS_FXSAVEQ
))
187 return check_insn(fxrstorq
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
189 /* See comment in fpu_fxsave() below. */
190 return check_insn(rex64
/fxrstor (%[fx
]), "=m" (*fx
), [fx
] "R" (fx
),
194 static inline int fxrstor_user(struct i387_fxsave_struct __user
*fx
)
196 if (config_enabled(CONFIG_X86_32
))
197 return user_insn(fxrstor
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
198 else if (config_enabled(CONFIG_AS_FXSAVEQ
))
199 return user_insn(fxrstorq
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
201 /* See comment in fpu_fxsave() below. */
202 return user_insn(rex64
/fxrstor (%[fx
]), "=m" (*fx
), [fx
] "R" (fx
),
206 static inline int frstor_checking(struct i387_fsave_struct
*fx
)
208 return check_insn(frstor
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
211 static inline int frstor_user(struct i387_fsave_struct __user
*fx
)
213 return user_insn(frstor
%[fx
], "=m" (*fx
), [fx
] "m" (*fx
));
216 static inline void fpu_fxsave(struct fpu
*fpu
)
218 if (config_enabled(CONFIG_X86_32
))
219 asm volatile( "fxsave %[fx]" : [fx
] "=m" (fpu
->state
->fxsave
));
220 else if (config_enabled(CONFIG_AS_FXSAVEQ
))
221 asm volatile("fxsaveq %[fx]" : [fx
] "=m" (fpu
->state
->fxsave
));
223 /* Using "rex64; fxsave %0" is broken because, if the memory
224 * operand uses any extended registers for addressing, a second
225 * REX prefix will be generated (to the assembler, rex64
226 * followed by semicolon is a separate instruction), and hence
227 * the 64-bitness is lost.
229 * Using "fxsaveq %0" would be the ideal choice, but is only
230 * supported starting with gas 2.16.
232 * Using, as a workaround, the properly prefixed form below
233 * isn't accepted by any binutils version so far released,
234 * complaining that the same type of prefix is used twice if
235 * an extended register is needed for addressing (fix submitted
236 * to mainline 2005-11-21).
238 * asm volatile("rex64/fxsave %0" : "=m" (fpu->state->fxsave));
240 * This, however, we can work around by forcing the compiler to
241 * select an addressing mode that doesn't require extended
244 asm volatile( "rex64/fxsave (%[fx])"
245 : "=m" (fpu
->state
->fxsave
)
246 : [fx
] "R" (&fpu
->state
->fxsave
));
251 * These must be called with preempt disabled. Returns
252 * 'true' if the FPU state is still intact.
254 static inline int fpu_save_init(struct fpu
*fpu
)
257 xsave_state(&fpu
->state
->xsave
);
260 * xsave header may indicate the init state of the FP.
262 if (!(fpu
->state
->xsave
.xsave_hdr
.xstate_bv
& XSTATE_FP
))
264 } else if (use_fxsr()) {
267 asm volatile("fnsave %[fx]; fwait"
268 : [fx
] "=m" (fpu
->state
->fsave
));
273 * If exceptions are pending, we need to clear them so
274 * that we don't randomly get exceptions later.
276 * FIXME! Is this perhaps only true for the old-style
277 * irq13 case? Maybe we could leave the x87 state
280 if (unlikely(fpu
->state
->fxsave
.swd
& X87_FSW_ES
)) {
281 asm volatile("fnclex");
287 static inline int fpu_restore_checking(struct fpu
*fpu
)
290 return fpu_xrstor_checking(&fpu
->state
->xsave
);
292 return fxrstor_checking(&fpu
->state
->fxsave
);
294 return frstor_checking(&fpu
->state
->fsave
);
297 static inline int restore_fpu_checking(struct fpu
*fpu
)
300 * AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is
301 * pending. Clear the x87 state here by setting it to fixed values.
302 * "m" is a random variable that should be in L1.
304 if (unlikely(static_cpu_has_bug_safe(X86_BUG_FXSAVE_LEAK
))) {
308 "fildl %P[addr]" /* set F?P to defined value */
309 : : [addr
] "m" (fpu
->has_fpu
));
312 return fpu_restore_checking(fpu
);
315 /* Must be paired with an 'stts' after! */
316 static inline void __thread_clear_has_fpu(struct fpu
*fpu
)
319 this_cpu_write(fpu_fpregs_owner_ctx
, NULL
);
322 /* Must be paired with a 'clts' before! */
323 static inline void __thread_set_has_fpu(struct fpu
*fpu
)
326 this_cpu_write(fpu_fpregs_owner_ctx
, fpu
);
330 * Encapsulate the CR0.TS handling together with the
333 * These generally need preemption protection to work,
334 * do try to avoid using these on their own.
336 static inline void __thread_fpu_end(struct fpu
*fpu
)
338 __thread_clear_has_fpu(fpu
);
339 if (!use_eager_fpu())
343 static inline void __thread_fpu_begin(struct fpu
*fpu
)
345 if (!use_eager_fpu())
347 __thread_set_has_fpu(fpu
);
350 static inline void drop_fpu(struct fpu
*fpu
)
353 * Forget coprocessor state..
359 /* Ignore delayed exceptions from user space */
360 asm volatile("1: fwait\n"
362 _ASM_EXTABLE(1b
, 2b
));
363 __thread_fpu_end(fpu
);
366 fpu
->fpstate_active
= 0;
371 static inline void restore_init_xstate(void)
374 xrstor_state(init_xstate_buf
, -1);
376 fxrstor_checking(&init_xstate_buf
->i387
);
380 * Reset the FPU state in the eager case and drop it in the lazy case (later use
383 static inline void fpu_reset_state(struct fpu
*fpu
)
385 if (!use_eager_fpu())
388 restore_init_xstate();
392 * FPU state switching for scheduling.
394 * This is a two-stage process:
396 * - switch_fpu_prepare() saves the old state and
397 * sets the new state of the CR0.TS bit. This is
398 * done within the context of the old process.
400 * - switch_fpu_finish() restores the new state as
403 typedef struct { int preload
; } fpu_switch_t
;
405 static inline fpu_switch_t
406 switch_fpu_prepare(struct fpu
*old_fpu
, struct fpu
*new_fpu
, int cpu
)
411 * If the task has used the math, pre-load the FPU on xsave processors
412 * or if the past 5 consecutive context-switches used math.
414 fpu
.preload
= new_fpu
->fpstate_active
&&
415 (use_eager_fpu() || new_fpu
->counter
> 5);
417 if (old_fpu
->has_fpu
) {
418 if (!fpu_save_init(old_fpu
))
419 old_fpu
->last_cpu
= -1;
421 old_fpu
->last_cpu
= cpu
;
423 /* But leave fpu_fpregs_owner_ctx! */
424 old_fpu
->has_fpu
= 0;
426 /* Don't change CR0.TS if we just switch! */
429 __thread_set_has_fpu(new_fpu
);
430 prefetch(new_fpu
->state
);
431 } else if (!use_eager_fpu())
434 old_fpu
->counter
= 0;
435 old_fpu
->last_cpu
= -1;
438 if (fpu_want_lazy_restore(new_fpu
, cpu
))
441 prefetch(new_fpu
->state
);
442 __thread_fpu_begin(new_fpu
);
449 * By the time this gets called, we've already cleared CR0.TS and
450 * given the process the FPU if we are going to preload the FPU
451 * state - all we need to do is to conditionally restore the register
454 static inline void switch_fpu_finish(struct fpu
*new_fpu
, fpu_switch_t fpu_switch
)
456 if (fpu_switch
.preload
) {
457 if (unlikely(restore_fpu_checking(new_fpu
)))
458 fpu_reset_state(new_fpu
);
463 * Signal frame handlers...
465 extern int save_xstate_sig(void __user
*buf
, void __user
*fx
, int size
);
466 extern int __restore_xstate_sig(void __user
*buf
, void __user
*fx
, int size
);
468 static inline int xstate_sigframe_size(void)
470 return use_xsave() ? xstate_size
+ FP_XSTATE_MAGIC2_SIZE
: xstate_size
;
473 static inline int restore_xstate_sig(void __user
*buf
, int ia32_frame
)
475 void __user
*buf_fx
= buf
;
476 int size
= xstate_sigframe_size();
478 if (ia32_frame
&& use_fxsr()) {
479 buf_fx
= buf
+ sizeof(struct i387_fsave_struct
);
480 size
+= sizeof(struct i387_fsave_struct
);
483 return __restore_xstate_sig(buf
, buf_fx
, size
);
487 * Needs to be preemption-safe.
489 * NOTE! user_fpu_begin() must be used only immediately before restoring
490 * the save state. It does not do any saving/restoring on its own. In
491 * lazy FPU mode, it is just an optimization to avoid a #NM exception,
492 * the task can lose the FPU right after preempt_enable().
494 static inline void user_fpu_begin(void)
496 struct fpu
*fpu
= ¤t
->thread
.fpu
;
500 __thread_fpu_begin(fpu
);
505 * i387 state interaction
507 static inline unsigned short get_fpu_cwd(struct task_struct
*tsk
)
510 return tsk
->thread
.fpu
.state
->fxsave
.cwd
;
512 return (unsigned short)tsk
->thread
.fpu
.state
->fsave
.cwd
;
516 static inline unsigned short get_fpu_swd(struct task_struct
*tsk
)
519 return tsk
->thread
.fpu
.state
->fxsave
.swd
;
521 return (unsigned short)tsk
->thread
.fpu
.state
->fsave
.swd
;
525 static inline unsigned short get_fpu_mxcsr(struct task_struct
*tsk
)
528 return tsk
->thread
.fpu
.state
->fxsave
.mxcsr
;
530 return MXCSR_DEFAULT
;
534 extern void fpstate_cache_init(void);
536 extern int fpstate_alloc(struct fpu
*fpu
);
537 extern void fpstate_free(struct fpu
*fpu
);
538 extern int fpu__copy(struct fpu
*dst_fpu
, struct fpu
*src_fpu
);
540 static inline unsigned long
541 alloc_mathframe(unsigned long sp
, int ia32_frame
, unsigned long *buf_fx
,
544 unsigned long frame_size
= xstate_sigframe_size();
546 *buf_fx
= sp
= round_down(sp
- frame_size
, 64);
547 if (ia32_frame
&& use_fxsr()) {
548 frame_size
+= sizeof(struct i387_fsave_struct
);
549 sp
-= sizeof(struct i387_fsave_struct
);