1 // SPDX-License-Identifier: GPL-2.0
3 * FPU signal frame handling routines.
6 #include <linux/compat.h>
8 #include <linux/pagemap.h>
10 #include <asm/fpu/internal.h>
11 #include <asm/fpu/signal.h>
12 #include <asm/fpu/regset.h>
13 #include <asm/fpu/xstate.h>
15 #include <asm/sigframe.h>
16 #include <asm/trace/fpu.h>
18 static struct _fpx_sw_bytes fx_sw_reserved __ro_after_init
;
19 static struct _fpx_sw_bytes fx_sw_reserved_ia32 __ro_after_init
;
22 * Check for the presence of extended state information in the
23 * user fpstate pointer in the sigcontext.
25 static inline int check_xstate_in_sigframe(struct fxregs_state __user
*fxbuf
,
26 struct _fpx_sw_bytes
*fx_sw
)
28 int min_xstate_size
= sizeof(struct fxregs_state
) +
29 sizeof(struct xstate_header
);
30 void __user
*fpstate
= fxbuf
;
33 if (__copy_from_user(fx_sw
, &fxbuf
->sw_reserved
[0], sizeof(*fx_sw
)))
36 /* Check for the first magic field and other error scenarios. */
37 if (fx_sw
->magic1
!= FP_XSTATE_MAGIC1
||
38 fx_sw
->xstate_size
< min_xstate_size
||
39 fx_sw
->xstate_size
> fpu_user_xstate_size
||
40 fx_sw
->xstate_size
> fx_sw
->extended_size
)
44 * Check for the presence of second magic word at the end of memory
45 * layout. This detects the case where the user just copied the legacy
46 * fpstate layout with out copying the extended state information
47 * in the memory layout.
49 if (__get_user(magic2
, (__u32 __user
*)(fpstate
+ fx_sw
->xstate_size
)))
52 if (likely(magic2
== FP_XSTATE_MAGIC2
))
55 trace_x86_fpu_xstate_check_failed(¤t
->thread
.fpu
);
57 /* Set the parameters for fx only state */
59 fx_sw
->xstate_size
= sizeof(struct fxregs_state
);
60 fx_sw
->xfeatures
= XFEATURE_MASK_FPSSE
;
65 * Signal frame handlers.
67 static inline int save_fsave_header(struct task_struct
*tsk
, void __user
*buf
)
70 struct xregs_state
*xsave
= &tsk
->thread
.fpu
.state
.xsave
;
71 struct user_i387_ia32_struct env
;
72 struct _fpstate_32 __user
*fp
= buf
;
75 if (!test_thread_flag(TIF_NEED_FPU_LOAD
))
76 fxsave(&tsk
->thread
.fpu
.state
.fxsave
);
79 convert_from_fxsr(&env
, tsk
);
81 if (__copy_to_user(buf
, &env
, sizeof(env
)) ||
82 __put_user(xsave
->i387
.swd
, &fp
->status
) ||
83 __put_user(X86_FXSR_MAGIC
, &fp
->magic
))
86 struct fregs_state __user
*fp
= buf
;
88 if (__get_user(swd
, &fp
->swd
) || __put_user(swd
, &fp
->status
))
95 static inline int save_xstate_epilog(void __user
*buf
, int ia32_frame
)
97 struct xregs_state __user
*x
= buf
;
98 struct _fpx_sw_bytes
*sw_bytes
;
102 /* Setup the bytes not touched by the [f]xsave and reserved for SW. */
103 sw_bytes
= ia32_frame
? &fx_sw_reserved_ia32
: &fx_sw_reserved
;
104 err
= __copy_to_user(&x
->i387
.sw_reserved
, sw_bytes
, sizeof(*sw_bytes
));
109 err
|= __put_user(FP_XSTATE_MAGIC2
,
110 (__u32 __user
*)(buf
+ fpu_user_xstate_size
));
113 * Read the xfeatures which we copied (directly from the cpu or
114 * from the state in task struct) to the user buffers.
116 err
|= __get_user(xfeatures
, (__u32 __user
*)&x
->header
.xfeatures
);
119 * For legacy compatible, we always set FP/SSE bits in the bit
120 * vector while saving the state to the user context. This will
121 * enable us capturing any changes(during sigreturn) to
122 * the FP/SSE bits by the legacy applications which don't touch
123 * xfeatures in the xsave header.
125 * xsave aware apps can change the xfeatures in the xsave
126 * header as well as change any contents in the memory layout.
127 * xrestore as part of sigreturn will capture all the changes.
129 xfeatures
|= XFEATURE_MASK_FPSSE
;
131 err
|= __put_user(xfeatures
, (__u32 __user
*)&x
->header
.xfeatures
);
136 static inline int copy_fpregs_to_sigframe(struct xregs_state __user
*buf
)
141 err
= xsave_to_user_sigframe(buf
);
143 err
= fxsave_to_user_sigframe((struct fxregs_state __user
*) buf
);
145 err
= fnsave_to_user_sigframe((struct fregs_state __user
*) buf
);
147 if (unlikely(err
) && __clear_user(buf
, fpu_user_xstate_size
))
153 * Save the fpu, extended register state to the user signal frame.
155 * 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save
157 * 'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'.
159 * buf == buf_fx for 64-bit frames and 32-bit fsave frame.
160 * buf != buf_fx for 32-bit frames with fxstate.
162 * Try to save it directly to the user frame with disabled page fault handler.
163 * If this fails then do the slow path where the FPU state is first saved to
164 * task's fpu->state and then copy it to the user frame pointed to by the
165 * aligned pointer 'buf_fx'.
167 * If this is a 32-bit frame with fxstate, put a fsave header before
168 * the aligned state at 'buf_fx'.
170 * For [f]xsave state, update the SW reserved fields in the [f]xsave frame
171 * indicating the absence/presence of the extended state to the user.
173 int copy_fpstate_to_sigframe(void __user
*buf
, void __user
*buf_fx
, int size
)
175 struct task_struct
*tsk
= current
;
176 int ia32_fxstate
= (buf
!= buf_fx
);
179 ia32_fxstate
&= (IS_ENABLED(CONFIG_X86_32
) ||
180 IS_ENABLED(CONFIG_IA32_EMULATION
));
182 if (!static_cpu_has(X86_FEATURE_FPU
)) {
183 struct user_i387_ia32_struct fp
;
184 fpregs_soft_get(current
, NULL
, (struct membuf
){.p
= &fp
,
185 .left
= sizeof(fp
)});
186 return copy_to_user(buf
, &fp
, sizeof(fp
)) ? -EFAULT
: 0;
189 if (!access_ok(buf
, size
))
193 * Load the FPU registers if they are not valid for the current task.
194 * With a valid FPU state we can attempt to save the state directly to
195 * userland's stack frame which will likely succeed. If it does not,
196 * resolve the fault in the user memory and try again.
199 if (test_thread_flag(TIF_NEED_FPU_LOAD
))
200 fpregs_restore_userregs();
203 ret
= copy_fpregs_to_sigframe(buf_fx
);
208 if (!fault_in_pages_writeable(buf_fx
, fpu_user_xstate_size
))
213 /* Save the fsave header for the 32-bit frames. */
214 if ((ia32_fxstate
|| !use_fxsr()) && save_fsave_header(tsk
, buf
))
217 if (use_fxsr() && save_xstate_epilog(buf_fx
, ia32_fxstate
))
223 static int __restore_fpregs_from_user(void __user
*buf
, u64 xrestore
,
227 u64 init_bv
= xfeatures_mask_uabi() & ~xrestore
;
230 if (likely(!fx_only
))
231 ret
= xrstor_from_user_sigframe(buf
, xrestore
);
233 ret
= fxrstor_from_user_sigframe(buf
);
235 if (!ret
&& unlikely(init_bv
))
236 os_xrstor(&init_fpstate
.xsave
, init_bv
);
238 } else if (use_fxsr()) {
239 return fxrstor_from_user_sigframe(buf
);
241 return frstor_from_user_sigframe(buf
);
246 * Attempt to restore the FPU registers directly from user memory.
247 * Pagefaults are handled and any errors returned are fatal.
249 static int restore_fpregs_from_user(void __user
*buf
, u64 xrestore
,
250 bool fx_only
, unsigned int size
)
252 struct fpu
*fpu
= ¤t
->thread
.fpu
;
258 ret
= __restore_fpregs_from_user(buf
, xrestore
, fx_only
);
263 * The above did an FPU restore operation, restricted to
264 * the user portion of the registers, and failed, but the
265 * microcode might have modified the FPU registers
268 * If the FPU registers do not belong to current, then
269 * invalidate the FPU register state otherwise the task
270 * might preempt current and return to user space with
271 * corrupted FPU registers.
273 if (test_thread_flag(TIF_NEED_FPU_LOAD
))
274 __cpu_invalidate_fpregs_state();
277 /* Try to handle #PF, but anything else is fatal. */
281 ret
= fault_in_pages_readable(buf
, size
);
288 * Restore supervisor states: previous context switch etc has done
289 * XSAVES and saved the supervisor states in the kernel buffer from
290 * which they can be restored now.
292 * It would be optimal to handle this with a single XRSTORS, but
293 * this does not work because the rest of the FPU registers have
294 * been restored from a user buffer directly.
296 if (test_thread_flag(TIF_NEED_FPU_LOAD
) && xfeatures_mask_supervisor())
297 os_xrstor(&fpu
->state
.xsave
, xfeatures_mask_supervisor());
299 fpregs_mark_activate();
304 static int __fpu_restore_sig(void __user
*buf
, void __user
*buf_fx
,
307 int state_size
= fpu_kernel_xstate_size
;
308 struct task_struct
*tsk
= current
;
309 struct fpu
*fpu
= &tsk
->thread
.fpu
;
310 struct user_i387_ia32_struct env
;
311 u64 user_xfeatures
= 0;
312 bool fx_only
= false;
316 struct _fpx_sw_bytes fx_sw_user
;
318 ret
= check_xstate_in_sigframe(buf_fx
, &fx_sw_user
);
322 fx_only
= !fx_sw_user
.magic1
;
323 state_size
= fx_sw_user
.xstate_size
;
324 user_xfeatures
= fx_sw_user
.xfeatures
;
326 user_xfeatures
= XFEATURE_MASK_FPSSE
;
329 if (likely(!ia32_fxstate
)) {
331 * Attempt to restore the FPU registers directly from user
332 * memory. For that to succeed, the user access cannot cause page
333 * faults. If it does, fall back to the slow path below, going
334 * through the kernel buffer with the enabled pagefault handler.
336 return restore_fpregs_from_user(buf_fx
, user_xfeatures
, fx_only
,
341 * Copy the legacy state because the FP portion of the FX frame has
342 * to be ignored for histerical raisins. The legacy state is folded
343 * in once the larger state has been copied.
345 ret
= __copy_from_user(&env
, buf
, sizeof(env
));
350 * By setting TIF_NEED_FPU_LOAD it is ensured that our xstate is
351 * not modified on context switch and that the xstate is considered
352 * to be loaded again on return to userland (overriding last_cpu avoids
356 if (!test_thread_flag(TIF_NEED_FPU_LOAD
)) {
358 * If supervisor states are available then save the
359 * hardware state in current's fpstate so that the
360 * supervisor state is preserved. Save the full state for
361 * simplicity. There is no point in optimizing this by only
362 * saving the supervisor states and then shuffle them to
363 * the right place in memory. It's ia32 mode. Shrug.
365 if (xfeatures_mask_supervisor())
366 os_xsave(&fpu
->state
.xsave
);
367 set_thread_flag(TIF_NEED_FPU_LOAD
);
369 __fpu_invalidate_fpregs_state(fpu
);
370 __cpu_invalidate_fpregs_state();
373 if (use_xsave() && !fx_only
) {
374 ret
= copy_sigframe_from_user_to_xstate(&fpu
->state
.xsave
, buf_fx
);
378 if (__copy_from_user(&fpu
->state
.fxsave
, buf_fx
,
379 sizeof(fpu
->state
.fxsave
)))
382 /* Reject invalid MXCSR values. */
383 if (fpu
->state
.fxsave
.mxcsr
& ~mxcsr_feature_mask
)
386 /* Enforce XFEATURE_MASK_FPSSE when XSAVE is enabled */
388 fpu
->state
.xsave
.header
.xfeatures
|= XFEATURE_MASK_FPSSE
;
391 /* Fold the legacy FP storage */
392 convert_to_fxsr(&fpu
->state
.fxsave
, &env
);
397 * Remove all UABI feature bits not set in user_xfeatures
398 * from the memory xstate header which makes the full
399 * restore below bring them into init state. This works for
400 * fx_only mode as well because that has only FP and SSE
401 * set in user_xfeatures.
403 * Preserve supervisor states!
405 u64 mask
= user_xfeatures
| xfeatures_mask_supervisor();
407 fpu
->state
.xsave
.header
.xfeatures
&= mask
;
408 ret
= os_xrstor_safe(&fpu
->state
.xsave
, xfeatures_mask_all
);
410 ret
= fxrstor_safe(&fpu
->state
.fxsave
);
414 fpregs_mark_activate();
419 static inline int xstate_sigframe_size(void)
421 return use_xsave() ? fpu_user_xstate_size
+ FP_XSTATE_MAGIC2_SIZE
:
422 fpu_user_xstate_size
;
426 * Restore FPU state from a sigframe:
428 int fpu__restore_sig(void __user
*buf
, int ia32_frame
)
430 unsigned int size
= xstate_sigframe_size();
431 struct fpu
*fpu
= ¤t
->thread
.fpu
;
432 void __user
*buf_fx
= buf
;
433 bool ia32_fxstate
= false;
436 if (unlikely(!buf
)) {
437 fpu__clear_user_states(fpu
);
441 ia32_frame
&= (IS_ENABLED(CONFIG_X86_32
) ||
442 IS_ENABLED(CONFIG_IA32_EMULATION
));
445 * Only FXSR enabled systems need the FX state quirk.
446 * FRSTOR does not need it and can use the fast path.
448 if (ia32_frame
&& use_fxsr()) {
449 buf_fx
= buf
+ sizeof(struct fregs_state
);
450 size
+= sizeof(struct fregs_state
);
454 if (!access_ok(buf
, size
)) {
459 if (!IS_ENABLED(CONFIG_X86_64
) && !cpu_feature_enabled(X86_FEATURE_FPU
)) {
460 ret
= fpregs_soft_set(current
, NULL
, 0,
461 sizeof(struct user_i387_ia32_struct
),
464 ret
= __fpu_restore_sig(buf
, buf_fx
, ia32_fxstate
);
469 fpu__clear_user_states(fpu
);
474 fpu__alloc_mathframe(unsigned long sp
, int ia32_frame
,
475 unsigned long *buf_fx
, unsigned long *size
)
477 unsigned long frame_size
= xstate_sigframe_size();
479 *buf_fx
= sp
= round_down(sp
- frame_size
, 64);
480 if (ia32_frame
&& use_fxsr()) {
481 frame_size
+= sizeof(struct fregs_state
);
482 sp
-= sizeof(struct fregs_state
);
490 unsigned long fpu__get_fpstate_size(void)
492 unsigned long ret
= xstate_sigframe_size();
495 * This space is needed on (most) 32-bit kernels, or when a 32-bit
496 * app is running on a 64-bit kernel. To keep things simple, just
497 * assume the worst case and always include space for 'freg_state',
498 * even for 64-bit apps on 64-bit kernels. This wastes a bit of
499 * space, but keeps the code simple.
501 if ((IS_ENABLED(CONFIG_IA32_EMULATION
) ||
502 IS_ENABLED(CONFIG_X86_32
)) && use_fxsr())
503 ret
+= sizeof(struct fregs_state
);
509 * Prepare the SW reserved portion of the fxsave memory layout, indicating
510 * the presence of the extended state information in the memory layout
511 * pointed by the fpstate pointer in the sigcontext.
512 * This will be saved when ever the FP and extended state context is
513 * saved on the user stack during the signal handler delivery to the user.
515 void fpu__init_prepare_fx_sw_frame(void)
517 int size
= fpu_user_xstate_size
+ FP_XSTATE_MAGIC2_SIZE
;
519 fx_sw_reserved
.magic1
= FP_XSTATE_MAGIC1
;
520 fx_sw_reserved
.extended_size
= size
;
521 fx_sw_reserved
.xfeatures
= xfeatures_mask_uabi();
522 fx_sw_reserved
.xstate_size
= fpu_user_xstate_size
;
524 if (IS_ENABLED(CONFIG_IA32_EMULATION
) ||
525 IS_ENABLED(CONFIG_X86_32
)) {
526 int fsave_header_size
= sizeof(struct fregs_state
);
528 fx_sw_reserved_ia32
= fx_sw_reserved
;
529 fx_sw_reserved_ia32
.extended_size
= size
+ fsave_header_size
;