x86/fpu: Provide struct fpstate

BugLink: https://bugs.launchpad.net/bugs/1967750
New xfeatures will not longer be automatically stored in the regular XSAVE
buffer in thread_struct::fpu.

The kernel will provide the default sized buffer for storing the regular
features up to AVX512 in thread_struct::fpu and if a task requests to use
one of the new features then the register storage has to be extended.

The state will be accessed via a pointer in thread_struct::fpu which
defaults to the builtin storage and can be switched when extended storage
is required.

To avoid conditionals all over the code, create a new container for the
register storage which will gain other information, e.g. size, feature
masks etc., later. For now it just contains the register storage, which
gives it exactly the same layout as the exiting fpu::state.

Stick fpu::state and the new fpu::__fpstate into an anonymous union and
initialize the pointer. Add build time checks to validate that both are
at the same place and have the same size.

This allows step by step conversion of all users.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Borislav Petkov <bp@suse.de>
Link: https://lkml.kernel.org/r/20211013145322.234458659@linutronix.de
(cherry picked from commit 87d0e5be0fac322f4415128def9f16a71a267a40)
Acked-by: Dimitri John Ledkov <dimitri.ledkov@canonical.com>
Signed-off-by: Andrea Righi <andrea.righi@canonical.com>

diff --git a/arch/x86/include/asm/fpu/types.h b/arch/x86/include/asm/fpu/types.h
index f5a38a5f3ae11ec02154fb7898a435ce54b65d4d..3bb6277efbb5494162cdd9097147bc761b1dc74e 100644 (file)
--- a/arch/x86/include/asm/fpu/types.h
+++ b/arch/x86/include/asm/fpu/types.h
@@ -309,6 +309,13 @@ union fpregs_state {
        u8 __padding[PAGE_SIZE];
 };
 
+struct fpstate {
+       /* @regs: The register state union for all supported formats */
+       union fpregs_state              regs;
+
+       /* @regs is dynamically sized! Don't add anything after @regs! */
+} __aligned(64);
+
 /*
  * Highest level per task FPU state data structure that
  * contains the FPU register state plus various FPU
@@ -336,6 +343,14 @@ struct fpu {
         */
        unsigned long                   avx512_timestamp;
 
+       /*
+        * @fpstate:
+        *
+        * Pointer to the active struct fpstate. Initialized to
+        * point at @__fpstate below.
+        */
+       struct fpstate                  *fpstate;
+
        /*
         * @state:
         *
@@ -345,7 +360,10 @@ struct fpu {
         * copy. If the task context-switches away then they get
         * saved here and represent the FPU state.
         */
-       union fpregs_state              state;
+       union {
+               struct fpstate                  __fpstate;
+               union fpregs_state              state;
+       };
        /*
         * WARNING: 'state' is dynamically-sized.  Do not put
         * anything after it here.

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 577f342dbfb27fbd16c1cb38482c969a61887958..f6cf20ebc2807edb09575f93c035249c3edf430d 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -538,11 +538,11 @@ struct thread_struct {
         */
 };
 
-/* Whitelist the FPU state from the task_struct for hardened usercopy. */
+/* Whitelist the FPU register state from the task_struct for hardened usercopy. */
 static inline void arch_thread_struct_whitelist(unsigned long *offset,
                                                unsigned long *size)
 {
-       *offset = offsetof(struct thread_struct, fpu.state);
+       *offset = offsetof(struct thread_struct, fpu.__fpstate.regs);
        *size = fpu_kernel_xstate_size;
 }
 

diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index ac540a7d410e512d0a090736e826153c20645f29..d7643115a7ee4141b4e0f39334f813ea6ded34de 100644 (file)
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -337,10 +337,17 @@ void fpstate_init_user(union fpregs_state *state)
                fpstate_init_fstate(&state->fsave);
 }
 
+void fpstate_reset(struct fpu *fpu)
+{
+       /* Set the fpstate pointer to the default fpstate */
+       fpu->fpstate = &fpu->__fpstate;
+}
+
 #if IS_ENABLED(CONFIG_KVM)
 void fpu_init_fpstate_user(struct fpu *fpu)
 {
-       fpstate_init_user(&fpu->state);
+       fpstate_reset(fpu);
+       fpstate_init_user(&fpu->fpstate->regs);
 }
 EXPORT_SYMBOL_GPL(fpu_init_fpstate_user);
 #endif
@@ -354,6 +361,8 @@ int fpu_clone(struct task_struct *dst)
        /* The new task's FPU state cannot be valid in the hardware. */
        dst_fpu->last_cpu = -1;
 
+       fpstate_reset(dst_fpu);
+
        if (!cpu_feature_enabled(X86_FEATURE_FPU))
                return 0;
 

diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c
index 23791355ca67d8d9c41e9ab5c24487d5fb1aa134..31ecbfba9ff7ceb42033ee80dbea0ad19e4a1db4 100644 (file)
--- a/arch/x86/kernel/fpu/init.c
+++ b/arch/x86/kernel/fpu/init.c
@@ -165,7 +165,7 @@ static void __init fpu__init_task_struct_size(void)
         * Subtract off the static size of the register state.
         * It potentially has a bunch of padding.
         */
-       task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state);
+       task_size -= sizeof(current->thread.fpu.__fpstate.regs);
 
        /*
         * Add back the dynamically-calculated register state
@@ -180,10 +180,14 @@ static void __init fpu__init_task_struct_size(void)
         * you hit a compile error here, check the structure to
         * see if something got added to the end.
         */
-       CHECK_MEMBER_AT_END_OF(struct fpu, state);
+       CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate);
        CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
        CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
 
+       BUILD_BUG_ON(sizeof(struct fpstate) != sizeof(union fpregs_state));
+       BUILD_BUG_ON(offsetof(struct thread_struct, fpu.state) !=
+                    offsetof(struct thread_struct, fpu.__fpstate));
+
        arch_task_struct_size = task_size;
 }
 
@@ -220,6 +224,7 @@ static void __init fpu__init_system_xstate_size_legacy(void)
  */
 void __init fpu__init_system(struct cpuinfo_x86 *c)
 {
+       fpstate_reset(&current->thread.fpu);
        fpu__init_system_early_generic(c);
 
        /*

diff --git a/arch/x86/kernel/fpu/internal.h b/arch/x86/kernel/fpu/internal.h
index 479f2db6e16066f1599812dad5abe9abf43cf217..63bd75fe95a8596aee518dbc23badcdec141a7e6 100644 (file)
--- a/arch/x86/kernel/fpu/internal.h
+++ b/arch/x86/kernel/fpu/internal.h
@@ -26,5 +26,6 @@ extern void fpu__init_prepare_fx_sw_frame(void);
 
 /* Used in init.c */
 extern void fpstate_init_user(union fpregs_state *state);
+extern void fpstate_reset(struct fpu *fpu);
 
 #endif