x86/cpu: Sanitize X86_FEATURE_OSPKE

X86_FEATURE_OSPKE is enabled first on the boot CPU and the feature flag is
set. Secondary CPUs have to enable CR4.PKE as well and set their per CPU
feature flag. That's ineffective because all call sites have checks for
boot_cpu_data.

Make it smarter and force the feature flag when PKU is enabled on the boot
cpu which allows then to use cpu_feature_enabled(X86_FEATURE_OSPKE) all
over the place. That either compiles the code out when PKEY support is
disabled in Kconfig or uses a static_cpu_has() for the feature check which
makes a significant difference in hotpaths, e.g. context switch.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Borislav Petkov <bp@suse.de>
Link: https://lkml.kernel.org/r/20210623121455.305113644@linutronix.de

diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h
index 2ff9b98812b7637de7fedadc25aa60d20634e19a..4128f647c7552c1e05f4b6b40052dc84510f86bd 100644 (file)
--- a/arch/x86/include/asm/pkeys.h
+++ b/arch/x86/include/asm/pkeys.h
@@ -9,14 +9,14 @@
  * will be necessary to ensure that the types that store key
  * numbers and masks have sufficient capacity.
  */
-#define arch_max_pkey() (boot_cpu_has(X86_FEATURE_OSPKE) ? 16 : 1)
+#define arch_max_pkey() (cpu_feature_enabled(X86_FEATURE_OSPKE) ? 16 : 1)
 
 extern int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
                unsigned long init_val);
 
 static inline bool arch_pkeys_enabled(void)
 {
-       return boot_cpu_has(X86_FEATURE_OSPKE);
+       return cpu_feature_enabled(X86_FEATURE_OSPKE);
 }
 
 /*
@@ -26,7 +26,7 @@ static inline bool arch_pkeys_enabled(void)
 extern int __execute_only_pkey(struct mm_struct *mm);
 static inline int execute_only_pkey(struct mm_struct *mm)
 {
-       if (!boot_cpu_has(X86_FEATURE_OSPKE))
+       if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
                return ARCH_DEFAULT_PKEY;
 
        return __execute_only_pkey(mm);
@@ -37,7 +37,7 @@ extern int __arch_override_mprotect_pkey(struct vm_area_struct *vma,
 static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma,
                int prot, int pkey)
 {
-       if (!boot_cpu_has(X86_FEATURE_OSPKE))
+       if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
                return 0;
 
        return __arch_override_mprotect_pkey(vma, prot, pkey);

diff --git a/arch/x86/include/asm/pkru.h b/arch/x86/include/asm/pkru.h
index 35ffcfd6e4037f4bd10d0e2bb992f96615c69988..ec8dd2878dc997e114a6fc803786be9a583300a9 100644 (file)
--- a/arch/x86/include/asm/pkru.h
+++ b/arch/x86/include/asm/pkru.h
@@ -32,7 +32,7 @@ static inline bool __pkru_allows_write(u32 pkru, u16 pkey)
 
 static inline u32 read_pkru(void)
 {
-       if (boot_cpu_has(X86_FEATURE_OSPKE))
+       if (cpu_feature_enabled(X86_FEATURE_OSPKE))
                return rdpkru();
        return 0;
 }
@@ -41,7 +41,7 @@ static inline void write_pkru(u32 pkru)
 {
        struct pkru_state *pk;
 
-       if (!boot_cpu_has(X86_FEATURE_OSPKE))
+       if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
                return;
 
        pk = get_xsave_addr(&current->thread.fpu.state.xsave, XFEATURE_PKRU);

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index f875dea49d7e550521d698a77546a6e4b84cf540..dbfb335ffac4e01e3160cf346d833aa6e846167a 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -466,22 +466,20 @@ static bool pku_disabled;
 
 static __always_inline void setup_pku(struct cpuinfo_x86 *c)
 {
-       /* check the boot processor, plus compile options for PKU: */
-       if (!cpu_feature_enabled(X86_FEATURE_PKU))
-               return;
-       /* checks the actual processor's cpuid bits: */
-       if (!cpu_has(c, X86_FEATURE_PKU))
-               return;
-       if (pku_disabled)
+       if (c == &boot_cpu_data) {
+               if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
+                       return;
+               /*
+                * Setting CR4.PKE will cause the X86_FEATURE_OSPKE cpuid
+                * bit to be set.  Enforce it.
+                */
+               setup_force_cpu_cap(X86_FEATURE_OSPKE);
+
+       } else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
                return;
+       }
 
        cr4_set_bits(X86_CR4_PKE);
-       /*
-        * Setting X86_CR4_PKE will cause the X86_FEATURE_OSPKE
-        * cpuid bit to be set.  We need to ensure that we
-        * update that bit in this CPU's "cpu_info".
-        */
-       set_cpu_cap(c, X86_FEATURE_OSPKE);
 }
 
 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS

diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index 8762b1a8966a5e25ae111448b9a3823fe114f06b..3866954354a45db2a38fc65a7671d9fe59750312 100644 (file)
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -311,7 +311,7 @@ static inline void restore_fpregs_from_init_fpstate(u64 features_mask)
        else
                frstor(&init_fpstate.fsave);
 
-       if (boot_cpu_has(X86_FEATURE_OSPKE))
+       if (cpu_feature_enabled(X86_FEATURE_OSPKE))
                copy_init_pkru_to_fpregs();
 }
 

diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c
index 735d44c3efb6f4927493c7a2ccf21252bed6d97a..bc71609ba65a3b9ec85cb8bef462127edacd93ee 100644 (file)
--- a/arch/x86/kernel/fpu/xstate.c
+++ b/arch/x86/kernel/fpu/xstate.c
@@ -921,7 +921,7 @@ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
         * This check implies XSAVE support.  OSPKE only gets
         * set if we enable XSAVE and we enable PKU in XCR0.
         */
-       if (!boot_cpu_has(X86_FEATURE_OSPKE))
+       if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
                return -EINVAL;
 
        /*

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 4651ab08e6e15ad34004e6d1676a863aba4a1fab..40a963809203d10033562cc5a4a2908e365c4766 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -137,7 +137,7 @@ void __show_regs(struct pt_regs *regs, enum show_regs_mode mode,
                       log_lvl, d3, d6, d7);
        }
 
-       if (boot_cpu_has(X86_FEATURE_OSPKE))
+       if (cpu_feature_enabled(X86_FEATURE_OSPKE))
                printk("%sPKRU: %08x\n", log_lvl, read_pkru());
 }
 

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 6bda7f67d737e9bb88102b295ff013d210a4a377..f33a61a432ce4feff89f376f5e66837bfa7ff526 100644 (file)
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -875,7 +875,7 @@ static inline bool bad_area_access_from_pkeys(unsigned long error_code,
        /* This code is always called on the current mm */
        bool foreign = false;
 
-       if (!boot_cpu_has(X86_FEATURE_OSPKE))
+       if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
                return false;
        if (error_code & X86_PF_PK)
                return true;