noexec=on: enable non-executable mappings (default)
noexec=off: disable non-executable mappings
+ noibrs [X86]
+ Don't use indirect branch restricted speculation (IBRS)
+ feature when running in secure environment,
+ to avoid performance overhead.
+
+ noibpb [X86]
+ Don't use indirect branch prediction barrier (IBPB)
+ feature when running in secure environment,
+ to avoid performance overhead.
+
nosmap [X86]
Disable SMAP (Supervisor Mode Access Prevention)
even if it is supported by processor.
mb();
}
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibrs_inuse)
native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
__monitor((void *)¤t_thread_info()->flags, 0, 0);
if (!need_resched())
__mwait(eax, ecx);
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibrs_inuse)
native_wrmsrl(MSR_IA32_SPEC_CTRL, FEATURE_ENABLE_IBRS);
}
current_clr_polling();
#ifdef __ASSEMBLY__
+.extern use_ibrs
+.extern use_ibpb
+
#define __ASM_ENABLE_IBRS \
pushq %rax; \
pushq %rcx; \
add $(32*8), %rsp;
.macro ENABLE_IBRS
-ALTERNATIVE "", __stringify(__ASM_ENABLE_IBRS), X86_FEATURE_SPEC_CTRL
+ testl $1, use_ibrs
+ jz 10f
+ __ASM_ENABLE_IBRS
+ jmp 20f
+10:
+ lfence
+20:
.endm
.macro ENABLE_IBRS_CLOBBER
-ALTERNATIVE "", __stringify(__ASM_ENABLE_IBRS_CLOBBER), X86_FEATURE_SPEC_CTRL
+ testl $1, use_ibrs
+ jz 11f
+ __ASM_ENABLE_IBRS_CLOBBER
+ jmp 21f
+11:
+ lfence
+21:
.endm
.macro DISABLE_IBRS
-ALTERNATIVE "", __stringify(__ASM_DISABLE_IBRS), X86_FEATURE_SPEC_CTRL
+ testl $1, use_ibrs
+ jz 9f
+ __ASM_DISABLE_IBRS
+9:
.endm
.macro STUFF_RSB
init_intel_misc_features(c);
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (boot_cpu_has(X86_FEATURE_SPEC_CTRL)) {
printk_once(KERN_INFO "FEATURE SPEC_CTRL Present\n");
- else
+ set_ibrs_supported();
+ set_ibpb_supported();
+ if (ibrs_inuse)
+ sysctl_ibrs_enabled = 1;
+ if (ibpb_inuse)
+ sysctl_ibpb_enabled = 1;
+ } else {
printk_once(KERN_INFO "FEATURE SPEC_CTRL Not Present\n");
+ }
}
#ifdef CONFIG_X86_32
}
if (!ret)
perf_check_microcode();
+
+ if (boot_cpu_has(X86_FEATURE_SPEC_CTRL)) {
+ printk_once(KERN_INFO "FEATURE SPEC_CTRL Present\n");
+ set_ibrs_supported();
+ set_ibpb_supported();
+ if (ibrs_inuse)
+ sysctl_ibrs_enabled = 1;
+ if (ibpb_inuse)
+ sysctl_ibpb_enabled = 1;
+ }
+
mutex_unlock(µcode_mutex);
put_online_cpus();
mb(); /* quirk */
}
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibrs_inuse)
native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
__monitor((void *)¤t_thread_info()->flags, 0, 0);
if (!need_resched()) {
__sti_mwait(0, 0);
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibrs_inuse)
native_wrmsrl(MSR_IA32_SPEC_CTRL, FEATURE_ENABLE_IBRS);
} else {
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibrs_inuse)
native_wrmsrl(MSR_IA32_SPEC_CTRL, FEATURE_ENABLE_IBRS);
local_irq_enable();
}
play_dead_common();
tboot_shutdown(TB_SHUTDOWN_WFS);
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibrs_inuse)
native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
mwait_play_dead(); /* Only returns on failure */
if (cpuidle_play_dead())
hlt_play_dead();
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibrs_inuse)
native_wrmsrl(MSR_IA32_SPEC_CTRL, FEATURE_ENABLE_IBRS);
}
if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
vmcs_load(vmx->loaded_vmcs->vmcs);
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibpb_inuse)
native_wrmsrl(MSR_IA32_PRED_CMD, FEATURE_SET_IBPB);
}
atomic_switch_perf_msrs(vmx);
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ if (ibrs_inuse)
add_atomic_switch_msr(vmx, MSR_IA32_SPEC_CTRL,
vcpu->arch.spec_ctrl, FEATURE_ENABLE_IBRS);
for (;;) {
delay = min_t(u64, MWAITX_MAX_LOOPS, loops);
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
- (delay > IBRS_DISABLE_THRESHOLD))
+ if (ibrs_inuse && (delay > IBRS_DISABLE_THRESHOLD))
native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
/*
*/
__mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE);
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
- (delay > IBRS_DISABLE_THRESHOLD))
+ if (ibrs_inuse && (delay > IBRS_DISABLE_THRESHOLD))
native_wrmsrl(MSR_IA32_SPEC_CTRL, FEATURE_ENABLE_IBRS);
end = rdtsc_ordered();
bool need_flush;
/* Null tsk means switching to kernel, so that's safe */
- if (boot_cpu_has(X86_FEATURE_SPEC_CTRL) && tsk &&
+ if (ibpb_inuse && tsk &&
___ptrace_may_access(tsk, current, PTRACE_MODE_IBPB))
native_wrmsrl(MSR_IA32_PRED_CMD, FEATURE_SET_IBPB);
int smp_call_function_single_async(int cpu, struct call_single_data *csd);
+#ifdef CONFIG_X86
+/* indicate usage of IBRS to control execution speculation */
+extern int use_ibrs;
+extern u32 sysctl_ibrs_enabled;
+extern struct mutex spec_ctrl_mutex;
+#define ibrs_supported (use_ibrs & 0x2)
+#define ibrs_disabled (use_ibrs & 0x4)
+static inline void set_ibrs_inuse(void)
+{
+ if (ibrs_supported)
+ use_ibrs |= 0x1;
+}
+static inline void clear_ibrs_inuse(void)
+{
+ use_ibrs &= ~0x1;
+}
+static inline int check_ibrs_inuse(void)
+{
+ if (use_ibrs & 0x1)
+ return 1;
+ else
+ /* rmb to prevent wrong speculation for security */
+ rmb();
+ return 0;
+}
+static inline void set_ibrs_supported(void)
+{
+ use_ibrs |= 0x2;
+ if (!ibrs_disabled)
+ set_ibrs_inuse();
+}
+static inline void set_ibrs_disabled(void)
+{
+ use_ibrs |= 0x4;
+ if (check_ibrs_inuse())
+ clear_ibrs_inuse();
+}
+static inline void clear_ibrs_disabled(void)
+{
+ use_ibrs &= ~0x4;
+ set_ibrs_inuse();
+}
+#define ibrs_inuse (check_ibrs_inuse())
+
+/* indicate usage of IBPB to control execution speculation */
+extern int use_ibpb;
+extern u32 sysctl_ibpb_enabled;
+#define ibpb_supported (use_ibpb & 0x2)
+#define ibpb_disabled (use_ibpb & 0x4)
+static inline void set_ibpb_inuse(void)
+{
+ if (ibpb_supported)
+ use_ibpb |= 0x1;
+}
+static inline void clear_ibpb_inuse(void)
+{
+ use_ibpb &= ~0x1;
+}
+static inline int check_ibpb_inuse(void)
+{
+ if (use_ibpb & 0x1)
+ return 1;
+ else
+ /* rmb to prevent wrong speculation for security */
+ rmb();
+ return 0;
+}
+static inline void set_ibpb_supported(void)
+{
+ use_ibpb |= 0x2;
+ if (!ibpb_disabled)
+ set_ibpb_inuse();
+}
+static inline void set_ibpb_disabled(void)
+{
+ use_ibpb |= 0x4;
+ if (check_ibpb_inuse())
+ clear_ibpb_inuse();
+}
+static inline void clear_ibpb_disabled(void)
+{
+ use_ibpb &= ~0x4;
+ set_ibpb_inuse();
+}
+#define ibpb_inuse (check_ibpb_inuse())
+#endif
+
#ifdef CONFIG_SMP
#include <linux/preempt.h>
unsigned int setup_max_cpus = NR_CPUS;
EXPORT_SYMBOL(setup_max_cpus);
+#ifdef CONFIG_X86
+/*
+ * use IBRS
+ * bit 0 = indicate if ibrs is currently in use
+ * bit 1 = indicate if system supports ibrs
+ * bit 2 = indicate if admin disables ibrs
+*/
+
+int use_ibrs;
+EXPORT_SYMBOL(use_ibrs);
+
+/*
+ * use IBRS
+ * bit 0 = indicate if ibpb is currently in use
+ * bit 1 = indicate if system supports ibpb
+ * bit 2 = indicate if admin disables ibpb
+*/
+int use_ibpb;
+EXPORT_SYMBOL(use_ibpb);
+#endif
/*
* Setup routine for controlling SMP activation
early_param("nosmp", nosmp);
+#ifdef CONFIG_X86
+static int __init noibrs(char *str)
+{
+ set_ibrs_disabled();
+
+ return 0;
+}
+
+early_param("noibrs", noibrs);
+
+static int __init noibpb(char *str)
+{
+ set_ibpb_disabled();
+
+ return 0;
+}
+
+early_param("noibpb", noibpb);
+#endif
+
+
/* this is hard limit */
static int __init nrcpus(char *str)
{
#include <asm/processor.h>
#ifdef CONFIG_X86
+#include <asm/msr.h>
#include <asm/nmi.h>
#include <asm/stacktrace.h>
#include <asm/io.h>
void __user *buffer, size_t *lenp, loff_t *ppos);
#endif
+#ifdef CONFIG_X86
+int proc_dointvec_ibrs_ctrl(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos);
+int proc_dointvec_ibpb_ctrl(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos);
+int proc_dointvec_ibrs_dump(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos);
+#endif
+
#ifdef CONFIG_MAGIC_SYSRQ
/* Note: sysrq code uses it's own private copy */
static int __sysrq_enabled = CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE;
int sysctl_legacy_va_layout;
#endif
+u32 sysctl_ibrs_dump = 0;
+u32 sysctl_ibrs_enabled = 0;
+EXPORT_SYMBOL(sysctl_ibrs_enabled);
+u32 sysctl_ibpb_enabled = 0;
+EXPORT_SYMBOL(sysctl_ibpb_enabled);
+
/* The default sysctl tables: */
static struct ctl_table sysctl_base_table[] = {
.extra1 = &zero,
.extra2 = &one,
},
+#endif
+#ifdef CONFIG_X86
+ {
+ .procname = "ibrs_enabled",
+ .data = &sysctl_ibrs_enabled,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_ibrs_ctrl,
+ .extra1 = &zero,
+ .extra2 = &two,
+ },
+ {
+ .procname = "ibpb_enabled",
+ .data = &sysctl_ibpb_enabled,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_ibpb_ctrl,
+ .extra1 = &zero,
+ .extra2 = &one,
+ },
+ {
+ .procname = "ibrs_dump",
+ .data = &sysctl_ibrs_dump,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_ibrs_dump,
+ .extra1 = &zero,
+ .extra2 = &one,
+ },
#endif
{ }
};
do_proc_dointvec_minmax_conv, ¶m);
}
+#ifdef CONFIG_X86
+int proc_dointvec_ibrs_dump(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+ int ret;
+ unsigned int cpu;
+
+ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+ printk("sysctl_ibrs_enabled = %u, sysctl_ibpb_enabled = %u\n", sysctl_ibrs_enabled, sysctl_ibpb_enabled);
+ printk("use_ibrs = %d, use_ibpb = %d\n", use_ibrs, use_ibpb);
+ for_each_online_cpu(cpu) {
+ u64 val;
+
+ if (boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+ rdmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, &val);
+ else
+ val = 0;
+ printk("read cpu %d ibrs val %lu\n", cpu, (unsigned long) val);
+ }
+ return ret;
+}
+
+int proc_dointvec_ibrs_ctrl(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+ int ret;
+ unsigned int cpu;
+
+ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+ pr_debug("sysctl_ibrs_enabled = %u, sysctl_ibpb_enabled = %u\n", sysctl_ibrs_enabled, sysctl_ibpb_enabled);
+ pr_debug("before:use_ibrs = %d, use_ibpb = %d\n", use_ibrs, use_ibpb);
+ if (sysctl_ibrs_enabled == 0) {
+ /* always set IBRS off */
+ set_ibrs_disabled();
+ if (ibrs_supported) {
+ for_each_online_cpu(cpu)
+ wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, 0x0);
+ }
+ } else if (sysctl_ibrs_enabled == 2) {
+ /* always set IBRS on, even in user space */
+ clear_ibrs_disabled();
+ if (ibrs_supported) {
+ for_each_online_cpu(cpu)
+ wrmsrl_on_cpu(cpu, MSR_IA32_SPEC_CTRL, FEATURE_ENABLE_IBRS);
+ } else {
+ sysctl_ibrs_enabled = 0;
+ }
+ } else if (sysctl_ibrs_enabled == 1) {
+ /* use IBRS in kernel */
+ clear_ibrs_disabled();
+ if (!ibrs_inuse)
+ /* platform don't support ibrs */
+ sysctl_ibrs_enabled = 0;
+ }
+ pr_debug("after:use_ibrs = %d, use_ibpb = %d\n", use_ibrs, use_ibpb);
+ return ret;
+}
+
+int proc_dointvec_ibpb_ctrl(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+ int ret;
+
+ ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+ pr_debug("sysctl_ibrs_enabled = %u, sysctl_ibpb_enabled = %u\n", sysctl_ibrs_enabled, sysctl_ibpb_enabled);
+ pr_debug("before:use_ibrs = %d, use_ibpb = %d\n", use_ibrs, use_ibpb);
+ if (sysctl_ibpb_enabled == 0)
+ set_ibpb_disabled();
+ else if (sysctl_ibpb_enabled == 1) {
+ clear_ibpb_disabled();
+ if (!ibpb_inuse)
+ /* platform don't support ibpb */
+ sysctl_ibpb_enabled = 0;
+ }
+ pr_debug("after:use_ibrs = %d, use_ibpb = %d\n", use_ibrs, use_ibpb);
+ return ret;
+}
+#endif
+
+
struct do_proc_douintvec_minmax_conv_param {
unsigned int *min;
unsigned int *max;