s390: add options to change branch prediction behaviour for the kernel

BugLink: http://bugs.launchpad.net/bugs/1754580
Add the PPA instruction to the system entry and exit path to switch
the kernel to a different branch prediction behaviour. The instructions
are added via CPU alternatives and can be disabled with the "nospec"
or the "nobp=0" kernel parameter. If the default behaviour selected
with CONFIG_KERNEL_NOBP is set to "n" then the "nobp=1" parameter can be
used to enable the changed kernel branch prediction.

Acked-by: Cornelia Huck <cohuck@redhat.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
(cherry picked from commit d768bd892fc8f066cd3aa000eb1867bcf32db0ee)
Signed-off-by: Seth Forshee <seth.forshee@canonical.com>

diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 5493895675cef79a43d69d1802033814f6ddfa2f..333390c80e936260781d6bd952f71e2fb8fabe43 100644 (file)
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -541,6 +541,23 @@ config ARCH_RANDOM
 
          If unsure, say Y.
 
+config KERNEL_NOBP
+       def_bool n
+       prompt "Enable modified branch prediction for the kernel by default"
+       help
+         If this option is selected the kernel will switch to a modified
+         branch prediction mode if the firmware interface is available.
+         The modified branch prediction mode improves the behaviour in
+         regard to speculative execution.
+
+         With the option enabled the kernel parameter "nobp=0" or "nospec"
+         can be used to run the kernel in the normal branch prediction mode.
+
+         With the option disabled the modified branch prediction mode is
+         enabled with the "nobp=1" kernel parameter.
+
+         If unsure, say N.
+
 endmenu
 
 menu "Memory setup"

diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index bfbfad4822897246bebc2e0214981de37b926a9d..5f37f9ceef5ebeb2ba7ee2aef71835c874558d51 100644 (file)
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -91,6 +91,7 @@ void cpu_detect_mhz_feature(void);
 extern const struct seq_operations cpuinfo_op;
 extern int sysctl_ieee_emulation_warnings;
 extern void execve_tail(void);
+extern void __bpon(void);
 
 /*
  * User space process size: 2GB for 31 bit, 4TB or 8PT for 64 bit.

diff --git a/arch/s390/kernel/alternative.c b/arch/s390/kernel/alternative.c
index 1abf4f35d059e55bc5906af05ffc97e498a80b1a..22476135f738440a56372b6ac065f7c9cb8b3930 100644 (file)
--- a/arch/s390/kernel/alternative.c
+++ b/arch/s390/kernel/alternative.c
@@ -15,6 +15,29 @@ static int __init disable_alternative_instructions(char *str)
 
 early_param("noaltinstr", disable_alternative_instructions);
 
+static int __init nobp_setup_early(char *str)
+{
+       bool enabled;
+       int rc;
+
+       rc = kstrtobool(str, &enabled);
+       if (rc)
+               return rc;
+       if (enabled && test_facility(82))
+               __set_facility(82, S390_lowcore.alt_stfle_fac_list);
+       else
+               __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
+       return 0;
+}
+early_param("nobp", nobp_setup_early);
+
+static int __init nospec_setup_early(char *str)
+{
+       __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
+       return 0;
+}
+early_param("nospec", nospec_setup_early);
+
 struct brcl_insn {
        u16 opc;
        s32 disp;

diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index 510f2183a7e73d915b9ed1d82762945fad242fcf..ac707a9f729eac9cd095ac521e9e2dfc9e0673de 100644 (file)
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -196,6 +196,8 @@ static noinline __init void setup_facility_list(void)
        memcpy(S390_lowcore.alt_stfle_fac_list,
               S390_lowcore.stfle_fac_list,
               sizeof(S390_lowcore.alt_stfle_fac_list));
+       if (!IS_ENABLED(CONFIG_KERNEL_NOBP))
+               __clear_facility(82, S390_lowcore.alt_stfle_fac_list);
 }
 
 static __init void detect_diag9c(void)

diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index daf45ae769b7f74aa4721aacb5de14d55ff3d828..e65e4551ee25869102cce93f325ff90ec21c085b 100644 (file)
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -159,6 +159,34 @@ _PIF_WORK  = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
                tm      off+\addr, \mask
        .endm
 
+       .macro BPOFF
+       .pushsection .altinstr_replacement, "ax"
+660:   .long   0xb2e8c000
+       .popsection
+661:   .long   0x47000000
+       .pushsection .altinstructions, "a"
+       .long 661b - .
+       .long 660b - .
+       .word 82
+       .byte 4
+       .byte 4
+       .popsection
+       .endm
+
+       .macro BPON
+       .pushsection .altinstr_replacement, "ax"
+662:   .long   0xb2e8d000
+       .popsection
+663:   .long   0x47000000
+       .pushsection .altinstructions, "a"
+       .long 663b - .
+       .long 662b - .
+       .word 82
+       .byte 4
+       .byte 4
+       .popsection
+       .endm
+
        .section .kprobes.text, "ax"
 .Ldummy:
        /*
@@ -171,6 +199,11 @@ _PIF_WORK  = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
         */
        nop     0
 
+ENTRY(__bpon)
+       .globl __bpon
+       BPON
+       br      %r14
+
 /*
  * Scheduler resume function, called by switch_to
  *  gpr2 = (task_struct *) prev
@@ -226,8 +259,11 @@ ENTRY(sie64a)
        jnz     .Lsie_skip
        TSTMSK  __LC_CPU_FLAGS,_CIF_FPU
        jo      .Lsie_skip                      # exit if fp/vx regs changed
+       BPON
 .Lsie_entry:
        sie     0(%r14)
+.Lsie_exit:
+       BPOFF
 .Lsie_skip:
        ni      __SIE_PROG0C+3(%r14),0xfe       # no longer in SIE
        lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
@@ -279,6 +315,7 @@ ENTRY(system_call)
        stpt    __LC_SYNC_ENTER_TIMER
 .Lsysc_stmg:
        stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
+       BPOFF
        lg      %r12,__LC_CURRENT
        lghi    %r13,__TASK_thread
        lghi    %r14,_PIF_SYSCALL
@@ -325,6 +362,7 @@ ENTRY(system_call)
        jnz     .Lsysc_work                     # check for work
        TSTMSK  __LC_CPU_FLAGS,_CIF_WORK
        jnz     .Lsysc_work
+       BPON
 .Lsysc_restore:
        lg      %r14,__LC_VDSO_PER_CPU
        lmg     %r0,%r10,__PT_R0(%r11)
@@ -530,6 +568,7 @@ ENTRY(kernel_thread_starter)
 
 ENTRY(pgm_check_handler)
        stpt    __LC_SYNC_ENTER_TIMER
+       BPOFF
        stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
        lg      %r10,__LC_LAST_BREAK
        lg      %r12,__LC_CURRENT
@@ -637,6 +676,7 @@ ENTRY(pgm_check_handler)
 ENTRY(io_int_handler)
        STCK    __LC_INT_CLOCK
        stpt    __LC_ASYNC_ENTER_TIMER
+       BPOFF
        stmg    %r8,%r15,__LC_SAVE_AREA_ASYNC
        lg      %r12,__LC_CURRENT
        larl    %r13,cleanup_critical
@@ -687,9 +727,13 @@ ENTRY(io_int_handler)
        lg      %r14,__LC_VDSO_PER_CPU
        lmg     %r0,%r10,__PT_R0(%r11)
        mvc     __LC_RETURN_PSW(16),__PT_PSW(%r11)
+       tm      __PT_PSW+1(%r11),0x01   # returning to user ?
+       jno     .Lio_exit_kernel
+       BPON
 .Lio_exit_timer:
        stpt    __LC_EXIT_TIMER
        mvc     __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
+.Lio_exit_kernel:
        lmg     %r11,%r15,__PT_R11(%r11)
        lpswe   __LC_RETURN_PSW
 .Lio_done:
@@ -860,6 +904,7 @@ ENTRY(io_int_handler)
 ENTRY(ext_int_handler)
        STCK    __LC_INT_CLOCK
        stpt    __LC_ASYNC_ENTER_TIMER
+       BPOFF
        stmg    %r8,%r15,__LC_SAVE_AREA_ASYNC
        lg      %r12,__LC_CURRENT
        larl    %r13,cleanup_critical
@@ -908,6 +953,7 @@ ENTRY(psw_idle)
 .Lpsw_idle_stcctm:
 #endif
        oi      __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
+       BPON
        STCK    __CLOCK_IDLE_ENTER(%r2)
        stpt    __TIMER_IDLE_ENTER(%r2)
 .Lpsw_idle_lpsw:
@@ -1008,6 +1054,7 @@ load_fpu_regs:
  */
 ENTRY(mcck_int_handler)
        STCK    __LC_MCCK_CLOCK
+       BPOFF
        la      %r1,4095                # validate r1
        spt     __LC_CPU_TIMER_SAVE_AREA-4095(%r1)      # validate cpu timer
        sckc    __LC_CLOCK_COMPARATOR                   # validate comparator
@@ -1118,6 +1165,7 @@ ENTRY(mcck_int_handler)
        mvc     __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
        tm      __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
        jno     0f
+       BPON
        stpt    __LC_EXIT_TIMER
        mvc     __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
 0:     lmg     %r11,%r15,__PT_R11(%r11)

diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index 8ecb8726ac4762582a6ced188444ed14ea48cf46..443b9b88b2db48148c69cf9def885901bf50c754 100644 (file)
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -547,6 +547,7 @@ static struct kset *ipl_kset;
 
 static void __ipl_run(void *unused)
 {
+       __bpon();
        diag308(DIAG308_LOAD_CLEAR, NULL);
        if (MACHINE_IS_VM)
                __cpcmd("IPL", NULL, 0, NULL);

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index fc28c95716473471ac3f2c94a9aac8928fe7b29e..2fd7d609dae087a406cd2db4e6fa53684b981070 100644 (file)
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -319,6 +319,7 @@ static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
        mem_assign_absolute(lc->restart_fn, (unsigned long) func);
        mem_assign_absolute(lc->restart_data, (unsigned long) data);
        mem_assign_absolute(lc->restart_source, source_cpu);
+       __bpon();
        asm volatile(
                "0:     sigp    0,%0,%2 # sigp restart to target cpu\n"
                "       brc     2,0b    # busy, try again\n"
@@ -903,6 +904,7 @@ void __cpu_die(unsigned int cpu)
 void __noreturn cpu_die(void)
 {
        idle_task_exit();
+       __bpon();
        pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
        for (;;) ;
 }