s390: use expoline thunks in the BPF JIT

The BPF JIT need safe guarding against spectre v2 in the sk_load_xxx
assembler stubs and the indirect branches generated by the JIT itself
need to be converted to expolines.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/net/bpf_jit.S b/arch/s390/net/bpf_jit.S
index 25bb4643c4f46cc3bcbf904e9fc257a3145f4221..9f794869c1b090a9a6589d573e77739562265cf5 100644 (file)
--- a/arch/s390/net/bpf_jit.S
+++ b/arch/s390/net/bpf_jit.S
@@ -9,6 +9,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/nospec-insn.h>
 #include "bpf_jit.h"
 
 /*
@@ -54,7 +55,7 @@ ENTRY(sk_load_##NAME##_pos);                                          \
        clg     %r3,STK_OFF_HLEN(%r15); /* Offset + SIZE > hlen? */     \
        jh      sk_load_##NAME##_slow;                                  \
        LOAD    %r14,-SIZE(%r3,%r12);   /* Get data from skb */         \
-       b       OFF_OK(%r6);            /* Return */                    \
+       B_EX    OFF_OK,%r6;             /* Return */                    \
                                                                        \
 sk_load_##NAME##_slow:;                                                        \
        lgr     %r2,%r7;                /* Arg1 = skb pointer */        \
@@ -64,11 +65,14 @@ sk_load_##NAME##_slow:;                                                     \
        brasl   %r14,skb_copy_bits;     /* Get data from skb */         \
        LOAD    %r14,STK_OFF_TMP(%r15); /* Load from temp bufffer */    \
        ltgr    %r2,%r2;                /* Set cc to (%r2 != 0) */      \
-       br      %r6;                    /* Return */
+       BR_EX   %r6;                    /* Return */
 
 sk_load_common(word, 4, llgf)  /* r14 = *(u32 *) (skb->data+offset) */
 sk_load_common(half, 2, llgh)  /* r14 = *(u16 *) (skb->data+offset) */
 
+       GEN_BR_THUNK %r6
+       GEN_B_THUNK OFF_OK,%r6
+
 /*
  * Load 1 byte from SKB (optimized version)
  */
@@ -80,7 +84,7 @@ ENTRY(sk_load_byte_pos)
        clg     %r3,STK_OFF_HLEN(%r15)  # Offset >= hlen?
        jnl     sk_load_byte_slow
        llgc    %r14,0(%r3,%r12)        # Get byte from skb
-       b       OFF_OK(%r6)             # Return OK
+       B_EX    OFF_OK,%r6              # Return OK
 
 sk_load_byte_slow:
        lgr     %r2,%r7                 # Arg1 = skb pointer
@@ -90,7 +94,7 @@ sk_load_byte_slow:
        brasl   %r14,skb_copy_bits      # Get data from skb
        llgc    %r14,STK_OFF_TMP(%r15)  # Load result from temp buffer
        ltgr    %r2,%r2                 # Set cc to (%r2 != 0)
-       br      %r6                     # Return cc
+       BR_EX   %r6                     # Return cc
 
 #define sk_negative_common(NAME, SIZE, LOAD)                           \
 sk_load_##NAME##_slow_neg:;                                            \
@@ -104,7 +108,7 @@ sk_load_##NAME##_slow_neg:;                                         \
        jz      bpf_error;                                              \
        LOAD    %r14,0(%r2);            /* Get data from pointer */     \
        xr      %r3,%r3;                /* Set cc to zero */            \
-       br      %r6;                    /* Return cc */
+       BR_EX   %r6;                    /* Return cc */
 
 sk_negative_common(word, 4, llgf)
 sk_negative_common(half, 2, llgh)
@@ -113,4 +117,4 @@ sk_negative_common(byte, 1, llgc)
 bpf_error:
 # force a return 0 from jit handler
        ltgr    %r15,%r15       # Set condition code
-       br      %r6
+       BR_EX   %r6

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 78a19c93b3802ff48b4c27a90d6428100fc5566f..dd2bcf0e7d00d45406c83e1254879e4f16db36fb 100644 (file)
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -25,6 +25,8 @@
 #include <linux/bpf.h>
 #include <asm/cacheflush.h>
 #include <asm/dis.h>
+#include <asm/facility.h>
+#include <asm/nospec-branch.h>
 #include <asm/set_memory.h>
 #include "bpf_jit.h"
 
@@ -41,6 +43,8 @@ struct bpf_jit {
        int base_ip;            /* Base address for literal pool */
        int ret0_ip;            /* Address of return 0 */
        int exit_ip;            /* Address of exit */
+       int r1_thunk_ip;        /* Address of expoline thunk for 'br %r1' */
+       int r14_thunk_ip;       /* Address of expoline thunk for 'br %r14' */
        int tail_call_start;    /* Tail call start offset */
        int labels[1];          /* Labels for local jumps */
 };
@@ -250,6 +254,19 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
        REG_SET_SEEN(b2);                                       \
 })
 
+#define EMIT6_PCREL_RILB(op, b, target)                                \
+({                                                             \
+       int rel = (target - jit->prg) / 2;                      \
+       _EMIT6(op | reg_high(b) << 16 | rel >> 16, rel & 0xffff);       \
+       REG_SET_SEEN(b);                                        \
+})
+
+#define EMIT6_PCREL_RIL(op, target)                            \
+({                                                             \
+       int rel = (target - jit->prg) / 2;                      \
+       _EMIT6(op | rel >> 16, rel & 0xffff);                   \
+})
+
 #define _EMIT6_IMM(op, imm)                                    \
 ({                                                             \
        unsigned int __imm = (imm);                             \
@@ -469,8 +486,45 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
        EMIT4(0xb9040000, REG_2, BPF_REG_0);
        /* Restore registers */
        save_restore_regs(jit, REGS_RESTORE, stack_depth);
+       if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {
+               jit->r14_thunk_ip = jit->prg;
+               /* Generate __s390_indirect_jump_r14 thunk */
+               if (test_facility(35)) {
+                       /* exrl %r0,.+10 */
+                       EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
+               } else {
+                       /* larl %r1,.+14 */
+                       EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);
+                       /* ex 0,0(%r1) */
+                       EMIT4_DISP(0x44000000, REG_0, REG_1, 0);
+               }
+               /* j . */
+               EMIT4_PCREL(0xa7f40000, 0);
+       }
        /* br %r14 */
        _EMIT2(0x07fe);
+
+       if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable &&
+           (jit->seen & SEEN_FUNC)) {
+               jit->r1_thunk_ip = jit->prg;
+               /* Generate __s390_indirect_jump_r1 thunk */
+               if (test_facility(35)) {
+                       /* exrl %r0,.+10 */
+                       EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
+                       /* j . */
+                       EMIT4_PCREL(0xa7f40000, 0);
+                       /* br %r1 */
+                       _EMIT2(0x07f1);
+               } else {
+                       /* larl %r1,.+14 */
+                       EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);
+                       /* ex 0,S390_lowcore.br_r1_tampoline */
+                       EMIT4_DISP(0x44000000, REG_0, REG_0,
+                                  offsetof(struct lowcore, br_r1_trampoline));
+                       /* j . */
+                       EMIT4_PCREL(0xa7f40000, 0);
+               }
+       }
 }
 
 /*
@@ -966,8 +1020,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
                /* lg %w1,<d(imm)>(%l) */
                EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,
                              EMIT_CONST_U64(func));
-               /* basr %r14,%w1 */
-               EMIT2(0x0d00, REG_14, REG_W1);
+               if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {
+                       /* brasl %r14,__s390_indirect_jump_r1 */
+                       EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
+               } else {
+                       /* basr %r14,%w1 */
+                       EMIT2(0x0d00, REG_14, REG_W1);
+               }
                /* lgr %b0,%r2: load return value into %b0 */
                EMIT4(0xb9040000, BPF_REG_0, REG_2);
                if ((jit->seen & SEEN_SKB) &&