bpf, x86: Use kvmalloc_array instead kmalloc_array in bpf_jit_comp

[mirror_ubuntu-focal-kernel.git] / arch / x86 / net / bpf_jit_comp32.c

diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index 393d251798c0fa51ad7dda280aa38d8abc702120..0fcba32077c87705accd544d511395dca8eeae58 100644 (file)
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -1847,14 +1847,16 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                        case BPF_B:
                        case BPF_H:
                        case BPF_W:
-                               if (!bpf_prog->aux->verifier_zext)
+                               if (bpf_prog->aux->verifier_zext)
                                        break;
                                if (dstk) {
                                        EMIT3(0xC7, add_1reg(0x40, IA32_EBP),
                                              STACK_VAR(dst_hi));
                                        EMIT(0x0, 4);
                                } else {
-                                       EMIT3(0xC7, add_1reg(0xC0, dst_hi), 0);
+                                       /* xor dst_hi,dst_hi */
+                                       EMIT2(0x33,
+                                             add_2reg(0xC0, dst_hi, dst_hi));
                                }
                                break;
                        case BPF_DW:
@@ -2013,8 +2015,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                case BPF_JMP | BPF_JSET | BPF_X:
                case BPF_JMP32 | BPF_JSET | BPF_X: {
                        bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
-                       u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
-                       u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+                       u8 dreg_lo = IA32_EAX;
+                       u8 dreg_hi = IA32_EDX;
                        u8 sreg_lo = sstk ? IA32_ECX : src_lo;
                        u8 sreg_hi = sstk ? IA32_EBX : src_hi;
 
@@ -2026,6 +2028,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                                              add_2reg(0x40, IA32_EBP,
                                                       IA32_EDX),
                                              STACK_VAR(dst_hi));
+                       } else {
+                               /* mov dreg_lo,dst_lo */
+                               EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo));
+                               if (is_jmp64)
+                                       /* mov dreg_hi,dst_hi */
+                                       EMIT2(0x89,
+                                             add_2reg(0xC0, dreg_hi, dst_hi));
                        }
 
                        if (sstk) {
@@ -2039,17 +2048,19 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                        }
                        /* and dreg_lo,sreg_lo */
                        EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo));
-                       /* and dreg_hi,sreg_hi */
-                       EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
-                       /* or dreg_lo,dreg_hi */
-                       EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
+                       if (is_jmp64) {
+                               /* and dreg_hi,sreg_hi */
+                               EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi));
+                               /* or dreg_lo,dreg_hi */
+                               EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi));
+                       }
                        goto emit_cond_jmp;
                }
                case BPF_JMP | BPF_JSET | BPF_K:
                case BPF_JMP32 | BPF_JSET | BPF_K: {
                        bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP;
-                       u8 dreg_lo = dstk ? IA32_EAX : dst_lo;
-                       u8 dreg_hi = dstk ? IA32_EDX : dst_hi;
+                       u8 dreg_lo = IA32_EAX;
+                       u8 dreg_hi = IA32_EDX;
                        u8 sreg_lo = IA32_ECX;
                        u8 sreg_hi = IA32_EBX;
                        u32 hi;
@@ -2062,6 +2073,13 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                                              add_2reg(0x40, IA32_EBP,
                                                       IA32_EDX),
                                              STACK_VAR(dst_hi));
+                       } else {
+                               /* mov dreg_lo,dst_lo */
+                               EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo));
+                               if (is_jmp64)
+                                       /* mov dreg_hi,dst_hi */
+                                       EMIT2(0x89,
+                                             add_2reg(0xC0, dreg_hi, dst_hi));
                        }
 
                        /* mov ecx,imm32 */
@@ -2260,7 +2278,16 @@ notyet:
                }
 
                if (image) {
-                       if (unlikely(proglen + ilen > oldproglen)) {
+                       /*
+                        * When populating the image, assert that:
+                        *
+                        *  i) We do not write beyond the allocated space, and
+                        * ii) addrs[i] did not change from the prior run, in order
+                        *     to validate assumptions made for computing branch
+                        *     displacements.
+                        */
+                       if (unlikely(proglen + ilen > oldproglen ||
+                                    proglen + ilen != addrs[i])) {
                                pr_err("bpf_jit: fatal error\n");
                                return -EFAULT;
                        }