kernel/bpf/dispatcher.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2019 Intel Corporation. */
   3
   4 #include <linux/hash.h>
   5 #include <linux/bpf.h>
   6 #include <linux/filter.h>
   7
   8 /* The BPF dispatcher is a multiway branch code generator. The
   9  * dispatcher is a mechanism to avoid the performance penalty of an
  10  * indirect call, which is expensive when retpolines are enabled. A
  11  * dispatch client registers a BPF program into the dispatcher, and if
  12  * there is available room in the dispatcher a direct call to the BPF
  13  * program will be generated. All calls to the BPF programs called via
  14  * the dispatcher will then be a direct call, instead of an
  15  * indirect. The dispatcher hijacks a trampoline function it via the
  16  * __fentry__ of the trampoline. The trampoline function has the
  17  * following signature:
  18  *
  19  * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi,
  20  *                         unsigned int (*bpf_func)(const void *,
  21  *                                                  const struct bpf_insn *));
  22  */
  23
  24 static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog(
  25         struct bpf_dispatcher *d, struct bpf_prog *prog)
  26 {
  27         int i;
  28
  29         for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
  30                 if (prog == d->progs[i].prog)
  31                         return &d->progs[i];
  32         }
  33         return NULL;
  34 }
  35
  36 static struct bpf_dispatcher_prog *bpf_dispatcher_find_free(
  37         struct bpf_dispatcher *d)
  38 {
  39         return bpf_dispatcher_find_prog(d, NULL);
  40 }
  41
  42 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d,
  43                                     struct bpf_prog *prog)
  44 {
  45         struct bpf_dispatcher_prog *entry;
  46
  47         if (!prog)
  48                 return false;
  49
  50         entry = bpf_dispatcher_find_prog(d, prog);
  51         if (entry) {
  52                 refcount_inc(&entry->users);
  53                 return false;
  54         }
  55
  56         entry = bpf_dispatcher_find_free(d);
  57         if (!entry)
  58                 return false;
  59
  60         bpf_prog_inc(prog);
  61         entry->prog = prog;
  62         refcount_set(&entry->users, 1);
  63         d->num_progs++;
  64         return true;
  65 }
  66
  67 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
  68                                        struct bpf_prog *prog)
  69 {
  70         struct bpf_dispatcher_prog *entry;
  71
  72         if (!prog)
  73                 return false;
  74
  75         entry = bpf_dispatcher_find_prog(d, prog);
  76         if (!entry)
  77                 return false;
  78
  79         if (refcount_dec_and_test(&entry->users)) {
  80                 entry->prog = NULL;
  81                 bpf_prog_put(prog);
  82                 d->num_progs--;
  83                 return true;
  84         }
  85         return false;
  86 }
  87
  88 int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
  89 {
  90         return -ENOTSUPP;
  91 }
  92
  93 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)
  94 {
  95         s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
  96         int i;
  97
  98         for (i = 0; i < BPF_DISPATCHER_MAX; i++) {
  99                 if (d->progs[i].prog)
 100                         *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
 101         }
 102         return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs);
 103 }
 104
 105 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
 106 {
 107         void *old, *new;
 108         u32 noff;
 109         int err;
 110
 111         if (!prev_num_progs) {
 112                 old = NULL;
 113                 noff = 0;
 114         } else {
 115                 old = d->image + d->image_off;
 116                 noff = d->image_off ^ (PAGE_SIZE / 2);
 117         }
 118
 119         new = d->num_progs ? d->image + noff : NULL;
 120         if (new) {
 121                 if (bpf_dispatcher_prepare(d, new))
 122                         return;
 123         }
 124
 125         err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new);
 126         if (err || !new)
 127                 return;
 128
 129         d->image_off = noff;
 130 }
 131
 132 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
 133                                 struct bpf_prog *to)
 134 {
 135         bool changed = false;
 136         int prev_num_progs;
 137
 138         if (from == to)
 139                 return;
 140
 141         mutex_lock(&d->mutex);
 142         if (!d->image) {
 143                 d->image = bpf_jit_alloc_exec_page();
 144                 if (!d->image)
 145                         goto out;
 146                 bpf_image_ksym_add(d->image, &d->ksym);
 147         }
 148
 149         prev_num_progs = d->num_progs;
 150         changed |= bpf_dispatcher_remove_prog(d, from);
 151         changed |= bpf_dispatcher_add_prog(d, to);
 152
 153         if (!changed)
 154                 goto out;
 155
 156         bpf_dispatcher_update(d, prev_num_progs);
 157 out:
 158         mutex_unlock(&d->mutex);
 159 }