libbpf: Implement enum value-based CO-RE relocations

Implement two relocations of a new enumerator value-based CO-RE relocation
kind: ENUMVAL_EXISTS and ENUMVAL_VALUE.

First, ENUMVAL_EXISTS, allows to detect the presence of a named enumerator
value in the target (kernel) BTF. This is useful to do BPF helper/map/program
type support detection from BPF program side. bpf_core_enum_value_exists()
macro helper is provided to simplify built-in usage.

Second, ENUMVAL_VALUE, allows to capture enumerator integer value and relocate
it according to the target BTF, if it changes. This is useful to have
a guarantee against intentional or accidental re-ordering/re-numbering of some
of the internal (non-UAPI) enumerations, where kernel developers don't care
about UAPI backwards compatiblity concerns. bpf_core_enum_value() allows to
capture this succinctly and use correct enum values in code.

LLVM uses ldimm64 instruction to capture enumerator value-based relocations,
so add support for ldimm64 instruction patching as well.

Signed-off-by: Andrii Nakryiko <andriin@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Yonghong Song <yhs@fb.com>
Link: https://lore.kernel.org/bpf/20200819194519.3375898-5-andriin@fb.com

diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h
index 684bfb86361af35100125c844d53e0dd2eaf67fe..bbcefb3ff5a572fa7ed5c0ff44b23d0e364bca2c 100644 (file)
--- a/tools/lib/bpf/bpf_core_read.h
+++ b/tools/lib/bpf/bpf_core_read.h
@@ -31,6 +31,12 @@ enum bpf_type_info_kind {
        BPF_TYPE_SIZE = 1,              /* type size in target kernel */
 };
 
+/* second argument to __builtin_preserve_enum_value() built-in */
+enum bpf_enum_value_kind {
+       BPF_ENUMVAL_EXISTS = 0,         /* enum value existence in kernel */
+       BPF_ENUMVAL_VALUE = 1,          /* enum value value relocation */
+};
+
 #define __CORE_RELO(src, field, info)                                        \
        __builtin_preserve_field_info((src)->field, BPF_FIELD_##info)
 
@@ -150,6 +156,28 @@ enum bpf_type_info_kind {
 #define bpf_core_type_size(type)                                           \
        __builtin_preserve_type_info(*(typeof(type) *)0, BPF_TYPE_SIZE)
 
+/*
+ * Convenience macro to check that provided enumerator value is defined in
+ * a target kernel.
+ * Returns:
+ *    1, if specified enum type and its enumerator value are present in target
+ *    kernel's BTF;
+ *    0, if no matching enum and/or enum value within that enum is found.
+ */
+#define bpf_core_enum_value_exists(enum_type, enum_value)                  \
+       __builtin_preserve_enum_value(*(typeof(enum_type) *)enum_value, BPF_ENUMVAL_EXISTS)
+
+/*
+ * Convenience macro to get the integer value of an enumerator value in
+ * a target kernel.
+ * Returns:
+ *    64-bit value, if specified enum type and its enumerator value are
+ *    present in target kernel's BTF;
+ *    0, if no matching enum and/or enum value within that enum is found.
+ */
+#define bpf_core_enum_value(enum_type, enum_value)                         \
+       __builtin_preserve_enum_value(*(typeof(enum_type) *)enum_value, BPF_ENUMVAL_VALUE)
+
 /*
  * bpf_core_read() abstracts away bpf_probe_read_kernel() call and captures
  * offset relocation for source address using __builtin_preserve_access_index()

diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 882759dfc33e996c56770b9a0a999d120dc5b593..77d420c020944fb316bbdf189ba5fc14d6cb2edc 100644 (file)
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -4115,6 +4115,8 @@ static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
        case BPF_TYPE_ID_TARGET: return "target_type_id";
        case BPF_TYPE_EXISTS: return "type_exists";
        case BPF_TYPE_SIZE: return "type_size";
+       case BPF_ENUMVAL_EXISTS: return "enumval_exists";
+       case BPF_ENUMVAL_VALUE: return "enumval_value";
        default: return "unknown";
        }
 }
@@ -4147,6 +4149,17 @@ static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
        }
 }
 
+static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
+{
+       switch (kind) {
+       case BPF_ENUMVAL_EXISTS:
+       case BPF_ENUMVAL_VALUE:
+               return true;
+       default:
+               return false;
+       }
+}
+
 /*
  * Turn bpf_core_relo into a low- and high-level spec representation,
  * validating correctness along the way, as well as calculating resulting
@@ -4180,6 +4193,9 @@ static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
  * Type-based relocations (TYPE_EXISTS/TYPE_SIZE,
  * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their
  * spec and raw_spec are kept empty.
+ *
+ * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access
+ * string to specify enumerator's value index that need to be relocated.
  */
 static int bpf_core_parse_spec(const struct btf *btf,
                               __u32 type_id,
@@ -4224,16 +4240,25 @@ static int bpf_core_parse_spec(const struct btf *btf,
        if (spec->raw_len == 0)
                return -EINVAL;
 
-       /* first spec value is always reloc type array index */
        t = skip_mods_and_typedefs(btf, type_id, &id);
        if (!t)
                return -EINVAL;
 
        access_idx = spec->raw_spec[0];
-       spec->spec[0].type_id = id;
-       spec->spec[0].idx = access_idx;
+       acc = &spec->spec[0];
+       acc->type_id = id;
+       acc->idx = access_idx;
        spec->len++;
 
+       if (core_relo_is_enumval_based(relo_kind)) {
+               if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
+                       return -EINVAL;
+
+               /* record enumerator name in a first accessor */
+               acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off);
+               return 0;
+       }
+
        if (!core_relo_is_field_based(relo_kind))
                return -EINVAL;
 
@@ -4676,6 +4701,39 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
        local_acc = &local_spec->spec[0];
        targ_acc = &targ_spec->spec[0];
 
+       if (core_relo_is_enumval_based(local_spec->relo_kind)) {
+               size_t local_essent_len, targ_essent_len;
+               const struct btf_enum *e;
+               const char *targ_name;
+
+               /* has to resolve to an enum */
+               targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
+               if (!btf_is_enum(targ_type))
+                       return 0;
+
+               local_essent_len = bpf_core_essential_name_len(local_acc->name);
+
+               for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) {
+                       targ_name = btf__name_by_offset(targ_spec->btf, e->name_off);
+                       targ_essent_len = bpf_core_essential_name_len(targ_name);
+                       if (targ_essent_len != local_essent_len)
+                               continue;
+                       if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {
+                               targ_acc->type_id = targ_id;
+                               targ_acc->idx = i;
+                               targ_acc->name = targ_name;
+                               targ_spec->len++;
+                               targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
+                               targ_spec->raw_len++;
+                               return 1;
+                       }
+               }
+               return 0;
+       }
+
+       if (!core_relo_is_field_based(local_spec->relo_kind))
+               return -EINVAL;
+
        for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
                targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
                                                   &targ_id);
@@ -4880,6 +4938,31 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
        return 0;
 }
 
+static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
+                                     const struct bpf_core_spec *spec,
+                                     __u32 *val)
+{
+       const struct btf_type *t;
+       const struct btf_enum *e;
+
+       switch (relo->kind) {
+       case BPF_ENUMVAL_EXISTS:
+               *val = spec ? 1 : 0;
+               break;
+       case BPF_ENUMVAL_VALUE:
+               if (!spec)
+                       return -EUCLEAN; /* request instruction poisoning */
+               t = btf__type_by_id(spec->btf, spec->spec[0].type_id);
+               e = btf_enum(t) + spec->spec[0].idx;
+               *val = e->val;
+               break;
+       default:
+               return -EOPNOTSUPP;
+       }
+
+       return 0;
+}
+
 struct bpf_core_relo_res
 {
        /* expected value in the instruction, unless validate == false */
@@ -4918,6 +5001,9 @@ static int bpf_core_calc_relo(const struct bpf_program *prog,
        } else if (core_relo_is_type_based(relo->kind)) {
                err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
                err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
+       } else if (core_relo_is_enumval_based(relo->kind)) {
+               err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
+               err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
        }
 
        if (err == -EUCLEAN) {
@@ -4954,6 +5040,11 @@ static void bpf_core_poison_insn(struct bpf_program *prog, int relo_idx,
        insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */
 }
 
+static bool is_ldimm64(struct bpf_insn *insn)
+{
+       return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
+}
+
 /*
  * Patch relocatable BPF instruction.
  *
@@ -4966,6 +5057,7 @@ static void bpf_core_poison_insn(struct bpf_program *prog, int relo_idx,
  * Currently three kinds of BPF instructions are supported:
  * 1. rX = <imm> (assignment with immediate operand);
  * 2. rX += <imm> (arithmetic operations with immediate operand);
+ * 3. rX = <imm64> (load with 64-bit immediate value).
  */
 static int bpf_core_patch_insn(struct bpf_program *prog,
                               const struct bpf_core_relo *relo,
@@ -4984,6 +5076,11 @@ static int bpf_core_patch_insn(struct bpf_program *prog,
        class = BPF_CLASS(insn->code);
 
        if (res->poison) {
+               /* poison second part of ldimm64 to avoid confusing error from
+                * verifier about "unknown opcode 00"
+                */
+               if (is_ldimm64(insn))
+                       bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);
                bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);
                return 0;
        }
@@ -5012,7 +5109,7 @@ static int bpf_core_patch_insn(struct bpf_program *prog,
        case BPF_ST:
        case BPF_STX:
                if (res->validate && insn->off != orig_val) {
-                       pr_warn("prog '%s': relo #%d: unexpected insn #%d (LD/LDX/ST/STX) value: got %u, exp %u -> %u\n",
+                       pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n",
                                bpf_program__title(prog, false), relo_idx,
                                insn_idx, insn->off, orig_val, new_val);
                        return -EINVAL;
@@ -5029,8 +5126,36 @@ static int bpf_core_patch_insn(struct bpf_program *prog,
                         bpf_program__title(prog, false), relo_idx, insn_idx,
                         orig_val, new_val);
                break;
+       case BPF_LD: {
+               __u64 imm;
+
+               if (!is_ldimm64(insn) ||
+                   insn[0].src_reg != 0 || insn[0].off != 0 ||
+                   insn_idx + 1 >= prog->insns_cnt ||
+                   insn[1].code != 0 || insn[1].dst_reg != 0 ||
+                   insn[1].src_reg != 0 || insn[1].off != 0) {
+                       pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",
+                               bpf_program__title(prog, false), relo_idx, insn_idx);
+                       return -EINVAL;
+               }
+
+               imm = insn[0].imm + ((__u64)insn[1].imm << 32);
+               if (res->validate && imm != orig_val) {
+                       pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n",
+                               bpf_program__title(prog, false), relo_idx,
+                               insn_idx, imm, orig_val, new_val);
+                       return -EINVAL;
+               }
+
+               insn[0].imm = new_val;
+               insn[1].imm = 0; /* currently only 32-bit values are supported */
+               pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n",
+                        bpf_program__title(prog, false), relo_idx, insn_idx,
+                        imm, new_val);
+               break;
+       }
        default:
-               pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",
+               pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
                        bpf_program__title(prog, false), relo_idx,
                        insn_idx, insn->code, insn->src_reg, insn->dst_reg,
                        insn->off, insn->imm);
@@ -5047,6 +5172,7 @@ static int bpf_core_patch_insn(struct bpf_program *prog,
 static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
 {
        const struct btf_type *t;
+       const struct btf_enum *e;
        const char *s;
        __u32 type_id;
        int i;
@@ -5060,6 +5186,15 @@ static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
        if (core_relo_is_type_based(spec->relo_kind))
                return;
 
+       if (core_relo_is_enumval_based(spec->relo_kind)) {
+               t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
+               e = btf_enum(t) + spec->raw_spec[0];
+               s = btf__name_by_offset(spec->btf, e->name_off);
+
+               libbpf_print(level, "::%s = %u", s, e->val);
+               return;
+       }
+
        if (core_relo_is_field_based(spec->relo_kind)) {
                for (i = 0; i < spec->len; i++) {
                        if (spec->spec[i].name)

diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index edd3511aa2423f780e7af92e3793ba683980e462..61dff515a2f0f32b073ce231122cf55ac82a9d13 100644 (file)
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -242,6 +242,8 @@ enum bpf_core_relo_kind {
        BPF_TYPE_ID_TARGET = 7,         /* type ID in target kernel */
        BPF_TYPE_EXISTS = 8,            /* type existence in target kernel */
        BPF_TYPE_SIZE = 9,              /* type size in bytes */
+       BPF_ENUMVAL_EXISTS = 10,        /* enum value existence in target kernel */
+       BPF_ENUMVAL_VALUE = 11,         /* enum value integer value */
 };
 
 /* The minimum bpf_core_relo checked by the loader