bpf: net: Use precomputed btf_id for bpf iterators

One additional field btf_id is added to struct
bpf_ctx_arg_aux to store the precomputed btf_ids.
The btf_id is computed at build time with
BTF_ID_LIST or BTF_ID_LIST_GLOBAL macro definitions.
All existing bpf iterators are changed to used
pre-compute btf_ids.

Signed-off-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20200720163403.1393551-1-yhs@fb.com

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 1df1c0fd3f2849599f8f639e291748ef6858163b..bae557ff2da8be6736e47477e0f482db794f85ce 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -668,6 +668,7 @@ struct bpf_jit_poke_descriptor {
 struct bpf_ctx_arg_aux {
        u32 offset;
        enum bpf_reg_type reg_type;
+       u32 btf_id;
 };
 
 struct bpf_prog_aux {

diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 315cde73421b71a77263a07e955cb5908a2154ac..ee36b7f609368416bdd31056479389880b4835ac 100644 (file)
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -3817,16 +3817,17 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
                return true;
 
        /* this is a pointer to another type */
-       info->reg_type = PTR_TO_BTF_ID;
        for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
                const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
 
                if (ctx_arg_info->offset == off) {
                        info->reg_type = ctx_arg_info->reg_type;
-                       break;
+                       info->btf_id = ctx_arg_info->btf_id;
+                       return true;
                }
        }
 
+       info->reg_type = PTR_TO_BTF_ID;
        if (tgt_prog) {
                ret = btf_translate_to_vmlinux(log, btf, t, tgt_prog->type, arg);
                if (ret > 0) {

diff --git a/kernel/bpf/map_iter.c b/kernel/bpf/map_iter.c
index c69071e334bf6e2c7460715b6e416d9fb6e956f8..8a7af11b411f189b2a4dbbf2e248d83ad2f986df 100644 (file)
--- a/kernel/bpf/map_iter.c
+++ b/kernel/bpf/map_iter.c
@@ -4,6 +4,7 @@
 #include <linux/fs.h>
 #include <linux/filter.h>
 #include <linux/kernel.h>
+#include <linux/btf_ids.h>
 
 struct bpf_iter_seq_map_info {
        u32 mid;
@@ -81,7 +82,10 @@ static const struct seq_operations bpf_map_seq_ops = {
        .show   = bpf_map_seq_show,
 };
 
-static const struct bpf_iter_reg bpf_map_reg_info = {
+BTF_ID_LIST(btf_bpf_map_id)
+BTF_ID(struct, bpf_map)
+
+static struct bpf_iter_reg bpf_map_reg_info = {
        .target                 = "bpf_map",
        .seq_ops                = &bpf_map_seq_ops,
        .init_seq_private       = NULL,
@@ -96,6 +100,7 @@ static const struct bpf_iter_reg bpf_map_reg_info = {
 
 static int __init bpf_map_iter_init(void)
 {
+       bpf_map_reg_info.ctx_arg_info[0].btf_id = *btf_bpf_map_id;
        return bpf_iter_reg_target(&bpf_map_reg_info);
 }
 

diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c
index 4dbf2b6035f87528fe0063dc60234c2181fb0072..2feecf095609029eeaad91994731ca9bd936d592 100644 (file)
--- a/kernel/bpf/task_iter.c
+++ b/kernel/bpf/task_iter.c
@@ -7,6 +7,7 @@
 #include <linux/fs.h>
 #include <linux/fdtable.h>
 #include <linux/filter.h>
+#include <linux/btf_ids.h>
 
 struct bpf_iter_seq_task_common {
        struct pid_namespace *ns;
@@ -312,7 +313,11 @@ static const struct seq_operations task_file_seq_ops = {
        .show   = task_file_seq_show,
 };
 
-static const struct bpf_iter_reg task_reg_info = {
+BTF_ID_LIST(btf_task_file_ids)
+BTF_ID(struct, task_struct)
+BTF_ID(struct, file)
+
+static struct bpf_iter_reg task_reg_info = {
        .target                 = "task",
        .seq_ops                = &task_seq_ops,
        .init_seq_private       = init_seq_pidns,
@@ -325,7 +330,7 @@ static const struct bpf_iter_reg task_reg_info = {
        },
 };
 
-static const struct bpf_iter_reg task_file_reg_info = {
+static struct bpf_iter_reg task_file_reg_info = {
        .target                 = "task_file",
        .seq_ops                = &task_file_seq_ops,
        .init_seq_private       = init_seq_pidns,
@@ -344,10 +349,13 @@ static int __init task_iter_init(void)
 {
        int ret;
 
+       task_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
        ret = bpf_iter_reg_target(&task_reg_info);
        if (ret)
                return ret;
 
+       task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
+       task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1];
        return bpf_iter_reg_target(&task_file_reg_info);
 }
 late_initcall(task_iter_init);

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 116c11a0aaed6262c496e186f01e214d818a977f..a7f1b41482f888897f32dd85336734ca2a3a17bd 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -76,6 +76,7 @@
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/inetdevice.h>
+#include <linux/btf_ids.h>
 
 #include <crypto/hash.h>
 #include <linux/scatterlist.h>
@@ -2954,7 +2955,7 @@ static void bpf_iter_fini_tcp(void *priv_data)
        bpf_iter_fini_seq_net(priv_data);
 }
 
-static const struct bpf_iter_reg tcp_reg_info = {
+static struct bpf_iter_reg tcp_reg_info = {
        .target                 = "tcp",
        .seq_ops                = &bpf_iter_tcp_seq_ops,
        .init_seq_private       = bpf_iter_init_tcp,
@@ -2969,6 +2970,7 @@ static const struct bpf_iter_reg tcp_reg_info = {
 
 static void __init bpf_iter_register(void)
 {
+       tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
        if (bpf_iter_reg_target(&tcp_reg_info))
                pr_warn("Warning: could not register bpf iterator tcp\n");
 }

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index b738c63d7a777989cf3637625ee4b78ca8e6b6ac..b5231ab350e0d8d2e699cd527e4d27b2c6ecc7c5 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -106,6 +106,7 @@
 #include <net/xfrm.h>
 #include <trace/events/udp.h>
 #include <linux/static_key.h>
+#include <linux/btf_ids.h>
 #include <trace/events/skb.h>
 #include <net/busy_poll.h>
 #include "udp_impl.h"
@@ -3232,7 +3233,7 @@ static void bpf_iter_fini_udp(void *priv_data)
        bpf_iter_fini_seq_net(priv_data);
 }
 
-static const struct bpf_iter_reg udp_reg_info = {
+static struct bpf_iter_reg udp_reg_info = {
        .target                 = "udp",
        .seq_ops                = &bpf_iter_udp_seq_ops,
        .init_seq_private       = bpf_iter_init_udp,
@@ -3247,6 +3248,7 @@ static const struct bpf_iter_reg udp_reg_info = {
 
 static void __init bpf_iter_register(void)
 {
+       udp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UDP];
        if (bpf_iter_reg_target(&udp_reg_info))
                pr_warn("Warning: could not register bpf iterator udp\n");
 }

diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 427b81cbc1644c89c35d23f8787a463373b0395e..33f5efbad0a98d8b2d3c01da8482547fad32bbff 100644 (file)
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -61,6 +61,7 @@
 #include <net/l3mdev.h>
 #include <net/ip.h>
 #include <linux/uaccess.h>
+#include <linux/btf_ids.h>
 
 #ifdef CONFIG_SYSCTL
 #include <linux/sysctl.h>
@@ -6423,7 +6424,10 @@ void __init ip6_route_init_special_entries(void)
 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
 DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt)
 
-static const struct bpf_iter_reg ipv6_route_reg_info = {
+BTF_ID_LIST(btf_fib6_info_id)
+BTF_ID(struct, fib6_info)
+
+static struct bpf_iter_reg ipv6_route_reg_info = {
        .target                 = "ipv6_route",
        .seq_ops                = &ipv6_route_seq_ops,
        .init_seq_private       = bpf_iter_init_seq_net,
@@ -6438,6 +6442,7 @@ static const struct bpf_iter_reg ipv6_route_reg_info = {
 
 static int __init bpf_iter_register(void)
 {
+       ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id;
        return bpf_iter_reg_target(&ipv6_route_reg_info);
 }
 

diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index 4f2c3b14ddbfa343aef7cec54cfd42e2a79109c2..3cd58f0c2de43678ddd4a02cd35a4a1804004127 100644 (file)
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -60,6 +60,7 @@
 #include <linux/genetlink.h>
 #include <linux/net_namespace.h>
 #include <linux/nospec.h>
+#include <linux/btf_ids.h>
 
 #include <net/net_namespace.h>
 #include <net/netns/generic.h>
@@ -2803,7 +2804,10 @@ static const struct rhashtable_params netlink_rhashtable_params = {
 };
 
 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
-static const struct bpf_iter_reg netlink_reg_info = {
+BTF_ID_LIST(btf_netlink_sock_id)
+BTF_ID(struct, netlink_sock)
+
+static struct bpf_iter_reg netlink_reg_info = {
        .target                 = "netlink",
        .seq_ops                = &netlink_seq_ops,
        .init_seq_private       = bpf_iter_init_seq_net,
@@ -2818,6 +2822,7 @@ static const struct bpf_iter_reg netlink_reg_info = {
 
 static int __init bpf_iter_register(void)
 {
+       netlink_reg_info.ctx_arg_info[0].btf_id = *btf_netlink_sock_id;
        return bpf_iter_reg_target(&netlink_reg_info);
 }
 #endif