bpf: Add map_meta_equal map ops

Some properties of the inner map is used in the verification time.
When an inner map is inserted to an outer map at runtime,
bpf_map_meta_equal() is currently used to ensure those properties
of the inserting inner map stays the same as the verification
time.

In particular, the current bpf_map_meta_equal() checks max_entries which
turns out to be too restrictive for most of the maps which do not use
max_entries during the verification time.  It limits the use case that
wants to replace a smaller inner map with a larger inner map.  There are
some maps do use max_entries during verification though.  For example,
the map_gen_lookup in array_map_ops uses the max_entries to generate
the inline lookup code.

To accommodate differences between maps, the map_meta_equal is added
to bpf_map_ops.  Each map-type can decide what to check when its
map is used as an inner map during runtime.

Also, some map types cannot be used as an inner map and they are
currently black listed in bpf_map_meta_alloc() in map_in_map.c.
It is not unusual that the new map types may not aware that such
blacklist exists.  This patch enforces an explicit opt-in
and only allows a map to be used as an inner map if it has
implemented the map_meta_equal ops.  It is based on the
discussion in [1].

All maps that support inner map has its map_meta_equal points
to bpf_map_meta_equal in this patch.  A later patch will
relax the max_entries check for most maps.  bpf_types.h
counts 28 map types.  This patch adds 23 ".map_meta_equal"
by using coccinelle.  -5 for
	BPF_MAP_TYPE_PROG_ARRAY
	BPF_MAP_TYPE_(PERCPU)_CGROUP_STORAGE
	BPF_MAP_TYPE_STRUCT_OPS
	BPF_MAP_TYPE_ARRAY_OF_MAPS
	BPF_MAP_TYPE_HASH_OF_MAPS

The "if (inner_map->inner_map_meta)" check in bpf_map_meta_alloc()
is moved such that the same error is returned.

[1]: https://lore.kernel.org/bpf/20200522022342.899756-1-kafai@fb.com/

Signed-off-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20200828011806.1970400-1-kafai@fb.com

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a6131d95e31e42bafb3246edacce46a052e6c1be..dbba82a8008788a56cc9920c3f957133e1b44f45 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -112,6 +112,19 @@ struct bpf_map_ops {
        void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
                                           void *owner, u32 size);
        struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
+
+       /* map_meta_equal must be implemented for maps that can be
+        * used as an inner map.  It is a runtime check to ensure
+        * an inner map can be inserted to an outer map.
+        *
+        * Some properties of the inner map has been used during the
+        * verification time.  When inserting an inner map at the runtime,
+        * map_meta_equal has to ensure the inserting map has the same
+        * properties that the verifier has used earlier.
+        */
+       bool (*map_meta_equal)(const struct bpf_map *meta0,
+                              const struct bpf_map *meta1);
+
        /* BTF name and id of struct allocated by map_alloc */
        const char * const map_btf_name;
        int *map_btf_id;
@@ -235,6 +248,9 @@ int map_check_no_btf(const struct bpf_map *map,
                     const struct btf_type *key_type,
                     const struct btf_type *value_type);
 
+bool bpf_map_meta_equal(const struct bpf_map *meta0,
+                       const struct bpf_map *meta1);
+
 extern const struct bpf_map_ops bpf_map_offload_ops;
 
 /* function argument constraints */

diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 8ff419b632a65b0fef9a1392dd525d6ce5c0d41f..40d1f7f9430700d3cdcb767ab0b218e71868214b 100644 (file)
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -625,6 +625,7 @@ static const struct bpf_iter_seq_info iter_seq_info = {
 
 static int array_map_btf_id;
 const struct bpf_map_ops array_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = array_map_alloc_check,
        .map_alloc = array_map_alloc,
        .map_free = array_map_free,
@@ -647,6 +648,7 @@ const struct bpf_map_ops array_map_ops = {
 
 static int percpu_array_map_btf_id;
 const struct bpf_map_ops percpu_array_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = array_map_alloc_check,
        .map_alloc = array_map_alloc,
        .map_free = array_map_free,
@@ -1003,6 +1005,11 @@ static void prog_array_map_free(struct bpf_map *map)
        fd_array_map_free(map);
 }
 
+/* prog_array->aux->{type,jited} is a runtime binding.
+ * Doing static check alone in the verifier is not enough.
+ * Thus, prog_array_map cannot be used as an inner_map
+ * and map_meta_equal is not implemented.
+ */
 static int prog_array_map_btf_id;
 const struct bpf_map_ops prog_array_map_ops = {
        .map_alloc_check = fd_array_map_alloc_check,
@@ -1101,6 +1108,7 @@ static void perf_event_fd_array_release(struct bpf_map *map,
 
 static int perf_event_array_map_btf_id;
 const struct bpf_map_ops perf_event_array_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = fd_array_map_alloc_check,
        .map_alloc = array_map_alloc,
        .map_free = fd_array_map_free,
@@ -1137,6 +1145,7 @@ static void cgroup_fd_array_free(struct bpf_map *map)
 
 static int cgroup_array_map_btf_id;
 const struct bpf_map_ops cgroup_array_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = fd_array_map_alloc_check,
        .map_alloc = array_map_alloc,
        .map_free = cgroup_fd_array_free,

diff --git a/kernel/bpf/bpf_inode_storage.c b/kernel/bpf/bpf_inode_storage.c
index f3a44e929447e375627d4dca6ac579092e82cfaf..75be02799c0f1a9cb9149a4218b6437fe98c3cb3 100644 (file)
--- a/kernel/bpf/bpf_inode_storage.c
+++ b/kernel/bpf/bpf_inode_storage.c
@@ -235,6 +235,7 @@ static void inode_storage_map_free(struct bpf_map *map)
 
 static int inode_storage_map_btf_id;
 const struct bpf_map_ops inode_storage_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = bpf_local_storage_map_alloc_check,
        .map_alloc = inode_storage_map_alloc,
        .map_free = inode_storage_map_free,

diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index f1c46529929bd397b7f23230e69eddae54eb79b9..8d2a8623d2a7a664dc4e42dcaede1280da6a532e 100644 (file)
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -658,6 +658,7 @@ static int cpu_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
 
 static int cpu_map_btf_id;
 const struct bpf_map_ops cpu_map_ops = {
+       .map_meta_equal         = bpf_map_meta_equal,
        .map_alloc              = cpu_map_alloc,
        .map_free               = cpu_map_free,
        .map_delete_elem        = cpu_map_delete_elem,

diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 10abb06065bb23f14a9d7a306ab25b5538e91153..a42052b85c357837688f38aa45ac86fe4c04656f 100644 (file)
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -751,6 +751,7 @@ static int dev_map_hash_update_elem(struct bpf_map *map, void *key, void *value,
 
 static int dev_map_btf_id;
 const struct bpf_map_ops dev_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = dev_map_alloc,
        .map_free = dev_map_free,
        .map_get_next_key = dev_map_get_next_key,
@@ -764,6 +765,7 @@ const struct bpf_map_ops dev_map_ops = {
 
 static int dev_map_hash_map_btf_id;
 const struct bpf_map_ops dev_map_hash_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = dev_map_alloc,
        .map_free = dev_map_free,
        .map_get_next_key = dev_map_hash_get_next_key,

diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 78dfff6a501b96c013b0e861ecc662be358d018e..ad80f45774e75be1bfdd4765c197561a1395ef97 100644 (file)
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -1810,6 +1810,7 @@ static const struct bpf_iter_seq_info iter_seq_info = {
 
 static int htab_map_btf_id;
 const struct bpf_map_ops htab_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = htab_map_alloc_check,
        .map_alloc = htab_map_alloc,
        .map_free = htab_map_free,
@@ -1827,6 +1828,7 @@ const struct bpf_map_ops htab_map_ops = {
 
 static int htab_lru_map_btf_id;
 const struct bpf_map_ops htab_lru_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = htab_map_alloc_check,
        .map_alloc = htab_map_alloc,
        .map_free = htab_map_free,
@@ -1947,6 +1949,7 @@ static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,
 
 static int htab_percpu_map_btf_id;
 const struct bpf_map_ops htab_percpu_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = htab_map_alloc_check,
        .map_alloc = htab_map_alloc,
        .map_free = htab_map_free,
@@ -1963,6 +1966,7 @@ const struct bpf_map_ops htab_percpu_map_ops = {
 
 static int htab_lru_percpu_map_btf_id;
 const struct bpf_map_ops htab_lru_percpu_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = htab_map_alloc_check,
        .map_alloc = htab_map_alloc,
        .map_free = htab_map_free,

diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index 44474bf3ab7ad21f658236158e8e6402c434a428..00e32f2ec3e60ce89ebaa3df8338e713d8374a89 100644 (file)
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -732,6 +732,7 @@ static int trie_check_btf(const struct bpf_map *map,
 
 static int trie_map_btf_id;
 const struct bpf_map_ops trie_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = trie_alloc,
        .map_free = trie_free,
        .map_get_next_key = trie_get_next_key,

diff --git a/kernel/bpf/map_in_map.c b/kernel/bpf/map_in_map.c
index 17738c93bec8c9a7917b9b5647dd2b8640b972ad..e97a22dd32324b122dd1201cc8d4a4e4b99f8042 100644 (file)
--- a/kernel/bpf/map_in_map.c
+++ b/kernel/bpf/map_in_map.c
@@ -17,23 +17,17 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd)
        if (IS_ERR(inner_map))
                return inner_map;
 
-       /* prog_array->aux->{type,jited} is a runtime binding.
-        * Doing static check alone in the verifier is not enough.
-        */
-       if (inner_map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
-           inner_map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
-           inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ||
-           inner_map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
-               fdput(f);
-               return ERR_PTR(-ENOTSUPP);
-       }
-
        /* Does not support >1 level map-in-map */
        if (inner_map->inner_map_meta) {
                fdput(f);
                return ERR_PTR(-EINVAL);
        }
 
+       if (!inner_map->ops->map_meta_equal) {
+               fdput(f);
+               return ERR_PTR(-ENOTSUPP);
+       }
+
        if (map_value_has_spin_lock(inner_map)) {
                fdput(f);
                return ERR_PTR(-ENOTSUPP);
@@ -89,7 +83,7 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map,
                         struct file *map_file /* not used */,
                         int ufd)
 {
-       struct bpf_map *inner_map;
+       struct bpf_map *inner_map, *inner_map_meta;
        struct fd f;
 
        f = fdget(ufd);
@@ -97,7 +91,8 @@ void *bpf_map_fd_get_ptr(struct bpf_map *map,
        if (IS_ERR(inner_map))
                return inner_map;
 
-       if (bpf_map_meta_equal(map->inner_map_meta, inner_map))
+       inner_map_meta = map->inner_map_meta;
+       if (inner_map_meta->ops->map_meta_equal(inner_map_meta, inner_map))
                bpf_map_inc(inner_map);
        else
                inner_map = ERR_PTR(-EINVAL);

diff --git a/kernel/bpf/map_in_map.h b/kernel/bpf/map_in_map.h
index a507bf6ef8b93af4bf3644e23cc7e7474e82df75..bcb7534afb3c0d073222a20bc7d38538e986ee27 100644 (file)
--- a/kernel/bpf/map_in_map.h
+++ b/kernel/bpf/map_in_map.h
@@ -11,8 +11,6 @@ struct bpf_map;
 
 struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd);
 void bpf_map_meta_free(struct bpf_map *map_meta);
-bool bpf_map_meta_equal(const struct bpf_map *meta0,
-                       const struct bpf_map *meta1);
 void *bpf_map_fd_get_ptr(struct bpf_map *map, struct file *map_file,
                         int ufd);
 void bpf_map_fd_put_ptr(void *ptr);

diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c
index 44184f82916a938f6f487a54672fc7b42490b748..0ee2347ba510de8d839933facafc5a9a7bba4623 100644 (file)
--- a/kernel/bpf/queue_stack_maps.c
+++ b/kernel/bpf/queue_stack_maps.c
@@ -257,6 +257,7 @@ static int queue_stack_map_get_next_key(struct bpf_map *map, void *key,
 
 static int queue_map_btf_id;
 const struct bpf_map_ops queue_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = queue_stack_map_alloc_check,
        .map_alloc = queue_stack_map_alloc,
        .map_free = queue_stack_map_free,
@@ -273,6 +274,7 @@ const struct bpf_map_ops queue_map_ops = {
 
 static int stack_map_btf_id;
 const struct bpf_map_ops stack_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = queue_stack_map_alloc_check,
        .map_alloc = queue_stack_map_alloc,
        .map_free = queue_stack_map_free,

diff --git a/kernel/bpf/reuseport_array.c b/kernel/bpf/reuseport_array.c
index 90b29c5b1da781643830c6fd11564e0b556443cf..5a2ba11824938b028f57afb988404d3ed3f97deb 100644 (file)
--- a/kernel/bpf/reuseport_array.c
+++ b/kernel/bpf/reuseport_array.c
@@ -351,6 +351,7 @@ static int reuseport_array_get_next_key(struct bpf_map *map, void *key,
 
 static int reuseport_array_map_btf_id;
 const struct bpf_map_ops reuseport_array_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = reuseport_array_alloc_check,
        .map_alloc = reuseport_array_alloc,
        .map_free = reuseport_array_free,

diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c
index 002f8a5c9e511e3e688307b22791ee6e56e2510f..31cb04a4dd2d569ba94076ea4705436d4365a557 100644 (file)
--- a/kernel/bpf/ringbuf.c
+++ b/kernel/bpf/ringbuf.c
@@ -287,6 +287,7 @@ static __poll_t ringbuf_map_poll(struct bpf_map *map, struct file *filp,
 
 static int ringbuf_map_btf_id;
 const struct bpf_map_ops ringbuf_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = ringbuf_map_alloc,
        .map_free = ringbuf_map_free,
        .map_mmap = ringbuf_map_mmap,

diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index cfed0ac44d38c64b77cd7430a350f170ded15278..a2fa006f430ef8d800356ece500d66c4ef63c061 100644 (file)
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -839,6 +839,7 @@ static void stack_map_free(struct bpf_map *map)
 
 static int stack_trace_map_btf_id;
 const struct bpf_map_ops stack_trace_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = stack_map_alloc,
        .map_free = stack_map_free,
        .map_get_next_key = stack_map_get_next_key,

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 5443cea86cefa03ec47a32f265b7eeb40a8a4482..b86b1155b74817d91467f2ede0e1a2bf8f047ffd 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -90,6 +90,7 @@ int bpf_check_uarg_tail_zero(void __user *uaddr,
 }
 
 const struct bpf_map_ops bpf_map_offload_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = bpf_map_offload_map_alloc,
        .map_free = bpf_map_offload_map_free,
        .map_check_btf = map_check_no_btf,

diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
index 55fae03b4cc32f5ffbb3ed91b7d5108efddc36c8..a0d1a3265b71880960e0afaf85572b1e9fbda639 100644 (file)
--- a/net/core/bpf_sk_storage.c
+++ b/net/core/bpf_sk_storage.c
@@ -335,6 +335,7 @@ sk_storage_ptr(void *owner)
 
 static int sk_storage_map_btf_id;
 const struct bpf_map_ops sk_storage_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = bpf_local_storage_map_alloc_check,
        .map_alloc = sk_storage_map_alloc,
        .map_free = sk_storage_map_free,

diff --git a/net/core/sock_map.c b/net/core/sock_map.c
index d6c6e1e312fce4d9a819986c078ecc6aad701e1d..078386d7d9a24d6fa52d0a82eeeb5ef6b38ab438 100644 (file)
--- a/net/core/sock_map.c
+++ b/net/core/sock_map.c
@@ -705,6 +705,7 @@ const struct bpf_func_proto bpf_msg_redirect_map_proto = {
 
 static int sock_map_btf_id;
 const struct bpf_map_ops sock_map_ops = {
+       .map_meta_equal         = bpf_map_meta_equal,
        .map_alloc              = sock_map_alloc,
        .map_free               = sock_map_free,
        .map_get_next_key       = sock_map_get_next_key,
@@ -1200,6 +1201,7 @@ const struct bpf_func_proto bpf_msg_redirect_hash_proto = {
 
 static int sock_hash_map_btf_id;
 const struct bpf_map_ops sock_hash_ops = {
+       .map_meta_equal         = bpf_map_meta_equal,
        .map_alloc              = sock_hash_alloc,
        .map_free               = sock_hash_free,
        .map_get_next_key       = sock_hash_get_next_key,

diff --git a/net/xdp/xskmap.c b/net/xdp/xskmap.c
index 8367adbbe9df2fba1345c54c06b253ef2565ecb2..f45f29f041510cf7c067f9bc9d8f1cc2505c0afc 100644 (file)
--- a/net/xdp/xskmap.c
+++ b/net/xdp/xskmap.c
@@ -256,6 +256,7 @@ void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs,
 
 static int xsk_map_btf_id;
 const struct bpf_map_ops xsk_map_ops = {
+       .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = xsk_map_alloc,
        .map_free = xsk_map_free,
        .map_get_next_key = xsk_map_get_next_key,