bpf: introduce the bpf_get_local_storage() helper function

The bpf_get_local_storage() helper function is used
to get a pointer to the bpf local storage from a bpf program.

It takes a pointer to a storage map and flags as arguments.
Right now it accepts only cgroup storage maps, and flags
argument has to be 0. Further it can be extended to support
other types of local storage: e.g. thread local storage etc.

Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index ca4ac2a39defd6f5b8ffaf773b28dc48e510c6e1..cd8790d2c6ed6c4f66859a59525b2b16db39276f 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -788,6 +788,8 @@ extern const struct bpf_func_proto bpf_sock_map_update_proto;
 extern const struct bpf_func_proto bpf_sock_hash_update_proto;
 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
 
+extern const struct bpf_func_proto bpf_get_local_storage_proto;
+
 /* Shared helpers among cBPF and eBPF. */
 void bpf_user_rnd_init_once(void);
 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index b10118ee5afeea687b65d19ddb659e201182abfd..dd5758dc35d305e6b5a3faaa98e7390ac8984560 100644 (file)
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -2095,6 +2095,24 @@ union bpf_attr {
  *     Return
  *             A 64-bit integer containing the current cgroup id based
  *             on the cgroup within which the current task is running.
+ *
+ * void* get_local_storage(void *map, u64 flags)
+ *     Description
+ *             Get the pointer to the local storage area.
+ *             The type and the size of the local storage is defined
+ *             by the *map* argument.
+ *             The *flags* meaning is specific for each map type,
+ *             and has to be 0 for cgroup local storage.
+ *
+ *             Depending on the bpf program type, a local storage area
+ *             can be shared between multiple instances of the bpf program,
+ *             running simultaneously.
+ *
+ *             A user should care about the synchronization by himself.
+ *             For example, by using the BPF_STX_XADD instruction to alter
+ *             the shared data.
+ *     Return
+ *             Pointer to the local storage area.
  */
 #define __BPF_FUNC_MAPPER(FN)          \
        FN(unspec),                     \
@@ -2177,7 +2195,8 @@ union bpf_attr {
        FN(rc_repeat),                  \
        FN(rc_keydown),                 \
        FN(skb_cgroup_id),              \
-       FN(get_current_cgroup_id),
+       FN(get_current_cgroup_id),      \
+       FN(get_local_storage),
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
  * function eBPF program intends to call

diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index ddfa6cc13e57a344f727b7d9bc825db031ed57b5..0a4fe5a7dc913f30982d3905105c4d19247b70b7 100644 (file)
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -684,6 +684,8 @@ cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
                return &bpf_map_delete_elem_proto;
        case BPF_FUNC_get_current_uid_gid:
                return &bpf_get_current_uid_gid_proto;
+       case BPF_FUNC_get_local_storage:
+               return &bpf_get_local_storage_proto;
        case BPF_FUNC_trace_printk:
                if (capable(CAP_SYS_ADMIN))
                        return bpf_get_trace_printk_proto();

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 9abcf25ebf9f70c71b0cc4a79a0c8e228690a129..4d09e610777fc3bb9bdaa2b540ac1145977f8394 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1795,6 +1795,7 @@ const struct bpf_func_proto bpf_get_current_comm_proto __weak;
 const struct bpf_func_proto bpf_sock_map_update_proto __weak;
 const struct bpf_func_proto bpf_sock_hash_update_proto __weak;
 const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak;
+const struct bpf_func_proto bpf_get_local_storage_proto __weak;
 
 const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void)
 {

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 73065e2d23c2810757e660bf9ad51266e5480803..1991466b8327cf51b32c89ed3c10de4a23083b11 100644 (file)
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -193,4 +193,24 @@ const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
 };
+
+DECLARE_PER_CPU(void*, bpf_cgroup_storage);
+
+BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
+{
+       /* map and flags arguments are not used now,
+        * but provide an ability to extend the API
+        * for other types of local storages.
+        * verifier checks that their values are correct.
+        */
+       return (unsigned long) this_cpu_read(bpf_cgroup_storage);
+}
+
+const struct bpf_func_proto bpf_get_local_storage_proto = {
+       .func           = bpf_get_local_storage,
+       .gpl_only       = false,
+       .ret_type       = RET_PTR_TO_MAP_VALUE,
+       .arg1_type      = ARG_CONST_MAP_PTR,
+       .arg2_type      = ARG_ANYTHING,
+};
 #endif

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 1ede16c8bb4081e06fde4181466b7d7ecb130ddf..587468a9c37d2ea739caf7a18b428196c1b148a9 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2127,6 +2127,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
                    func_id != BPF_FUNC_current_task_under_cgroup)
                        goto error;
                break;
+       case BPF_MAP_TYPE_CGROUP_STORAGE:
+               if (func_id != BPF_FUNC_get_local_storage)
+                       goto error;
+               break;
        /* devmap returns a pointer to a live net_device ifindex that we cannot
         * allow to be modified from bpf side. So do not allow lookup elements
         * for now.
@@ -2209,6 +2213,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
                if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
                        goto error;
                break;
+       case BPF_FUNC_get_local_storage:
+               if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE)
+                       goto error;
+               break;
        default:
                break;
        }
@@ -2533,6 +2541,16 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
        }
 
        regs = cur_regs(env);
+
+       /* check that flags argument in get_local_storage(map, flags) is 0,
+        * this is required because get_local_storage() can't return an error.
+        */
+       if (func_id == BPF_FUNC_get_local_storage &&
+           !register_is_null(&regs[BPF_REG_2])) {
+               verbose(env, "get_local_storage() doesn't support non-zero flags\n");
+               return -EINVAL;
+       }
+
        /* reset caller saved regs */
        for (i = 0; i < CALLER_SAVED_REGS; i++) {
                mark_reg_not_init(env, regs, caller_saved[i]);

diff --git a/net/core/filter.c b/net/core/filter.c
index 9bb9a4488e25a76fa1b844ebf724fceb441d7518..9f73aae2f089a11ba78962db7c8a5364aab78ba7 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4820,6 +4820,8 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
         */
        case BPF_FUNC_get_current_uid_gid:
                return &bpf_get_current_uid_gid_proto;
+       case BPF_FUNC_get_local_storage:
+               return &bpf_get_local_storage_proto;
        default:
                return bpf_base_func_proto(func_id);
        }
@@ -4844,6 +4846,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
                }
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_cookie_sock_addr_proto;
+       case BPF_FUNC_get_local_storage:
+               return &bpf_get_local_storage_proto;
        default:
                return bpf_base_func_proto(func_id);
        }
@@ -4866,6 +4870,17 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
        }
 }
 
+static const struct bpf_func_proto *
+cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+       switch (func_id) {
+       case BPF_FUNC_get_local_storage:
+               return &bpf_get_local_storage_proto;
+       default:
+               return sk_filter_func_proto(func_id, prog);
+       }
+}
+
 static const struct bpf_func_proto *
 tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
@@ -4988,6 +5003,8 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
                return &bpf_sock_hash_update_proto;
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_cookie_sock_ops_proto;
+       case BPF_FUNC_get_local_storage:
+               return &bpf_get_local_storage_proto;
        default:
                return bpf_base_func_proto(func_id);
        }
@@ -5007,6 +5024,8 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
                return &bpf_msg_cork_bytes_proto;
        case BPF_FUNC_msg_pull_data:
                return &bpf_msg_pull_data_proto;
+       case BPF_FUNC_get_local_storage:
+               return &bpf_get_local_storage_proto;
        default:
                return bpf_base_func_proto(func_id);
        }
@@ -5034,6 +5053,8 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
                return &bpf_sk_redirect_map_proto;
        case BPF_FUNC_sk_redirect_hash:
                return &bpf_sk_redirect_hash_proto;
+       case BPF_FUNC_get_local_storage:
+               return &bpf_get_local_storage_proto;
        default:
                return bpf_base_func_proto(func_id);
        }
@@ -6838,7 +6859,7 @@ const struct bpf_prog_ops xdp_prog_ops = {
 };
 
 const struct bpf_verifier_ops cg_skb_verifier_ops = {
-       .get_func_proto         = sk_filter_func_proto,
+       .get_func_proto         = cg_skb_func_proto,
        .is_valid_access        = sk_filter_is_valid_access,
        .convert_ctx_access     = bpf_convert_ctx_access,
 };