bpf: Adjust BPF stack helper functions to accommodate skip > 0

[mirror_ubuntu-jammy-kernel.git] / kernel / bpf / stackmap.c

diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index 0dcaed4d3f4cec61af155571e1b5ee79e0fb7d94..fc0f77f91224bfaa316b3e9e30b12b55e4b6d8ca 100644 (file)
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -219,7 +219,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
 }
 
 static struct perf_callchain_entry *
-get_callchain_entry_for_task(struct task_struct *task, u32 init_nr)
+get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
 {
 #ifdef CONFIG_STACKTRACE
        struct perf_callchain_entry *entry;
@@ -230,9 +230,8 @@ get_callchain_entry_for_task(struct task_struct *task, u32 init_nr)
        if (!entry)
                return NULL;
 
-       entry->nr = init_nr +
-               stack_trace_save_tsk(task, (unsigned long *)(entry->ip + init_nr),
-                                    sysctl_perf_event_max_stack - init_nr, 0);
+       entry->nr = stack_trace_save_tsk(task, (unsigned long *)entry->ip,
+                                        max_depth, 0);
 
        /* stack_trace_save_tsk() works on unsigned long array, while
         * perf_callchain_entry uses u64 array. For 32-bit systems, it is
@@ -244,7 +243,7 @@ get_callchain_entry_for_task(struct task_struct *task, u32 init_nr)
                int i;
 
                /* copy data from the end to avoid using extra buffer */
-               for (i = entry->nr - 1; i >= (int)init_nr; i--)
+               for (i = entry->nr - 1; i >= 0; i--)
                        to[i] = (u64)(from[i]);
        }
 
@@ -261,27 +260,19 @@ static long __bpf_get_stackid(struct bpf_map *map,
 {
        struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
        struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
-       u32 max_depth = map->value_size / stack_map_data_size(map);
-       /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
-       u32 init_nr = sysctl_perf_event_max_stack - max_depth;
        u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
        u32 hash, id, trace_nr, trace_len;
        bool user = flags & BPF_F_USER_STACK;
        u64 *ips;
        bool hash_matches;
 
-       /* get_perf_callchain() guarantees that trace->nr >= init_nr
-        * and trace-nr <= sysctl_perf_event_max_stack, so trace_nr <= max_depth
-        */
-       trace_nr = trace->nr - init_nr;
-
-       if (trace_nr <= skip)
+       if (trace->nr <= skip)
                /* skipping more than usable stack trace */
                return -EFAULT;
 
-       trace_nr -= skip;
+       trace_nr = trace->nr - skip;
        trace_len = trace_nr * sizeof(u64);
-       ips = trace->ip + skip + init_nr;
+       ips = trace->ip + skip;
        hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
        id = hash & (smap->n_buckets - 1);
        bucket = READ_ONCE(smap->buckets[id]);
@@ -338,8 +329,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
           u64, flags)
 {
        u32 max_depth = map->value_size / stack_map_data_size(map);
-       /* stack_map_alloc() checks that max_depth <= sysctl_perf_event_max_stack */
-       u32 init_nr = sysctl_perf_event_max_stack - max_depth;
+       u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
        bool user = flags & BPF_F_USER_STACK;
        struct perf_callchain_entry *trace;
        bool kernel = !user;
@@ -348,8 +338,12 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
                               BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
                return -EINVAL;
 
-       trace = get_perf_callchain(regs, init_nr, kernel, user,
-                                  sysctl_perf_event_max_stack, false, false);
+       max_depth += skip;
+       if (max_depth > sysctl_perf_event_max_stack)
+               max_depth = sysctl_perf_event_max_stack;
+
+       trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
+                                  false, false);
 
        if (unlikely(!trace))
                /* couldn't fetch the stack trace */
@@ -440,7 +434,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
                            struct perf_callchain_entry *trace_in,
                            void *buf, u32 size, u64 flags)
 {
-       u32 init_nr, trace_nr, copy_len, elem_size, num_elem;
+       u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
        bool user_build_id = flags & BPF_F_USER_BUILD_ID;
        u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
        bool user = flags & BPF_F_USER_STACK;
@@ -465,30 +459,28 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
                goto err_fault;
 
        num_elem = size / elem_size;
-       if (sysctl_perf_event_max_stack < num_elem)
-               init_nr = 0;
-       else
-               init_nr = sysctl_perf_event_max_stack - num_elem;
+       max_depth = num_elem + skip;
+       if (sysctl_perf_event_max_stack < max_depth)
+               max_depth = sysctl_perf_event_max_stack;
 
        if (trace_in)
                trace = trace_in;
        else if (kernel && task)
-               trace = get_callchain_entry_for_task(task, init_nr);
+               trace = get_callchain_entry_for_task(task, max_depth);
        else
-               trace = get_perf_callchain(regs, init_nr, kernel, user,
-                                          sysctl_perf_event_max_stack,
+               trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
                                           false, false);
        if (unlikely(!trace))
                goto err_fault;
 
-       trace_nr = trace->nr - init_nr;
-       if (trace_nr < skip)
+       if (trace->nr < skip)
                goto err_fault;
 
-       trace_nr -= skip;
+       trace_nr = trace->nr - skip;
        trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
        copy_len = trace_nr * elem_size;
-       ips = trace->ip + skip + init_nr;
+
+       ips = trace->ip + skip;
        if (user && user_build_id)
                stack_map_get_build_id_offset(buf, ips, trace_nr, user);
        else