bpf: Use this_cpu_{inc|dec|inc_return} for bpf_task_storage_busy

Now migrate_disable() does not disable preemption and under some
architectures (e.g. arm64) __this_cpu_{inc|dec|inc_return} are neither
preemption-safe nor IRQ-safe, so for fully preemptible kernel concurrent
lookups or updates on the same task local storage and on the same CPU
may make bpf_task_storage_busy be imbalanced, and
bpf_task_storage_trylock() on the specific cpu will always fail.

Fixing it by using this_cpu_{inc|dec|inc_return} when manipulating
bpf_task_storage_busy.

Fixes: bc235cdb423a ("bpf: Prevent deadlock from recursive bpf_task_storage_[get|delete]")
Signed-off-by: Hou Tao <houtao1@huawei.com>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/r/20220901061938.3789460-2-houtao@huaweicloud.com
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>

diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index 4ee2e7286c23ab7f9ac805df7a0e7ee9d0b8f731..802fc15b0d73cd9ecfe5bf66f7f4d5a8576006f7 100644 (file)
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -555,11 +555,11 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
                                struct bpf_local_storage_elem, map_node))) {
                        if (busy_counter) {
                                migrate_disable();
-                               __this_cpu_inc(*busy_counter);
+                               this_cpu_inc(*busy_counter);
                        }
                        bpf_selem_unlink(selem, false);
                        if (busy_counter) {
-                               __this_cpu_dec(*busy_counter);
+                               this_cpu_dec(*busy_counter);
                                migrate_enable();
                        }
                        cond_resched_rcu();

diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c
index e9014dc626820a3af282e85e6bead87fcb3c3788..6f290623347e01d7bc2984f8ff0e8d7c82e1c840 100644 (file)
--- a/kernel/bpf/bpf_task_storage.c
+++ b/kernel/bpf/bpf_task_storage.c
@@ -26,20 +26,20 @@ static DEFINE_PER_CPU(int, bpf_task_storage_busy);
 static void bpf_task_storage_lock(void)
 {
        migrate_disable();
-       __this_cpu_inc(bpf_task_storage_busy);
+       this_cpu_inc(bpf_task_storage_busy);
 }
 
 static void bpf_task_storage_unlock(void)
 {
-       __this_cpu_dec(bpf_task_storage_busy);
+       this_cpu_dec(bpf_task_storage_busy);
        migrate_enable();
 }
 
 static bool bpf_task_storage_trylock(void)
 {
        migrate_disable();
-       if (unlikely(__this_cpu_inc_return(bpf_task_storage_busy) != 1)) {
-               __this_cpu_dec(bpf_task_storage_busy);
+       if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) {
+               this_cpu_dec(bpf_task_storage_busy);
                migrate_enable();
                return false;
        }