]>
Commit | Line | Data |
---|---|---|
92acdc58 DB |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef __LINUX_COOKIE_H | |
3 | #define __LINUX_COOKIE_H | |
4 | ||
5 | #include <linux/atomic.h> | |
6 | #include <linux/percpu.h> | |
7 | #include <asm/local.h> | |
8 | ||
9 | struct pcpu_gen_cookie { | |
10 | local_t nesting; | |
11 | u64 last; | |
12 | } __aligned(16); | |
13 | ||
14 | struct gen_cookie { | |
15 | struct pcpu_gen_cookie __percpu *local; | |
16 | atomic64_t forward_last ____cacheline_aligned_in_smp; | |
17 | atomic64_t reverse_last; | |
18 | }; | |
19 | ||
20 | #define COOKIE_LOCAL_BATCH 4096 | |
21 | ||
22 | #define DEFINE_COOKIE(name) \ | |
23 | static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name); \ | |
24 | static struct gen_cookie name = { \ | |
25 | .local = &__##name, \ | |
26 | .forward_last = ATOMIC64_INIT(0), \ | |
27 | .reverse_last = ATOMIC64_INIT(0), \ | |
28 | } | |
29 | ||
30 | static __always_inline u64 gen_cookie_next(struct gen_cookie *gc) | |
31 | { | |
32 | struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local); | |
33 | u64 val; | |
34 | ||
35 | if (likely(local_inc_return(&local->nesting) == 1)) { | |
36 | val = local->last; | |
37 | if (__is_defined(CONFIG_SMP) && | |
38 | unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) { | |
39 | s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH, | |
40 | &gc->forward_last); | |
41 | val = next - COOKIE_LOCAL_BATCH; | |
42 | } | |
43 | local->last = ++val; | |
44 | } else { | |
45 | val = atomic64_dec_return(&gc->reverse_last); | |
46 | } | |
47 | local_dec(&local->nesting); | |
48 | return val; | |
49 | } | |
50 | ||
51 | #endif /* __LINUX_COOKIE_H */ |