include/linux/cookie.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __LINUX_COOKIE_H
   3 #define __LINUX_COOKIE_H
   4
   5 #include <linux/atomic.h>
   6 #include <linux/percpu.h>
   7 #include <asm/local.h>
   8
   9 struct pcpu_gen_cookie {
  10         local_t nesting;
  11         u64 last;
  12 } __aligned(16);
  13
  14 struct gen_cookie {
  15         struct pcpu_gen_cookie __percpu *local;
  16         atomic64_t forward_last ____cacheline_aligned_in_smp;
  17         atomic64_t reverse_last;
  18 };
  19
  20 #define COOKIE_LOCAL_BATCH      4096
  21
  22 #define DEFINE_COOKIE(name)                                             \
  23         static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name);        \
  24         static struct gen_cookie name = {                               \
  25                 .local          = &__##name,                            \
  26                 .forward_last   = ATOMIC64_INIT(0),                     \
  27                 .reverse_last   = ATOMIC64_INIT(0),                     \
  28         }
  29
  30 static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)
  31 {
  32         struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);
  33         u64 val;
  34
  35         if (likely(local_inc_return(&local->nesting) == 1)) {
  36                 val = local->last;
  37                 if (__is_defined(CONFIG_SMP) &&
  38                     unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {
  39                         s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,
  40                                                        &gc->forward_last);
  41                         val = next - COOKIE_LOCAL_BATCH;
  42                 }
  43                 local->last = ++val;
  44         } else {
  45                 val = atomic64_dec_return(&gc->reverse_last);
  46         }
  47         local_dec(&local->nesting);
  48         return val;
  49 }
  50
  51 #endif /* __LINUX_COOKIE_H */