1 #ifndef _LINUX_JUMP_LABEL_H
2 #define _LINUX_JUMP_LABEL_H
7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
10 * Jump labels provide an interface to generate dynamic branches using
11 * self-modifying code. Assuming toolchain and architecture support the result
12 * of a "if (static_key_false(&key))" statement is a unconditional branch (which
13 * defaults to false - and the true block is placed out of line).
15 * However at runtime we can change the branch target using
16 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
17 * object and for as long as there are references all branches referring to
18 * that particular key will point to the (out of line) true block.
20 * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
21 * must be considered absolute slow paths (machine wide synchronization etc.).
22 * OTOH, since the affected branches are unconditional their runtime overhead
23 * will be absolutely minimal, esp. in the default (off) case where the total
24 * effect is a single NOP of appropriate size. The on case will patch in a jump
25 * to the out-of-line block.
27 * When the control is directly exposed to userspace it is prudent to delay the
28 * decrement to avoid high frequency code modifications which can (and do)
29 * cause significant performance degradation. Struct static_key_deferred and
30 * static_key_slow_dec_deferred() provide for this.
32 * Lacking toolchain and or architecture support, it falls back to a simple
35 * struct static_key my_key = STATIC_KEY_INIT_TRUE;
37 * if (static_key_true(&my_key)) {
40 * will result in the true case being in-line and starts the key with a single
41 * reference. Mixing static_key_true() and static_key_false() on the same key is not
44 * Not initializing the key (static data is initialized to 0s anyway) is the
45 * same as using STATIC_KEY_INIT_FALSE and static_key_false() is
46 * equivalent with static_branch().
50 #include <linux/types.h>
51 #include <linux/compiler.h>
52 #include <linux/workqueue.h>
54 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
58 /* Set lsb bit to 1 if branch is default true, 0 ot */
59 struct jump_entry
*entries
;
61 struct static_key_mod
*next
;
65 struct static_key_deferred
{
66 struct static_key key
;
67 unsigned long timeout
;
68 struct delayed_work work
;
71 # include <asm/jump_label.h>
72 # define HAVE_JUMP_LABEL
73 #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
75 enum jump_label_type
{
76 JUMP_LABEL_DISABLE
= 0,
82 #ifdef HAVE_JUMP_LABEL
84 #define JUMP_LABEL_TRUE_BRANCH 1UL
87 inline struct jump_entry
*jump_label_get_entries(struct static_key
*key
)
89 return (struct jump_entry
*)((unsigned long)key
->entries
90 & ~JUMP_LABEL_TRUE_BRANCH
);
93 static inline bool jump_label_get_branch_default(struct static_key
*key
)
95 if ((unsigned long)key
->entries
& JUMP_LABEL_TRUE_BRANCH
)
100 static __always_inline
bool static_key_false(struct static_key
*key
)
102 return arch_static_branch(key
);
105 static __always_inline
bool static_key_true(struct static_key
*key
)
107 return !static_key_false(key
);
110 /* Deprecated. Please use 'static_key_false() instead. */
111 static __always_inline
bool static_branch(struct static_key
*key
)
113 return arch_static_branch(key
);
116 extern struct jump_entry __start___jump_table
[];
117 extern struct jump_entry __stop___jump_table
[];
119 extern void jump_label_init(void);
120 extern void jump_label_lock(void);
121 extern void jump_label_unlock(void);
122 extern void arch_jump_label_transform(struct jump_entry
*entry
,
123 enum jump_label_type type
);
124 extern void arch_jump_label_transform_static(struct jump_entry
*entry
,
125 enum jump_label_type type
);
126 extern int jump_label_text_reserved(void *start
, void *end
);
127 extern void static_key_slow_inc(struct static_key
*key
);
128 extern void static_key_slow_dec(struct static_key
*key
);
129 extern void static_key_slow_dec_deferred(struct static_key_deferred
*key
);
130 extern void jump_label_apply_nops(struct module
*mod
);
132 jump_label_rate_limit(struct static_key_deferred
*key
, unsigned long rl
);
134 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
135 { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
136 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
137 { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
139 #else /* !HAVE_JUMP_LABEL */
141 #include <linux/atomic.h>
147 static __always_inline
void jump_label_init(void)
151 struct static_key_deferred
{
152 struct static_key key
;
155 static __always_inline
bool static_key_false(struct static_key
*key
)
157 if (unlikely(atomic_read(&key
->enabled
)) > 0)
162 static __always_inline
bool static_key_true(struct static_key
*key
)
164 if (likely(atomic_read(&key
->enabled
)) > 0)
169 /* Deprecated. Please use 'static_key_false() instead. */
170 static __always_inline
bool static_branch(struct static_key
*key
)
172 if (unlikely(atomic_read(&key
->enabled
)) > 0)
177 static inline void static_key_slow_inc(struct static_key
*key
)
179 atomic_inc(&key
->enabled
);
182 static inline void static_key_slow_dec(struct static_key
*key
)
184 atomic_dec(&key
->enabled
);
187 static inline void static_key_slow_dec_deferred(struct static_key_deferred
*key
)
189 static_key_slow_dec(&key
->key
);
192 static inline int jump_label_text_reserved(void *start
, void *end
)
197 static inline void jump_label_lock(void) {}
198 static inline void jump_label_unlock(void) {}
200 static inline int jump_label_apply_nops(struct module
*mod
)
206 jump_label_rate_limit(struct static_key_deferred
*key
,
211 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
212 { .enabled = ATOMIC_INIT(1) })
213 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
214 { .enabled = ATOMIC_INIT(0) })
216 #endif /* HAVE_JUMP_LABEL */
218 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
219 #define jump_label_enabled static_key_enabled
221 static inline bool static_key_enabled(struct static_key
*key
)
223 return (atomic_read(&key
->enabled
) > 0);
226 #endif /* _LINUX_JUMP_LABEL_H */