1 #ifndef _LINUX_JUMP_LABEL_H
2 #define _LINUX_JUMP_LABEL_H
7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
12 * The use of 'struct static_key' directly, is now DEPRECATED. In addition
13 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
15 * struct static_key false = STATIC_KEY_INIT_FALSE;
16 * struct static_key true = STATIC_KEY_INIT_TRUE;
20 * The updated API replacements are:
22 * DEFINE_STATIC_KEY_TRUE(key);
23 * DEFINE_STATIC_KEY_FALSE(key);
24 * DEFINE_STATIC_KEY_ARRAY_TRUE(keys, count);
25 * DEFINE_STATIC_KEY_ARRAY_FALSE(keys, count);
26 * static_branch_likely()
27 * static_branch_unlikely()
29 * Jump labels provide an interface to generate dynamic branches using
30 * self-modifying code. Assuming toolchain and architecture support, if we
31 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
32 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
33 * (which defaults to false - and the true block is placed out of line).
34 * Similarly, we can define an initially true key via
35 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
36 * "if (static_branch_unlikely(&key))", in which case we will generate an
37 * unconditional branch to the out-of-line true branch. Keys that are
38 * initially true or false can be using in both static_branch_unlikely()
39 * and static_branch_likely() statements.
41 * At runtime we can change the branch target by setting the key
42 * to true via a call to static_branch_enable(), or false using
43 * static_branch_disable(). If the direction of the branch is switched by
44 * these calls then we run-time modify the branch target via a
45 * no-op -> jump or jump -> no-op conversion. For example, for an
46 * initially false key that is used in an "if (static_branch_unlikely(&key))"
47 * statement, setting the key to true requires us to patch in a jump
48 * to the out-of-line of true branch.
50 * In addition to static_branch_{enable,disable}, we can also reference count
51 * the key or branch direction via static_branch_{inc,dec}. Thus,
52 * static_branch_inc() can be thought of as a 'make more true' and
53 * static_branch_dec() as a 'make more false'.
55 * Since this relies on modifying code, the branch modifying functions
56 * must be considered absolute slow paths (machine wide synchronization etc.).
57 * OTOH, since the affected branches are unconditional, their runtime overhead
58 * will be absolutely minimal, esp. in the default (off) case where the total
59 * effect is a single NOP of appropriate size. The on case will patch in a jump
60 * to the out-of-line block.
62 * When the control is directly exposed to userspace, it is prudent to delay the
63 * decrement to avoid high frequency code modifications which can (and do)
64 * cause significant performance degradation. Struct static_key_deferred and
65 * static_key_slow_dec_deferred() provide for this.
67 * Lacking toolchain and or architecture support, static keys fall back to a
68 * simple conditional branch.
70 * Additional babbling in: Documentation/static-keys.txt
73 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
74 # define HAVE_JUMP_LABEL
79 #include <linux/types.h>
80 #include <linux/compiler.h>
82 extern bool static_key_initialized
;
84 #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \
85 "%s used before call to jump_label_init", \
88 #ifdef HAVE_JUMP_LABEL
92 /* Set lsb bit to 1 if branch is default true, 0 ot */
93 struct jump_entry
*entries
;
95 struct static_key_mod
*next
;
103 #endif /* HAVE_JUMP_LABEL */
104 #endif /* __ASSEMBLY__ */
106 #ifdef HAVE_JUMP_LABEL
107 #include <asm/jump_label.h>
112 enum jump_label_type
{
119 #ifdef HAVE_JUMP_LABEL
121 #define JUMP_TYPE_FALSE 0UL
122 #define JUMP_TYPE_TRUE 1UL
123 #define JUMP_TYPE_MASK 1UL
125 static __always_inline
bool static_key_false(struct static_key
*key
)
127 return arch_static_branch(key
, false);
130 static __always_inline
bool static_key_true(struct static_key
*key
)
132 return !arch_static_branch(key
, true);
135 extern struct jump_entry __start___jump_table
[];
136 extern struct jump_entry __stop___jump_table
[];
138 extern void jump_label_init(void);
139 extern void jump_label_lock(void);
140 extern void jump_label_unlock(void);
141 extern void arch_jump_label_transform(struct jump_entry
*entry
,
142 enum jump_label_type type
);
143 extern void arch_jump_label_transform_static(struct jump_entry
*entry
,
144 enum jump_label_type type
);
145 extern int jump_label_text_reserved(void *start
, void *end
);
146 extern void static_key_slow_inc(struct static_key
*key
);
147 extern void static_key_slow_dec(struct static_key
*key
);
148 extern void jump_label_apply_nops(struct module
*mod
);
149 extern int static_key_count(struct static_key
*key
);
150 extern void static_key_enable(struct static_key
*key
);
151 extern void static_key_disable(struct static_key
*key
);
154 * We should be using ATOMIC_INIT() for initializing .enabled, but
155 * the inclusion of atomic.h is problematic for inclusion of jump_label.h
156 * in 'low-level' headers. Thus, we are initializing .enabled with a
157 * raw value, but have added a BUILD_BUG_ON() to catch any issues in
158 * jump_label_init() see: kernel/jump_label.c.
160 #define STATIC_KEY_INIT_TRUE \
161 { .enabled = { 1 }, \
162 .entries = (void *)JUMP_TYPE_TRUE }
163 #define STATIC_KEY_INIT_FALSE \
164 { .enabled = { 0 }, \
165 .entries = (void *)JUMP_TYPE_FALSE }
167 #else /* !HAVE_JUMP_LABEL */
169 #include <linux/atomic.h>
170 #include <linux/bug.h>
172 static inline int static_key_count(struct static_key
*key
)
174 return atomic_read(&key
->enabled
);
177 static __always_inline
void jump_label_init(void)
179 static_key_initialized
= true;
182 static __always_inline
bool static_key_false(struct static_key
*key
)
184 if (unlikely(static_key_count(key
) > 0))
189 static __always_inline
bool static_key_true(struct static_key
*key
)
191 if (likely(static_key_count(key
) > 0))
196 static inline void static_key_slow_inc(struct static_key
*key
)
198 STATIC_KEY_CHECK_USE();
199 atomic_inc(&key
->enabled
);
202 static inline void static_key_slow_dec(struct static_key
*key
)
204 STATIC_KEY_CHECK_USE();
205 atomic_dec(&key
->enabled
);
208 static inline int jump_label_text_reserved(void *start
, void *end
)
213 static inline void jump_label_lock(void) {}
214 static inline void jump_label_unlock(void) {}
216 static inline int jump_label_apply_nops(struct module
*mod
)
221 static inline void static_key_enable(struct static_key
*key
)
223 int count
= static_key_count(key
);
225 WARN_ON_ONCE(count
< 0 || count
> 1);
228 static_key_slow_inc(key
);
231 static inline void static_key_disable(struct static_key
*key
)
233 int count
= static_key_count(key
);
235 WARN_ON_ONCE(count
< 0 || count
> 1);
238 static_key_slow_dec(key
);
241 #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
242 #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
244 #endif /* HAVE_JUMP_LABEL */
246 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
247 #define jump_label_enabled static_key_enabled
249 /* -------------------------------------------------------------------------- */
252 * Two type wrappers around static_key, such that we can use compile time
253 * type differentiation to emit the right code.
255 * All the below code is macros in order to play type games.
258 struct static_key_true
{
259 struct static_key key
;
262 struct static_key_false
{
263 struct static_key key
;
266 #define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
267 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
269 #define DEFINE_STATIC_KEY_TRUE(name) \
270 struct static_key_true name = STATIC_KEY_TRUE_INIT
272 #define DEFINE_STATIC_KEY_FALSE(name) \
273 struct static_key_false name = STATIC_KEY_FALSE_INIT
275 #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
276 struct static_key_true name[count] = { \
277 [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
280 #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
281 struct static_key_false name[count] = { \
282 [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
285 extern bool ____wrong_branch_error(void);
287 #define static_key_enabled(x) \
289 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
290 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
291 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
292 ____wrong_branch_error(); \
293 static_key_count((struct static_key *)x) > 0; \
296 #ifdef HAVE_JUMP_LABEL
299 * Combine the right initial value (type) with the right branch order
300 * to generate the desired result.
303 * type\branch| likely (1) | unlikely (0)
304 * -----------+-----------------------+------------------
306 * true (1) | ... | ...
308 * | <br-stmts> | 1: ...
314 * -----------+-----------------------+------------------
316 * false (0) | ... | ...
318 * | <br-stmts> | 1: ...
324 * -----------+-----------------------+------------------
326 * The initial value is encoded in the LSB of static_key::entries,
327 * type: 0 = false, 1 = true.
329 * The branch type is encoded in the LSB of jump_entry::key,
330 * branch: 0 = unlikely, 1 = likely.
332 * This gives the following logic table:
334 * enabled type branch instuction
335 * -----------------------------+-----------
346 * Which gives the following functions:
348 * dynamic: instruction = enabled ^ branch
349 * static: instruction = type ^ branch
351 * See jump_label_type() / jump_label_init_type().
354 #define static_branch_likely(x) \
357 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
358 branch = !arch_static_branch(&(x)->key, true); \
359 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
360 branch = !arch_static_branch_jump(&(x)->key, true); \
362 branch = ____wrong_branch_error(); \
366 #define static_branch_unlikely(x) \
369 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
370 branch = arch_static_branch_jump(&(x)->key, false); \
371 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
372 branch = arch_static_branch(&(x)->key, false); \
374 branch = ____wrong_branch_error(); \
378 #else /* !HAVE_JUMP_LABEL */
380 #define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
381 #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
383 #endif /* HAVE_JUMP_LABEL */
386 * Advanced usage; refcount, branch is enabled when: count != 0
389 #define static_branch_inc(x) static_key_slow_inc(&(x)->key)
390 #define static_branch_dec(x) static_key_slow_dec(&(x)->key)
393 * Normal usage; boolean enable/disable.
396 #define static_branch_enable(x) static_key_enable(&(x)->key)
397 #define static_branch_disable(x) static_key_disable(&(x)->key)
399 #endif /* _LINUX_JUMP_LABEL_H */
401 #endif /* __ASSEMBLY__ */