1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
6 * Copyright (C) 2011 Peter Zijlstra
9 #include <linux/memory.h>
10 #include <linux/uaccess.h>
11 #include <linux/module.h>
12 #include <linux/list.h>
13 #include <linux/slab.h>
14 #include <linux/sort.h>
15 #include <linux/err.h>
16 #include <linux/static_key.h>
17 #include <linux/jump_label_ratelimit.h>
18 #include <linux/bug.h>
19 #include <linux/cpu.h>
20 #include <asm/sections.h>
22 /* mutex to protect coming/going of the the jump_label table */
23 static DEFINE_MUTEX(jump_label_mutex
);
25 void jump_label_lock(void)
27 mutex_lock(&jump_label_mutex
);
30 void jump_label_unlock(void)
32 mutex_unlock(&jump_label_mutex
);
35 static int jump_label_cmp(const void *a
, const void *b
)
37 const struct jump_entry
*jea
= a
;
38 const struct jump_entry
*jeb
= b
;
40 if (jump_entry_key(jea
) < jump_entry_key(jeb
))
43 if (jump_entry_key(jea
) > jump_entry_key(jeb
))
49 static void jump_label_swap(void *a
, void *b
, int size
)
51 long delta
= (unsigned long)a
- (unsigned long)b
;
52 struct jump_entry
*jea
= a
;
53 struct jump_entry
*jeb
= b
;
54 struct jump_entry tmp
= *jea
;
56 jea
->code
= jeb
->code
- delta
;
57 jea
->target
= jeb
->target
- delta
;
58 jea
->key
= jeb
->key
- delta
;
60 jeb
->code
= tmp
.code
+ delta
;
61 jeb
->target
= tmp
.target
+ delta
;
62 jeb
->key
= tmp
.key
+ delta
;
66 jump_label_sort_entries(struct jump_entry
*start
, struct jump_entry
*stop
)
71 if (IS_ENABLED(CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE
))
72 swapfn
= jump_label_swap
;
74 size
= (((unsigned long)stop
- (unsigned long)start
)
75 / sizeof(struct jump_entry
));
76 sort(start
, size
, sizeof(struct jump_entry
), jump_label_cmp
, swapfn
);
79 static void jump_label_update(struct static_key
*key
);
82 * There are similar definitions for the !CONFIG_JUMP_LABEL case in jump_label.h.
83 * The use of 'atomic_read()' requires atomic.h and its problematic for some
84 * kernel headers such as kernel.h and others. Since static_key_count() is not
85 * used in the branch statements as it is for the !CONFIG_JUMP_LABEL case its ok
86 * to have it be a function here. Similarly, for 'static_key_enable()' and
87 * 'static_key_disable()', which require bug.h. This should allow jump_label.h
88 * to be included from most/all places for CONFIG_JUMP_LABEL.
90 int static_key_count(struct static_key
*key
)
93 * -1 means the first static_key_slow_inc() is in progress.
94 * static_key_enabled() must return true, so return 1 here.
96 int n
= atomic_read(&key
->enabled
);
98 return n
>= 0 ? n
: 1;
100 EXPORT_SYMBOL_GPL(static_key_count
);
102 void static_key_slow_inc_cpuslocked(struct static_key
*key
)
106 STATIC_KEY_CHECK_USE(key
);
107 lockdep_assert_cpus_held();
110 * Careful if we get concurrent static_key_slow_inc() calls;
111 * later calls must wait for the first one to _finish_ the
112 * jump_label_update() process. At the same time, however,
113 * the jump_label_update() call below wants to see
114 * static_key_enabled(&key) for jumps to be updated properly.
116 * So give a special meaning to negative key->enabled: it sends
117 * static_key_slow_inc() down the slow path, and it is non-zero
118 * so it counts as "enabled" in jump_label_update(). Note that
119 * atomic_inc_unless_negative() checks >= 0, so roll our own.
121 for (v
= atomic_read(&key
->enabled
); v
> 0; v
= v1
) {
122 v1
= atomic_cmpxchg(&key
->enabled
, v
, v
+ 1);
128 if (atomic_read(&key
->enabled
) == 0) {
129 atomic_set(&key
->enabled
, -1);
130 jump_label_update(key
);
132 * Ensure that if the above cmpxchg loop observes our positive
133 * value, it must also observe all the text changes.
135 atomic_set_release(&key
->enabled
, 1);
137 atomic_inc(&key
->enabled
);
142 void static_key_slow_inc(struct static_key
*key
)
145 static_key_slow_inc_cpuslocked(key
);
148 EXPORT_SYMBOL_GPL(static_key_slow_inc
);
150 void static_key_enable_cpuslocked(struct static_key
*key
)
152 STATIC_KEY_CHECK_USE(key
);
153 lockdep_assert_cpus_held();
155 if (atomic_read(&key
->enabled
) > 0) {
156 WARN_ON_ONCE(atomic_read(&key
->enabled
) != 1);
161 if (atomic_read(&key
->enabled
) == 0) {
162 atomic_set(&key
->enabled
, -1);
163 jump_label_update(key
);
165 * See static_key_slow_inc().
167 atomic_set_release(&key
->enabled
, 1);
171 EXPORT_SYMBOL_GPL(static_key_enable_cpuslocked
);
173 void static_key_enable(struct static_key
*key
)
176 static_key_enable_cpuslocked(key
);
179 EXPORT_SYMBOL_GPL(static_key_enable
);
181 void static_key_disable_cpuslocked(struct static_key
*key
)
183 STATIC_KEY_CHECK_USE(key
);
184 lockdep_assert_cpus_held();
186 if (atomic_read(&key
->enabled
) != 1) {
187 WARN_ON_ONCE(atomic_read(&key
->enabled
) != 0);
192 if (atomic_cmpxchg(&key
->enabled
, 1, 0))
193 jump_label_update(key
);
196 EXPORT_SYMBOL_GPL(static_key_disable_cpuslocked
);
198 void static_key_disable(struct static_key
*key
)
201 static_key_disable_cpuslocked(key
);
204 EXPORT_SYMBOL_GPL(static_key_disable
);
206 static bool static_key_slow_try_dec(struct static_key
*key
)
210 val
= atomic_fetch_add_unless(&key
->enabled
, -1, 1);
215 * The negative count check is valid even when a negative
216 * key->enabled is in use by static_key_slow_inc(); a
217 * __static_key_slow_dec() before the first static_key_slow_inc()
218 * returns is unbalanced, because all other static_key_slow_inc()
219 * instances block while the update is in progress.
221 WARN(val
< 0, "jump label: negative count!\n");
225 static void __static_key_slow_dec_cpuslocked(struct static_key
*key
)
227 lockdep_assert_cpus_held();
229 if (static_key_slow_try_dec(key
))
233 if (atomic_dec_and_test(&key
->enabled
))
234 jump_label_update(key
);
238 static void __static_key_slow_dec(struct static_key
*key
)
241 __static_key_slow_dec_cpuslocked(key
);
245 void jump_label_update_timeout(struct work_struct
*work
)
247 struct static_key_deferred
*key
=
248 container_of(work
, struct static_key_deferred
, work
.work
);
249 __static_key_slow_dec(&key
->key
);
251 EXPORT_SYMBOL_GPL(jump_label_update_timeout
);
253 void static_key_slow_dec(struct static_key
*key
)
255 STATIC_KEY_CHECK_USE(key
);
256 __static_key_slow_dec(key
);
258 EXPORT_SYMBOL_GPL(static_key_slow_dec
);
260 void static_key_slow_dec_cpuslocked(struct static_key
*key
)
262 STATIC_KEY_CHECK_USE(key
);
263 __static_key_slow_dec_cpuslocked(key
);
266 void __static_key_slow_dec_deferred(struct static_key
*key
,
267 struct delayed_work
*work
,
268 unsigned long timeout
)
270 STATIC_KEY_CHECK_USE(key
);
272 if (static_key_slow_try_dec(key
))
275 schedule_delayed_work(work
, timeout
);
277 EXPORT_SYMBOL_GPL(__static_key_slow_dec_deferred
);
279 void __static_key_deferred_flush(void *key
, struct delayed_work
*work
)
281 STATIC_KEY_CHECK_USE(key
);
282 flush_delayed_work(work
);
284 EXPORT_SYMBOL_GPL(__static_key_deferred_flush
);
286 void jump_label_rate_limit(struct static_key_deferred
*key
,
289 STATIC_KEY_CHECK_USE(key
);
291 INIT_DELAYED_WORK(&key
->work
, jump_label_update_timeout
);
293 EXPORT_SYMBOL_GPL(jump_label_rate_limit
);
295 static int addr_conflict(struct jump_entry
*entry
, void *start
, void *end
)
297 if (jump_entry_code(entry
) <= (unsigned long)end
&&
298 jump_entry_code(entry
) + JUMP_LABEL_NOP_SIZE
> (unsigned long)start
)
304 static int __jump_label_text_reserved(struct jump_entry
*iter_start
,
305 struct jump_entry
*iter_stop
, void *start
, void *end
)
307 struct jump_entry
*iter
;
310 while (iter
< iter_stop
) {
311 if (addr_conflict(iter
, start
, end
))
320 * Update code which is definitely not currently executing.
321 * Architectures which need heavyweight synchronization to modify
322 * running code can override this to make the non-live update case
325 void __weak __init_or_module
arch_jump_label_transform_static(struct jump_entry
*entry
,
326 enum jump_label_type type
)
328 arch_jump_label_transform(entry
, type
);
331 static inline struct jump_entry
*static_key_entries(struct static_key
*key
)
333 WARN_ON_ONCE(key
->type
& JUMP_TYPE_LINKED
);
334 return (struct jump_entry
*)(key
->type
& ~JUMP_TYPE_MASK
);
337 static inline bool static_key_type(struct static_key
*key
)
339 return key
->type
& JUMP_TYPE_TRUE
;
342 static inline bool static_key_linked(struct static_key
*key
)
344 return key
->type
& JUMP_TYPE_LINKED
;
347 static inline void static_key_clear_linked(struct static_key
*key
)
349 key
->type
&= ~JUMP_TYPE_LINKED
;
352 static inline void static_key_set_linked(struct static_key
*key
)
354 key
->type
|= JUMP_TYPE_LINKED
;
358 * A 'struct static_key' uses a union such that it either points directly
359 * to a table of 'struct jump_entry' or to a linked list of modules which in
360 * turn point to 'struct jump_entry' tables.
362 * The two lower bits of the pointer are used to keep track of which pointer
363 * type is in use and to store the initial branch direction, we use an access
364 * function which preserves these bits.
366 static void static_key_set_entries(struct static_key
*key
,
367 struct jump_entry
*entries
)
371 WARN_ON_ONCE((unsigned long)entries
& JUMP_TYPE_MASK
);
372 type
= key
->type
& JUMP_TYPE_MASK
;
373 key
->entries
= entries
;
377 static enum jump_label_type
jump_label_type(struct jump_entry
*entry
)
379 struct static_key
*key
= jump_entry_key(entry
);
380 bool enabled
= static_key_enabled(key
);
381 bool branch
= jump_entry_is_branch(entry
);
383 /* See the comment in linux/jump_label.h */
384 return enabled
^ branch
;
387 static bool jump_label_can_update(struct jump_entry
*entry
, bool init
)
390 * Cannot update code that was in an init text area.
392 if (!init
&& jump_entry_is_init(entry
))
395 if (!kernel_text_address(jump_entry_code(entry
))) {
396 WARN_ONCE(1, "can't patch jump_label at %pS", (void *)jump_entry_code(entry
));
403 static void __jump_label_update(struct static_key
*key
,
404 struct jump_entry
*entry
,
405 struct jump_entry
*stop
,
408 for (; (entry
< stop
) && (jump_entry_key(entry
) == key
); entry
++) {
409 if (jump_label_can_update(entry
, init
))
410 arch_jump_label_transform(entry
, jump_label_type(entry
));
414 void __init
jump_label_init(void)
416 struct jump_entry
*iter_start
= __start___jump_table
;
417 struct jump_entry
*iter_stop
= __stop___jump_table
;
418 struct static_key
*key
= NULL
;
419 struct jump_entry
*iter
;
422 * Since we are initializing the static_key.enabled field with
423 * with the 'raw' int values (to avoid pulling in atomic.h) in
424 * jump_label.h, let's make sure that is safe. There are only two
425 * cases to check since we initialize to 0 or 1.
427 BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
428 BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);
430 if (static_key_initialized
)
435 jump_label_sort_entries(iter_start
, iter_stop
);
437 for (iter
= iter_start
; iter
< iter_stop
; iter
++) {
438 struct static_key
*iterk
;
441 if (jump_label_type(iter
) == JUMP_LABEL_NOP
)
442 arch_jump_label_transform_static(iter
, JUMP_LABEL_NOP
);
444 if (init_section_contains((void *)jump_entry_code(iter
), 1))
445 jump_entry_set_init(iter
);
447 iterk
= jump_entry_key(iter
);
452 static_key_set_entries(key
, iter
);
454 static_key_initialized
= true;
459 #ifdef CONFIG_MODULES
461 static enum jump_label_type
jump_label_init_type(struct jump_entry
*entry
)
463 struct static_key
*key
= jump_entry_key(entry
);
464 bool type
= static_key_type(key
);
465 bool branch
= jump_entry_is_branch(entry
);
467 /* See the comment in linux/jump_label.h */
468 return type
^ branch
;
471 struct static_key_mod
{
472 struct static_key_mod
*next
;
473 struct jump_entry
*entries
;
477 static inline struct static_key_mod
*static_key_mod(struct static_key
*key
)
479 WARN_ON_ONCE(!static_key_linked(key
));
480 return (struct static_key_mod
*)(key
->type
& ~JUMP_TYPE_MASK
);
484 * key->type and key->next are the same via union.
485 * This sets key->next and preserves the type bits.
487 * See additional comments above static_key_set_entries().
489 static void static_key_set_mod(struct static_key
*key
,
490 struct static_key_mod
*mod
)
494 WARN_ON_ONCE((unsigned long)mod
& JUMP_TYPE_MASK
);
495 type
= key
->type
& JUMP_TYPE_MASK
;
500 static int __jump_label_mod_text_reserved(void *start
, void *end
)
505 mod
= __module_text_address((unsigned long)start
);
506 WARN_ON_ONCE(__module_text_address((unsigned long)end
) != mod
);
513 return __jump_label_text_reserved(mod
->jump_entries
,
514 mod
->jump_entries
+ mod
->num_jump_entries
,
518 static void __jump_label_mod_update(struct static_key
*key
)
520 struct static_key_mod
*mod
;
522 for (mod
= static_key_mod(key
); mod
; mod
= mod
->next
) {
523 struct jump_entry
*stop
;
527 * NULL if the static_key is defined in a module
528 * that does not use it
535 stop
= __stop___jump_table
;
537 stop
= m
->jump_entries
+ m
->num_jump_entries
;
538 __jump_label_update(key
, mod
->entries
, stop
,
539 m
&& m
->state
== MODULE_STATE_COMING
);
544 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
545 * @mod: module to patch
547 * Allow for run-time selection of the optimal nops. Before the module
548 * loads patch these with arch_get_jump_label_nop(), which is specified by
549 * the arch specific jump label code.
551 void jump_label_apply_nops(struct module
*mod
)
553 struct jump_entry
*iter_start
= mod
->jump_entries
;
554 struct jump_entry
*iter_stop
= iter_start
+ mod
->num_jump_entries
;
555 struct jump_entry
*iter
;
557 /* if the module doesn't have jump label entries, just return */
558 if (iter_start
== iter_stop
)
561 for (iter
= iter_start
; iter
< iter_stop
; iter
++) {
562 /* Only write NOPs for arch_branch_static(). */
563 if (jump_label_init_type(iter
) == JUMP_LABEL_NOP
)
564 arch_jump_label_transform_static(iter
, JUMP_LABEL_NOP
);
568 static int jump_label_add_module(struct module
*mod
)
570 struct jump_entry
*iter_start
= mod
->jump_entries
;
571 struct jump_entry
*iter_stop
= iter_start
+ mod
->num_jump_entries
;
572 struct jump_entry
*iter
;
573 struct static_key
*key
= NULL
;
574 struct static_key_mod
*jlm
, *jlm2
;
576 /* if the module doesn't have jump label entries, just return */
577 if (iter_start
== iter_stop
)
580 jump_label_sort_entries(iter_start
, iter_stop
);
582 for (iter
= iter_start
; iter
< iter_stop
; iter
++) {
583 struct static_key
*iterk
;
585 if (within_module_init(jump_entry_code(iter
), mod
))
586 jump_entry_set_init(iter
);
588 iterk
= jump_entry_key(iter
);
593 if (within_module((unsigned long)key
, mod
)) {
594 static_key_set_entries(key
, iter
);
597 jlm
= kzalloc(sizeof(struct static_key_mod
), GFP_KERNEL
);
600 if (!static_key_linked(key
)) {
601 jlm2
= kzalloc(sizeof(struct static_key_mod
),
608 jlm2
->mod
= __module_address((unsigned long)key
);
610 jlm2
->entries
= static_key_entries(key
);
612 static_key_set_mod(key
, jlm2
);
613 static_key_set_linked(key
);
617 jlm
->next
= static_key_mod(key
);
618 static_key_set_mod(key
, jlm
);
619 static_key_set_linked(key
);
621 /* Only update if we've changed from our initial state */
622 if (jump_label_type(iter
) != jump_label_init_type(iter
))
623 __jump_label_update(key
, iter
, iter_stop
, true);
629 static void jump_label_del_module(struct module
*mod
)
631 struct jump_entry
*iter_start
= mod
->jump_entries
;
632 struct jump_entry
*iter_stop
= iter_start
+ mod
->num_jump_entries
;
633 struct jump_entry
*iter
;
634 struct static_key
*key
= NULL
;
635 struct static_key_mod
*jlm
, **prev
;
637 for (iter
= iter_start
; iter
< iter_stop
; iter
++) {
638 if (jump_entry_key(iter
) == key
)
641 key
= jump_entry_key(iter
);
643 if (within_module((unsigned long)key
, mod
))
646 /* No memory during module load */
647 if (WARN_ON(!static_key_linked(key
)))
651 jlm
= static_key_mod(key
);
653 while (jlm
&& jlm
->mod
!= mod
) {
658 /* No memory during module load */
662 if (prev
== &key
->next
)
663 static_key_set_mod(key
, jlm
->next
);
669 jlm
= static_key_mod(key
);
670 /* if only one etry is left, fold it back into the static_key */
671 if (jlm
->next
== NULL
) {
672 static_key_set_entries(key
, jlm
->entries
);
673 static_key_clear_linked(key
);
680 jump_label_module_notify(struct notifier_block
*self
, unsigned long val
,
683 struct module
*mod
= data
;
690 case MODULE_STATE_COMING
:
691 ret
= jump_label_add_module(mod
);
693 WARN(1, "Failed to allocate memory: jump_label may not work properly.\n");
694 jump_label_del_module(mod
);
697 case MODULE_STATE_GOING
:
698 jump_label_del_module(mod
);
705 return notifier_from_errno(ret
);
708 static struct notifier_block jump_label_module_nb
= {
709 .notifier_call
= jump_label_module_notify
,
710 .priority
= 1, /* higher than tracepoints */
713 static __init
int jump_label_init_module(void)
715 return register_module_notifier(&jump_label_module_nb
);
717 early_initcall(jump_label_init_module
);
719 #endif /* CONFIG_MODULES */
722 * jump_label_text_reserved - check if addr range is reserved
723 * @start: start text addr
724 * @end: end text addr
726 * checks if the text addr located between @start and @end
727 * overlaps with any of the jump label patch addresses. Code
728 * that wants to modify kernel text should first verify that
729 * it does not overlap with any of the jump label addresses.
730 * Caller must hold jump_label_mutex.
732 * returns 1 if there is an overlap, 0 otherwise
734 int jump_label_text_reserved(void *start
, void *end
)
736 int ret
= __jump_label_text_reserved(__start___jump_table
,
737 __stop___jump_table
, start
, end
);
742 #ifdef CONFIG_MODULES
743 ret
= __jump_label_mod_text_reserved(start
, end
);
748 static void jump_label_update(struct static_key
*key
)
750 struct jump_entry
*stop
= __stop___jump_table
;
751 struct jump_entry
*entry
;
752 #ifdef CONFIG_MODULES
755 if (static_key_linked(key
)) {
756 __jump_label_mod_update(key
);
761 mod
= __module_address((unsigned long)key
);
763 stop
= mod
->jump_entries
+ mod
->num_jump_entries
;
766 entry
= static_key_entries(key
);
767 /* if there are no users, entry can be NULL */
769 __jump_label_update(key
, entry
, stop
,
770 system_state
< SYSTEM_RUNNING
);
773 #ifdef CONFIG_STATIC_KEYS_SELFTEST
774 static DEFINE_STATIC_KEY_TRUE(sk_true
);
775 static DEFINE_STATIC_KEY_FALSE(sk_false
);
777 static __init
int jump_label_test(void)
781 for (i
= 0; i
< 2; i
++) {
782 WARN_ON(static_key_enabled(&sk_true
.key
) != true);
783 WARN_ON(static_key_enabled(&sk_false
.key
) != false);
785 WARN_ON(!static_branch_likely(&sk_true
));
786 WARN_ON(!static_branch_unlikely(&sk_true
));
787 WARN_ON(static_branch_likely(&sk_false
));
788 WARN_ON(static_branch_unlikely(&sk_false
));
790 static_branch_disable(&sk_true
);
791 static_branch_enable(&sk_false
);
793 WARN_ON(static_key_enabled(&sk_true
.key
) == true);
794 WARN_ON(static_key_enabled(&sk_false
.key
) == false);
796 WARN_ON(static_branch_likely(&sk_true
));
797 WARN_ON(static_branch_unlikely(&sk_true
));
798 WARN_ON(!static_branch_likely(&sk_false
));
799 WARN_ON(!static_branch_unlikely(&sk_false
));
801 static_branch_enable(&sk_true
);
802 static_branch_disable(&sk_false
);
807 early_initcall(jump_label_test
);
808 #endif /* STATIC_KEYS_SELFTEST */