4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
5 * Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com>
8 #include <linux/memory.h>
9 #include <linux/uaccess.h>
10 #include <linux/module.h>
11 #include <linux/list.h>
12 #include <linux/slab.h>
13 #include <linux/sort.h>
14 #include <linux/err.h>
15 #include <linux/jump_label.h>
17 #ifdef HAVE_JUMP_LABEL
19 /* mutex to protect coming/going of the the jump_label table */
20 static DEFINE_MUTEX(jump_label_mutex
);
22 void jump_label_lock(void)
24 mutex_lock(&jump_label_mutex
);
27 void jump_label_unlock(void)
29 mutex_unlock(&jump_label_mutex
);
32 bool jump_label_enabled(struct jump_label_key
*key
)
34 return !!atomic_read(&key
->enabled
);
37 static int jump_label_cmp(const void *a
, const void *b
)
39 const struct jump_entry
*jea
= a
;
40 const struct jump_entry
*jeb
= b
;
42 if (jea
->key
< jeb
->key
)
45 if (jea
->key
> jeb
->key
)
52 jump_label_sort_entries(struct jump_entry
*start
, struct jump_entry
*stop
)
56 size
= (((unsigned long)stop
- (unsigned long)start
)
57 / sizeof(struct jump_entry
));
58 sort(start
, size
, sizeof(struct jump_entry
), jump_label_cmp
, NULL
);
61 static void jump_label_update(struct jump_label_key
*key
, int enable
);
63 void jump_label_inc(struct jump_label_key
*key
)
65 if (atomic_inc_not_zero(&key
->enabled
))
69 if (atomic_add_return(1, &key
->enabled
) == 1)
70 jump_label_update(key
, JUMP_LABEL_ENABLE
);
74 void jump_label_dec(struct jump_label_key
*key
)
76 if (!atomic_dec_and_mutex_lock(&key
->enabled
, &jump_label_mutex
))
79 jump_label_update(key
, JUMP_LABEL_DISABLE
);
83 static int addr_conflict(struct jump_entry
*entry
, void *start
, void *end
)
85 if (entry
->code
<= (unsigned long)end
&&
86 entry
->code
+ JUMP_LABEL_NOP_SIZE
> (unsigned long)start
)
92 static int __jump_label_text_reserved(struct jump_entry
*iter_start
,
93 struct jump_entry
*iter_stop
, void *start
, void *end
)
95 struct jump_entry
*iter
;
98 while (iter
< iter_stop
) {
99 if (addr_conflict(iter
, start
, end
))
108 * Update code which is definitely not currently executing.
109 * Architectures which need heavyweight synchronization to modify
110 * running code can override this to make the non-live update case
113 void __weak
arch_jump_label_transform_static(struct jump_entry
*entry
,
114 enum jump_label_type type
)
116 arch_jump_label_transform(entry
, type
);
119 static void __jump_label_update(struct jump_label_key
*key
,
120 struct jump_entry
*entry
,
121 struct jump_entry
*stop
, int enable
)
123 for (; (entry
< stop
) &&
124 (entry
->key
== (jump_label_t
)(unsigned long)key
);
127 * entry->code set to 0 invalidates module init text sections
128 * kernel_text_address() verifies we are not in core kernel
129 * init code, see jump_label_invalidate_module_init().
131 if (entry
->code
&& kernel_text_address(entry
->code
))
132 arch_jump_label_transform(entry
, enable
);
136 static __init
int jump_label_init(void)
138 struct jump_entry
*iter_start
= __start___jump_table
;
139 struct jump_entry
*iter_stop
= __stop___jump_table
;
140 struct jump_label_key
*key
= NULL
;
141 struct jump_entry
*iter
;
144 jump_label_sort_entries(iter_start
, iter_stop
);
146 for (iter
= iter_start
; iter
< iter_stop
; iter
++) {
147 struct jump_label_key
*iterk
;
149 iterk
= (struct jump_label_key
*)(unsigned long)iter
->key
;
150 arch_jump_label_transform_static(iter
, jump_label_enabled(iterk
) ?
151 JUMP_LABEL_ENABLE
: JUMP_LABEL_DISABLE
);
157 #ifdef CONFIG_MODULES
165 early_initcall(jump_label_init
);
167 #ifdef CONFIG_MODULES
169 struct jump_label_mod
{
170 struct jump_label_mod
*next
;
171 struct jump_entry
*entries
;
175 static int __jump_label_mod_text_reserved(void *start
, void *end
)
179 mod
= __module_text_address((unsigned long)start
);
183 WARN_ON_ONCE(__module_text_address((unsigned long)end
) != mod
);
185 return __jump_label_text_reserved(mod
->jump_entries
,
186 mod
->jump_entries
+ mod
->num_jump_entries
,
190 static void __jump_label_mod_update(struct jump_label_key
*key
, int enable
)
192 struct jump_label_mod
*mod
= key
->next
;
195 struct module
*m
= mod
->mod
;
197 __jump_label_update(key
, mod
->entries
,
198 m
->jump_entries
+ m
->num_jump_entries
,
205 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
206 * @mod: module to patch
208 * Allow for run-time selection of the optimal nops. Before the module
209 * loads patch these with arch_get_jump_label_nop(), which is specified by
210 * the arch specific jump label code.
212 void jump_label_apply_nops(struct module
*mod
)
214 struct jump_entry
*iter_start
= mod
->jump_entries
;
215 struct jump_entry
*iter_stop
= iter_start
+ mod
->num_jump_entries
;
216 struct jump_entry
*iter
;
218 /* if the module doesn't have jump label entries, just return */
219 if (iter_start
== iter_stop
)
222 for (iter
= iter_start
; iter
< iter_stop
; iter
++)
223 arch_jump_label_transform_static(iter
, JUMP_LABEL_DISABLE
);
226 static int jump_label_add_module(struct module
*mod
)
228 struct jump_entry
*iter_start
= mod
->jump_entries
;
229 struct jump_entry
*iter_stop
= iter_start
+ mod
->num_jump_entries
;
230 struct jump_entry
*iter
;
231 struct jump_label_key
*key
= NULL
;
232 struct jump_label_mod
*jlm
;
234 /* if the module doesn't have jump label entries, just return */
235 if (iter_start
== iter_stop
)
238 jump_label_sort_entries(iter_start
, iter_stop
);
240 for (iter
= iter_start
; iter
< iter_stop
; iter
++) {
241 if (iter
->key
== (jump_label_t
)(unsigned long)key
)
244 key
= (struct jump_label_key
*)(unsigned long)iter
->key
;
246 if (__module_address(iter
->key
) == mod
) {
247 atomic_set(&key
->enabled
, 0);
253 jlm
= kzalloc(sizeof(struct jump_label_mod
), GFP_KERNEL
);
259 jlm
->next
= key
->next
;
262 if (jump_label_enabled(key
))
263 __jump_label_update(key
, iter
, iter_stop
,
270 static void jump_label_del_module(struct module
*mod
)
272 struct jump_entry
*iter_start
= mod
->jump_entries
;
273 struct jump_entry
*iter_stop
= iter_start
+ mod
->num_jump_entries
;
274 struct jump_entry
*iter
;
275 struct jump_label_key
*key
= NULL
;
276 struct jump_label_mod
*jlm
, **prev
;
278 for (iter
= iter_start
; iter
< iter_stop
; iter
++) {
279 if (iter
->key
== (jump_label_t
)(unsigned long)key
)
282 key
= (struct jump_label_key
*)(unsigned long)iter
->key
;
284 if (__module_address(iter
->key
) == mod
)
290 while (jlm
&& jlm
->mod
!= mod
) {
302 static void jump_label_invalidate_module_init(struct module
*mod
)
304 struct jump_entry
*iter_start
= mod
->jump_entries
;
305 struct jump_entry
*iter_stop
= iter_start
+ mod
->num_jump_entries
;
306 struct jump_entry
*iter
;
308 for (iter
= iter_start
; iter
< iter_stop
; iter
++) {
309 if (within_module_init(iter
->code
, mod
))
315 jump_label_module_notify(struct notifier_block
*self
, unsigned long val
,
318 struct module
*mod
= data
;
322 case MODULE_STATE_COMING
:
324 ret
= jump_label_add_module(mod
);
326 jump_label_del_module(mod
);
329 case MODULE_STATE_GOING
:
331 jump_label_del_module(mod
);
334 case MODULE_STATE_LIVE
:
336 jump_label_invalidate_module_init(mod
);
341 return notifier_from_errno(ret
);
344 struct notifier_block jump_label_module_nb
= {
345 .notifier_call
= jump_label_module_notify
,
346 .priority
= 1, /* higher than tracepoints */
349 static __init
int jump_label_init_module(void)
351 return register_module_notifier(&jump_label_module_nb
);
353 early_initcall(jump_label_init_module
);
355 #endif /* CONFIG_MODULES */
358 * jump_label_text_reserved - check if addr range is reserved
359 * @start: start text addr
360 * @end: end text addr
362 * checks if the text addr located between @start and @end
363 * overlaps with any of the jump label patch addresses. Code
364 * that wants to modify kernel text should first verify that
365 * it does not overlap with any of the jump label addresses.
366 * Caller must hold jump_label_mutex.
368 * returns 1 if there is an overlap, 0 otherwise
370 int jump_label_text_reserved(void *start
, void *end
)
372 int ret
= __jump_label_text_reserved(__start___jump_table
,
373 __stop___jump_table
, start
, end
);
378 #ifdef CONFIG_MODULES
379 ret
= __jump_label_mod_text_reserved(start
, end
);
384 static void jump_label_update(struct jump_label_key
*key
, int enable
)
386 struct jump_entry
*entry
= key
->entries
, *stop
= __stop___jump_table
;
388 #ifdef CONFIG_MODULES
389 struct module
*mod
= __module_address((jump_label_t
)key
);
391 __jump_label_mod_update(key
, enable
);
394 stop
= mod
->jump_entries
+ mod
->num_jump_entries
;
396 /* if there are no users, entry can be NULL */
398 __jump_label_update(key
, entry
, stop
, enable
);