2 * core.c - Kernel Live Patching Core
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23 #include <linux/module.h>
24 #include <linux/kernel.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/ftrace.h>
28 #include <linux/list.h>
29 #include <linux/kallsyms.h>
30 #include <linux/livepatch.h>
33 * The klp_mutex protects the klp_patches list and state transitions of any
34 * structure reachable from the patches list. References to any structure must
35 * be obtained under mutex protection.
38 static DEFINE_MUTEX(klp_mutex
);
39 static LIST_HEAD(klp_patches
);
41 static struct kobject
*klp_root_kobj
;
43 static bool klp_is_module(struct klp_object
*obj
)
48 static bool klp_is_object_loaded(struct klp_object
*obj
)
50 return !obj
->name
|| obj
->mod
;
53 /* sets obj->mod if object is not vmlinux and module is found */
54 static void klp_find_object_module(struct klp_object
*obj
)
56 if (!klp_is_module(obj
))
59 mutex_lock(&module_mutex
);
61 * We don't need to take a reference on the module here because we have
62 * the klp_mutex, which is also taken by the module notifier. This
63 * prevents any module from unloading until we release the klp_mutex.
65 obj
->mod
= find_module(obj
->name
);
66 mutex_unlock(&module_mutex
);
69 /* klp_mutex must be held by caller */
70 static bool klp_is_patch_registered(struct klp_patch
*patch
)
72 struct klp_patch
*mypatch
;
74 list_for_each_entry(mypatch
, &klp_patches
, list
)
81 static bool klp_initialized(void)
91 * If count == 0, the symbol was not found. If count == 1, a unique
92 * match was found and addr is set. If count > 1, there is
93 * unresolvable ambiguity among "count" number of symbols with the same
94 * name in the same object.
99 static int klp_find_callback(void *data
, const char *name
,
100 struct module
*mod
, unsigned long addr
)
102 struct klp_find_arg
*args
= data
;
104 if ((mod
&& !args
->objname
) || (!mod
&& args
->objname
))
107 if (strcmp(args
->name
, name
))
110 if (args
->objname
&& strcmp(args
->objname
, mod
->name
))
114 * args->addr might be overwritten if another match is found
115 * but klp_find_object_symbol() handles this and only returns the
116 * addr if count == 1.
124 static int klp_find_object_symbol(const char *objname
, const char *name
,
127 struct klp_find_arg args
= {
134 kallsyms_on_each_symbol(klp_find_callback
, &args
);
137 pr_err("symbol '%s' not found in symbol table\n", name
);
138 else if (args
.count
> 1)
139 pr_err("unresolvable ambiguity (%lu matches) on symbol '%s' in object '%s'\n",
140 args
.count
, name
, objname
);
150 struct klp_verify_args
{
152 const unsigned long addr
;
155 static int klp_verify_callback(void *data
, const char *name
,
156 struct module
*mod
, unsigned long addr
)
158 struct klp_verify_args
*args
= data
;
161 !strcmp(args
->name
, name
) &&
168 static int klp_verify_vmlinux_symbol(const char *name
, unsigned long addr
)
170 struct klp_verify_args args
= {
175 if (kallsyms_on_each_symbol(klp_verify_callback
, &args
))
178 pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?",
183 static int klp_find_verify_func_addr(struct klp_object
*obj
,
184 struct klp_func
*func
)
188 #if defined(CONFIG_RANDOMIZE_BASE)
189 /* KASLR is enabled, disregard old_addr from user */
193 if (!func
->old_addr
|| klp_is_module(obj
))
194 ret
= klp_find_object_symbol(obj
->name
, func
->old_name
,
197 ret
= klp_verify_vmlinux_symbol(func
->old_name
,
204 * external symbols are located outside the parent object (where the parent
205 * object is either vmlinux or the kmod being patched).
207 static int klp_find_external_symbol(struct module
*pmod
, const char *name
,
210 const struct kernel_symbol
*sym
;
212 /* first, check if it's an exported symbol */
214 sym
= find_symbol(name
, NULL
, NULL
, true, true);
221 /* otherwise check if it's in another .o within the patch module */
222 return klp_find_object_symbol(pmod
->name
, name
, addr
);
225 static int klp_write_object_relocations(struct module
*pmod
,
226 struct klp_object
*obj
)
229 struct klp_reloc
*reloc
;
231 if (WARN_ON(!klp_is_object_loaded(obj
)))
234 if (WARN_ON(!obj
->relocs
))
237 for (reloc
= obj
->relocs
; reloc
->name
; reloc
++) {
238 if (!klp_is_module(obj
)) {
239 ret
= klp_verify_vmlinux_symbol(reloc
->name
,
244 /* module, reloc->val needs to be discovered */
246 ret
= klp_find_external_symbol(pmod
,
250 ret
= klp_find_object_symbol(obj
->mod
->name
,
256 ret
= klp_write_module_reloc(pmod
, reloc
->type
, reloc
->loc
,
257 reloc
->val
+ reloc
->addend
);
259 pr_err("relocation failed for symbol '%s' at 0x%016lx (%d)\n",
260 reloc
->name
, reloc
->val
, ret
);
268 static void notrace
klp_ftrace_handler(unsigned long ip
,
269 unsigned long parent_ip
,
270 struct ftrace_ops
*ops
,
271 struct pt_regs
*regs
)
273 struct klp_func
*func
= ops
->private;
275 regs
->ip
= (unsigned long)func
->new_func
;
278 static int klp_disable_func(struct klp_func
*func
)
282 if (WARN_ON(func
->state
!= KLP_ENABLED
))
285 if (WARN_ON(!func
->old_addr
))
288 ret
= unregister_ftrace_function(func
->fops
);
290 pr_err("failed to unregister ftrace handler for function '%s' (%d)\n",
291 func
->old_name
, ret
);
295 ret
= ftrace_set_filter_ip(func
->fops
, func
->old_addr
, 1, 0);
297 pr_warn("function unregister succeeded but failed to clear the filter\n");
299 func
->state
= KLP_DISABLED
;
304 static int klp_enable_func(struct klp_func
*func
)
308 if (WARN_ON(!func
->old_addr
))
311 if (WARN_ON(func
->state
!= KLP_DISABLED
))
314 ret
= ftrace_set_filter_ip(func
->fops
, func
->old_addr
, 0, 0);
316 pr_err("failed to set ftrace filter for function '%s' (%d)\n",
317 func
->old_name
, ret
);
321 ret
= register_ftrace_function(func
->fops
);
323 pr_err("failed to register ftrace handler for function '%s' (%d)\n",
324 func
->old_name
, ret
);
325 ftrace_set_filter_ip(func
->fops
, func
->old_addr
, 1, 0);
327 func
->state
= KLP_ENABLED
;
333 static int klp_disable_object(struct klp_object
*obj
)
335 struct klp_func
*func
;
338 for (func
= obj
->funcs
; func
->old_name
; func
++) {
339 if (func
->state
!= KLP_ENABLED
)
342 ret
= klp_disable_func(func
);
347 obj
->state
= KLP_DISABLED
;
352 static int klp_enable_object(struct klp_object
*obj
)
354 struct klp_func
*func
;
357 if (WARN_ON(obj
->state
!= KLP_DISABLED
))
360 if (WARN_ON(!klp_is_object_loaded(obj
)))
363 for (func
= obj
->funcs
; func
->old_name
; func
++) {
364 ret
= klp_enable_func(func
);
368 obj
->state
= KLP_ENABLED
;
373 WARN_ON(klp_disable_object(obj
));
377 static int __klp_disable_patch(struct klp_patch
*patch
)
379 struct klp_object
*obj
;
382 pr_notice("disabling patch '%s'\n", patch
->mod
->name
);
384 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
385 if (obj
->state
!= KLP_ENABLED
)
388 ret
= klp_disable_object(obj
);
393 patch
->state
= KLP_DISABLED
;
399 * klp_disable_patch() - disables a registered patch
400 * @patch: The registered, enabled patch to be disabled
402 * Unregisters the patched functions from ftrace.
404 * Return: 0 on success, otherwise error
406 int klp_disable_patch(struct klp_patch
*patch
)
410 mutex_lock(&klp_mutex
);
412 if (!klp_is_patch_registered(patch
)) {
417 if (patch
->state
== KLP_DISABLED
) {
422 ret
= __klp_disable_patch(patch
);
425 mutex_unlock(&klp_mutex
);
428 EXPORT_SYMBOL_GPL(klp_disable_patch
);
430 static int __klp_enable_patch(struct klp_patch
*patch
)
432 struct klp_object
*obj
;
435 if (WARN_ON(patch
->state
!= KLP_DISABLED
))
438 pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n");
439 add_taint(TAINT_LIVEPATCH
, LOCKDEP_STILL_OK
);
441 pr_notice("enabling patch '%s'\n", patch
->mod
->name
);
443 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
444 klp_find_object_module(obj
);
446 if (!klp_is_object_loaded(obj
))
449 ret
= klp_enable_object(obj
);
454 patch
->state
= KLP_ENABLED
;
459 WARN_ON(__klp_disable_patch(patch
));
464 * klp_enable_patch() - enables a registered patch
465 * @patch: The registered, disabled patch to be enabled
467 * Performs the needed symbol lookups and code relocations,
468 * then registers the patched functions with ftrace.
470 * Return: 0 on success, otherwise error
472 int klp_enable_patch(struct klp_patch
*patch
)
476 mutex_lock(&klp_mutex
);
478 if (!klp_is_patch_registered(patch
)) {
483 ret
= __klp_enable_patch(patch
);
486 mutex_unlock(&klp_mutex
);
489 EXPORT_SYMBOL_GPL(klp_enable_patch
);
494 * /sys/kernel/livepatch
495 * /sys/kernel/livepatch/<patch>
496 * /sys/kernel/livepatch/<patch>/enabled
497 * /sys/kernel/livepatch/<patch>/<object>
498 * /sys/kernel/livepatch/<patch>/<object>/<func>
501 static ssize_t
enabled_store(struct kobject
*kobj
, struct kobj_attribute
*attr
,
502 const char *buf
, size_t count
)
504 struct klp_patch
*patch
;
508 ret
= kstrtoul(buf
, 10, &val
);
512 if (val
!= KLP_DISABLED
&& val
!= KLP_ENABLED
)
515 patch
= container_of(kobj
, struct klp_patch
, kobj
);
517 mutex_lock(&klp_mutex
);
519 if (val
== patch
->state
) {
520 /* already in requested state */
525 if (val
== KLP_ENABLED
) {
526 ret
= __klp_enable_patch(patch
);
530 ret
= __klp_disable_patch(patch
);
535 mutex_unlock(&klp_mutex
);
540 mutex_unlock(&klp_mutex
);
544 static ssize_t
enabled_show(struct kobject
*kobj
,
545 struct kobj_attribute
*attr
, char *buf
)
547 struct klp_patch
*patch
;
549 patch
= container_of(kobj
, struct klp_patch
, kobj
);
550 return snprintf(buf
, PAGE_SIZE
-1, "%d\n", patch
->state
);
553 static struct kobj_attribute enabled_kobj_attr
= __ATTR_RW(enabled
);
554 static struct attribute
*klp_patch_attrs
[] = {
555 &enabled_kobj_attr
.attr
,
559 static void klp_kobj_release_patch(struct kobject
*kobj
)
562 * Once we have a consistency model we'll need to module_put() the
563 * patch module here. See klp_register_patch() for more details.
567 static struct kobj_type klp_ktype_patch
= {
568 .release
= klp_kobj_release_patch
,
569 .sysfs_ops
= &kobj_sysfs_ops
,
570 .default_attrs
= klp_patch_attrs
,
573 static void klp_kobj_release_func(struct kobject
*kobj
)
575 struct klp_func
*func
;
577 func
= container_of(kobj
, struct klp_func
, kobj
);
581 static struct kobj_type klp_ktype_func
= {
582 .release
= klp_kobj_release_func
,
583 .sysfs_ops
= &kobj_sysfs_ops
,
587 * Free all functions' kobjects in the array up to some limit. When limit is
588 * NULL, all kobjects are freed.
590 static void klp_free_funcs_limited(struct klp_object
*obj
,
591 struct klp_func
*limit
)
593 struct klp_func
*func
;
595 for (func
= obj
->funcs
; func
->old_name
&& func
!= limit
; func
++)
596 kobject_put(&func
->kobj
);
599 /* Clean up when a patched object is unloaded */
600 static void klp_free_object_loaded(struct klp_object
*obj
)
602 struct klp_func
*func
;
606 for (func
= obj
->funcs
; func
->old_name
; func
++)
611 * Free all objects' kobjects in the array up to some limit. When limit is
612 * NULL, all kobjects are freed.
614 static void klp_free_objects_limited(struct klp_patch
*patch
,
615 struct klp_object
*limit
)
617 struct klp_object
*obj
;
619 for (obj
= patch
->objs
; obj
->funcs
&& obj
!= limit
; obj
++) {
620 klp_free_funcs_limited(obj
, NULL
);
621 kobject_put(obj
->kobj
);
625 static void klp_free_patch(struct klp_patch
*patch
)
627 klp_free_objects_limited(patch
, NULL
);
628 if (!list_empty(&patch
->list
))
629 list_del(&patch
->list
);
630 kobject_put(&patch
->kobj
);
633 static int klp_init_func(struct klp_object
*obj
, struct klp_func
*func
)
635 struct ftrace_ops
*ops
;
638 ops
= kzalloc(sizeof(*ops
), GFP_KERNEL
);
643 ops
->func
= klp_ftrace_handler
;
644 ops
->flags
= FTRACE_OPS_FL_SAVE_REGS
| FTRACE_OPS_FL_DYNAMIC
;
646 func
->state
= KLP_DISABLED
;
648 ret
= kobject_init_and_add(&func
->kobj
, &klp_ktype_func
,
649 obj
->kobj
, func
->old_name
);
658 /* parts of the initialization that is done only when the object is loaded */
659 static int klp_init_object_loaded(struct klp_patch
*patch
,
660 struct klp_object
*obj
)
662 struct klp_func
*func
;
666 ret
= klp_write_object_relocations(patch
->mod
, obj
);
671 for (func
= obj
->funcs
; func
->old_name
; func
++) {
672 ret
= klp_find_verify_func_addr(obj
, func
);
680 static int klp_init_object(struct klp_patch
*patch
, struct klp_object
*obj
)
682 struct klp_func
*func
;
689 obj
->state
= KLP_DISABLED
;
691 klp_find_object_module(obj
);
693 name
= klp_is_module(obj
) ? obj
->name
: "vmlinux";
694 obj
->kobj
= kobject_create_and_add(name
, &patch
->kobj
);
698 for (func
= obj
->funcs
; func
->old_name
; func
++) {
699 ret
= klp_init_func(obj
, func
);
704 if (klp_is_object_loaded(obj
)) {
705 ret
= klp_init_object_loaded(patch
, obj
);
713 klp_free_funcs_limited(obj
, func
);
714 kobject_put(obj
->kobj
);
718 static int klp_init_patch(struct klp_patch
*patch
)
720 struct klp_object
*obj
;
726 mutex_lock(&klp_mutex
);
728 patch
->state
= KLP_DISABLED
;
730 ret
= kobject_init_and_add(&patch
->kobj
, &klp_ktype_patch
,
731 klp_root_kobj
, patch
->mod
->name
);
735 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
736 ret
= klp_init_object(patch
, obj
);
741 list_add(&patch
->list
, &klp_patches
);
743 mutex_unlock(&klp_mutex
);
748 klp_free_objects_limited(patch
, obj
);
749 kobject_put(&patch
->kobj
);
751 mutex_unlock(&klp_mutex
);
756 * klp_unregister_patch() - unregisters a patch
757 * @patch: Disabled patch to be unregistered
759 * Frees the data structures and removes the sysfs interface.
761 * Return: 0 on success, otherwise error
763 int klp_unregister_patch(struct klp_patch
*patch
)
767 mutex_lock(&klp_mutex
);
769 if (!klp_is_patch_registered(patch
)) {
774 if (patch
->state
== KLP_ENABLED
) {
779 klp_free_patch(patch
);
782 mutex_unlock(&klp_mutex
);
785 EXPORT_SYMBOL_GPL(klp_unregister_patch
);
788 * klp_register_patch() - registers a patch
789 * @patch: Patch to be registered
791 * Initializes the data structure associated with the patch and
792 * creates the sysfs interface.
794 * Return: 0 on success, otherwise error
796 int klp_register_patch(struct klp_patch
*patch
)
800 if (!klp_initialized())
803 if (!patch
|| !patch
->mod
)
807 * A reference is taken on the patch module to prevent it from being
808 * unloaded. Right now, we don't allow patch modules to unload since
809 * there is currently no method to determine if a thread is still
810 * running in the patched code contained in the patch module once
811 * the ftrace registration is successful.
813 if (!try_module_get(patch
->mod
))
816 ret
= klp_init_patch(patch
);
818 module_put(patch
->mod
);
822 EXPORT_SYMBOL_GPL(klp_register_patch
);
824 static void klp_module_notify_coming(struct klp_patch
*patch
,
825 struct klp_object
*obj
)
827 struct module
*pmod
= patch
->mod
;
828 struct module
*mod
= obj
->mod
;
831 ret
= klp_init_object_loaded(patch
, obj
);
835 if (patch
->state
== KLP_DISABLED
)
838 pr_notice("applying patch '%s' to loading module '%s'\n",
839 pmod
->name
, mod
->name
);
841 ret
= klp_enable_object(obj
);
846 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
847 pmod
->name
, mod
->name
, ret
);
850 static void klp_module_notify_going(struct klp_patch
*patch
,
851 struct klp_object
*obj
)
853 struct module
*pmod
= patch
->mod
;
854 struct module
*mod
= obj
->mod
;
857 if (patch
->state
== KLP_DISABLED
)
860 pr_notice("reverting patch '%s' on unloading module '%s'\n",
861 pmod
->name
, mod
->name
);
863 ret
= klp_disable_object(obj
);
865 pr_warn("failed to revert patch '%s' on module '%s' (%d)\n",
866 pmod
->name
, mod
->name
, ret
);
869 klp_free_object_loaded(obj
);
872 static int klp_module_notify(struct notifier_block
*nb
, unsigned long action
,
875 struct module
*mod
= data
;
876 struct klp_patch
*patch
;
877 struct klp_object
*obj
;
879 if (action
!= MODULE_STATE_COMING
&& action
!= MODULE_STATE_GOING
)
882 mutex_lock(&klp_mutex
);
884 list_for_each_entry(patch
, &klp_patches
, list
) {
885 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
886 if (!klp_is_module(obj
) || strcmp(obj
->name
, mod
->name
))
889 if (action
== MODULE_STATE_COMING
) {
891 klp_module_notify_coming(patch
, obj
);
892 } else /* MODULE_STATE_GOING */
893 klp_module_notify_going(patch
, obj
);
899 mutex_unlock(&klp_mutex
);
904 static struct notifier_block klp_module_nb
= {
905 .notifier_call
= klp_module_notify
,
906 .priority
= INT_MIN
+1, /* called late but before ftrace notifier */
909 static int klp_init(void)
913 ret
= register_module_notifier(&klp_module_nb
);
917 klp_root_kobj
= kobject_create_and_add("livepatch", kernel_kobj
);
918 if (!klp_root_kobj
) {
926 unregister_module_notifier(&klp_module_nb
);
930 module_init(klp_init
);