2 * core.c - Kernel Live Patching Core
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23 #include <linux/module.h>
24 #include <linux/kernel.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/ftrace.h>
28 #include <linux/list.h>
29 #include <linux/kallsyms.h>
30 #include <linux/livepatch.h>
33 * struct klp_ops - structure for tracking registered ftrace ops structs
35 * A single ftrace_ops is shared between all enabled replacement functions
36 * (klp_func structs) which have the same old_addr. This allows the switch
37 * between function versions to happen instantaneously by updating the klp_ops
38 * struct's func_stack list. The winner is the klp_func at the top of the
39 * func_stack (front of the list).
41 * @node: node for the global klp_ops list
42 * @func_stack: list head for the stack of klp_func's (active func is on top)
43 * @fops: registered ftrace ops struct
46 struct list_head node
;
47 struct list_head func_stack
;
48 struct ftrace_ops fops
;
52 * The klp_mutex protects the global lists and state transitions of any
53 * structure reachable from them. References to any structure must be obtained
54 * under mutex protection (except in klp_ftrace_handler(), which uses RCU to
55 * ensure it gets consistent data).
57 static DEFINE_MUTEX(klp_mutex
);
59 static LIST_HEAD(klp_patches
);
60 static LIST_HEAD(klp_ops
);
62 static struct kobject
*klp_root_kobj
;
64 static struct klp_ops
*klp_find_ops(unsigned long old_addr
)
67 struct klp_func
*func
;
69 list_for_each_entry(ops
, &klp_ops
, node
) {
70 func
= list_first_entry(&ops
->func_stack
, struct klp_func
,
72 if (func
->old_addr
== old_addr
)
79 static bool klp_is_module(struct klp_object
*obj
)
84 static bool klp_is_object_loaded(struct klp_object
*obj
)
86 return !obj
->name
|| obj
->mod
;
89 /* sets obj->mod if object is not vmlinux and module is found */
90 static void klp_find_object_module(struct klp_object
*obj
)
92 if (!klp_is_module(obj
))
95 mutex_lock(&module_mutex
);
97 * We don't need to take a reference on the module here because we have
98 * the klp_mutex, which is also taken by the module notifier. This
99 * prevents any module from unloading until we release the klp_mutex.
101 obj
->mod
= find_module(obj
->name
);
102 mutex_unlock(&module_mutex
);
105 /* klp_mutex must be held by caller */
106 static bool klp_is_patch_registered(struct klp_patch
*patch
)
108 struct klp_patch
*mypatch
;
110 list_for_each_entry(mypatch
, &klp_patches
, list
)
111 if (mypatch
== patch
)
117 static bool klp_initialized(void)
119 return klp_root_kobj
;
122 struct klp_find_arg
{
127 * If count == 0, the symbol was not found. If count == 1, a unique
128 * match was found and addr is set. If count > 1, there is
129 * unresolvable ambiguity among "count" number of symbols with the same
130 * name in the same object.
135 static int klp_find_callback(void *data
, const char *name
,
136 struct module
*mod
, unsigned long addr
)
138 struct klp_find_arg
*args
= data
;
140 if ((mod
&& !args
->objname
) || (!mod
&& args
->objname
))
143 if (strcmp(args
->name
, name
))
146 if (args
->objname
&& strcmp(args
->objname
, mod
->name
))
150 * args->addr might be overwritten if another match is found
151 * but klp_find_object_symbol() handles this and only returns the
152 * addr if count == 1.
160 static int klp_find_object_symbol(const char *objname
, const char *name
,
163 struct klp_find_arg args
= {
170 kallsyms_on_each_symbol(klp_find_callback
, &args
);
173 pr_err("symbol '%s' not found in symbol table\n", name
);
174 else if (args
.count
> 1)
175 pr_err("unresolvable ambiguity (%lu matches) on symbol '%s' in object '%s'\n",
176 args
.count
, name
, objname
);
186 struct klp_verify_args
{
188 const unsigned long addr
;
191 static int klp_verify_callback(void *data
, const char *name
,
192 struct module
*mod
, unsigned long addr
)
194 struct klp_verify_args
*args
= data
;
197 !strcmp(args
->name
, name
) &&
204 static int klp_verify_vmlinux_symbol(const char *name
, unsigned long addr
)
206 struct klp_verify_args args
= {
211 if (kallsyms_on_each_symbol(klp_verify_callback
, &args
))
214 pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?\n",
219 static int klp_find_verify_func_addr(struct klp_object
*obj
,
220 struct klp_func
*func
)
224 #if defined(CONFIG_RANDOMIZE_BASE)
225 /* KASLR is enabled, disregard old_addr from user */
229 if (!func
->old_addr
|| klp_is_module(obj
))
230 ret
= klp_find_object_symbol(obj
->name
, func
->old_name
,
233 ret
= klp_verify_vmlinux_symbol(func
->old_name
,
240 * external symbols are located outside the parent object (where the parent
241 * object is either vmlinux or the kmod being patched).
243 static int klp_find_external_symbol(struct module
*pmod
, const char *name
,
246 const struct kernel_symbol
*sym
;
248 /* first, check if it's an exported symbol */
250 sym
= find_symbol(name
, NULL
, NULL
, true, true);
257 /* otherwise check if it's in another .o within the patch module */
258 return klp_find_object_symbol(pmod
->name
, name
, addr
);
261 static int klp_write_object_relocations(struct module
*pmod
,
262 struct klp_object
*obj
)
265 struct klp_reloc
*reloc
;
267 if (WARN_ON(!klp_is_object_loaded(obj
)))
270 if (WARN_ON(!obj
->relocs
))
273 for (reloc
= obj
->relocs
; reloc
->name
; reloc
++) {
274 if (!klp_is_module(obj
)) {
275 ret
= klp_verify_vmlinux_symbol(reloc
->name
,
280 /* module, reloc->val needs to be discovered */
282 ret
= klp_find_external_symbol(pmod
,
286 ret
= klp_find_object_symbol(obj
->mod
->name
,
292 ret
= klp_write_module_reloc(pmod
, reloc
->type
, reloc
->loc
,
293 reloc
->val
+ reloc
->addend
);
295 pr_err("relocation failed for symbol '%s' at 0x%016lx (%d)\n",
296 reloc
->name
, reloc
->val
, ret
);
304 static void notrace
klp_ftrace_handler(unsigned long ip
,
305 unsigned long parent_ip
,
306 struct ftrace_ops
*fops
,
307 struct pt_regs
*regs
)
310 struct klp_func
*func
;
312 ops
= container_of(fops
, struct klp_ops
, fops
);
315 func
= list_first_or_null_rcu(&ops
->func_stack
, struct klp_func
,
319 if (WARN_ON_ONCE(!func
))
322 klp_arch_set_pc(regs
, (unsigned long)func
->new_func
);
325 static int klp_disable_func(struct klp_func
*func
)
330 if (WARN_ON(func
->state
!= KLP_ENABLED
))
333 if (WARN_ON(!func
->old_addr
))
336 ops
= klp_find_ops(func
->old_addr
);
340 if (list_is_singular(&ops
->func_stack
)) {
341 ret
= unregister_ftrace_function(&ops
->fops
);
343 pr_err("failed to unregister ftrace handler for function '%s' (%d)\n",
344 func
->old_name
, ret
);
348 ret
= ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 1, 0);
350 pr_warn("function unregister succeeded but failed to clear the filter\n");
352 list_del_rcu(&func
->stack_node
);
353 list_del(&ops
->node
);
356 list_del_rcu(&func
->stack_node
);
359 func
->state
= KLP_DISABLED
;
364 static int klp_enable_func(struct klp_func
*func
)
369 if (WARN_ON(!func
->old_addr
))
372 if (WARN_ON(func
->state
!= KLP_DISABLED
))
375 ops
= klp_find_ops(func
->old_addr
);
377 ops
= kzalloc(sizeof(*ops
), GFP_KERNEL
);
381 ops
->fops
.func
= klp_ftrace_handler
;
382 ops
->fops
.flags
= FTRACE_OPS_FL_SAVE_REGS
|
383 FTRACE_OPS_FL_DYNAMIC
|
384 FTRACE_OPS_FL_IPMODIFY
;
386 list_add(&ops
->node
, &klp_ops
);
388 INIT_LIST_HEAD(&ops
->func_stack
);
389 list_add_rcu(&func
->stack_node
, &ops
->func_stack
);
391 ret
= ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 0, 0);
393 pr_err("failed to set ftrace filter for function '%s' (%d)\n",
394 func
->old_name
, ret
);
398 ret
= register_ftrace_function(&ops
->fops
);
400 pr_err("failed to register ftrace handler for function '%s' (%d)\n",
401 func
->old_name
, ret
);
402 ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 1, 0);
408 list_add_rcu(&func
->stack_node
, &ops
->func_stack
);
411 func
->state
= KLP_ENABLED
;
416 list_del_rcu(&func
->stack_node
);
417 list_del(&ops
->node
);
422 static int klp_disable_object(struct klp_object
*obj
)
424 struct klp_func
*func
;
427 for (func
= obj
->funcs
; func
->old_name
; func
++) {
428 if (func
->state
!= KLP_ENABLED
)
431 ret
= klp_disable_func(func
);
436 obj
->state
= KLP_DISABLED
;
441 static int klp_enable_object(struct klp_object
*obj
)
443 struct klp_func
*func
;
446 if (WARN_ON(obj
->state
!= KLP_DISABLED
))
449 if (WARN_ON(!klp_is_object_loaded(obj
)))
452 for (func
= obj
->funcs
; func
->old_name
; func
++) {
453 ret
= klp_enable_func(func
);
457 obj
->state
= KLP_ENABLED
;
462 WARN_ON(klp_disable_object(obj
));
466 static int __klp_disable_patch(struct klp_patch
*patch
)
468 struct klp_object
*obj
;
471 /* enforce stacking: only the last enabled patch can be disabled */
472 if (!list_is_last(&patch
->list
, &klp_patches
) &&
473 list_next_entry(patch
, list
)->state
== KLP_ENABLED
)
476 pr_notice("disabling patch '%s'\n", patch
->mod
->name
);
478 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
479 if (obj
->state
!= KLP_ENABLED
)
482 ret
= klp_disable_object(obj
);
487 patch
->state
= KLP_DISABLED
;
493 * klp_disable_patch() - disables a registered patch
494 * @patch: The registered, enabled patch to be disabled
496 * Unregisters the patched functions from ftrace.
498 * Return: 0 on success, otherwise error
500 int klp_disable_patch(struct klp_patch
*patch
)
504 mutex_lock(&klp_mutex
);
506 if (!klp_is_patch_registered(patch
)) {
511 if (patch
->state
== KLP_DISABLED
) {
516 ret
= __klp_disable_patch(patch
);
519 mutex_unlock(&klp_mutex
);
522 EXPORT_SYMBOL_GPL(klp_disable_patch
);
524 static int __klp_enable_patch(struct klp_patch
*patch
)
526 struct klp_object
*obj
;
529 if (WARN_ON(patch
->state
!= KLP_DISABLED
))
532 /* enforce stacking: only the first disabled patch can be enabled */
533 if (patch
->list
.prev
!= &klp_patches
&&
534 list_prev_entry(patch
, list
)->state
== KLP_DISABLED
)
537 pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n");
538 add_taint(TAINT_LIVEPATCH
, LOCKDEP_STILL_OK
);
540 pr_notice("enabling patch '%s'\n", patch
->mod
->name
);
542 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
543 klp_find_object_module(obj
);
545 if (!klp_is_object_loaded(obj
))
548 ret
= klp_enable_object(obj
);
553 patch
->state
= KLP_ENABLED
;
558 WARN_ON(__klp_disable_patch(patch
));
563 * klp_enable_patch() - enables a registered patch
564 * @patch: The registered, disabled patch to be enabled
566 * Performs the needed symbol lookups and code relocations,
567 * then registers the patched functions with ftrace.
569 * Return: 0 on success, otherwise error
571 int klp_enable_patch(struct klp_patch
*patch
)
575 mutex_lock(&klp_mutex
);
577 if (!klp_is_patch_registered(patch
)) {
582 ret
= __klp_enable_patch(patch
);
585 mutex_unlock(&klp_mutex
);
588 EXPORT_SYMBOL_GPL(klp_enable_patch
);
593 * /sys/kernel/livepatch
594 * /sys/kernel/livepatch/<patch>
595 * /sys/kernel/livepatch/<patch>/enabled
596 * /sys/kernel/livepatch/<patch>/<object>
597 * /sys/kernel/livepatch/<patch>/<object>/<func>
600 static ssize_t
enabled_store(struct kobject
*kobj
, struct kobj_attribute
*attr
,
601 const char *buf
, size_t count
)
603 struct klp_patch
*patch
;
607 ret
= kstrtoul(buf
, 10, &val
);
611 if (val
!= KLP_DISABLED
&& val
!= KLP_ENABLED
)
614 patch
= container_of(kobj
, struct klp_patch
, kobj
);
616 mutex_lock(&klp_mutex
);
618 if (val
== patch
->state
) {
619 /* already in requested state */
624 if (val
== KLP_ENABLED
) {
625 ret
= __klp_enable_patch(patch
);
629 ret
= __klp_disable_patch(patch
);
634 mutex_unlock(&klp_mutex
);
639 mutex_unlock(&klp_mutex
);
643 static ssize_t
enabled_show(struct kobject
*kobj
,
644 struct kobj_attribute
*attr
, char *buf
)
646 struct klp_patch
*patch
;
648 patch
= container_of(kobj
, struct klp_patch
, kobj
);
649 return snprintf(buf
, PAGE_SIZE
-1, "%d\n", patch
->state
);
652 static struct kobj_attribute enabled_kobj_attr
= __ATTR_RW(enabled
);
653 static struct attribute
*klp_patch_attrs
[] = {
654 &enabled_kobj_attr
.attr
,
658 static void klp_kobj_release_patch(struct kobject
*kobj
)
661 * Once we have a consistency model we'll need to module_put() the
662 * patch module here. See klp_register_patch() for more details.
666 static struct kobj_type klp_ktype_patch
= {
667 .release
= klp_kobj_release_patch
,
668 .sysfs_ops
= &kobj_sysfs_ops
,
669 .default_attrs
= klp_patch_attrs
,
672 static void klp_kobj_release_func(struct kobject
*kobj
)
676 static struct kobj_type klp_ktype_func
= {
677 .release
= klp_kobj_release_func
,
678 .sysfs_ops
= &kobj_sysfs_ops
,
682 * Free all functions' kobjects in the array up to some limit. When limit is
683 * NULL, all kobjects are freed.
685 static void klp_free_funcs_limited(struct klp_object
*obj
,
686 struct klp_func
*limit
)
688 struct klp_func
*func
;
690 for (func
= obj
->funcs
; func
->old_name
&& func
!= limit
; func
++)
691 kobject_put(&func
->kobj
);
694 /* Clean up when a patched object is unloaded */
695 static void klp_free_object_loaded(struct klp_object
*obj
)
697 struct klp_func
*func
;
701 for (func
= obj
->funcs
; func
->old_name
; func
++)
706 * Free all objects' kobjects in the array up to some limit. When limit is
707 * NULL, all kobjects are freed.
709 static void klp_free_objects_limited(struct klp_patch
*patch
,
710 struct klp_object
*limit
)
712 struct klp_object
*obj
;
714 for (obj
= patch
->objs
; obj
->funcs
&& obj
!= limit
; obj
++) {
715 klp_free_funcs_limited(obj
, NULL
);
716 kobject_put(obj
->kobj
);
720 static void klp_free_patch(struct klp_patch
*patch
)
722 klp_free_objects_limited(patch
, NULL
);
723 if (!list_empty(&patch
->list
))
724 list_del(&patch
->list
);
725 kobject_put(&patch
->kobj
);
728 static int klp_init_func(struct klp_object
*obj
, struct klp_func
*func
)
730 INIT_LIST_HEAD(&func
->stack_node
);
731 func
->state
= KLP_DISABLED
;
733 return kobject_init_and_add(&func
->kobj
, &klp_ktype_func
,
734 obj
->kobj
, func
->old_name
);
737 /* parts of the initialization that is done only when the object is loaded */
738 static int klp_init_object_loaded(struct klp_patch
*patch
,
739 struct klp_object
*obj
)
741 struct klp_func
*func
;
745 ret
= klp_write_object_relocations(patch
->mod
, obj
);
750 for (func
= obj
->funcs
; func
->old_name
; func
++) {
751 ret
= klp_find_verify_func_addr(obj
, func
);
759 static int klp_init_object(struct klp_patch
*patch
, struct klp_object
*obj
)
761 struct klp_func
*func
;
768 obj
->state
= KLP_DISABLED
;
770 klp_find_object_module(obj
);
772 name
= klp_is_module(obj
) ? obj
->name
: "vmlinux";
773 obj
->kobj
= kobject_create_and_add(name
, &patch
->kobj
);
777 for (func
= obj
->funcs
; func
->old_name
; func
++) {
778 ret
= klp_init_func(obj
, func
);
783 if (klp_is_object_loaded(obj
)) {
784 ret
= klp_init_object_loaded(patch
, obj
);
792 klp_free_funcs_limited(obj
, func
);
793 kobject_put(obj
->kobj
);
797 static int klp_init_patch(struct klp_patch
*patch
)
799 struct klp_object
*obj
;
805 mutex_lock(&klp_mutex
);
807 patch
->state
= KLP_DISABLED
;
809 ret
= kobject_init_and_add(&patch
->kobj
, &klp_ktype_patch
,
810 klp_root_kobj
, patch
->mod
->name
);
814 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
815 ret
= klp_init_object(patch
, obj
);
820 list_add_tail(&patch
->list
, &klp_patches
);
822 mutex_unlock(&klp_mutex
);
827 klp_free_objects_limited(patch
, obj
);
828 kobject_put(&patch
->kobj
);
830 mutex_unlock(&klp_mutex
);
835 * klp_unregister_patch() - unregisters a patch
836 * @patch: Disabled patch to be unregistered
838 * Frees the data structures and removes the sysfs interface.
840 * Return: 0 on success, otherwise error
842 int klp_unregister_patch(struct klp_patch
*patch
)
846 mutex_lock(&klp_mutex
);
848 if (!klp_is_patch_registered(patch
)) {
853 if (patch
->state
== KLP_ENABLED
) {
858 klp_free_patch(patch
);
861 mutex_unlock(&klp_mutex
);
864 EXPORT_SYMBOL_GPL(klp_unregister_patch
);
867 * klp_register_patch() - registers a patch
868 * @patch: Patch to be registered
870 * Initializes the data structure associated with the patch and
871 * creates the sysfs interface.
873 * Return: 0 on success, otherwise error
875 int klp_register_patch(struct klp_patch
*patch
)
879 if (!klp_initialized())
882 if (!patch
|| !patch
->mod
)
886 * A reference is taken on the patch module to prevent it from being
887 * unloaded. Right now, we don't allow patch modules to unload since
888 * there is currently no method to determine if a thread is still
889 * running in the patched code contained in the patch module once
890 * the ftrace registration is successful.
892 if (!try_module_get(patch
->mod
))
895 ret
= klp_init_patch(patch
);
897 module_put(patch
->mod
);
901 EXPORT_SYMBOL_GPL(klp_register_patch
);
903 static void klp_module_notify_coming(struct klp_patch
*patch
,
904 struct klp_object
*obj
)
906 struct module
*pmod
= patch
->mod
;
907 struct module
*mod
= obj
->mod
;
910 ret
= klp_init_object_loaded(patch
, obj
);
914 if (patch
->state
== KLP_DISABLED
)
917 pr_notice("applying patch '%s' to loading module '%s'\n",
918 pmod
->name
, mod
->name
);
920 ret
= klp_enable_object(obj
);
925 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
926 pmod
->name
, mod
->name
, ret
);
929 static void klp_module_notify_going(struct klp_patch
*patch
,
930 struct klp_object
*obj
)
932 struct module
*pmod
= patch
->mod
;
933 struct module
*mod
= obj
->mod
;
936 if (patch
->state
== KLP_DISABLED
)
939 pr_notice("reverting patch '%s' on unloading module '%s'\n",
940 pmod
->name
, mod
->name
);
942 ret
= klp_disable_object(obj
);
944 pr_warn("failed to revert patch '%s' on module '%s' (%d)\n",
945 pmod
->name
, mod
->name
, ret
);
948 klp_free_object_loaded(obj
);
951 static int klp_module_notify(struct notifier_block
*nb
, unsigned long action
,
954 struct module
*mod
= data
;
955 struct klp_patch
*patch
;
956 struct klp_object
*obj
;
958 if (action
!= MODULE_STATE_COMING
&& action
!= MODULE_STATE_GOING
)
961 mutex_lock(&klp_mutex
);
963 list_for_each_entry(patch
, &klp_patches
, list
) {
964 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
965 if (!klp_is_module(obj
) || strcmp(obj
->name
, mod
->name
))
968 if (action
== MODULE_STATE_COMING
) {
970 klp_module_notify_coming(patch
, obj
);
971 } else /* MODULE_STATE_GOING */
972 klp_module_notify_going(patch
, obj
);
978 mutex_unlock(&klp_mutex
);
983 static struct notifier_block klp_module_nb
= {
984 .notifier_call
= klp_module_notify
,
985 .priority
= INT_MIN
+1, /* called late but before ftrace notifier */
988 static int klp_init(void)
992 ret
= klp_check_compiler_support();
994 pr_info("Your compiler is too old; turning off.\n");
998 ret
= register_module_notifier(&klp_module_nb
);
1002 klp_root_kobj
= kobject_create_and_add("livepatch", kernel_kobj
);
1003 if (!klp_root_kobj
) {
1011 unregister_module_notifier(&klp_module_nb
);
1015 module_init(klp_init
);