2 * core.c - Kernel Live Patching Core
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, see <http://www.gnu.org/licenses/>.
21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23 #include <linux/module.h>
24 #include <linux/kernel.h>
25 #include <linux/mutex.h>
26 #include <linux/slab.h>
27 #include <linux/ftrace.h>
28 #include <linux/list.h>
29 #include <linux/kallsyms.h>
30 #include <linux/livepatch.h>
33 * struct klp_ops - structure for tracking registered ftrace ops structs
35 * A single ftrace_ops is shared between all enabled replacement functions
36 * (klp_func structs) which have the same old_addr. This allows the switch
37 * between function versions to happen instantaneously by updating the klp_ops
38 * struct's func_stack list. The winner is the klp_func at the top of the
39 * func_stack (front of the list).
41 * @node: node for the global klp_ops list
42 * @func_stack: list head for the stack of klp_func's (active func is on top)
43 * @fops: registered ftrace ops struct
46 struct list_head node
;
47 struct list_head func_stack
;
48 struct ftrace_ops fops
;
52 * The klp_mutex protects the global lists and state transitions of any
53 * structure reachable from them. References to any structure must be obtained
54 * under mutex protection (except in klp_ftrace_handler(), which uses RCU to
55 * ensure it gets consistent data).
57 static DEFINE_MUTEX(klp_mutex
);
59 static LIST_HEAD(klp_patches
);
60 static LIST_HEAD(klp_ops
);
62 static struct kobject
*klp_root_kobj
;
64 static struct klp_ops
*klp_find_ops(unsigned long old_addr
)
67 struct klp_func
*func
;
69 list_for_each_entry(ops
, &klp_ops
, node
) {
70 func
= list_first_entry(&ops
->func_stack
, struct klp_func
,
72 if (func
->old_addr
== old_addr
)
79 static bool klp_is_module(struct klp_object
*obj
)
84 static bool klp_is_object_loaded(struct klp_object
*obj
)
86 return !obj
->name
|| obj
->mod
;
89 /* sets obj->mod if object is not vmlinux and module is found */
90 static void klp_find_object_module(struct klp_object
*obj
)
94 if (!klp_is_module(obj
))
97 mutex_lock(&module_mutex
);
99 * We do not want to block removal of patched modules and therefore
100 * we do not take a reference here. The patches are removed by
101 * a going module handler instead.
103 mod
= find_module(obj
->name
);
105 * Do not mess work of the module coming and going notifiers.
106 * Note that the patch might still be needed before the going handler
107 * is called. Module functions can be called even in the GOING state
108 * until mod->exit() finishes. This is especially important for
109 * patches that modify semantic of the functions.
111 if (mod
&& mod
->klp_alive
)
114 mutex_unlock(&module_mutex
);
117 /* klp_mutex must be held by caller */
118 static bool klp_is_patch_registered(struct klp_patch
*patch
)
120 struct klp_patch
*mypatch
;
122 list_for_each_entry(mypatch
, &klp_patches
, list
)
123 if (mypatch
== patch
)
129 static bool klp_initialized(void)
131 return klp_root_kobj
;
134 struct klp_find_arg
{
139 * If count == 0, the symbol was not found. If count == 1, a unique
140 * match was found and addr is set. If count > 1, there is
141 * unresolvable ambiguity among "count" number of symbols with the same
142 * name in the same object.
147 static int klp_find_callback(void *data
, const char *name
,
148 struct module
*mod
, unsigned long addr
)
150 struct klp_find_arg
*args
= data
;
152 if ((mod
&& !args
->objname
) || (!mod
&& args
->objname
))
155 if (strcmp(args
->name
, name
))
158 if (args
->objname
&& strcmp(args
->objname
, mod
->name
))
162 * args->addr might be overwritten if another match is found
163 * but klp_find_object_symbol() handles this and only returns the
164 * addr if count == 1.
172 static int klp_find_object_symbol(const char *objname
, const char *name
,
175 struct klp_find_arg args
= {
182 kallsyms_on_each_symbol(klp_find_callback
, &args
);
185 pr_err("symbol '%s' not found in symbol table\n", name
);
186 else if (args
.count
> 1)
187 pr_err("unresolvable ambiguity (%lu matches) on symbol '%s' in object '%s'\n",
188 args
.count
, name
, objname
);
198 struct klp_verify_args
{
200 const unsigned long addr
;
203 static int klp_verify_callback(void *data
, const char *name
,
204 struct module
*mod
, unsigned long addr
)
206 struct klp_verify_args
*args
= data
;
209 !strcmp(args
->name
, name
) &&
216 static int klp_verify_vmlinux_symbol(const char *name
, unsigned long addr
)
218 struct klp_verify_args args
= {
223 if (kallsyms_on_each_symbol(klp_verify_callback
, &args
))
226 pr_err("symbol '%s' not found at specified address 0x%016lx, kernel mismatch?\n",
231 static int klp_find_verify_func_addr(struct klp_object
*obj
,
232 struct klp_func
*func
)
236 #if defined(CONFIG_RANDOMIZE_BASE)
237 /* KASLR is enabled, disregard old_addr from user */
241 if (!func
->old_addr
|| klp_is_module(obj
))
242 ret
= klp_find_object_symbol(obj
->name
, func
->old_name
,
245 ret
= klp_verify_vmlinux_symbol(func
->old_name
,
252 * external symbols are located outside the parent object (where the parent
253 * object is either vmlinux or the kmod being patched).
255 static int klp_find_external_symbol(struct module
*pmod
, const char *name
,
258 const struct kernel_symbol
*sym
;
260 /* first, check if it's an exported symbol */
262 sym
= find_symbol(name
, NULL
, NULL
, true, true);
270 /* otherwise check if it's in another .o within the patch module */
271 return klp_find_object_symbol(pmod
->name
, name
, addr
);
274 static int klp_write_object_relocations(struct module
*pmod
,
275 struct klp_object
*obj
)
278 struct klp_reloc
*reloc
;
280 if (WARN_ON(!klp_is_object_loaded(obj
)))
283 if (WARN_ON(!obj
->relocs
))
286 for (reloc
= obj
->relocs
; reloc
->name
; reloc
++) {
287 if (!klp_is_module(obj
)) {
288 ret
= klp_verify_vmlinux_symbol(reloc
->name
,
293 /* module, reloc->val needs to be discovered */
295 ret
= klp_find_external_symbol(pmod
,
299 ret
= klp_find_object_symbol(obj
->mod
->name
,
305 ret
= klp_write_module_reloc(pmod
, reloc
->type
, reloc
->loc
,
306 reloc
->val
+ reloc
->addend
);
308 pr_err("relocation failed for symbol '%s' at 0x%016lx (%d)\n",
309 reloc
->name
, reloc
->val
, ret
);
317 static void notrace
klp_ftrace_handler(unsigned long ip
,
318 unsigned long parent_ip
,
319 struct ftrace_ops
*fops
,
320 struct pt_regs
*regs
)
323 struct klp_func
*func
;
325 ops
= container_of(fops
, struct klp_ops
, fops
);
328 func
= list_first_or_null_rcu(&ops
->func_stack
, struct klp_func
,
330 if (WARN_ON_ONCE(!func
))
333 klp_arch_set_pc(regs
, (unsigned long)func
->new_func
);
338 static void klp_disable_func(struct klp_func
*func
)
342 WARN_ON(func
->state
!= KLP_ENABLED
);
343 WARN_ON(!func
->old_addr
);
345 ops
= klp_find_ops(func
->old_addr
);
349 if (list_is_singular(&ops
->func_stack
)) {
350 WARN_ON(unregister_ftrace_function(&ops
->fops
));
351 WARN_ON(ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 1, 0));
353 list_del_rcu(&func
->stack_node
);
354 list_del(&ops
->node
);
357 list_del_rcu(&func
->stack_node
);
360 func
->state
= KLP_DISABLED
;
363 static int klp_enable_func(struct klp_func
*func
)
368 if (WARN_ON(!func
->old_addr
))
371 if (WARN_ON(func
->state
!= KLP_DISABLED
))
374 ops
= klp_find_ops(func
->old_addr
);
376 ops
= kzalloc(sizeof(*ops
), GFP_KERNEL
);
380 ops
->fops
.func
= klp_ftrace_handler
;
381 ops
->fops
.flags
= FTRACE_OPS_FL_SAVE_REGS
|
382 FTRACE_OPS_FL_DYNAMIC
|
383 FTRACE_OPS_FL_IPMODIFY
;
385 list_add(&ops
->node
, &klp_ops
);
387 INIT_LIST_HEAD(&ops
->func_stack
);
388 list_add_rcu(&func
->stack_node
, &ops
->func_stack
);
390 ret
= ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 0, 0);
392 pr_err("failed to set ftrace filter for function '%s' (%d)\n",
393 func
->old_name
, ret
);
397 ret
= register_ftrace_function(&ops
->fops
);
399 pr_err("failed to register ftrace handler for function '%s' (%d)\n",
400 func
->old_name
, ret
);
401 ftrace_set_filter_ip(&ops
->fops
, func
->old_addr
, 1, 0);
407 list_add_rcu(&func
->stack_node
, &ops
->func_stack
);
410 func
->state
= KLP_ENABLED
;
415 list_del_rcu(&func
->stack_node
);
416 list_del(&ops
->node
);
421 static void klp_disable_object(struct klp_object
*obj
)
423 struct klp_func
*func
;
425 for (func
= obj
->funcs
; func
->old_name
; func
++)
426 if (func
->state
== KLP_ENABLED
)
427 klp_disable_func(func
);
429 obj
->state
= KLP_DISABLED
;
432 static int klp_enable_object(struct klp_object
*obj
)
434 struct klp_func
*func
;
437 if (WARN_ON(obj
->state
!= KLP_DISABLED
))
440 if (WARN_ON(!klp_is_object_loaded(obj
)))
443 for (func
= obj
->funcs
; func
->old_name
; func
++) {
444 ret
= klp_enable_func(func
);
446 klp_disable_object(obj
);
450 obj
->state
= KLP_ENABLED
;
455 static int __klp_disable_patch(struct klp_patch
*patch
)
457 struct klp_object
*obj
;
459 /* enforce stacking: only the last enabled patch can be disabled */
460 if (!list_is_last(&patch
->list
, &klp_patches
) &&
461 list_next_entry(patch
, list
)->state
== KLP_ENABLED
)
464 pr_notice("disabling patch '%s'\n", patch
->mod
->name
);
466 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
467 if (obj
->state
== KLP_ENABLED
)
468 klp_disable_object(obj
);
471 patch
->state
= KLP_DISABLED
;
477 * klp_disable_patch() - disables a registered patch
478 * @patch: The registered, enabled patch to be disabled
480 * Unregisters the patched functions from ftrace.
482 * Return: 0 on success, otherwise error
484 int klp_disable_patch(struct klp_patch
*patch
)
488 mutex_lock(&klp_mutex
);
490 if (!klp_is_patch_registered(patch
)) {
495 if (patch
->state
== KLP_DISABLED
) {
500 ret
= __klp_disable_patch(patch
);
503 mutex_unlock(&klp_mutex
);
506 EXPORT_SYMBOL_GPL(klp_disable_patch
);
508 static int __klp_enable_patch(struct klp_patch
*patch
)
510 struct klp_object
*obj
;
513 if (WARN_ON(patch
->state
!= KLP_DISABLED
))
516 /* enforce stacking: only the first disabled patch can be enabled */
517 if (patch
->list
.prev
!= &klp_patches
&&
518 list_prev_entry(patch
, list
)->state
== KLP_DISABLED
)
521 pr_notice_once("tainting kernel with TAINT_LIVEPATCH\n");
522 add_taint(TAINT_LIVEPATCH
, LOCKDEP_STILL_OK
);
524 pr_notice("enabling patch '%s'\n", patch
->mod
->name
);
526 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
527 if (!klp_is_object_loaded(obj
))
530 ret
= klp_enable_object(obj
);
535 patch
->state
= KLP_ENABLED
;
540 WARN_ON(__klp_disable_patch(patch
));
545 * klp_enable_patch() - enables a registered patch
546 * @patch: The registered, disabled patch to be enabled
548 * Performs the needed symbol lookups and code relocations,
549 * then registers the patched functions with ftrace.
551 * Return: 0 on success, otherwise error
553 int klp_enable_patch(struct klp_patch
*patch
)
557 mutex_lock(&klp_mutex
);
559 if (!klp_is_patch_registered(patch
)) {
564 ret
= __klp_enable_patch(patch
);
567 mutex_unlock(&klp_mutex
);
570 EXPORT_SYMBOL_GPL(klp_enable_patch
);
575 * /sys/kernel/livepatch
576 * /sys/kernel/livepatch/<patch>
577 * /sys/kernel/livepatch/<patch>/enabled
578 * /sys/kernel/livepatch/<patch>/<object>
579 * /sys/kernel/livepatch/<patch>/<object>/<func>
582 static ssize_t
enabled_store(struct kobject
*kobj
, struct kobj_attribute
*attr
,
583 const char *buf
, size_t count
)
585 struct klp_patch
*patch
;
589 ret
= kstrtoul(buf
, 10, &val
);
593 if (val
!= KLP_DISABLED
&& val
!= KLP_ENABLED
)
596 patch
= container_of(kobj
, struct klp_patch
, kobj
);
598 mutex_lock(&klp_mutex
);
600 if (val
== patch
->state
) {
601 /* already in requested state */
606 if (val
== KLP_ENABLED
) {
607 ret
= __klp_enable_patch(patch
);
611 ret
= __klp_disable_patch(patch
);
616 mutex_unlock(&klp_mutex
);
621 mutex_unlock(&klp_mutex
);
625 static ssize_t
enabled_show(struct kobject
*kobj
,
626 struct kobj_attribute
*attr
, char *buf
)
628 struct klp_patch
*patch
;
630 patch
= container_of(kobj
, struct klp_patch
, kobj
);
631 return snprintf(buf
, PAGE_SIZE
-1, "%d\n", patch
->state
);
634 static struct kobj_attribute enabled_kobj_attr
= __ATTR_RW(enabled
);
635 static struct attribute
*klp_patch_attrs
[] = {
636 &enabled_kobj_attr
.attr
,
640 static void klp_kobj_release_patch(struct kobject
*kobj
)
643 * Once we have a consistency model we'll need to module_put() the
644 * patch module here. See klp_register_patch() for more details.
648 static struct kobj_type klp_ktype_patch
= {
649 .release
= klp_kobj_release_patch
,
650 .sysfs_ops
= &kobj_sysfs_ops
,
651 .default_attrs
= klp_patch_attrs
,
654 static void klp_kobj_release_func(struct kobject
*kobj
)
658 static struct kobj_type klp_ktype_func
= {
659 .release
= klp_kobj_release_func
,
660 .sysfs_ops
= &kobj_sysfs_ops
,
664 * Free all functions' kobjects in the array up to some limit. When limit is
665 * NULL, all kobjects are freed.
667 static void klp_free_funcs_limited(struct klp_object
*obj
,
668 struct klp_func
*limit
)
670 struct klp_func
*func
;
672 for (func
= obj
->funcs
; func
->old_name
&& func
!= limit
; func
++)
673 kobject_put(&func
->kobj
);
676 /* Clean up when a patched object is unloaded */
677 static void klp_free_object_loaded(struct klp_object
*obj
)
679 struct klp_func
*func
;
683 for (func
= obj
->funcs
; func
->old_name
; func
++)
688 * Free all objects' kobjects in the array up to some limit. When limit is
689 * NULL, all kobjects are freed.
691 static void klp_free_objects_limited(struct klp_patch
*patch
,
692 struct klp_object
*limit
)
694 struct klp_object
*obj
;
696 for (obj
= patch
->objs
; obj
->funcs
&& obj
!= limit
; obj
++) {
697 klp_free_funcs_limited(obj
, NULL
);
698 kobject_put(obj
->kobj
);
702 static void klp_free_patch(struct klp_patch
*patch
)
704 klp_free_objects_limited(patch
, NULL
);
705 if (!list_empty(&patch
->list
))
706 list_del(&patch
->list
);
707 kobject_put(&patch
->kobj
);
710 static int klp_init_func(struct klp_object
*obj
, struct klp_func
*func
)
712 INIT_LIST_HEAD(&func
->stack_node
);
713 func
->state
= KLP_DISABLED
;
715 return kobject_init_and_add(&func
->kobj
, &klp_ktype_func
,
716 obj
->kobj
, "%s", func
->old_name
);
719 /* parts of the initialization that is done only when the object is loaded */
720 static int klp_init_object_loaded(struct klp_patch
*patch
,
721 struct klp_object
*obj
)
723 struct klp_func
*func
;
727 ret
= klp_write_object_relocations(patch
->mod
, obj
);
732 for (func
= obj
->funcs
; func
->old_name
; func
++) {
733 ret
= klp_find_verify_func_addr(obj
, func
);
741 static int klp_init_object(struct klp_patch
*patch
, struct klp_object
*obj
)
743 struct klp_func
*func
;
750 obj
->state
= KLP_DISABLED
;
753 klp_find_object_module(obj
);
755 name
= klp_is_module(obj
) ? obj
->name
: "vmlinux";
756 obj
->kobj
= kobject_create_and_add(name
, &patch
->kobj
);
760 for (func
= obj
->funcs
; func
->old_name
; func
++) {
761 ret
= klp_init_func(obj
, func
);
766 if (klp_is_object_loaded(obj
)) {
767 ret
= klp_init_object_loaded(patch
, obj
);
775 klp_free_funcs_limited(obj
, func
);
776 kobject_put(obj
->kobj
);
780 static int klp_init_patch(struct klp_patch
*patch
)
782 struct klp_object
*obj
;
788 mutex_lock(&klp_mutex
);
790 patch
->state
= KLP_DISABLED
;
792 ret
= kobject_init_and_add(&patch
->kobj
, &klp_ktype_patch
,
793 klp_root_kobj
, "%s", patch
->mod
->name
);
797 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
798 ret
= klp_init_object(patch
, obj
);
803 list_add_tail(&patch
->list
, &klp_patches
);
805 mutex_unlock(&klp_mutex
);
810 klp_free_objects_limited(patch
, obj
);
811 kobject_put(&patch
->kobj
);
813 mutex_unlock(&klp_mutex
);
818 * klp_unregister_patch() - unregisters a patch
819 * @patch: Disabled patch to be unregistered
821 * Frees the data structures and removes the sysfs interface.
823 * Return: 0 on success, otherwise error
825 int klp_unregister_patch(struct klp_patch
*patch
)
829 mutex_lock(&klp_mutex
);
831 if (!klp_is_patch_registered(patch
)) {
836 if (patch
->state
== KLP_ENABLED
) {
841 klp_free_patch(patch
);
844 mutex_unlock(&klp_mutex
);
847 EXPORT_SYMBOL_GPL(klp_unregister_patch
);
850 * klp_register_patch() - registers a patch
851 * @patch: Patch to be registered
853 * Initializes the data structure associated with the patch and
854 * creates the sysfs interface.
856 * Return: 0 on success, otherwise error
858 int klp_register_patch(struct klp_patch
*patch
)
862 if (!klp_initialized())
865 if (!patch
|| !patch
->mod
)
869 * A reference is taken on the patch module to prevent it from being
870 * unloaded. Right now, we don't allow patch modules to unload since
871 * there is currently no method to determine if a thread is still
872 * running in the patched code contained in the patch module once
873 * the ftrace registration is successful.
875 if (!try_module_get(patch
->mod
))
878 ret
= klp_init_patch(patch
);
880 module_put(patch
->mod
);
884 EXPORT_SYMBOL_GPL(klp_register_patch
);
886 static void klp_module_notify_coming(struct klp_patch
*patch
,
887 struct klp_object
*obj
)
889 struct module
*pmod
= patch
->mod
;
890 struct module
*mod
= obj
->mod
;
893 ret
= klp_init_object_loaded(patch
, obj
);
897 if (patch
->state
== KLP_DISABLED
)
900 pr_notice("applying patch '%s' to loading module '%s'\n",
901 pmod
->name
, mod
->name
);
903 ret
= klp_enable_object(obj
);
908 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
909 pmod
->name
, mod
->name
, ret
);
912 static void klp_module_notify_going(struct klp_patch
*patch
,
913 struct klp_object
*obj
)
915 struct module
*pmod
= patch
->mod
;
916 struct module
*mod
= obj
->mod
;
918 if (patch
->state
== KLP_DISABLED
)
921 pr_notice("reverting patch '%s' on unloading module '%s'\n",
922 pmod
->name
, mod
->name
);
924 klp_disable_object(obj
);
927 klp_free_object_loaded(obj
);
930 static int klp_module_notify(struct notifier_block
*nb
, unsigned long action
,
933 struct module
*mod
= data
;
934 struct klp_patch
*patch
;
935 struct klp_object
*obj
;
937 if (action
!= MODULE_STATE_COMING
&& action
!= MODULE_STATE_GOING
)
940 mutex_lock(&klp_mutex
);
943 * Each module has to know that the notifier has been called.
944 * We never know what module will get patched by a new patch.
946 if (action
== MODULE_STATE_COMING
)
947 mod
->klp_alive
= true;
948 else /* MODULE_STATE_GOING */
949 mod
->klp_alive
= false;
951 list_for_each_entry(patch
, &klp_patches
, list
) {
952 for (obj
= patch
->objs
; obj
->funcs
; obj
++) {
953 if (!klp_is_module(obj
) || strcmp(obj
->name
, mod
->name
))
956 if (action
== MODULE_STATE_COMING
) {
958 klp_module_notify_coming(patch
, obj
);
959 } else /* MODULE_STATE_GOING */
960 klp_module_notify_going(patch
, obj
);
966 mutex_unlock(&klp_mutex
);
971 static struct notifier_block klp_module_nb
= {
972 .notifier_call
= klp_module_notify
,
973 .priority
= INT_MIN
+1, /* called late but before ftrace notifier */
976 static int klp_init(void)
980 ret
= klp_check_compiler_support();
982 pr_info("Your compiler is too old; turning off.\n");
986 ret
= register_module_notifier(&klp_module_nb
);
990 klp_root_kobj
= kobject_create_and_add("livepatch", kernel_kobj
);
991 if (!klp_root_kobj
) {
999 unregister_module_notifier(&klp_module_nb
);
1003 module_init(klp_init
);