kernel/livepatch/core.c

   1 /*
   2  * core.c - Kernel Live Patching Core
   3  *
   4  * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
   5  * Copyright (C) 2014 SUSE
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; either version 2
  10  * of the License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  22
  23 #include <linux/module.h>
  24 #include <linux/kernel.h>
  25 #include <linux/mutex.h>
  26 #include <linux/slab.h>
  27 #include <linux/list.h>
  28 #include <linux/kallsyms.h>
  29 #include <linux/livepatch.h>
  30 #include <linux/elf.h>
  31 #include <linux/moduleloader.h>
  32 #include <linux/completion.h>
  33 #include <asm/cacheflush.h>
  34 #include "core.h"
  35 #include "patch.h"
  36 #include "transition.h"
  37
  38 /*
  39  * klp_mutex is a coarse lock which serializes access to klp data.  All
  40  * accesses to klp-related variables and structures must have mutex protection,
  41  * except within the following functions which carefully avoid the need for it:
  42  *
  43  * - klp_ftrace_handler()
  44  * - klp_update_patch_state()
  45  */
  46 DEFINE_MUTEX(klp_mutex);
  47
  48 static LIST_HEAD(klp_patches);
  49
  50 static struct kobject *klp_root_kobj;
  51
  52 static bool klp_is_module(struct klp_object *obj)
  53 {
  54         return obj->name;
  55 }
  56
  57 /* sets obj->mod if object is not vmlinux and module is found */
  58 static void klp_find_object_module(struct klp_object *obj)
  59 {
  60         struct module *mod;
  61
  62         if (!klp_is_module(obj))
  63                 return;
  64
  65         mutex_lock(&module_mutex);
  66         /*
  67          * We do not want to block removal of patched modules and therefore
  68          * we do not take a reference here. The patches are removed by
  69          * klp_module_going() instead.
  70          */
  71         mod = find_module(obj->name);
  72         /*
  73          * Do not mess work of klp_module_coming() and klp_module_going().
  74          * Note that the patch might still be needed before klp_module_going()
  75          * is called. Module functions can be called even in the GOING state
  76          * until mod->exit() finishes. This is especially important for
  77          * patches that modify semantic of the functions.
  78          */
  79         if (mod && mod->klp_alive)
  80                 obj->mod = mod;
  81
  82         mutex_unlock(&module_mutex);
  83 }
  84
  85 static bool klp_is_patch_registered(struct klp_patch *patch)
  86 {
  87         struct klp_patch *mypatch;
  88
  89         list_for_each_entry(mypatch, &klp_patches, list)
  90                 if (mypatch == patch)
  91                         return true;
  92
  93         return false;
  94 }
  95
  96 static bool klp_initialized(void)
  97 {
  98         return !!klp_root_kobj;
  99 }
 100
 101 struct klp_find_arg {
 102         const char *objname;
 103         const char *name;
 104         unsigned long addr;
 105         unsigned long count;
 106         unsigned long pos;
 107 };
 108
 109 static int klp_find_callback(void *data, const char *name,
 110                              struct module *mod, unsigned long addr)
 111 {
 112         struct klp_find_arg *args = data;
 113
 114         if ((mod && !args->objname) || (!mod && args->objname))
 115                 return 0;
 116
 117         if (strcmp(args->name, name))
 118                 return 0;
 119
 120         if (args->objname && strcmp(args->objname, mod->name))
 121                 return 0;
 122
 123         args->addr = addr;
 124         args->count++;
 125
 126         /*
 127          * Finish the search when the symbol is found for the desired position
 128          * or the position is not defined for a non-unique symbol.
 129          */
 130         if ((args->pos && (args->count == args->pos)) ||
 131             (!args->pos && (args->count > 1)))
 132                 return 1;
 133
 134         return 0;
 135 }
 136
 137 static int klp_find_object_symbol(const char *objname, const char *name,
 138                                   unsigned long sympos, unsigned long *addr)
 139 {
 140         struct klp_find_arg args = {
 141                 .objname = objname,
 142                 .name = name,
 143                 .addr = 0,
 144                 .count = 0,
 145                 .pos = sympos,
 146         };
 147
 148         mutex_lock(&module_mutex);
 149         if (objname)
 150                 module_kallsyms_on_each_symbol(klp_find_callback, &args);
 151         else
 152                 kallsyms_on_each_symbol(klp_find_callback, &args);
 153         mutex_unlock(&module_mutex);
 154
 155         /*
 156          * Ensure an address was found. If sympos is 0, ensure symbol is unique;
 157          * otherwise ensure the symbol position count matches sympos.
 158          */
 159         if (args.addr == 0)
 160                 pr_err("symbol '%s' not found in symbol table\n", name);
 161         else if (args.count > 1 && sympos == 0) {
 162                 pr_err("unresolvable ambiguity for symbol '%s' in object '%s'\n",
 163                        name, objname);
 164         } else if (sympos != args.count && sympos > 0) {
 165                 pr_err("symbol position %lu for symbol '%s' in object '%s' not found\n",
 166                        sympos, name, objname ? objname : "vmlinux");
 167         } else {
 168                 *addr = args.addr;
 169                 return 0;
 170         }
 171
 172         *addr = 0;
 173         return -EINVAL;
 174 }
 175
 176 static int klp_resolve_symbols(Elf_Shdr *relasec, struct module *pmod)
 177 {
 178         int i, cnt, vmlinux, ret;
 179         char objname[MODULE_NAME_LEN];
 180         char symname[KSYM_NAME_LEN];
 181         char *strtab = pmod->core_kallsyms.strtab;
 182         Elf_Rela *relas;
 183         Elf_Sym *sym;
 184         unsigned long sympos, addr;
 185
 186         /*
 187          * Since the field widths for objname and symname in the sscanf()
 188          * call are hard-coded and correspond to MODULE_NAME_LEN and
 189          * KSYM_NAME_LEN respectively, we must make sure that MODULE_NAME_LEN
 190          * and KSYM_NAME_LEN have the values we expect them to have.
 191          *
 192          * Because the value of MODULE_NAME_LEN can differ among architectures,
 193          * we use the smallest/strictest upper bound possible (56, based on
 194          * the current definition of MODULE_NAME_LEN) to prevent overflows.
 195          */
 196         BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);
 197
 198         relas = (Elf_Rela *) relasec->sh_addr;
 199         /* For each rela in this klp relocation section */
 200         for (i = 0; i < relasec->sh_size / sizeof(Elf_Rela); i++) {
 201                 sym = pmod->core_kallsyms.symtab + ELF_R_SYM(relas[i].r_info);
 202                 if (sym->st_shndx != SHN_LIVEPATCH) {
 203                         pr_err("symbol %s is not marked as a livepatch symbol\n",
 204                                strtab + sym->st_name);
 205                         return -EINVAL;
 206                 }
 207
 208                 /* Format: .klp.sym.objname.symname,sympos */
 209                 cnt = sscanf(strtab + sym->st_name,
 210                              ".klp.sym.%55[^.].%127[^,],%lu",
 211                              objname, symname, &sympos);
 212                 if (cnt != 3) {
 213                         pr_err("symbol %s has an incorrectly formatted name\n",
 214                                strtab + sym->st_name);
 215                         return -EINVAL;
 216                 }
 217
 218                 /* klp_find_object_symbol() treats a NULL objname as vmlinux */
 219                 vmlinux = !strcmp(objname, "vmlinux");
 220                 ret = klp_find_object_symbol(vmlinux ? NULL : objname,
 221                                              symname, sympos, &addr);
 222                 if (ret)
 223                         return ret;
 224
 225                 sym->st_value = addr;
 226         }
 227
 228         return 0;
 229 }
 230
 231 static int klp_write_object_relocations(struct module *pmod,
 232                                         struct klp_object *obj)
 233 {
 234         int i, cnt, ret = 0;
 235         const char *objname, *secname;
 236         char sec_objname[MODULE_NAME_LEN];
 237         Elf_Shdr *sec;
 238
 239         if (WARN_ON(!klp_is_object_loaded(obj)))
 240                 return -EINVAL;
 241
 242         objname = klp_is_module(obj) ? obj->name : "vmlinux";
 243
 244         /* For each klp relocation section */
 245         for (i = 1; i < pmod->klp_info->hdr.e_shnum; i++) {
 246                 sec = pmod->klp_info->sechdrs + i;
 247                 secname = pmod->klp_info->secstrings + sec->sh_name;
 248                 if (!(sec->sh_flags & SHF_RELA_LIVEPATCH))
 249                         continue;
 250
 251                 /*
 252                  * Format: .klp.rela.sec_objname.section_name
 253                  * See comment in klp_resolve_symbols() for an explanation
 254                  * of the selected field width value.
 255                  */
 256                 cnt = sscanf(secname, ".klp.rela.%55[^.]", sec_objname);
 257                 if (cnt != 1) {
 258                         pr_err("section %s has an incorrectly formatted name\n",
 259                                secname);
 260                         ret = -EINVAL;
 261                         break;
 262                 }
 263
 264                 if (strcmp(objname, sec_objname))
 265                         continue;
 266
 267                 ret = klp_resolve_symbols(sec, pmod);
 268                 if (ret)
 269                         break;
 270
 271                 ret = apply_relocate_add(pmod->klp_info->sechdrs,
 272                                          pmod->core_kallsyms.strtab,
 273                                          pmod->klp_info->symndx, i, pmod);
 274                 if (ret)
 275                         break;
 276         }
 277
 278         return ret;
 279 }
 280
 281 /*
 282  * Sysfs Interface
 283  *
 284  * /sys/kernel/livepatch
 285  * /sys/kernel/livepatch/<patch>
 286  * /sys/kernel/livepatch/<patch>/enabled
 287  * /sys/kernel/livepatch/<patch>/transition
 288  * /sys/kernel/livepatch/<patch>/signal
 289  * /sys/kernel/livepatch/<patch>/force
 290  * /sys/kernel/livepatch/<patch>/<object>
 291  * /sys/kernel/livepatch/<patch>/<object>/<function,sympos>
 292  */
 293 static int __klp_disable_patch(struct klp_patch *patch);
 294 static int __klp_enable_patch(struct klp_patch *patch);
 295
 296 static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr,
 297                              const char *buf, size_t count)
 298 {
 299         struct klp_patch *patch;
 300         int ret;
 301         bool enabled;
 302
 303         ret = kstrtobool(buf, &enabled);
 304         if (ret)
 305                 return ret;
 306
 307         patch = container_of(kobj, struct klp_patch, kobj);
 308
 309         mutex_lock(&klp_mutex);
 310
 311         if (!klp_is_patch_registered(patch)) {
 312                 /*
 313                  * Module with the patch could either disappear meanwhile or is
 314                  * not properly initialized yet.
 315                  */
 316                 ret = -EINVAL;
 317                 goto err;
 318         }
 319
 320         if (patch->enabled == enabled) {
 321                 /* already in requested state */
 322                 ret = -EINVAL;
 323                 goto err;
 324         }
 325
 326         if (patch == klp_transition_patch) {
 327                 klp_reverse_transition();
 328         } else if (enabled) {
 329                 ret = __klp_enable_patch(patch);
 330                 if (ret)
 331                         goto err;
 332         } else {
 333                 ret = __klp_disable_patch(patch);
 334                 if (ret)
 335                         goto err;
 336         }
 337
 338         mutex_unlock(&klp_mutex);
 339
 340         return count;
 341
 342 err:
 343         mutex_unlock(&klp_mutex);
 344         return ret;
 345 }
 346
 347 static ssize_t enabled_show(struct kobject *kobj,
 348                             struct kobj_attribute *attr, char *buf)
 349 {
 350         struct klp_patch *patch;
 351
 352         patch = container_of(kobj, struct klp_patch, kobj);
 353         return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);
 354 }
 355
 356 static ssize_t transition_show(struct kobject *kobj,
 357                                struct kobj_attribute *attr, char *buf)
 358 {
 359         struct klp_patch *patch;
 360
 361         patch = container_of(kobj, struct klp_patch, kobj);
 362         return snprintf(buf, PAGE_SIZE-1, "%d\n",
 363                         patch == klp_transition_patch);
 364 }
 365
 366 static ssize_t signal_store(struct kobject *kobj, struct kobj_attribute *attr,
 367                             const char *buf, size_t count)
 368 {
 369         struct klp_patch *patch;
 370         int ret;
 371         bool val;
 372
 373         ret = kstrtobool(buf, &val);
 374         if (ret)
 375                 return ret;
 376
 377         if (!val)
 378                 return count;
 379
 380         mutex_lock(&klp_mutex);
 381
 382         patch = container_of(kobj, struct klp_patch, kobj);
 383         if (patch != klp_transition_patch) {
 384                 mutex_unlock(&klp_mutex);
 385                 return -EINVAL;
 386         }
 387
 388         klp_send_signals();
 389
 390         mutex_unlock(&klp_mutex);
 391
 392         return count;
 393 }
 394
 395 static ssize_t force_store(struct kobject *kobj, struct kobj_attribute *attr,
 396                            const char *buf, size_t count)
 397 {
 398         struct klp_patch *patch;
 399         int ret;
 400         bool val;
 401
 402         ret = kstrtobool(buf, &val);
 403         if (ret)
 404                 return ret;
 405
 406         if (!val)
 407                 return count;
 408
 409         mutex_lock(&klp_mutex);
 410
 411         patch = container_of(kobj, struct klp_patch, kobj);
 412         if (patch != klp_transition_patch) {
 413                 mutex_unlock(&klp_mutex);
 414                 return -EINVAL;
 415         }
 416
 417         klp_force_transition();
 418
 419         mutex_unlock(&klp_mutex);
 420
 421         return count;
 422 }
 423
 424 static struct kobj_attribute enabled_kobj_attr = __ATTR_RW(enabled);
 425 static struct kobj_attribute transition_kobj_attr = __ATTR_RO(transition);
 426 static struct kobj_attribute signal_kobj_attr = __ATTR_WO(signal);
 427 static struct kobj_attribute force_kobj_attr = __ATTR_WO(force);
 428 static struct attribute *klp_patch_attrs[] = {
 429         &enabled_kobj_attr.attr,
 430         &transition_kobj_attr.attr,
 431         &signal_kobj_attr.attr,
 432         &force_kobj_attr.attr,
 433         NULL
 434 };
 435
 436 static void klp_kobj_release_patch(struct kobject *kobj)
 437 {
 438         struct klp_patch *patch;
 439
 440         patch = container_of(kobj, struct klp_patch, kobj);
 441         complete(&patch->finish);
 442 }
 443
 444 static struct kobj_type klp_ktype_patch = {
 445         .release = klp_kobj_release_patch,
 446         .sysfs_ops = &kobj_sysfs_ops,
 447         .default_attrs = klp_patch_attrs,
 448 };
 449
 450 static void klp_kobj_release_object(struct kobject *kobj)
 451 {
 452 }
 453
 454 static struct kobj_type klp_ktype_object = {
 455         .release = klp_kobj_release_object,
 456         .sysfs_ops = &kobj_sysfs_ops,
 457 };
 458
 459 static void klp_kobj_release_func(struct kobject *kobj)
 460 {
 461 }
 462
 463 static struct kobj_type klp_ktype_func = {
 464         .release = klp_kobj_release_func,
 465         .sysfs_ops = &kobj_sysfs_ops,
 466 };
 467
 468 static void klp_free_funcs(struct klp_object *obj)
 469 {
 470         struct klp_func *func;
 471
 472         klp_for_each_func(obj, func) {
 473                 /* Might be called from klp_init_patch() error path. */
 474                 if (func->kobj_added)
 475                         kobject_put(&func->kobj);
 476         }
 477 }
 478
 479 /* Clean up when a patched object is unloaded */
 480 static void klp_free_object_loaded(struct klp_object *obj)
 481 {
 482         struct klp_func *func;
 483
 484         obj->mod = NULL;
 485
 486         klp_for_each_func(obj, func)
 487                 func->old_func = NULL;
 488 }
 489
 490 static void klp_free_objects(struct klp_patch *patch)
 491 {
 492         struct klp_object *obj;
 493
 494         klp_for_each_object(patch, obj) {
 495                 klp_free_funcs(obj);
 496
 497                 /* Might be called from klp_init_patch() error path. */
 498                 if (obj->kobj_added)
 499                         kobject_put(&obj->kobj);
 500         }
 501 }
 502
 503 /*
 504  * This function implements the free operations that can be called safely
 505  * under klp_mutex.
 506  *
 507  * The operation must be completed by calling klp_free_patch_finish()
 508  * outside klp_mutex.
 509  */
 510 static void klp_free_patch_start(struct klp_patch *patch)
 511 {
 512         if (!list_empty(&patch->list))
 513                 list_del(&patch->list);
 514
 515         klp_free_objects(patch);
 516 }
 517
 518 /*
 519  * This function implements the free part that must be called outside
 520  * klp_mutex.
 521  *
 522  * It must be called after klp_free_patch_start(). And it has to be
 523  * the last function accessing the livepatch structures when the patch
 524  * gets disabled.
 525  */
 526 static void klp_free_patch_finish(struct klp_patch *patch)
 527 {
 528         /*
 529          * Avoid deadlock with enabled_store() sysfs callback by
 530          * calling this outside klp_mutex. It is safe because
 531          * this is called when the patch gets disabled and it
 532          * cannot get enabled again.
 533          */
 534         if (patch->kobj_added) {
 535                 kobject_put(&patch->kobj);
 536                 wait_for_completion(&patch->finish);
 537         }
 538 }
 539
 540 static int klp_init_func(struct klp_object *obj, struct klp_func *func)
 541 {
 542         int ret;
 543
 544         if (!func->old_name || !func->new_func)
 545                 return -EINVAL;
 546
 547         if (strlen(func->old_name) >= KSYM_NAME_LEN)
 548                 return -EINVAL;
 549
 550         INIT_LIST_HEAD(&func->stack_node);
 551         func->patched = false;
 552         func->transition = false;
 553
 554         /* The format for the sysfs directory is <function,sympos> where sympos
 555          * is the nth occurrence of this symbol in kallsyms for the patched
 556          * object. If the user selects 0 for old_sympos, then 1 will be used
 557          * since a unique symbol will be the first occurrence.
 558          */
 559         ret = kobject_init_and_add(&func->kobj, &klp_ktype_func,
 560                                    &obj->kobj, "%s,%lu", func->old_name,
 561                                    func->old_sympos ? func->old_sympos : 1);
 562         if (!ret)
 563                 func->kobj_added = true;
 564
 565         return ret;
 566 }
 567
 568 /* Arches may override this to finish any remaining arch-specific tasks */
 569 void __weak arch_klp_init_object_loaded(struct klp_patch *patch,
 570                                         struct klp_object *obj)
 571 {
 572 }
 573
 574 /* parts of the initialization that is done only when the object is loaded */
 575 static int klp_init_object_loaded(struct klp_patch *patch,
 576                                   struct klp_object *obj)
 577 {
 578         struct klp_func *func;
 579         int ret;
 580
 581         module_disable_ro(patch->mod);
 582         ret = klp_write_object_relocations(patch->mod, obj);
 583         if (ret) {
 584                 module_enable_ro(patch->mod, true);
 585                 return ret;
 586         }
 587
 588         arch_klp_init_object_loaded(patch, obj);
 589         module_enable_ro(patch->mod, true);
 590
 591         klp_for_each_func(obj, func) {
 592                 ret = klp_find_object_symbol(obj->name, func->old_name,
 593                                              func->old_sympos,
 594                                              (unsigned long *)&func->old_func);
 595                 if (ret)
 596                         return ret;
 597
 598                 ret = kallsyms_lookup_size_offset((unsigned long)func->old_func,
 599                                                   &func->old_size, NULL);
 600                 if (!ret) {
 601                         pr_err("kallsyms size lookup failed for '%s'\n",
 602                                func->old_name);
 603                         return -ENOENT;
 604                 }
 605
 606                 ret = kallsyms_lookup_size_offset((unsigned long)func->new_func,
 607                                                   &func->new_size, NULL);
 608                 if (!ret) {
 609                         pr_err("kallsyms size lookup failed for '%s' replacement\n",
 610                                func->old_name);
 611                         return -ENOENT;
 612                 }
 613         }
 614
 615         return 0;
 616 }
 617
 618 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)
 619 {
 620         struct klp_func *func;
 621         int ret;
 622         const char *name;
 623
 624         if (klp_is_module(obj) && strlen(obj->name) >= MODULE_NAME_LEN)
 625                 return -EINVAL;
 626
 627         obj->patched = false;
 628         obj->mod = NULL;
 629
 630         klp_find_object_module(obj);
 631
 632         name = klp_is_module(obj) ? obj->name : "vmlinux";
 633         ret = kobject_init_and_add(&obj->kobj, &klp_ktype_object,
 634                                    &patch->kobj, "%s", name);
 635         if (ret)
 636                 return ret;
 637         obj->kobj_added = true;
 638
 639         klp_for_each_func(obj, func) {
 640                 ret = klp_init_func(obj, func);
 641                 if (ret)
 642                         return ret;
 643         }
 644
 645         if (klp_is_object_loaded(obj))
 646                 ret = klp_init_object_loaded(patch, obj);
 647
 648         return ret;
 649 }
 650
 651 static int klp_init_patch_early(struct klp_patch *patch)
 652 {
 653         struct klp_object *obj;
 654         struct klp_func *func;
 655
 656         if (!patch->objs)
 657                 return -EINVAL;
 658
 659         INIT_LIST_HEAD(&patch->list);
 660         patch->kobj_added = false;
 661         patch->enabled = false;
 662         init_completion(&patch->finish);
 663
 664         klp_for_each_object(patch, obj) {
 665                 if (!obj->funcs)
 666                         return -EINVAL;
 667
 668                 obj->kobj_added = false;
 669
 670                 klp_for_each_func(obj, func)
 671                         func->kobj_added = false;
 672         }
 673
 674         return 0;
 675 }
 676
 677 static int klp_init_patch(struct klp_patch *patch)
 678 {
 679         struct klp_object *obj;
 680         int ret;
 681
 682         mutex_lock(&klp_mutex);
 683
 684         ret = klp_init_patch_early(patch);
 685         if (ret) {
 686                 mutex_unlock(&klp_mutex);
 687                 return ret;
 688         }
 689
 690         ret = kobject_init_and_add(&patch->kobj, &klp_ktype_patch,
 691                                    klp_root_kobj, "%s", patch->mod->name);
 692         if (ret) {
 693                 mutex_unlock(&klp_mutex);
 694                 return ret;
 695         }
 696         patch->kobj_added = true;
 697
 698         klp_for_each_object(patch, obj) {
 699                 ret = klp_init_object(patch, obj);
 700                 if (ret)
 701                         goto free;
 702         }
 703
 704         list_add_tail(&patch->list, &klp_patches);
 705
 706         mutex_unlock(&klp_mutex);
 707
 708         return 0;
 709
 710 free:
 711         klp_free_patch_start(patch);
 712
 713         mutex_unlock(&klp_mutex);
 714
 715         klp_free_patch_finish(patch);
 716
 717         return ret;
 718 }
 719
 720 /**
 721  * klp_unregister_patch() - unregisters a patch
 722  * @patch:      Disabled patch to be unregistered
 723  *
 724  * Frees the data structures and removes the sysfs interface.
 725  *
 726  * Return: 0 on success, otherwise error
 727  */
 728 int klp_unregister_patch(struct klp_patch *patch)
 729 {
 730         int ret;
 731
 732         mutex_lock(&klp_mutex);
 733
 734         if (!klp_is_patch_registered(patch)) {
 735                 ret = -EINVAL;
 736                 goto err;
 737         }
 738
 739         if (patch->enabled) {
 740                 ret = -EBUSY;
 741                 goto err;
 742         }
 743
 744         klp_free_patch_start(patch);
 745
 746         mutex_unlock(&klp_mutex);
 747
 748         klp_free_patch_finish(patch);
 749
 750         return 0;
 751 err:
 752         mutex_unlock(&klp_mutex);
 753         return ret;
 754 }
 755 EXPORT_SYMBOL_GPL(klp_unregister_patch);
 756
 757 /**
 758  * klp_register_patch() - registers a patch
 759  * @patch:      Patch to be registered
 760  *
 761  * Initializes the data structure associated with the patch and
 762  * creates the sysfs interface.
 763  *
 764  * There is no need to take the reference on the patch module here. It is done
 765  * later when the patch is enabled.
 766  *
 767  * Return: 0 on success, otherwise error
 768  */
 769 int klp_register_patch(struct klp_patch *patch)
 770 {
 771         if (!patch || !patch->mod)
 772                 return -EINVAL;
 773
 774         if (!is_livepatch_module(patch->mod)) {
 775                 pr_err("module %s is not marked as a livepatch module\n",
 776                        patch->mod->name);
 777                 return -EINVAL;
 778         }
 779
 780         if (!klp_initialized())
 781                 return -ENODEV;
 782
 783         if (!klp_have_reliable_stack()) {
 784                 pr_err("This architecture doesn't have support for the livepatch consistency model.\n");
 785                 return -ENOSYS;
 786         }
 787
 788         return klp_init_patch(patch);
 789 }
 790 EXPORT_SYMBOL_GPL(klp_register_patch);
 791
 792 static int __klp_disable_patch(struct klp_patch *patch)
 793 {
 794         struct klp_object *obj;
 795
 796         if (WARN_ON(!patch->enabled))
 797                 return -EINVAL;
 798
 799         if (klp_transition_patch)
 800                 return -EBUSY;
 801
 802         /* enforce stacking: only the last enabled patch can be disabled */
 803         if (!list_is_last(&patch->list, &klp_patches) &&
 804             list_next_entry(patch, list)->enabled)
 805                 return -EBUSY;
 806
 807         klp_init_transition(patch, KLP_UNPATCHED);
 808
 809         klp_for_each_object(patch, obj)
 810                 if (obj->patched)
 811                         klp_pre_unpatch_callback(obj);
 812
 813         /*
 814          * Enforce the order of the func->transition writes in
 815          * klp_init_transition() and the TIF_PATCH_PENDING writes in
 816          * klp_start_transition().  In the rare case where klp_ftrace_handler()
 817          * is called shortly after klp_update_patch_state() switches the task,
 818          * this ensures the handler sees that func->transition is set.
 819          */
 820         smp_wmb();
 821
 822         klp_start_transition();
 823         klp_try_complete_transition();
 824         patch->enabled = false;
 825
 826         return 0;
 827 }
 828
 829 /**
 830  * klp_disable_patch() - disables a registered patch
 831  * @patch:      The registered, enabled patch to be disabled
 832  *
 833  * Unregisters the patched functions from ftrace.
 834  *
 835  * Return: 0 on success, otherwise error
 836  */
 837 int klp_disable_patch(struct klp_patch *patch)
 838 {
 839         int ret;
 840
 841         mutex_lock(&klp_mutex);
 842
 843         if (!klp_is_patch_registered(patch)) {
 844                 ret = -EINVAL;
 845                 goto err;
 846         }
 847
 848         if (!patch->enabled) {
 849                 ret = -EINVAL;
 850                 goto err;
 851         }
 852
 853         ret = __klp_disable_patch(patch);
 854
 855 err:
 856         mutex_unlock(&klp_mutex);
 857         return ret;
 858 }
 859 EXPORT_SYMBOL_GPL(klp_disable_patch);
 860
 861 static int __klp_enable_patch(struct klp_patch *patch)
 862 {
 863         struct klp_object *obj;
 864         int ret;
 865
 866         if (klp_transition_patch)
 867                 return -EBUSY;
 868
 869         if (WARN_ON(patch->enabled))
 870                 return -EINVAL;
 871
 872         /* enforce stacking: only the first disabled patch can be enabled */
 873         if (patch->list.prev != &klp_patches &&
 874             !list_prev_entry(patch, list)->enabled)
 875                 return -EBUSY;
 876
 877         /*
 878          * A reference is taken on the patch module to prevent it from being
 879          * unloaded.
 880          */
 881         if (!try_module_get(patch->mod))
 882                 return -ENODEV;
 883
 884         pr_notice("enabling patch '%s'\n", patch->mod->name);
 885
 886         klp_init_transition(patch, KLP_PATCHED);
 887
 888         /*
 889          * Enforce the order of the func->transition writes in
 890          * klp_init_transition() and the ops->func_stack writes in
 891          * klp_patch_object(), so that klp_ftrace_handler() will see the
 892          * func->transition updates before the handler is registered and the
 893          * new funcs become visible to the handler.
 894          */
 895         smp_wmb();
 896
 897         klp_for_each_object(patch, obj) {
 898                 if (!klp_is_object_loaded(obj))
 899                         continue;
 900
 901                 ret = klp_pre_patch_callback(obj);
 902                 if (ret) {
 903                         pr_warn("pre-patch callback failed for object '%s'\n",
 904                                 klp_is_module(obj) ? obj->name : "vmlinux");
 905                         goto err;
 906                 }
 907
 908                 ret = klp_patch_object(obj);
 909                 if (ret) {
 910                         pr_warn("failed to patch object '%s'\n",
 911                                 klp_is_module(obj) ? obj->name : "vmlinux");
 912                         goto err;
 913                 }
 914         }
 915
 916         klp_start_transition();
 917         klp_try_complete_transition();
 918         patch->enabled = true;
 919
 920         return 0;
 921 err:
 922         pr_warn("failed to enable patch '%s'\n", patch->mod->name);
 923
 924         klp_cancel_transition();
 925         return ret;
 926 }
 927
 928 /**
 929  * klp_enable_patch() - enables a registered patch
 930  * @patch:      The registered, disabled patch to be enabled
 931  *
 932  * Performs the needed symbol lookups and code relocations,
 933  * then registers the patched functions with ftrace.
 934  *
 935  * Return: 0 on success, otherwise error
 936  */
 937 int klp_enable_patch(struct klp_patch *patch)
 938 {
 939         int ret;
 940
 941         mutex_lock(&klp_mutex);
 942
 943         if (!klp_is_patch_registered(patch)) {
 944                 ret = -EINVAL;
 945                 goto err;
 946         }
 947
 948         ret = __klp_enable_patch(patch);
 949
 950 err:
 951         mutex_unlock(&klp_mutex);
 952         return ret;
 953 }
 954 EXPORT_SYMBOL_GPL(klp_enable_patch);
 955
 956 /*
 957  * Remove parts of patches that touch a given kernel module. The list of
 958  * patches processed might be limited. When limit is NULL, all patches
 959  * will be handled.
 960  */
 961 static void klp_cleanup_module_patches_limited(struct module *mod,
 962                                                struct klp_patch *limit)
 963 {
 964         struct klp_patch *patch;
 965         struct klp_object *obj;
 966
 967         list_for_each_entry(patch, &klp_patches, list) {
 968                 if (patch == limit)
 969                         break;
 970
 971                 klp_for_each_object(patch, obj) {
 972                         if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
 973                                 continue;
 974
 975                         /*
 976                          * Only unpatch the module if the patch is enabled or
 977                          * is in transition.
 978                          */
 979                         if (patch->enabled || patch == klp_transition_patch) {
 980
 981                                 if (patch != klp_transition_patch)
 982                                         klp_pre_unpatch_callback(obj);
 983
 984                                 pr_notice("reverting patch '%s' on unloading module '%s'\n",
 985                                           patch->mod->name, obj->mod->name);
 986                                 klp_unpatch_object(obj);
 987
 988                                 klp_post_unpatch_callback(obj);
 989                         }
 990
 991                         klp_free_object_loaded(obj);
 992                         break;
 993                 }
 994         }
 995 }
 996
 997 int klp_module_coming(struct module *mod)
 998 {
 999         int ret;
1000         struct klp_patch *patch;
1001         struct klp_object *obj;
1002
1003         if (WARN_ON(mod->state != MODULE_STATE_COMING))
1004                 return -EINVAL;
1005
1006         mutex_lock(&klp_mutex);
1007         /*
1008          * Each module has to know that klp_module_coming()
1009          * has been called. We never know what module will
1010          * get patched by a new patch.
1011          */
1012         mod->klp_alive = true;
1013
1014         list_for_each_entry(patch, &klp_patches, list) {
1015                 klp_for_each_object(patch, obj) {
1016                         if (!klp_is_module(obj) || strcmp(obj->name, mod->name))
1017                                 continue;
1018
1019                         obj->mod = mod;
1020
1021                         ret = klp_init_object_loaded(patch, obj);
1022                         if (ret) {
1023                                 pr_warn("failed to initialize patch '%s' for module '%s' (%d)\n",
1024                                         patch->mod->name, obj->mod->name, ret);
1025                                 goto err;
1026                         }
1027
1028                         /*
1029                          * Only patch the module if the patch is enabled or is
1030                          * in transition.
1031                          */
1032                         if (!patch->enabled && patch != klp_transition_patch)
1033                                 break;
1034
1035                         pr_notice("applying patch '%s' to loading module '%s'\n",
1036                                   patch->mod->name, obj->mod->name);
1037
1038                         ret = klp_pre_patch_callback(obj);
1039                         if (ret) {
1040                                 pr_warn("pre-patch callback failed for object '%s'\n",
1041                                         obj->name);
1042                                 goto err;
1043                         }
1044
1045                         ret = klp_patch_object(obj);
1046                         if (ret) {
1047                                 pr_warn("failed to apply patch '%s' to module '%s' (%d)\n",
1048                                         patch->mod->name, obj->mod->name, ret);
1049
1050                                 klp_post_unpatch_callback(obj);
1051                                 goto err;
1052                         }
1053
1054                         if (patch != klp_transition_patch)
1055                                 klp_post_patch_callback(obj);
1056
1057                         break;
1058                 }
1059         }
1060
1061         mutex_unlock(&klp_mutex);
1062
1063         return 0;
1064
1065 err:
1066         /*
1067          * If a patch is unsuccessfully applied, return
1068          * error to the module loader.
1069          */
1070         pr_warn("patch '%s' failed for module '%s', refusing to load module '%s'\n",
1071                 patch->mod->name, obj->mod->name, obj->mod->name);
1072         mod->klp_alive = false;
1073         klp_cleanup_module_patches_limited(mod, patch);
1074         mutex_unlock(&klp_mutex);
1075
1076         return ret;
1077 }
1078
1079 void klp_module_going(struct module *mod)
1080 {
1081         if (WARN_ON(mod->state != MODULE_STATE_GOING &&
1082                     mod->state != MODULE_STATE_COMING))
1083                 return;
1084
1085         mutex_lock(&klp_mutex);
1086         /*
1087          * Each module has to know that klp_module_going()
1088          * has been called. We never know what module will
1089          * get patched by a new patch.
1090          */
1091         mod->klp_alive = false;
1092
1093         klp_cleanup_module_patches_limited(mod, NULL);
1094
1095         mutex_unlock(&klp_mutex);
1096 }
1097
1098 static int __init klp_init(void)
1099 {
1100         int ret;
1101
1102         ret = klp_check_compiler_support();
1103         if (ret) {
1104                 pr_info("Your compiler is too old; turning off.\n");
1105                 return -EINVAL;
1106         }
1107
1108         klp_root_kobj = kobject_create_and_add("livepatch", kernel_kobj);
1109         if (!klp_root_kobj)
1110                 return -ENOMEM;
1111
1112         return 0;
1113 }
1114
1115 module_init(klp_init);