arch/powerpc/kernel/ftrace.c

   1 /*
   2  * Code for replacing ftrace calls with jumps.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  *
   6  * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
   7  *
   8  * Added function graph tracer code, taken from x86 that was written
   9  * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
  10  *
  11  */
  12
  13 #define pr_fmt(fmt) "ftrace-powerpc: " fmt
  14
  15 #include <linux/spinlock.h>
  16 #include <linux/hardirq.h>
  17 #include <linux/uaccess.h>
  18 #include <linux/module.h>
  19 #include <linux/ftrace.h>
  20 #include <linux/percpu.h>
  21 #include <linux/init.h>
  22 #include <linux/list.h>
  23
  24 #include <asm/cacheflush.h>
  25 #include <asm/code-patching.h>
  26 #include <asm/ftrace.h>
  27 #include <asm/syscall.h>
  28
  29
  30 #ifdef CONFIG_DYNAMIC_FTRACE
  31 static unsigned int
  32 ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
  33 {
  34         unsigned int op;
  35
  36         addr = ppc_function_entry((void *)addr);
  37
  38         /* if (link) set op to 'bl' else 'b' */
  39         op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
  40
  41         return op;
  42 }
  43
  44 static int
  45 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
  46 {
  47         unsigned int replaced;
  48
  49         /*
  50          * Note:
  51          * We are paranoid about modifying text, as if a bug was to happen, it
  52          * could cause us to read or write to someplace that could cause harm.
  53          * Carefully read and modify the code with probe_kernel_*(), and make
  54          * sure what we read is what we expected it to be before modifying it.
  55          */
  56
  57         /* read the text we want to modify */
  58         if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
  59                 return -EFAULT;
  60
  61         /* Make sure it is what we expect it to be */
  62         if (replaced != old) {
  63                 pr_err("%p: replaced (%#x) != old (%#x)",
  64                 (void *)ip, replaced, old);
  65                 return -EINVAL;
  66         }
  67
  68         /* replace the text with the new text */
  69         if (patch_instruction((unsigned int *)ip, new))
  70                 return -EPERM;
  71
  72         return 0;
  73 }
  74
  75 /*
  76  * Helper functions that are the same for both PPC64 and PPC32.
  77  */
  78 static int test_24bit_addr(unsigned long ip, unsigned long addr)
  79 {
  80         addr = ppc_function_entry((void *)addr);
  81
  82         /* use the create_branch to verify that this offset can be branched */
  83         return create_branch((unsigned int *)ip, addr, 0);
  84 }
  85
  86 #ifdef CONFIG_MODULES
  87
  88 static int is_bl_op(unsigned int op)
  89 {
  90         return (op & 0xfc000003) == 0x48000001;
  91 }
  92
  93 static unsigned long find_bl_target(unsigned long ip, unsigned int op)
  94 {
  95         static int offset;
  96
  97         offset = (op & 0x03fffffc);
  98         /* make it signed */
  99         if (offset & 0x02000000)
 100                 offset |= 0xfe000000;
 101
 102         return ip + (long)offset;
 103 }
 104
 105 #ifdef CONFIG_PPC64
 106 static int
 107 __ftrace_make_nop(struct module *mod,
 108                   struct dyn_ftrace *rec, unsigned long addr)
 109 {
 110         unsigned long entry, ptr, tramp;
 111         unsigned long ip = rec->ip;
 112         unsigned int op, pop;
 113
 114         /* read where this goes */
 115         if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
 116                 pr_err("Fetching opcode failed.\n");
 117                 return -EFAULT;
 118         }
 119
 120         /* Make sure that that this is still a 24bit jump */
 121         if (!is_bl_op(op)) {
 122                 pr_err("Not expected bl: opcode is %x\n", op);
 123                 return -EINVAL;
 124         }
 125
 126         /* lets find where the pointer goes */
 127         tramp = find_bl_target(ip, op);
 128
 129         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 130
 131         if (module_trampoline_target(mod, tramp, &ptr)) {
 132                 pr_err("Failed to get trampoline target\n");
 133                 return -EFAULT;
 134         }
 135
 136         pr_devel("trampoline target %lx", ptr);
 137
 138         entry = ppc_global_function_entry((void *)addr);
 139         /* This should match what was called */
 140         if (ptr != entry) {
 141                 pr_err("addr %lx does not match expected %lx\n", ptr, entry);
 142                 return -EINVAL;
 143         }
 144
 145 #ifdef CC_USING_MPROFILE_KERNEL
 146         /* When using -mkernel_profile there is no load to jump over */
 147         pop = PPC_INST_NOP;
 148
 149         if (probe_kernel_read(&op, (void *)(ip - 4), 4)) {
 150                 pr_err("Fetching instruction at %lx failed.\n", ip - 4);
 151                 return -EFAULT;
 152         }
 153
 154         /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */
 155         if (op != PPC_INST_MFLR && op != PPC_INST_STD_LR) {
 156                 pr_err("Unexpected instruction %08x around bl _mcount\n", op);
 157                 return -EINVAL;
 158         }
 159 #else
 160         /*
 161          * Our original call site looks like:
 162          *
 163          * bl <tramp>
 164          * ld r2,XX(r1)
 165          *
 166          * Milton Miller pointed out that we can not simply nop the branch.
 167          * If a task was preempted when calling a trace function, the nops
 168          * will remove the way to restore the TOC in r2 and the r2 TOC will
 169          * get corrupted.
 170          *
 171          * Use a b +8 to jump over the load.
 172          */
 173
 174         pop = PPC_INST_BRANCH | 8;      /* b +8 */
 175
 176         /*
 177          * Check what is in the next instruction. We can see ld r2,40(r1), but
 178          * on first pass after boot we will see mflr r0.
 179          */
 180         if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) {
 181                 pr_err("Fetching op failed.\n");
 182                 return -EFAULT;
 183         }
 184
 185         if (op != PPC_INST_LD_TOC) {
 186                 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op);
 187                 return -EINVAL;
 188         }
 189 #endif /* CC_USING_MPROFILE_KERNEL */
 190
 191         if (patch_instruction((unsigned int *)ip, pop)) {
 192                 pr_err("Patching NOP failed.\n");
 193                 return -EPERM;
 194         }
 195
 196         return 0;
 197 }
 198
 199 #else /* !PPC64 */
 200 static int
 201 __ftrace_make_nop(struct module *mod,
 202                   struct dyn_ftrace *rec, unsigned long addr)
 203 {
 204         unsigned int op;
 205         unsigned int jmp[4];
 206         unsigned long ip = rec->ip;
 207         unsigned long tramp;
 208
 209         if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
 210                 return -EFAULT;
 211
 212         /* Make sure that that this is still a 24bit jump */
 213         if (!is_bl_op(op)) {
 214                 pr_err("Not expected bl: opcode is %x\n", op);
 215                 return -EINVAL;
 216         }
 217
 218         /* lets find where the pointer goes */
 219         tramp = find_bl_target(ip, op);
 220
 221         /*
 222          * On PPC32 the trampoline looks like:
 223          *  0x3d, 0x80, 0x00, 0x00  lis r12,sym@ha
 224          *  0x39, 0x8c, 0x00, 0x00  addi r12,r12,sym@l
 225          *  0x7d, 0x89, 0x03, 0xa6  mtctr r12
 226          *  0x4e, 0x80, 0x04, 0x20  bctr
 227          */
 228
 229         pr_devel("ip:%lx jumps to %lx", ip, tramp);
 230
 231         /* Find where the trampoline jumps to */
 232         if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
 233                 pr_err("Failed to read %lx\n", tramp);
 234                 return -EFAULT;
 235         }
 236
 237         pr_devel(" %08x %08x ", jmp[0], jmp[1]);
 238
 239         /* verify that this is what we expect it to be */
 240         if (((jmp[0] & 0xffff0000) != 0x3d800000) ||
 241             ((jmp[1] & 0xffff0000) != 0x398c0000) ||
 242             (jmp[2] != 0x7d8903a6) ||
 243             (jmp[3] != 0x4e800420)) {
 244                 pr_err("Not a trampoline\n");
 245                 return -EINVAL;
 246         }
 247
 248         tramp = (jmp[1] & 0xffff) |
 249                 ((jmp[0] & 0xffff) << 16);
 250         if (tramp & 0x8000)
 251                 tramp -= 0x10000;
 252
 253         pr_devel(" %lx ", tramp);
 254
 255         if (tramp != addr) {
 256                 pr_err("Trampoline location %08lx does not match addr\n",
 257                        tramp);
 258                 return -EINVAL;
 259         }
 260
 261         op = PPC_INST_NOP;
 262
 263         if (patch_instruction((unsigned int *)ip, op))
 264                 return -EPERM;
 265
 266         return 0;
 267 }
 268 #endif /* PPC64 */
 269 #endif /* CONFIG_MODULES */
 270
 271 int ftrace_make_nop(struct module *mod,
 272                     struct dyn_ftrace *rec, unsigned long addr)
 273 {
 274         unsigned long ip = rec->ip;
 275         unsigned int old, new;
 276
 277         /*
 278          * If the calling address is more that 24 bits away,
 279          * then we had to use a trampoline to make the call.
 280          * Otherwise just update the call site.
 281          */
 282         if (test_24bit_addr(ip, addr)) {
 283                 /* within range */
 284                 old = ftrace_call_replace(ip, addr, 1);
 285                 new = PPC_INST_NOP;
 286                 return ftrace_modify_code(ip, old, new);
 287         }
 288
 289 #ifdef CONFIG_MODULES
 290         /*
 291          * Out of range jumps are called from modules.
 292          * We should either already have a pointer to the module
 293          * or it has been passed in.
 294          */
 295         if (!rec->arch.mod) {
 296                 if (!mod) {
 297                         pr_err("No module loaded addr=%lx\n", addr);
 298                         return -EFAULT;
 299                 }
 300                 rec->arch.mod = mod;
 301         } else if (mod) {
 302                 if (mod != rec->arch.mod) {
 303                         pr_err("Record mod %p not equal to passed in mod %p\n",
 304                                rec->arch.mod, mod);
 305                         return -EINVAL;
 306                 }
 307                 /* nothing to do if mod == rec->arch.mod */
 308         } else
 309                 mod = rec->arch.mod;
 310
 311         return __ftrace_make_nop(mod, rec, addr);
 312 #else
 313         /* We should not get here without modules */
 314         return -EINVAL;
 315 #endif /* CONFIG_MODULES */
 316 }
 317
 318 #ifdef CONFIG_MODULES
 319 #ifdef CONFIG_PPC64
 320 /*
 321  * Examine the existing instructions for __ftrace_make_call.
 322  * They should effectively be a NOP, and follow formal constraints,
 323  * depending on the ABI. Return false if they don't.
 324  */
 325 #ifndef CC_USING_MPROFILE_KERNEL
 326 static int
 327 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
 328 {
 329         /*
 330          * We expect to see:
 331          *
 332          * b +8
 333          * ld r2,XX(r1)
 334          *
 335          * The load offset is different depending on the ABI. For simplicity
 336          * just mask it out when doing the compare.
 337          */
 338         if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000))
 339                 return 0;
 340         return 1;
 341 }
 342 #else
 343 static int
 344 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
 345 {
 346         /* look for patched "NOP" on ppc64 with -mprofile-kernel */
 347         if (op0 != PPC_INST_NOP)
 348                 return 0;
 349         return 1;
 350 }
 351 #endif
 352
 353 static int
 354 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 355 {
 356         unsigned int op[2];
 357         void *ip = (void *)rec->ip;
 358
 359         /* read where this goes */
 360         if (probe_kernel_read(op, ip, sizeof(op)))
 361                 return -EFAULT;
 362
 363         if (!expected_nop_sequence(ip, op[0], op[1])) {
 364                 pr_err("Unexpected call sequence at %p: %x %x\n",
 365                 ip, op[0], op[1]);
 366                 return -EINVAL;
 367         }
 368
 369         /* If we never set up a trampoline to ftrace_caller, then bail */
 370         if (!rec->arch.mod->arch.tramp) {
 371                 pr_err("No ftrace trampoline\n");
 372                 return -EINVAL;
 373         }
 374
 375         /* Ensure branch is within 24 bits */
 376         if (!create_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
 377                 pr_err("Branch out of range\n");
 378                 return -EINVAL;
 379         }
 380
 381         if (patch_branch(ip, rec->arch.mod->arch.tramp, BRANCH_SET_LINK)) {
 382                 pr_err("REL24 out of range!\n");
 383                 return -EINVAL;
 384         }
 385
 386         return 0;
 387 }
 388
 389 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 390 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 391                         unsigned long addr)
 392 {
 393         return ftrace_make_call(rec, addr);
 394 }
 395 #endif
 396
 397 #else  /* !CONFIG_PPC64: */
 398 static int
 399 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 400 {
 401         unsigned int op;
 402         unsigned long ip = rec->ip;
 403
 404         /* read where this goes */
 405         if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
 406                 return -EFAULT;
 407
 408         /* It should be pointing to a nop */
 409         if (op != PPC_INST_NOP) {
 410                 pr_err("Expected NOP but have %x\n", op);
 411                 return -EINVAL;
 412         }
 413
 414         /* If we never set up a trampoline to ftrace_caller, then bail */
 415         if (!rec->arch.mod->arch.tramp) {
 416                 pr_err("No ftrace trampoline\n");
 417                 return -EINVAL;
 418         }
 419
 420         /* create the branch to the trampoline */
 421         op = create_branch((unsigned int *)ip,
 422                            rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
 423         if (!op) {
 424                 pr_err("REL24 out of range!\n");
 425                 return -EINVAL;
 426         }
 427
 428         pr_devel("write to %lx\n", rec->ip);
 429
 430         if (patch_instruction((unsigned int *)ip, op))
 431                 return -EPERM;
 432
 433         return 0;
 434 }
 435 #endif /* CONFIG_PPC64 */
 436 #endif /* CONFIG_MODULES */
 437
 438 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 439 {
 440         unsigned long ip = rec->ip;
 441         unsigned int old, new;
 442
 443         /*
 444          * If the calling address is more that 24 bits away,
 445          * then we had to use a trampoline to make the call.
 446          * Otherwise just update the call site.
 447          */
 448         if (test_24bit_addr(ip, addr)) {
 449                 /* within range */
 450                 old = PPC_INST_NOP;
 451                 new = ftrace_call_replace(ip, addr, 1);
 452                 return ftrace_modify_code(ip, old, new);
 453         }
 454
 455 #ifdef CONFIG_MODULES
 456         /*
 457          * Out of range jumps are called from modules.
 458          * Being that we are converting from nop, it had better
 459          * already have a module defined.
 460          */
 461         if (!rec->arch.mod) {
 462                 pr_err("No module loaded\n");
 463                 return -EINVAL;
 464         }
 465
 466         return __ftrace_make_call(rec, addr);
 467 #else
 468         /* We should not get here without modules */
 469         return -EINVAL;
 470 #endif /* CONFIG_MODULES */
 471 }
 472
 473 int ftrace_update_ftrace_func(ftrace_func_t func)
 474 {
 475         unsigned long ip = (unsigned long)(&ftrace_call);
 476         unsigned int old, new;
 477         int ret;
 478
 479         old = *(unsigned int *)&ftrace_call;
 480         new = ftrace_call_replace(ip, (unsigned long)func, 1);
 481         ret = ftrace_modify_code(ip, old, new);
 482
 483         return ret;
 484 }
 485
 486 static int __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 487 {
 488         unsigned long ftrace_addr = (unsigned long)FTRACE_ADDR;
 489         int ret;
 490
 491         ret = ftrace_update_record(rec, enable);
 492
 493         switch (ret) {
 494         case FTRACE_UPDATE_IGNORE:
 495                 return 0;
 496         case FTRACE_UPDATE_MAKE_CALL:
 497                 return ftrace_make_call(rec, ftrace_addr);
 498         case FTRACE_UPDATE_MAKE_NOP:
 499                 return ftrace_make_nop(NULL, rec, ftrace_addr);
 500         }
 501
 502         return 0;
 503 }
 504
 505 void ftrace_replace_code(int enable)
 506 {
 507         struct ftrace_rec_iter *iter;
 508         struct dyn_ftrace *rec;
 509         int ret;
 510
 511         for (iter = ftrace_rec_iter_start(); iter;
 512              iter = ftrace_rec_iter_next(iter)) {
 513                 rec = ftrace_rec_iter_record(iter);
 514                 ret = __ftrace_replace_code(rec, enable);
 515                 if (ret) {
 516                         ftrace_bug(ret, rec);
 517                         return;
 518                 }
 519         }
 520 }
 521
 522 /*
 523  * Use the default ftrace_modify_all_code, but without
 524  * stop_machine().
 525  */
 526 void arch_ftrace_update_code(int command)
 527 {
 528         ftrace_modify_all_code(command);
 529 }
 530
 531 int __init ftrace_dyn_arch_init(void)
 532 {
 533         return 0;
 534 }
 535 #endif /* CONFIG_DYNAMIC_FTRACE */
 536
 537 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 538
 539 #ifdef CONFIG_DYNAMIC_FTRACE
 540 extern void ftrace_graph_call(void);
 541 extern void ftrace_graph_stub(void);
 542
 543 int ftrace_enable_ftrace_graph_caller(void)
 544 {
 545         unsigned long ip = (unsigned long)(&ftrace_graph_call);
 546         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
 547         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
 548         unsigned int old, new;
 549
 550         old = ftrace_call_replace(ip, stub, 0);
 551         new = ftrace_call_replace(ip, addr, 0);
 552
 553         return ftrace_modify_code(ip, old, new);
 554 }
 555
 556 int ftrace_disable_ftrace_graph_caller(void)
 557 {
 558         unsigned long ip = (unsigned long)(&ftrace_graph_call);
 559         unsigned long addr = (unsigned long)(&ftrace_graph_caller);
 560         unsigned long stub = (unsigned long)(&ftrace_graph_stub);
 561         unsigned int old, new;
 562
 563         old = ftrace_call_replace(ip, addr, 0);
 564         new = ftrace_call_replace(ip, stub, 0);
 565
 566         return ftrace_modify_code(ip, old, new);
 567 }
 568 #endif /* CONFIG_DYNAMIC_FTRACE */
 569
 570 /*
 571  * Hook the return address and push it in the stack of return addrs
 572  * in current thread info. Return the address we want to divert to.
 573  */
 574 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip)
 575 {
 576         struct ftrace_graph_ent trace;
 577         unsigned long return_hooker;
 578
 579         if (unlikely(ftrace_graph_is_dead()))
 580                 goto out;
 581
 582         if (unlikely(atomic_read(&current->tracing_graph_pause)))
 583                 goto out;
 584
 585         return_hooker = ppc_function_entry(return_to_handler);
 586
 587         trace.func = ip;
 588         trace.depth = current->curr_ret_stack + 1;
 589
 590         /* Only trace if the calling function expects to */
 591         if (!ftrace_graph_entry(&trace))
 592                 goto out;
 593
 594         if (ftrace_push_return_trace(parent, ip, &trace.depth, 0,
 595                                      NULL) == -EBUSY)
 596                 goto out;
 597
 598         parent = return_hooker;
 599 out:
 600         return parent;
 601 }
 602 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 603
 604 #if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
 605 unsigned long __init arch_syscall_addr(int nr)
 606 {
 607         return sys_call_table[nr*2];
 608 }
 609 #endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */
 610
 611 #ifdef PPC64_ELF_ABI_v1
 612 char *arch_ftrace_match_adjust(char *str, const char *search)
 613 {
 614         if (str[0] == '.' && search[0] != '.')
 615                 return str + 1;
 616         else
 617                 return str;
 618 }
 619 #endif /* PPC64_ELF_ABI_v1 */