arch/x86/kernel/ftrace.c

   1 /*
   2  * Code for replacing ftrace calls with jumps.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  *
   6  * Thanks goes to Ingo Molnar, for suggesting the idea.
   7  * Mathieu Desnoyers, for suggesting postponing the modifications.
   8  * Arjan van de Ven, for keeping me straight, and explaining to me
   9  * the dangers of modifying code on the run.
  10  */
  11
  12 #include <linux/spinlock.h>
  13 #include <linux/hardirq.h>
  14 #include <linux/uaccess.h>
  15 #include <linux/ftrace.h>
  16 #include <linux/percpu.h>
  17 #include <linux/sched.h>
  18 #include <linux/init.h>
  19 #include <linux/list.h>
  20
  21 #include <asm/ftrace.h>
  22 #include <linux/ftrace.h>
  23 #include <asm/nops.h>
  24 #include <asm/nmi.h>
  25
  26
  27
  28 #ifdef CONFIG_FUNCTION_RET_TRACER
  29
  30 /*
  31  * These functions are picked from those used on
  32  * this page for dynamic ftrace. They have been
  33  * simplified to ignore all traces in NMI context.
  34  */
  35 static atomic_t in_nmi;
  36
  37 void ftrace_nmi_enter(void)
  38 {
  39         atomic_inc(&in_nmi);
  40 }
  41
  42 void ftrace_nmi_exit(void)
  43 {
  44         atomic_dec(&in_nmi);
  45 }
  46
  47 /* Add a function return address to the trace stack on thread info.*/
  48 static int push_return_trace(unsigned long ret, unsigned long long time,
  49                                 unsigned long func)
  50 {
  51         int index;
  52         struct thread_info *ti = current_thread_info();
  53
  54         /* The return trace stack is full */
  55         if (ti->curr_ret_stack == FTRACE_RET_STACK_SIZE - 1)
  56                 return -EBUSY;
  57
  58         index = ++ti->curr_ret_stack;
  59         ti->ret_stack[index].ret = ret;
  60         ti->ret_stack[index].func = func;
  61         ti->ret_stack[index].calltime = time;
  62
  63         return 0;
  64 }
  65
  66 /* Retrieve a function return address to the trace stack on thread info.*/
  67 static void pop_return_trace(unsigned long *ret, unsigned long long *time,
  68                                 unsigned long *func)
  69 {
  70         int index;
  71
  72         struct thread_info *ti = current_thread_info();
  73         index = ti->curr_ret_stack;
  74         *ret = ti->ret_stack[index].ret;
  75         *func = ti->ret_stack[index].func;
  76         *time = ti->ret_stack[index].calltime;
  77         ti->curr_ret_stack--;
  78 }
  79
  80 /*
  81  * Send the trace to the ring-buffer.
  82  * @return the original return address.
  83  */
  84 unsigned long ftrace_return_to_handler(void)
  85 {
  86         struct ftrace_retfunc trace;
  87         pop_return_trace(&trace.ret, &trace.calltime, &trace.func);
  88         trace.rettime = cpu_clock(raw_smp_processor_id());
  89         ftrace_function_return(&trace);
  90
  91         return trace.ret;
  92 }
  93
  94 /*
  95  * Hook the return address and push it in the stack of return addrs
  96  * in current thread info.
  97  */
  98 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
  99 {
 100         unsigned long old;
 101         unsigned long long calltime;
 102         int faulted;
 103         unsigned long return_hooker = (unsigned long)
 104                                 &return_to_handler;
 105
 106         /* Nmi's are currently unsupported */
 107         if (atomic_read(&in_nmi))
 108                 return;
 109
 110         /*
 111          * Protect against fault, even if it shouldn't
 112          * happen. This tool is too much intrusive to
 113          * ignore such a protection.
 114          */
 115         asm volatile(
 116                 "1: movl (%[parent_old]), %[old]\n"
 117                 "2: movl %[return_hooker], (%[parent_replaced])\n"
 118                 "   movl $0, %[faulted]\n"
 119
 120                 ".section .fixup, \"ax\"\n"
 121                 "3: movl $1, %[faulted]\n"
 122                 ".previous\n"
 123
 124                 ".section __ex_table, \"a\"\n"
 125                 "   .long 1b, 3b\n"
 126                 "   .long 2b, 3b\n"
 127                 ".previous\n"
 128
 129                 : [parent_replaced] "=r" (parent), [old] "=r" (old),
 130                   [faulted] "=r" (faulted)
 131                 : [parent_old] "0" (parent), [return_hooker] "r" (return_hooker)
 132                 : "memory"
 133         );
 134
 135         if (WARN_ON(faulted)) {
 136                 unregister_ftrace_return();
 137                 return;
 138         }
 139
 140         if (WARN_ON(!__kernel_text_address(old))) {
 141                 unregister_ftrace_return();
 142                 *parent = old;
 143                 return;
 144         }
 145
 146         calltime = cpu_clock(raw_smp_processor_id());
 147
 148         if (push_return_trace(old, calltime, self_addr) == -EBUSY)
 149                 *parent = old;
 150 }
 151
 152 #endif
 153
 154 #ifdef CONFIG_DYNAMIC_FTRACE
 155
 156 union ftrace_code_union {
 157         char code[MCOUNT_INSN_SIZE];
 158         struct {
 159                 char e8;
 160                 int offset;
 161         } __attribute__((packed));
 162 };
 163
 164 static int ftrace_calc_offset(long ip, long addr)
 165 {
 166         return (int)(addr - ip);
 167 }
 168
 169 static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
 170 {
 171         static union ftrace_code_union calc;
 172
 173         calc.e8         = 0xe8;
 174         calc.offset     = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
 175
 176         /*
 177          * No locking needed, this must be called via kstop_machine
 178          * which in essence is like running on a uniprocessor machine.
 179          */
 180         return calc.code;
 181 }
 182
 183 /*
 184  * Modifying code must take extra care. On an SMP machine, if
 185  * the code being modified is also being executed on another CPU
 186  * that CPU will have undefined results and possibly take a GPF.
 187  * We use kstop_machine to stop other CPUS from exectuing code.
 188  * But this does not stop NMIs from happening. We still need
 189  * to protect against that. We separate out the modification of
 190  * the code to take care of this.
 191  *
 192  * Two buffers are added: An IP buffer and a "code" buffer.
 193  *
 194  * 1) Put the instruction pointer into the IP buffer
 195  *    and the new code into the "code" buffer.
 196  * 2) Set a flag that says we are modifying code
 197  * 3) Wait for any running NMIs to finish.
 198  * 4) Write the code
 199  * 5) clear the flag.
 200  * 6) Wait for any running NMIs to finish.
 201  *
 202  * If an NMI is executed, the first thing it does is to call
 203  * "ftrace_nmi_enter". This will check if the flag is set to write
 204  * and if it is, it will write what is in the IP and "code" buffers.
 205  *
 206  * The trick is, it does not matter if everyone is writing the same
 207  * content to the code location. Also, if a CPU is executing code
 208  * it is OK to write to that code location if the contents being written
 209  * are the same as what exists.
 210  */
 211
 212 static atomic_t in_nmi = ATOMIC_INIT(0);
 213 static int mod_code_status;             /* holds return value of text write */
 214 static int mod_code_write;              /* set when NMI should do the write */
 215 static void *mod_code_ip;               /* holds the IP to write to */
 216 static void *mod_code_newcode;          /* holds the text to write to the IP */
 217
 218 static unsigned nmi_wait_count;
 219 static atomic_t nmi_update_count = ATOMIC_INIT(0);
 220
 221 int ftrace_arch_read_dyn_info(char *buf, int size)
 222 {
 223         int r;
 224
 225         r = snprintf(buf, size, "%u %u",
 226                      nmi_wait_count,
 227                      atomic_read(&nmi_update_count));
 228         return r;
 229 }
 230
 231 static void ftrace_mod_code(void)
 232 {
 233         /*
 234          * Yes, more than one CPU process can be writing to mod_code_status.
 235          *    (and the code itself)
 236          * But if one were to fail, then they all should, and if one were
 237          * to succeed, then they all should.
 238          */
 239         mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
 240                                              MCOUNT_INSN_SIZE);
 241
 242 }
 243
 244 void ftrace_nmi_enter(void)
 245 {
 246         atomic_inc(&in_nmi);
 247         /* Must have in_nmi seen before reading write flag */
 248         smp_mb();
 249         if (mod_code_write) {
 250                 ftrace_mod_code();
 251                 atomic_inc(&nmi_update_count);
 252         }
 253 }
 254
 255 void ftrace_nmi_exit(void)
 256 {
 257         /* Finish all executions before clearing in_nmi */
 258         smp_wmb();
 259         atomic_dec(&in_nmi);
 260 }
 261
 262 static void wait_for_nmi(void)
 263 {
 264         int waited = 0;
 265
 266         while (atomic_read(&in_nmi)) {
 267                 waited = 1;
 268                 cpu_relax();
 269         }
 270
 271         if (waited)
 272                 nmi_wait_count++;
 273 }
 274
 275 static int
 276 do_ftrace_mod_code(unsigned long ip, void *new_code)
 277 {
 278         mod_code_ip = (void *)ip;
 279         mod_code_newcode = new_code;
 280
 281         /* The buffers need to be visible before we let NMIs write them */
 282         smp_wmb();
 283
 284         mod_code_write = 1;
 285
 286         /* Make sure write bit is visible before we wait on NMIs */
 287         smp_mb();
 288
 289         wait_for_nmi();
 290
 291         /* Make sure all running NMIs have finished before we write the code */
 292         smp_mb();
 293
 294         ftrace_mod_code();
 295
 296         /* Make sure the write happens before clearing the bit */
 297         smp_wmb();
 298
 299         mod_code_write = 0;
 300
 301         /* make sure NMIs see the cleared bit */
 302         smp_mb();
 303
 304         wait_for_nmi();
 305
 306         return mod_code_status;
 307 }
 308
 309
 310
 311
 312 static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];
 313
 314 static unsigned char *ftrace_nop_replace(void)
 315 {
 316         return ftrace_nop;
 317 }
 318
 319 static int
 320 ftrace_modify_code(unsigned long ip, unsigned char *old_code,
 321                    unsigned char *new_code)
 322 {
 323         unsigned char replaced[MCOUNT_INSN_SIZE];
 324
 325         /*
 326          * Note: Due to modules and __init, code can
 327          *  disappear and change, we need to protect against faulting
 328          *  as well as code changing. We do this by using the
 329          *  probe_kernel_* functions.
 330          *
 331          * No real locking needed, this code is run through
 332          * kstop_machine, or before SMP starts.
 333          */
 334
 335         /* read the text we want to modify */
 336         if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
 337                 return -EFAULT;
 338
 339         /* Make sure it is what we expect it to be */
 340         if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
 341                 return -EINVAL;
 342
 343         /* replace the text with the new text */
 344         if (do_ftrace_mod_code(ip, new_code))
 345                 return -EPERM;
 346
 347         sync_core();
 348
 349         return 0;
 350 }
 351
 352 int ftrace_make_nop(struct module *mod,
 353                     struct dyn_ftrace *rec, unsigned long addr)
 354 {
 355         unsigned char *new, *old;
 356         unsigned long ip = rec->ip;
 357
 358         old = ftrace_call_replace(ip, addr);
 359         new = ftrace_nop_replace();
 360
 361         return ftrace_modify_code(rec->ip, old, new);
 362 }
 363
 364 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 365 {
 366         unsigned char *new, *old;
 367         unsigned long ip = rec->ip;
 368
 369         old = ftrace_nop_replace();
 370         new = ftrace_call_replace(ip, addr);
 371
 372         return ftrace_modify_code(rec->ip, old, new);
 373 }
 374
 375 int ftrace_update_ftrace_func(ftrace_func_t func)
 376 {
 377         unsigned long ip = (unsigned long)(&ftrace_call);
 378         unsigned char old[MCOUNT_INSN_SIZE], *new;
 379         int ret;
 380
 381         memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
 382         new = ftrace_call_replace(ip, (unsigned long)func);
 383         ret = ftrace_modify_code(ip, old, new);
 384
 385         return ret;
 386 }
 387
 388 int __init ftrace_dyn_arch_init(void *data)
 389 {
 390         extern const unsigned char ftrace_test_p6nop[];
 391         extern const unsigned char ftrace_test_nop5[];
 392         extern const unsigned char ftrace_test_jmp[];
 393         int faulted = 0;
 394
 395         /*
 396          * There is no good nop for all x86 archs.
 397          * We will default to using the P6_NOP5, but first we
 398          * will test to make sure that the nop will actually
 399          * work on this CPU. If it faults, we will then
 400          * go to a lesser efficient 5 byte nop. If that fails
 401          * we then just use a jmp as our nop. This isn't the most
 402          * efficient nop, but we can not use a multi part nop
 403          * since we would then risk being preempted in the middle
 404          * of that nop, and if we enabled tracing then, it might
 405          * cause a system crash.
 406          *
 407          * TODO: check the cpuid to determine the best nop.
 408          */
 409         asm volatile (
 410                 "ftrace_test_jmp:"
 411                 "jmp ftrace_test_p6nop\n"
 412                 "nop\n"
 413                 "nop\n"
 414                 "nop\n"  /* 2 byte jmp + 3 bytes */
 415                 "ftrace_test_p6nop:"
 416                 P6_NOP5
 417                 "jmp 1f\n"
 418                 "ftrace_test_nop5:"
 419                 ".byte 0x66,0x66,0x66,0x66,0x90\n"
 420                 "1:"
 421                 ".section .fixup, \"ax\"\n"
 422                 "2:     movl $1, %0\n"
 423                 "       jmp ftrace_test_nop5\n"
 424                 "3:     movl $2, %0\n"
 425                 "       jmp 1b\n"
 426                 ".previous\n"
 427                 _ASM_EXTABLE(ftrace_test_p6nop, 2b)
 428                 _ASM_EXTABLE(ftrace_test_nop5, 3b)
 429                 : "=r"(faulted) : "0" (faulted));
 430
 431         switch (faulted) {
 432         case 0:
 433                 pr_info("ftrace: converting mcount calls to 0f 1f 44 00 00\n");
 434                 memcpy(ftrace_nop, ftrace_test_p6nop, MCOUNT_INSN_SIZE);
 435                 break;
 436         case 1:
 437                 pr_info("ftrace: converting mcount calls to 66 66 66 66 90\n");
 438                 memcpy(ftrace_nop, ftrace_test_nop5, MCOUNT_INSN_SIZE);
 439                 break;
 440         case 2:
 441                 pr_info("ftrace: converting mcount calls to jmp . + 5\n");
 442                 memcpy(ftrace_nop, ftrace_test_jmp, MCOUNT_INSN_SIZE);
 443                 break;
 444         }
 445
 446         /* The return code is retured via data */
 447         *(unsigned long *)data = 0;
 448
 449         return 0;
 450 }
 451 #endif