arch/x86/kernel/ftrace_32.S

   1 /*
   2  *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
   3  */
   4
   5 #include <linux/linkage.h>
   6 #include <asm/page_types.h>
   7 #include <asm/segment.h>
   8 #include <asm/export.h>
   9 #include <asm/ftrace.h>
  10
  11 #ifdef CC_USING_FENTRY
  12 # define function_hook  __fentry__
  13 EXPORT_SYMBOL(__fentry__)
  14 #else
  15 # define function_hook  mcount
  16 EXPORT_SYMBOL(mcount)
  17 #endif
  18
  19 #ifdef CONFIG_DYNAMIC_FTRACE
  20
  21 /* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */
  22 #if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER)
  23 # define USING_FRAME_POINTER
  24 #endif
  25
  26 #ifdef USING_FRAME_POINTER
  27 # define MCOUNT_FRAME                   1       /* using frame = true  */
  28 #else
  29 # define MCOUNT_FRAME                   0       /* using frame = false */
  30 #endif
  31
  32 ENTRY(function_hook)
  33         ret
  34 END(function_hook)
  35
  36 ENTRY(ftrace_caller)
  37
  38 #ifdef USING_FRAME_POINTER
  39 # ifdef CC_USING_FENTRY
  40         /*
  41          * Frame pointers are of ip followed by bp.
  42          * Since fentry is an immediate jump, we are left with
  43          * parent-ip, function-ip. We need to add a frame with
  44          * parent-ip followed by ebp.
  45          */
  46         pushl   4(%esp)                         /* parent ip */
  47         pushl   %ebp
  48         movl    %esp, %ebp
  49         pushl   2*4(%esp)                       /* function ip */
  50 # endif
  51         /* For mcount, the function ip is directly above */
  52         pushl   %ebp
  53         movl    %esp, %ebp
  54 #endif
  55         pushl   %eax
  56         pushl   %ecx
  57         pushl   %edx
  58         pushl   $0                              /* Pass NULL as regs pointer */
  59
  60 #ifdef USING_FRAME_POINTER
  61         /* Load parent ebp into edx */
  62         movl    4*4(%esp), %edx
  63 #else
  64         /* There's no frame pointer, load the appropriate stack addr instead */
  65         lea     4*4(%esp), %edx
  66 #endif
  67
  68         movl    (MCOUNT_FRAME+4)*4(%esp), %eax  /* load the rip */
  69         /* Get the parent ip */
  70         movl    4(%edx), %edx                   /* edx has ebp */
  71
  72         movl    function_trace_op, %ecx
  73         subl    $MCOUNT_INSN_SIZE, %eax
  74
  75 .globl ftrace_call
  76 ftrace_call:
  77         call    ftrace_stub
  78
  79         addl    $4, %esp                        /* skip NULL pointer */
  80         popl    %edx
  81         popl    %ecx
  82         popl    %eax
  83 #ifdef USING_FRAME_POINTER
  84         popl    %ebp
  85 # ifdef CC_USING_FENTRY
  86         addl    $4,%esp                         /* skip function ip */
  87         popl    %ebp                            /* this is the orig bp */
  88         addl    $4, %esp                        /* skip parent ip */
  89 # endif
  90 #endif
  91 .Lftrace_ret:
  92 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  93 .globl ftrace_graph_call
  94 ftrace_graph_call:
  95         jmp     ftrace_stub
  96 #endif
  97
  98 /* This is weak to keep gas from relaxing the jumps */
  99 WEAK(ftrace_stub)
 100         ret
 101 END(ftrace_caller)
 102
 103 ENTRY(ftrace_regs_caller)
 104         /*
 105          * i386 does not save SS and ESP when coming from kernel.
 106          * Instead, to get sp, &regs->sp is used (see ptrace.h).
 107          * Unfortunately, that means eflags must be at the same location
 108          * as the current return ip is. We move the return ip into the
 109          * regs->ip location, and move flags into the return ip location.
 110          */
 111         pushl   $__KERNEL_CS
 112         pushl   4(%esp)                         /* Save the return ip */
 113         pushl   $0                              /* Load 0 into orig_ax */
 114         pushl   %gs
 115         pushl   %fs
 116         pushl   %es
 117         pushl   %ds
 118         pushl   %eax
 119
 120         /* Get flags and place them into the return ip slot */
 121         pushf
 122         popl    %eax
 123         movl    %eax, 8*4(%esp)
 124
 125         pushl   %ebp
 126         pushl   %edi
 127         pushl   %esi
 128         pushl   %edx
 129         pushl   %ecx
 130         pushl   %ebx
 131
 132         movl    12*4(%esp), %eax                /* Load ip (1st parameter) */
 133         subl    $MCOUNT_INSN_SIZE, %eax         /* Adjust ip */
 134 #ifdef CC_USING_FENTRY
 135         movl    15*4(%esp), %edx                /* Load parent ip (2nd parameter) */
 136 #else
 137         movl    0x4(%ebp), %edx                 /* Load parent ip (2nd parameter) */
 138 #endif
 139         movl    function_trace_op, %ecx         /* Save ftrace_pos in 3rd parameter */
 140         pushl   %esp                            /* Save pt_regs as 4th parameter */
 141
 142 GLOBAL(ftrace_regs_call)
 143         call    ftrace_stub
 144
 145         addl    $4, %esp                        /* Skip pt_regs */
 146
 147         /* restore flags */
 148         push    14*4(%esp)
 149         popf
 150
 151         /* Move return ip back to its original location */
 152         movl    12*4(%esp), %eax
 153         movl    %eax, 14*4(%esp)
 154
 155         popl    %ebx
 156         popl    %ecx
 157         popl    %edx
 158         popl    %esi
 159         popl    %edi
 160         popl    %ebp
 161         popl    %eax
 162         popl    %ds
 163         popl    %es
 164         popl    %fs
 165         popl    %gs
 166
 167         /* use lea to not affect flags */
 168         lea     3*4(%esp), %esp                 /* Skip orig_ax, ip and cs */
 169
 170         jmp     .Lftrace_ret
 171 #else /* ! CONFIG_DYNAMIC_FTRACE */
 172
 173 ENTRY(function_hook)
 174         cmpl    $__PAGE_OFFSET, %esp
 175         jb      ftrace_stub                     /* Paging not enabled yet? */
 176
 177         cmpl    $ftrace_stub, ftrace_trace_function
 178         jnz     .Ltrace
 179 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 180         cmpl    $ftrace_stub, ftrace_graph_return
 181         jnz     ftrace_graph_caller
 182
 183         cmpl    $ftrace_graph_entry_stub, ftrace_graph_entry
 184         jnz     ftrace_graph_caller
 185 #endif
 186 .globl ftrace_stub
 187 ftrace_stub:
 188         ret
 189
 190         /* taken from glibc */
 191 .Ltrace:
 192         pushl   %eax
 193         pushl   %ecx
 194         pushl   %edx
 195         movl    0xc(%esp), %eax
 196         movl    0x4(%ebp), %edx
 197         subl    $MCOUNT_INSN_SIZE, %eax
 198
 199         call    *ftrace_trace_function
 200
 201         popl    %edx
 202         popl    %ecx
 203         popl    %eax
 204         jmp     ftrace_stub
 205 END(function_hook)
 206 #endif /* CONFIG_DYNAMIC_FTRACE */
 207
 208 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 209 ENTRY(ftrace_graph_caller)
 210         pushl   %eax
 211         pushl   %ecx
 212         pushl   %edx
 213         movl    3*4(%esp), %eax
 214         /* Even with frame pointers, fentry doesn't have one here */
 215 #ifdef CC_USING_FENTRY
 216         lea     4*4(%esp), %edx
 217         movl    $0, %ecx
 218 #else
 219         lea     0x4(%ebp), %edx
 220         movl    (%ebp), %ecx
 221 #endif
 222         subl    $MCOUNT_INSN_SIZE, %eax
 223         call    prepare_ftrace_return
 224         popl    %edx
 225         popl    %ecx
 226         popl    %eax
 227         ret
 228 END(ftrace_graph_caller)
 229
 230 .globl return_to_handler
 231 return_to_handler:
 232         pushl   %eax
 233         pushl   %edx
 234 #ifdef CC_USING_FENTRY
 235         movl    $0, %eax
 236 #else
 237         movl    %ebp, %eax
 238 #endif
 239         call    ftrace_return_to_handler
 240         movl    %eax, %ecx
 241         popl    %edx
 242         popl    %eax
 243         jmp     *%ecx
 244 #endif