[mirror_ubuntu-artful-kernel.git] / arch / x86 / kernel / ftrace_32.S

/*
 *  Copyright (C) 2017  Steven Rostedt, VMware Inc.
 */

#include <linux/linkage.h>
#include <asm/page_types.h>
#include <asm/segment.h>
#include <asm/export.h>
#include <asm/ftrace.h>

#ifdef CC_USING_FENTRY
# define function_hook	__fentry__
EXPORT_SYMBOL(__fentry__)
#else
# define function_hook	mcount
EXPORT_SYMBOL(mcount)
#endif

#ifdef CONFIG_DYNAMIC_FTRACE

/* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */
#if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER)
# define USING_FRAME_POINTER
#endif

#ifdef USING_FRAME_POINTER
# define MCOUNT_FRAME			1	/* using frame = true  */
#else
# define MCOUNT_FRAME			0	/* using frame = false */
#endif

ENTRY(function_hook)
	ret
END(function_hook)

ENTRY(ftrace_caller)

#ifdef USING_FRAME_POINTER
# ifdef CC_USING_FENTRY
	/*
	 * Frame pointers are of ip followed by bp.
	 * Since fentry is an immediate jump, we are left with
	 * parent-ip, function-ip. We need to add a frame with
	 * parent-ip followed by ebp.
	 */
	pushl	4(%esp)				/* parent ip */
	pushl	%ebp
	movl	%esp, %ebp
	pushl	2*4(%esp)			/* function ip */
# endif
	/* For mcount, the function ip is directly above */
	pushl	%ebp
	movl	%esp, %ebp
#endif
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	pushl	$0				/* Pass NULL as regs pointer */

#ifdef USING_FRAME_POINTER
	/* Load parent ebp into edx */
	movl	4*4(%esp), %edx
#else
	/* There's no frame pointer, load the appropriate stack addr instead */
	lea	4*4(%esp), %edx
#endif

	movl	(MCOUNT_FRAME+4)*4(%esp), %eax	/* load the rip */
	/* Get the parent ip */
	movl	4(%edx), %edx			/* edx has ebp */

	movl	function_trace_op, %ecx
	subl	$MCOUNT_INSN_SIZE, %eax

.globl ftrace_call
ftrace_call:
	call	ftrace_stub

	addl	$4, %esp			/* skip NULL pointer */
	popl	%edx
	popl	%ecx
	popl	%eax
#ifdef USING_FRAME_POINTER
	popl	%ebp
# ifdef CC_USING_FENTRY
	addl	$4,%esp				/* skip function ip */
	popl	%ebp				/* this is the orig bp */
	addl	$4, %esp			/* skip parent ip */
# endif
#endif
.Lftrace_ret:
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_call
ftrace_graph_call:
	jmp	ftrace_stub
#endif

/* This is weak to keep gas from relaxing the jumps */
WEAK(ftrace_stub)
	ret
END(ftrace_caller)

ENTRY(ftrace_regs_caller)
	/*
	 * i386 does not save SS and ESP when coming from kernel.
	 * Instead, to get sp, &regs->sp is used (see ptrace.h).
	 * Unfortunately, that means eflags must be at the same location
	 * as the current return ip is. We move the return ip into the
	 * regs->ip location, and move flags into the return ip location.
	 */
	pushl	$__KERNEL_CS
	pushl	4(%esp)				/* Save the return ip */
	pushl	$0				/* Load 0 into orig_ax */
	pushl	%gs
	pushl	%fs
	pushl	%es
	pushl	%ds
	pushl	%eax

	/* Get flags and place them into the return ip slot */
	pushf
	popl	%eax
	movl	%eax, 8*4(%esp)

	pushl	%ebp
	pushl	%edi
	pushl	%esi
	pushl	%edx
	pushl	%ecx
	pushl	%ebx

	movl	12*4(%esp), %eax		/* Load ip (1st parameter) */
	subl	$MCOUNT_INSN_SIZE, %eax		/* Adjust ip */
#ifdef CC_USING_FENTRY
	movl	15*4(%esp), %edx		/* Load parent ip (2nd parameter) */
#else
	movl	0x4(%ebp), %edx			/* Load parent ip (2nd parameter) */
#endif
	movl	function_trace_op, %ecx		/* Save ftrace_pos in 3rd parameter */
	pushl	%esp				/* Save pt_regs as 4th parameter */

GLOBAL(ftrace_regs_call)
	call	ftrace_stub

	addl	$4, %esp			/* Skip pt_regs */

	/* restore flags */
	push	14*4(%esp)
	popf

	/* Move return ip back to its original location */
	movl	12*4(%esp), %eax
	movl	%eax, 14*4(%esp)

	popl	%ebx
	popl	%ecx
	popl	%edx
	popl	%esi
	popl	%edi
	popl	%ebp
	popl	%eax
	popl	%ds
	popl	%es
	popl	%fs
	popl	%gs

	/* use lea to not affect flags */
	lea	3*4(%esp), %esp			/* Skip orig_ax, ip and cs */

	jmp	.Lftrace_ret
#else /* ! CONFIG_DYNAMIC_FTRACE */

ENTRY(function_hook)
	cmpl	$__PAGE_OFFSET, %esp
	jb	ftrace_stub			/* Paging not enabled yet? */

	cmpl	$ftrace_stub, ftrace_trace_function
	jnz	.Ltrace
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	cmpl	$ftrace_stub, ftrace_graph_return
	jnz	ftrace_graph_caller

	cmpl	$ftrace_graph_entry_stub, ftrace_graph_entry
	jnz	ftrace_graph_caller
#endif
.globl ftrace_stub
ftrace_stub:
	ret

	/* taken from glibc */
.Ltrace:
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	movl	0xc(%esp), %eax
	movl	0x4(%ebp), %edx
	subl	$MCOUNT_INSN_SIZE, %eax

	call	*ftrace_trace_function

	popl	%edx
	popl	%ecx
	popl	%eax
	jmp	ftrace_stub
END(function_hook)
#endif /* CONFIG_DYNAMIC_FTRACE */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
ENTRY(ftrace_graph_caller)
	pushl	%eax
	pushl	%ecx
	pushl	%edx
	movl	3*4(%esp), %eax
	/* Even with frame pointers, fentry doesn't have one here */
#ifdef CC_USING_FENTRY
	lea	4*4(%esp), %edx
	movl	$0, %ecx
#else
	lea	0x4(%ebp), %edx
	movl	(%ebp), %ecx
#endif
	subl	$MCOUNT_INSN_SIZE, %eax
	call	prepare_ftrace_return
	popl	%edx
	popl	%ecx
	popl	%eax
	ret
END(ftrace_graph_caller)

.globl return_to_handler
return_to_handler:
	pushl	%eax
	pushl	%edx
#ifdef CC_USING_FENTRY
	movl	$0, %eax
#else
	movl	%ebp, %eax
#endif
	call	ftrace_return_to_handler
	movl	%eax, %ecx
	popl	%edx
	popl	%eax
	jmp	*%ecx
#endif
Commit	Line	Data
3d82c59c SRV	1	/*
	2	* Copyright (C) 2017 Steven Rostedt, VMware Inc.
	3	*/
	4
	5	#include <linux/linkage.h>
	6	#include <asm/page_types.h>
	7	#include <asm/segment.h>
	8	#include <asm/export.h>
	9	#include <asm/ftrace.h>
	10
644e0e8d SRV	11	#ifdef CC_USING_FENTRY
	12	# define function_hook __fentry__
	13	EXPORT_SYMBOL(__fentry__)
	14	#else
	15	# define function_hook mcount
	16	EXPORT_SYMBOL(mcount)
	17	#endif
	18
3d82c59c SRV	19	#ifdef CONFIG_DYNAMIC_FTRACE
3d82c59c SRV	20
644e0e8d SRV	21	/* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */
	22	#if !defined(CC_USING_FENTRY) \|\| defined(CONFIG_FRAME_POINTER)
	23	# define USING_FRAME_POINTER
	24	#endif
	25
	26	#ifdef USING_FRAME_POINTER
	27	# define MCOUNT_FRAME 1 /* using frame = true */
	28	#else
	29	# define MCOUNT_FRAME 0 /* using frame = false */
	30	#endif
	31
	32	ENTRY(function_hook)
3d82c59c	33	ret
644e0e8d	34	END(function_hook)
3d82c59c SRV	35
3d82c59c SRV	36	ENTRY(ftrace_caller)
e6928e58	37
644e0e8d SRV	38	#ifdef USING_FRAME_POINTER
	39	# ifdef CC_USING_FENTRY
	40	/*
	41	* Frame pointers are of ip followed by bp.
	42	* Since fentry is an immediate jump, we are left with
	43	* parent-ip, function-ip. We need to add a frame with
	44	* parent-ip followed by ebp.
	45	*/
	46	pushl 4(%esp) /* parent ip */
e6928e58 SRV	47	pushl %ebp
e6928e58 SRV	48	movl %esp, %ebp
644e0e8d SRV	49	pushl 24(%esp) / function ip */
	50	# endif
	51	/* For mcount, the function ip is directly above */
	52	pushl %ebp
	53	movl %esp, %ebp
	54	#endif
3d82c59c SRV	55	pushl %eax
	56	pushl %ecx
	57	pushl %edx
	58	pushl $0 /* Pass NULL as regs pointer */
644e0e8d SRV	59
	60	#ifdef USING_FRAME_POINTER
	61	/* Load parent ebp into edx */
e6928e58	62	movl 4*4(%esp), %edx
644e0e8d SRV	63	#else
	64	/* There's no frame pointer, load the appropriate stack addr instead */
	65	lea 4*4(%esp), %edx
	66	#endif
	67
	68	movl (MCOUNT_FRAME+4)4(%esp), %eax / load the rip */
e6928e58	69	/* Get the parent ip */
644e0e8d SRV	70	movl 4(%edx), %edx /* edx has ebp */
644e0e8d SRV	71
3d82c59c SRV	72	movl function_trace_op, %ecx
	73	subl $MCOUNT_INSN_SIZE, %eax
	74
	75	.globl ftrace_call
	76	ftrace_call:
	77	call ftrace_stub
	78
	79	addl $4, %esp /* skip NULL pointer */
	80	popl %edx
	81	popl %ecx
	82	popl %eax
644e0e8d	83	#ifdef USING_FRAME_POINTER
e6928e58	84	popl %ebp
644e0e8d SRV	85	# ifdef CC_USING_FENTRY
	86	addl $4,%esp /* skip function ip */
	87	popl %ebp /* this is the orig bp */
	88	addl $4, %esp /* skip parent ip */
	89	# endif
	90	#endif
3d82c59c SRV	91	.Lftrace_ret:
	92	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	93	.globl ftrace_graph_call
	94	ftrace_graph_call:
	95	jmp ftrace_stub
	96	#endif
	97
	98	/* This is weak to keep gas from relaxing the jumps */
	99	WEAK(ftrace_stub)
	100	ret
	101	END(ftrace_caller)
	102
	103	ENTRY(ftrace_regs_caller)
3d82c59c SRV	104	/*
	105	* i386 does not save SS and ESP when coming from kernel.
	106	* Instead, to get sp, &regs->sp is used (see ptrace.h).
	107	* Unfortunately, that means eflags must be at the same location
	108	* as the current return ip is. We move the return ip into the
ff04b440	109	* regs->ip location, and move flags into the return ip location.
3d82c59c	110	*/
ff04b440 SRV	111	pushl $__KERNEL_CS
ff04b440 SRV	112	pushl 4(%esp) /* Save the return ip */
3d82c59c SRV	113	pushl $0 /* Load 0 into orig_ax */
	114	pushl %gs
	115	pushl %fs
	116	pushl %es
	117	pushl %ds
	118	pushl %eax
ff04b440 SRV	119
	120	/* Get flags and place them into the return ip slot */
	121	pushf
	122	popl %eax
	123	movl %eax, 8*4(%esp)
	124
3d82c59c SRV	125	pushl %ebp
	126	pushl %edi
	127	pushl %esi
	128	pushl %edx
	129	pushl %ecx
	130	pushl %ebx
	131
3d82c59c SRV	132	movl 124(%esp), %eax / Load ip (1st parameter) */
3d82c59c SRV	133	subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
dc912c30 SRV	134	#ifdef CC_USING_FENTRY
	135	movl 154(%esp), %edx / Load parent ip (2nd parameter) */
	136	#else
3d82c59c	137	movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */
dc912c30	138	#endif
3d82c59c SRV	139	movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
	140	pushl %esp /* Save pt_regs as 4th parameter */
	141
	142	GLOBAL(ftrace_regs_call)
	143	call ftrace_stub
	144
	145	addl $4, %esp /* Skip pt_regs */
ff04b440 SRV	146
	147	/* restore flags */
	148	push 14*4(%esp)
	149	popf
	150
	151	/* Move return ip back to its original location */
	152	movl 12*4(%esp), %eax
	153	movl %eax, 14*4(%esp)
3d82c59c SRV	154
	155	popl %ebx
	156	popl %ecx
	157	popl %edx
	158	popl %esi
	159	popl %edi
	160	popl %ebp
	161	popl %eax
	162	popl %ds
	163	popl %es
	164	popl %fs
	165	popl %gs
3d82c59c	166
ff04b440 SRV	167	/* use lea to not affect flags */
	168	lea 34(%esp), %esp / Skip orig_ax, ip and cs */
	169
	170	jmp .Lftrace_ret
3d82c59c SRV	171	#else /* ! CONFIG_DYNAMIC_FTRACE */
3d82c59c SRV	172
644e0e8d	173	ENTRY(function_hook)
3d82c59c SRV	174	cmpl $__PAGE_OFFSET, %esp
	175	jb ftrace_stub /* Paging not enabled yet? */
	176
	177	cmpl $ftrace_stub, ftrace_trace_function
	178	jnz .Ltrace
	179	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	180	cmpl $ftrace_stub, ftrace_graph_return
	181	jnz ftrace_graph_caller
	182
	183	cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
	184	jnz ftrace_graph_caller
	185	#endif
	186	.globl ftrace_stub
	187	ftrace_stub:
	188	ret
	189
	190	/* taken from glibc */
	191	.Ltrace:
	192	pushl %eax
	193	pushl %ecx
	194	pushl %edx
	195	movl 0xc(%esp), %eax
	196	movl 0x4(%ebp), %edx
	197	subl $MCOUNT_INSN_SIZE, %eax
	198
	199	call *ftrace_trace_function
	200
	201	popl %edx
	202	popl %ecx
	203	popl %eax
	204	jmp ftrace_stub
644e0e8d	205	END(function_hook)
3d82c59c	206	#endif /* CONFIG_DYNAMIC_FTRACE */
3d82c59c SRV	207
	208	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	209	ENTRY(ftrace_graph_caller)
	210	pushl %eax
	211	pushl %ecx
	212	pushl %edx
644e0e8d SRV	213	movl 3*4(%esp), %eax
	214	/* Even with frame pointers, fentry doesn't have one here */
	215	#ifdef CC_USING_FENTRY
	216	lea 4*4(%esp), %edx
	217	movl $0, %ecx
	218	#else
3d82c59c SRV	219	lea 0x4(%ebp), %edx
3d82c59c SRV	220	movl (%ebp), %ecx
644e0e8d	221	#endif
3d82c59c SRV	222	subl $MCOUNT_INSN_SIZE, %eax
	223	call prepare_ftrace_return
	224	popl %edx
	225	popl %ecx
	226	popl %eax
	227	ret
	228	END(ftrace_graph_caller)
	229
	230	.globl return_to_handler
	231	return_to_handler:
	232	pushl %eax
	233	pushl %edx
644e0e8d SRV	234	#ifdef CC_USING_FENTRY
	235	movl $0, %eax
	236	#else
3d82c59c	237	movl %ebp, %eax
644e0e8d	238	#endif
3d82c59c SRV	239	call ftrace_return_to_handler
	240	movl %eax, %ecx
	241	popl %edx
	242	popl %eax
	243	jmp *%ecx
	244	#endif