Merge branches 'tracing/ftrace', 'tracing/hw-branch-tracing' and 'tracing/ring-buffer...

[mirror_ubuntu-jammy-kernel.git] / arch / x86 / kernel / entry_64.S

/*
 *  linux/arch/x86_64/entry.S
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
 */

/*
 * entry.S contains the system-call and fault low-level handling routines.
 *
 * NOTE: This code handles signal-recognition, which happens every time
 * after an interrupt and after each system call.
 * 
 * Normal syscalls and interrupts don't save a full stack frame, this is 
 * only done for syscall tracing, signals or fork/exec et.al.
 * 
 * A note on terminology:        
 * - top of stack: Architecture defined interrupt frame from SS to RIP 
 * at the top of the kernel process stack.      
 * - partial stack frame: partially saved registers upto R11.
 * - full stack frame: Like partial stack frame, but all register saved. 
 *
 * Some macro usage:
 * - CFI macros are used to generate dwarf2 unwind information for better
 * backtraces. They don't change any code.
 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
 * There are unfortunately lots of special cases where some registers
 * not touched. The macro is a big mess that should be cleaned up.
 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
 * Gives a full stack frame.
 * - ENTRY/END Define functions in the symbol table.
 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
 * frame that is otherwise undefined after a SYSCALL
 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
 */

#include <linux/linkage.h>
#include <asm/segment.h>
#include <asm/cache.h>
#include <asm/errno.h>
#include <asm/dwarf2.h>
#include <asm/calling.h>
#include <asm/asm-offsets.h>
#include <asm/msr.h>
#include <asm/unistd.h>
#include <asm/thread_info.h>
#include <asm/hw_irq.h>
#include <asm/page.h>
#include <asm/irqflags.h>
#include <asm/paravirt.h>
#include <asm/ftrace.h>

/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
#include <linux/elf-em.h>
#define AUDIT_ARCH_X86_64       (EM_X86_64|__AUDIT_ARCH_64BIT|__AUDIT_ARCH_LE)
#define __AUDIT_ARCH_64BIT 0x80000000
#define __AUDIT_ARCH_LE    0x40000000

        .code64

#ifdef CONFIG_FUNCTION_TRACER
#ifdef CONFIG_DYNAMIC_FTRACE
ENTRY(mcount)
        retq
END(mcount)

ENTRY(ftrace_caller)
        cmpl $0, function_trace_stop
        jne  ftrace_stub

        MCOUNT_SAVE_FRAME

        movq 0x38(%rsp), %rdi
        movq 8(%rbp), %rsi
        subq $MCOUNT_INSN_SIZE, %rdi

.globl ftrace_call
ftrace_call:
        call ftrace_stub

        MCOUNT_RESTORE_FRAME

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.globl ftrace_graph_call
ftrace_graph_call:
        jmp ftrace_stub
#endif

.globl ftrace_stub
ftrace_stub:
        retq
END(ftrace_caller)

#else /* ! CONFIG_DYNAMIC_FTRACE */
ENTRY(mcount)
        cmpl $0, function_trace_stop
        jne  ftrace_stub

        cmpq $ftrace_stub, ftrace_trace_function
        jnz trace

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
        cmpq $ftrace_stub, ftrace_graph_return
        jnz ftrace_graph_caller

        cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
        jnz ftrace_graph_caller
#endif

.globl ftrace_stub
ftrace_stub:
        retq

trace:
        MCOUNT_SAVE_FRAME

        movq 0x38(%rsp), %rdi
        movq 8(%rbp), %rsi
        subq $MCOUNT_INSN_SIZE, %rdi

        call   *ftrace_trace_function

        MCOUNT_RESTORE_FRAME

        jmp ftrace_stub
END(mcount)
#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_FUNCTION_TRACER */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
ENTRY(ftrace_graph_caller)
        cmpl $0, function_trace_stop
        jne ftrace_stub

        MCOUNT_SAVE_FRAME

        leaq 8(%rbp), %rdi
        movq 0x38(%rsp), %rsi
        subq $MCOUNT_INSN_SIZE, %rsi

        call    prepare_ftrace_return

        MCOUNT_RESTORE_FRAME

        retq
END(ftrace_graph_caller)


.globl return_to_handler
return_to_handler:
        subq  $80, %rsp

        movq %rax, (%rsp)
        movq %rcx, 8(%rsp)
        movq %rdx, 16(%rsp)
        movq %rsi, 24(%rsp)
        movq %rdi, 32(%rsp)
        movq %r8, 40(%rsp)
        movq %r9, 48(%rsp)
        movq %r10, 56(%rsp)
        movq %r11, 64(%rsp)

        call ftrace_return_to_handler

        movq %rax, 72(%rsp)
        movq 64(%rsp), %r11
        movq 56(%rsp), %r10
        movq 48(%rsp), %r9
        movq 40(%rsp), %r8
        movq 32(%rsp), %rdi
        movq 24(%rsp), %rsi
        movq 16(%rsp), %rdx
        movq 8(%rsp), %rcx
        movq (%rsp), %rax
        addq $72, %rsp
        retq
#endif


#ifndef CONFIG_PREEMPT
#define retint_kernel retint_restore_args
#endif  

#ifdef CONFIG_PARAVIRT
ENTRY(native_usergs_sysret64)
        swapgs
        sysretq
#endif /* CONFIG_PARAVIRT */


.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
#ifdef CONFIG_TRACE_IRQFLAGS
        bt   $9,EFLAGS-\offset(%rsp)    /* interrupts off? */
        jnc  1f
        TRACE_IRQS_ON
1:
#endif
.endm

/*
 * C code is not supposed to know about undefined top of stack. Every time 
 * a C function with an pt_regs argument is called from the SYSCALL based 
 * fast path FIXUP_TOP_OF_STACK is needed.
 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
 * manipulation.
 */             
                
        /* %rsp:at FRAMEEND */ 
        .macro FIXUP_TOP_OF_STACK tmp
        movq    %gs:pda_oldrsp,\tmp
        movq    \tmp,RSP(%rsp)
        movq    $__USER_DS,SS(%rsp)
        movq    $__USER_CS,CS(%rsp)
        movq    $-1,RCX(%rsp)
        movq    R11(%rsp),\tmp  /* get eflags */
        movq    \tmp,EFLAGS(%rsp)
        .endm

        .macro RESTORE_TOP_OF_STACK tmp,offset=0
        movq   RSP-\offset(%rsp),\tmp
        movq   \tmp,%gs:pda_oldrsp
        movq   EFLAGS-\offset(%rsp),\tmp
        movq   \tmp,R11-\offset(%rsp)
        .endm

        .macro FAKE_STACK_FRAME child_rip
        /* push in order ss, rsp, eflags, cs, rip */
        xorl %eax, %eax
        pushq $__KERNEL_DS /* ss */
        CFI_ADJUST_CFA_OFFSET   8
        /*CFI_REL_OFFSET        ss,0*/
        pushq %rax /* rsp */
        CFI_ADJUST_CFA_OFFSET   8
        CFI_REL_OFFSET  rsp,0
        pushq $(1<<9) /* eflags - interrupts on */
        CFI_ADJUST_CFA_OFFSET   8
        /*CFI_REL_OFFSET        rflags,0*/
        pushq $__KERNEL_CS /* cs */
        CFI_ADJUST_CFA_OFFSET   8
        /*CFI_REL_OFFSET        cs,0*/
        pushq \child_rip /* rip */
        CFI_ADJUST_CFA_OFFSET   8
        CFI_REL_OFFSET  rip,0
        pushq   %rax /* orig rax */
        CFI_ADJUST_CFA_OFFSET   8
        .endm

        .macro UNFAKE_STACK_FRAME
        addq $8*6, %rsp
        CFI_ADJUST_CFA_OFFSET   -(6*8)
        .endm

        .macro  CFI_DEFAULT_STACK start=1
        .if \start
        CFI_STARTPROC   simple
        CFI_SIGNAL_FRAME
        CFI_DEF_CFA     rsp,SS+8
        .else
        CFI_DEF_CFA_OFFSET SS+8
        .endif
        CFI_REL_OFFSET  r15,R15
        CFI_REL_OFFSET  r14,R14
        CFI_REL_OFFSET  r13,R13
        CFI_REL_OFFSET  r12,R12
        CFI_REL_OFFSET  rbp,RBP
        CFI_REL_OFFSET  rbx,RBX
        CFI_REL_OFFSET  r11,R11
        CFI_REL_OFFSET  r10,R10
        CFI_REL_OFFSET  r9,R9
        CFI_REL_OFFSET  r8,R8
        CFI_REL_OFFSET  rax,RAX
        CFI_REL_OFFSET  rcx,RCX
        CFI_REL_OFFSET  rdx,RDX
        CFI_REL_OFFSET  rsi,RSI
        CFI_REL_OFFSET  rdi,RDI
        CFI_REL_OFFSET  rip,RIP
        /*CFI_REL_OFFSET        cs,CS*/
        /*CFI_REL_OFFSET        rflags,EFLAGS*/
        CFI_REL_OFFSET  rsp,RSP
        /*CFI_REL_OFFSET        ss,SS*/
        .endm
/*
 * A newly forked process directly context switches into this.
 */     
/* rdi: prev */ 
ENTRY(ret_from_fork)
        CFI_DEFAULT_STACK
        push kernel_eflags(%rip)
        CFI_ADJUST_CFA_OFFSET 8
        popf                            # reset kernel eflags
        CFI_ADJUST_CFA_OFFSET -8
        call schedule_tail
        GET_THREAD_INFO(%rcx)
        testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
        jnz rff_trace
rff_action:     
        RESTORE_REST
        testl $3,CS-ARGOFFSET(%rsp)     # from kernel_thread?
        je   int_ret_from_sys_call
        testl $_TIF_IA32,TI_flags(%rcx)
        jnz  int_ret_from_sys_call
        RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
        jmp ret_from_sys_call
rff_trace:
        movq %rsp,%rdi
        call syscall_trace_leave
        GET_THREAD_INFO(%rcx)   
        jmp rff_action
        CFI_ENDPROC
END(ret_from_fork)

/*
 * System call entry. Upto 6 arguments in registers are supported.
 *
 * SYSCALL does not save anything on the stack and does not change the
 * stack pointer.
 */
                
/*
 * Register setup:      
 * rax  system call number
 * rdi  arg0
 * rcx  return address for syscall/sysret, C arg3 
 * rsi  arg1
 * rdx  arg2    
 * r10  arg3    (--> moved to rcx for C)
 * r8   arg4
 * r9   arg5
 * r11  eflags for syscall/sysret, temporary for C
 * r12-r15,rbp,rbx saved by C code, not touched.                
 * 
 * Interrupts are off on entry.
 * Only called from user space.
 *
 * XXX  if we had a free scratch register we could save the RSP into the stack frame
 *      and report it properly in ps. Unfortunately we haven't.
 *
 * When user can change the frames always force IRET. That is because
 * it deals with uncanonical addresses better. SYSRET has trouble
 * with them due to bugs in both AMD and Intel CPUs.
 */                                     

ENTRY(system_call)
        CFI_STARTPROC   simple
        CFI_SIGNAL_FRAME
        CFI_DEF_CFA     rsp,PDA_STACKOFFSET
        CFI_REGISTER    rip,rcx
        /*CFI_REGISTER  rflags,r11*/
        SWAPGS_UNSAFE_STACK
        /*
         * A hypervisor implementation might want to use a label
         * after the swapgs, so that it can do the swapgs
         * for the guest and jump here on syscall.
         */
ENTRY(system_call_after_swapgs)

        movq    %rsp,%gs:pda_oldrsp 
        movq    %gs:pda_kernelstack,%rsp
        /*
         * No need to follow this irqs off/on section - it's straight
         * and short:
         */
        ENABLE_INTERRUPTS(CLBR_NONE)
        SAVE_ARGS 8,1
        movq  %rax,ORIG_RAX-ARGOFFSET(%rsp) 
        movq  %rcx,RIP-ARGOFFSET(%rsp)
        CFI_REL_OFFSET rip,RIP-ARGOFFSET
        GET_THREAD_INFO(%rcx)
        testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
        jnz tracesys
system_call_fastpath:
        cmpq $__NR_syscall_max,%rax
        ja badsys
        movq %r10,%rcx
        call *sys_call_table(,%rax,8)  # XXX:    rip relative
        movq %rax,RAX-ARGOFFSET(%rsp)
/*
 * Syscall return path ending with SYSRET (fast path)
 * Has incomplete stack frame and undefined top of stack. 
 */             
ret_from_sys_call:
        movl $_TIF_ALLWORK_MASK,%edi
        /* edi: flagmask */
sysret_check:           
        LOCKDEP_SYS_EXIT
        GET_THREAD_INFO(%rcx)
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        movl TI_flags(%rcx),%edx
        andl %edi,%edx
        jnz  sysret_careful 
        CFI_REMEMBER_STATE
        /*
         * sysretq will re-enable interrupts:
         */
        TRACE_IRQS_ON
        movq RIP-ARGOFFSET(%rsp),%rcx
        CFI_REGISTER    rip,rcx
        RESTORE_ARGS 0,-ARG_SKIP,1
        /*CFI_REGISTER  rflags,r11*/
        movq    %gs:pda_oldrsp, %rsp
        USERGS_SYSRET64

        CFI_RESTORE_STATE
        /* Handle reschedules */
        /* edx: work, edi: workmask */  
sysret_careful:
        bt $TIF_NEED_RESCHED,%edx
        jnc sysret_signal
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
        pushq %rdi
        CFI_ADJUST_CFA_OFFSET 8
        call schedule
        popq  %rdi
        CFI_ADJUST_CFA_OFFSET -8
        jmp sysret_check

        /* Handle a signal */ 
sysret_signal:
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
#ifdef CONFIG_AUDITSYSCALL
        bt $TIF_SYSCALL_AUDIT,%edx
        jc sysret_audit
#endif
        /* edx: work flags (arg3) */
        leaq do_notify_resume(%rip),%rax
        leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
        xorl %esi,%esi # oldset -> arg2
        call ptregscall_common
        movl $_TIF_WORK_MASK,%edi
        /* Use IRET because user could have changed frame. This
           works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp int_with_check
        
badsys:
        movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
        jmp ret_from_sys_call

#ifdef CONFIG_AUDITSYSCALL
        /*
         * Fast path for syscall audit without full syscall trace.
         * We just call audit_syscall_entry() directly, and then
         * jump back to the normal fast path.
         */
auditsys:
        movq %r10,%r9                   /* 6th arg: 4th syscall arg */
        movq %rdx,%r8                   /* 5th arg: 3rd syscall arg */
        movq %rsi,%rcx                  /* 4th arg: 2nd syscall arg */
        movq %rdi,%rdx                  /* 3rd arg: 1st syscall arg */
        movq %rax,%rsi                  /* 2nd arg: syscall number */
        movl $AUDIT_ARCH_X86_64,%edi    /* 1st arg: audit arch */
        call audit_syscall_entry
        LOAD_ARGS 0             /* reload call-clobbered registers */
        jmp system_call_fastpath

        /*
         * Return fast path for syscall audit.  Call audit_syscall_exit()
         * directly and then jump back to the fast path with TIF_SYSCALL_AUDIT
         * masked off.
         */
sysret_audit:
        movq %rax,%rsi          /* second arg, syscall return value */
        cmpq $0,%rax            /* is it < 0? */
        setl %al                /* 1 if so, 0 if not */
        movzbl %al,%edi         /* zero-extend that into %edi */
        inc %edi /* first arg, 0->1(AUDITSC_SUCCESS), 1->2(AUDITSC_FAILURE) */
        call audit_syscall_exit
        movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
        jmp sysret_check
#endif  /* CONFIG_AUDITSYSCALL */

        /* Do syscall tracing */
tracesys:                        
#ifdef CONFIG_AUDITSYSCALL
        testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
        jz auditsys
#endif
        SAVE_REST
        movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
        FIXUP_TOP_OF_STACK %rdi
        movq %rsp,%rdi
        call syscall_trace_enter
        /*
         * Reload arg registers from stack in case ptrace changed them.
         * We don't reload %rax because syscall_trace_enter() returned
         * the value it wants us to use in the table lookup.
         */
        LOAD_ARGS ARGOFFSET, 1
        RESTORE_REST
        cmpq $__NR_syscall_max,%rax
        ja   int_ret_from_sys_call      /* RAX(%rsp) set to -ENOSYS above */
        movq %r10,%rcx  /* fixup for C */
        call *sys_call_table(,%rax,8)
        movq %rax,RAX-ARGOFFSET(%rsp)
        /* Use IRET because user could have changed frame */
                
/* 
 * Syscall return path ending with IRET.
 * Has correct top of stack, but partial stack frame.
 */
        .globl int_ret_from_sys_call
        .globl int_with_check
int_ret_from_sys_call:
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        testl $3,CS-ARGOFFSET(%rsp)
        je retint_restore_args
        movl $_TIF_ALLWORK_MASK,%edi
        /* edi: mask to check */
int_with_check:
        LOCKDEP_SYS_EXIT_IRQ
        GET_THREAD_INFO(%rcx)
        movl TI_flags(%rcx),%edx
        andl %edi,%edx
        jnz   int_careful
        andl    $~TS_COMPAT,TI_status(%rcx)
        jmp   retint_swapgs

        /* Either reschedule or signal or syscall exit tracking needed. */
        /* First do a reschedule test. */
        /* edx: work, edi: workmask */
int_careful:
        bt $TIF_NEED_RESCHED,%edx
        jnc  int_very_careful
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
        pushq %rdi
        CFI_ADJUST_CFA_OFFSET 8
        call schedule
        popq %rdi
        CFI_ADJUST_CFA_OFFSET -8
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp int_with_check

        /* handle signals and tracing -- both require a full stack frame */
int_very_careful:
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
        SAVE_REST
        /* Check for syscall exit trace */      
        testl $_TIF_WORK_SYSCALL_EXIT,%edx
        jz int_signal
        pushq %rdi
        CFI_ADJUST_CFA_OFFSET 8
        leaq 8(%rsp),%rdi       # &ptregs -> arg1       
        call syscall_trace_leave
        popq %rdi
        CFI_ADJUST_CFA_OFFSET -8
        andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
        jmp int_restore_rest
        
int_signal:
        testl $_TIF_DO_NOTIFY_MASK,%edx
        jz 1f
        movq %rsp,%rdi          # &ptregs -> arg1
        xorl %esi,%esi          # oldset -> arg2
        call do_notify_resume
1:      movl $_TIF_WORK_MASK,%edi
int_restore_rest:
        RESTORE_REST
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp int_with_check
        CFI_ENDPROC
END(system_call)
                
/* 
 * Certain special system calls that need to save a complete full stack frame.
 */                                                             
        
        .macro PTREGSCALL label,func,arg
        .globl \label
\label:
        leaq    \func(%rip),%rax
        leaq    -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
        jmp     ptregscall_common
END(\label)
        .endm

        CFI_STARTPROC

        PTREGSCALL stub_clone, sys_clone, %r8
        PTREGSCALL stub_fork, sys_fork, %rdi
        PTREGSCALL stub_vfork, sys_vfork, %rdi
        PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
        PTREGSCALL stub_iopl, sys_iopl, %rsi

ENTRY(ptregscall_common)
        popq %r11
        CFI_ADJUST_CFA_OFFSET -8
        CFI_REGISTER rip, r11
        SAVE_REST
        movq %r11, %r15
        CFI_REGISTER rip, r15
        FIXUP_TOP_OF_STACK %r11
        call *%rax
        RESTORE_TOP_OF_STACK %r11
        movq %r15, %r11
        CFI_REGISTER rip, r11
        RESTORE_REST
        pushq %r11
        CFI_ADJUST_CFA_OFFSET 8
        CFI_REL_OFFSET rip, 0
        ret
        CFI_ENDPROC
END(ptregscall_common)
        
ENTRY(stub_execve)
        CFI_STARTPROC
        popq %r11
        CFI_ADJUST_CFA_OFFSET -8
        CFI_REGISTER rip, r11
        SAVE_REST
        FIXUP_TOP_OF_STACK %r11
        movq %rsp, %rcx
        call sys_execve
        RESTORE_TOP_OF_STACK %r11
        movq %rax,RAX(%rsp)
        RESTORE_REST
        jmp int_ret_from_sys_call
        CFI_ENDPROC
END(stub_execve)
        
/*
 * sigreturn is special because it needs to restore all registers on return.
 * This cannot be done with SYSRET, so use the IRET return path instead.
 */                
ENTRY(stub_rt_sigreturn)
        CFI_STARTPROC
        addq $8, %rsp
        CFI_ADJUST_CFA_OFFSET   -8
        SAVE_REST
        movq %rsp,%rdi
        FIXUP_TOP_OF_STACK %r11
        call sys_rt_sigreturn
        movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
        RESTORE_REST
        jmp int_ret_from_sys_call
        CFI_ENDPROC
END(stub_rt_sigreturn)

/*
 * initial frame state for interrupts and exceptions
 */
        .macro _frame ref
        CFI_STARTPROC simple
        CFI_SIGNAL_FRAME
        CFI_DEF_CFA rsp,SS+8-\ref
        /*CFI_REL_OFFSET ss,SS-\ref*/
        CFI_REL_OFFSET rsp,RSP-\ref
        /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
        /*CFI_REL_OFFSET cs,CS-\ref*/
        CFI_REL_OFFSET rip,RIP-\ref
        .endm

/* initial frame state for interrupts (and exceptions without error code) */
#define INTR_FRAME _frame RIP
/* initial frame state for exceptions with error code (and interrupts with
   vector already pushed) */
#define XCPT_FRAME _frame ORIG_RAX

/* 
 * Interrupt entry/exit.
 *
 * Interrupt entry points save only callee clobbered registers in fast path.
 *      
 * Entry runs with interrupts off.      
 */ 

/* 0(%rsp): interrupt number */ 
        .macro interrupt func
        cld
        SAVE_ARGS
        leaq -ARGOFFSET(%rsp),%rdi      # arg1 for handler
        pushq %rbp
        /*
         * Save rbp twice: One is for marking the stack frame, as usual, and the
         * other, to fill pt_regs properly. This is because bx comes right
         * before the last saved register in that structure, and not bp. If the
         * base pointer were in the place bx is today, this would not be needed.
         */
        movq %rbp, -8(%rsp)
        CFI_ADJUST_CFA_OFFSET   8
        CFI_REL_OFFSET          rbp, 0
        movq %rsp,%rbp
        CFI_DEF_CFA_REGISTER    rbp
        testl $3,CS(%rdi)
        je 1f
        SWAPGS
        /* irqcount is used to check if a CPU is already on an interrupt
           stack or not. While this is essentially redundant with preempt_count
           it is a little cheaper to use a separate counter in the PDA
           (short of moving irq_enter into assembly, which would be too
            much work) */
1:      incl    %gs:pda_irqcount
        cmoveq %gs:pda_irqstackptr,%rsp
        push    %rbp                    # backlink for old unwinder
        /*
         * We entered an interrupt context - irqs are off:
         */
        TRACE_IRQS_OFF
        call \func
        .endm

ENTRY(common_interrupt)
        XCPT_FRAME
        interrupt do_IRQ
        /* 0(%rsp): oldrsp-ARGOFFSET */
ret_from_intr:
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        decl %gs:pda_irqcount
        leaveq
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
exit_intr:
        GET_THREAD_INFO(%rcx)
        testl $3,CS-ARGOFFSET(%rsp)
        je retint_kernel
        
        /* Interrupt came from user space */
        /*
         * Has a correct top of stack, but a partial stack frame
         * %rcx: thread info. Interrupts off.
         */             
retint_with_reschedule:
        movl $_TIF_WORK_MASK,%edi
retint_check:
        LOCKDEP_SYS_EXIT_IRQ
        movl TI_flags(%rcx),%edx
        andl %edi,%edx
        CFI_REMEMBER_STATE
        jnz  retint_careful

retint_swapgs:          /* return to user-space */
        /*
         * The iretq could re-enable interrupts:
         */
        DISABLE_INTERRUPTS(CLBR_ANY)
        TRACE_IRQS_IRETQ
        SWAPGS
        jmp restore_args

retint_restore_args:    /* return to kernel space */
        DISABLE_INTERRUPTS(CLBR_ANY)
        /*
         * The iretq could re-enable interrupts:
         */
        TRACE_IRQS_IRETQ
restore_args:
        RESTORE_ARGS 0,8,0

irq_return:
        INTERRUPT_RETURN

        .section __ex_table, "a"
        .quad irq_return, bad_iret
        .previous

#ifdef CONFIG_PARAVIRT
ENTRY(native_iret)
        iretq

        .section __ex_table,"a"
        .quad native_iret, bad_iret
        .previous
#endif

        .section .fixup,"ax"
bad_iret:
        /*
         * The iret traps when the %cs or %ss being restored is bogus.
         * We've lost the original trap vector and error code.
         * #GPF is the most likely one to get for an invalid selector.
         * So pretend we completed the iret and took the #GPF in user mode.
         *
         * We are now running with the kernel GS after exception recovery.
         * But error_entry expects us to have user GS to match the user %cs,
         * so swap back.
         */
        pushq $0

        SWAPGS
        jmp general_protection

        .previous

        /* edi: workmask, edx: work */
retint_careful:
        CFI_RESTORE_STATE
        bt    $TIF_NEED_RESCHED,%edx
        jnc   retint_signal
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
        pushq %rdi
        CFI_ADJUST_CFA_OFFSET   8
        call  schedule
        popq %rdi               
        CFI_ADJUST_CFA_OFFSET   -8
        GET_THREAD_INFO(%rcx)
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        jmp retint_check
        
retint_signal:
        testl $_TIF_DO_NOTIFY_MASK,%edx
        jz    retint_swapgs
        TRACE_IRQS_ON
        ENABLE_INTERRUPTS(CLBR_NONE)
        SAVE_REST
        movq $-1,ORIG_RAX(%rsp)                         
        xorl %esi,%esi          # oldset
        movq %rsp,%rdi          # &pt_regs
        call do_notify_resume
        RESTORE_REST
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        GET_THREAD_INFO(%rcx)
        jmp retint_with_reschedule

#ifdef CONFIG_PREEMPT
        /* Returning to kernel space. Check if we need preemption */
        /* rcx:  threadinfo. interrupts off. */
ENTRY(retint_kernel)
        cmpl $0,TI_preempt_count(%rcx)
        jnz  retint_restore_args
        bt  $TIF_NEED_RESCHED,TI_flags(%rcx)
        jnc  retint_restore_args
        bt   $9,EFLAGS-ARGOFFSET(%rsp)  /* interrupts off? */
        jnc  retint_restore_args
        call preempt_schedule_irq
        jmp exit_intr
#endif  

        CFI_ENDPROC
END(common_interrupt)
        
/*
 * APIC interrupts.
 */             
        .macro apicinterrupt num,func
        INTR_FRAME
        pushq $~(\num)
        CFI_ADJUST_CFA_OFFSET 8
        interrupt \func
        jmp ret_from_intr
        CFI_ENDPROC
        .endm

ENTRY(thermal_interrupt)
        apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
END(thermal_interrupt)

ENTRY(threshold_interrupt)
        apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
END(threshold_interrupt)

#ifdef CONFIG_SMP       
ENTRY(reschedule_interrupt)
        apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
END(reschedule_interrupt)

        .macro INVALIDATE_ENTRY num
ENTRY(invalidate_interrupt\num)
        apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt 
END(invalidate_interrupt\num)
        .endm

        INVALIDATE_ENTRY 0
        INVALIDATE_ENTRY 1
        INVALIDATE_ENTRY 2
        INVALIDATE_ENTRY 3
        INVALIDATE_ENTRY 4
        INVALIDATE_ENTRY 5
        INVALIDATE_ENTRY 6
        INVALIDATE_ENTRY 7

ENTRY(call_function_interrupt)
        apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
END(call_function_interrupt)
ENTRY(call_function_single_interrupt)
        apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt
END(call_function_single_interrupt)
ENTRY(irq_move_cleanup_interrupt)
        apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
END(irq_move_cleanup_interrupt)
#endif

ENTRY(apic_timer_interrupt)
        apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
END(apic_timer_interrupt)

ENTRY(uv_bau_message_intr1)
        apicinterrupt 220,uv_bau_message_interrupt
END(uv_bau_message_intr1)

ENTRY(error_interrupt)
        apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
END(error_interrupt)

ENTRY(spurious_interrupt)
        apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
END(spurious_interrupt)
                                
/*
 * Exception entry points.
 */             
        .macro zeroentry sym
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        pushq $0        /* push error code/oldrax */ 
        CFI_ADJUST_CFA_OFFSET 8
        pushq %rax      /* push real oldrax to the rdi slot */ 
        CFI_ADJUST_CFA_OFFSET 8
        CFI_REL_OFFSET rax,0
        leaq  \sym(%rip),%rax
        jmp error_entry
        CFI_ENDPROC
        .endm   

        .macro errorentry sym
        XCPT_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        pushq %rax
        CFI_ADJUST_CFA_OFFSET 8
        CFI_REL_OFFSET rax,0
        leaq  \sym(%rip),%rax
        jmp error_entry
        CFI_ENDPROC
        .endm

        /* error code is on the stack already */
        /* handle NMI like exceptions that can happen everywhere */
        .macro paranoidentry sym, ist=0, irqtrace=1
        SAVE_ALL
        cld
        movl $1,%ebx
        movl  $MSR_GS_BASE,%ecx
        rdmsr
        testl %edx,%edx
        js    1f
        SWAPGS
        xorl  %ebx,%ebx
1:
        .if \ist
        movq    %gs:pda_data_offset, %rbp
        .endif
        .if \irqtrace
        TRACE_IRQS_OFF
        .endif
        movq %rsp,%rdi
        movq ORIG_RAX(%rsp),%rsi
        movq $-1,ORIG_RAX(%rsp)
        .if \ist
        subq    $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
        .endif
        call \sym
        .if \ist
        addq    $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
        .endif
        DISABLE_INTERRUPTS(CLBR_NONE)
        .if \irqtrace
        TRACE_IRQS_OFF
        .endif
        .endm

        /*
         * "Paranoid" exit path from exception stack.
         * Paranoid because this is used by NMIs and cannot take
         * any kernel state for granted.
         * We don't do kernel preemption checks here, because only
         * NMI should be common and it does not enable IRQs and
         * cannot get reschedule ticks.
         *
         * "trace" is 0 for the NMI handler only, because irq-tracing
         * is fundamentally NMI-unsafe. (we cannot change the soft and
         * hard flags at once, atomically)
         */
        .macro paranoidexit trace=1
        /* ebx: no swapgs flag */
paranoid_exit\trace:
        testl %ebx,%ebx                         /* swapgs needed? */
        jnz paranoid_restore\trace
        testl $3,CS(%rsp)
        jnz   paranoid_userspace\trace
paranoid_swapgs\trace:
        .if \trace
        TRACE_IRQS_IRETQ 0
        .endif
        SWAPGS_UNSAFE_STACK
paranoid_restore\trace:
        RESTORE_ALL 8
        jmp irq_return
paranoid_userspace\trace:
        GET_THREAD_INFO(%rcx)
        movl TI_flags(%rcx),%ebx
        andl $_TIF_WORK_MASK,%ebx
        jz paranoid_swapgs\trace
        movq %rsp,%rdi                  /* &pt_regs */
        call sync_regs
        movq %rax,%rsp                  /* switch stack for scheduling */
        testl $_TIF_NEED_RESCHED,%ebx
        jnz paranoid_schedule\trace
        movl %ebx,%edx                  /* arg3: thread flags */
        .if \trace
        TRACE_IRQS_ON
        .endif
        ENABLE_INTERRUPTS(CLBR_NONE)
        xorl %esi,%esi                  /* arg2: oldset */
        movq %rsp,%rdi                  /* arg1: &pt_regs */
        call do_notify_resume
        DISABLE_INTERRUPTS(CLBR_NONE)
        .if \trace
        TRACE_IRQS_OFF
        .endif
        jmp paranoid_userspace\trace
paranoid_schedule\trace:
        .if \trace
        TRACE_IRQS_ON
        .endif
        ENABLE_INTERRUPTS(CLBR_ANY)
        call schedule
        DISABLE_INTERRUPTS(CLBR_ANY)
        .if \trace
        TRACE_IRQS_OFF
        .endif
        jmp paranoid_userspace\trace
        CFI_ENDPROC
        .endm

/*
 * Exception entry point. This expects an error code/orig_rax on the stack
 * and the exception handler in %rax.   
 */                                             
KPROBE_ENTRY(error_entry)
        _frame RDI
        CFI_REL_OFFSET rax,0
        /* rdi slot contains rax, oldrax contains error code */
        cld     
        subq  $14*8,%rsp
        CFI_ADJUST_CFA_OFFSET   (14*8)
        movq %rsi,13*8(%rsp)
        CFI_REL_OFFSET  rsi,RSI
        movq 14*8(%rsp),%rsi    /* load rax from rdi slot */
        CFI_REGISTER    rax,rsi
        movq %rdx,12*8(%rsp)
        CFI_REL_OFFSET  rdx,RDX
        movq %rcx,11*8(%rsp)
        CFI_REL_OFFSET  rcx,RCX
        movq %rsi,10*8(%rsp)    /* store rax */ 
        CFI_REL_OFFSET  rax,RAX
        movq %r8, 9*8(%rsp)
        CFI_REL_OFFSET  r8,R8
        movq %r9, 8*8(%rsp)
        CFI_REL_OFFSET  r9,R9
        movq %r10,7*8(%rsp)
        CFI_REL_OFFSET  r10,R10
        movq %r11,6*8(%rsp)
        CFI_REL_OFFSET  r11,R11
        movq %rbx,5*8(%rsp) 
        CFI_REL_OFFSET  rbx,RBX
        movq %rbp,4*8(%rsp) 
        CFI_REL_OFFSET  rbp,RBP
        movq %r12,3*8(%rsp) 
        CFI_REL_OFFSET  r12,R12
        movq %r13,2*8(%rsp) 
        CFI_REL_OFFSET  r13,R13
        movq %r14,1*8(%rsp) 
        CFI_REL_OFFSET  r14,R14
        movq %r15,(%rsp) 
        CFI_REL_OFFSET  r15,R15
        xorl %ebx,%ebx  
        testl $3,CS(%rsp)
        je  error_kernelspace
error_swapgs:   
        SWAPGS
error_sti:
        TRACE_IRQS_OFF
        movq %rdi,RDI(%rsp)     
        CFI_REL_OFFSET  rdi,RDI
        movq %rsp,%rdi
        movq ORIG_RAX(%rsp),%rsi        /* get error code */ 
        movq $-1,ORIG_RAX(%rsp)
        call *%rax
        /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
error_exit:
        movl %ebx,%eax
        RESTORE_REST
        DISABLE_INTERRUPTS(CLBR_NONE)
        TRACE_IRQS_OFF
        GET_THREAD_INFO(%rcx)   
        testl %eax,%eax
        jne  retint_kernel
        LOCKDEP_SYS_EXIT_IRQ
        movl  TI_flags(%rcx),%edx
        movl  $_TIF_WORK_MASK,%edi
        andl  %edi,%edx
        jnz  retint_careful
        jmp retint_swapgs
        CFI_ENDPROC

error_kernelspace:
        incl %ebx
       /* There are two places in the kernel that can potentially fault with
          usergs. Handle them here. The exception handlers after
           iret run with kernel gs again, so don't set the user space flag.
           B stepping K8s sometimes report an truncated RIP for IRET 
           exceptions returning to compat mode. Check for these here too. */
        leaq irq_return(%rip),%rcx
        cmpq %rcx,RIP(%rsp)
        je   error_swapgs
        movl %ecx,%ecx  /* zero extend */
        cmpq %rcx,RIP(%rsp)
        je   error_swapgs
        cmpq $gs_change,RIP(%rsp)
        je   error_swapgs
        jmp  error_sti
KPROBE_END(error_entry)
        
       /* Reload gs selector with exception handling */
       /* edi:  new selector */ 
ENTRY(native_load_gs_index)
        CFI_STARTPROC
        pushf
        CFI_ADJUST_CFA_OFFSET 8
        DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
        SWAPGS
gs_change:     
        movl %edi,%gs   
2:      mfence          /* workaround */
        SWAPGS
        popf
        CFI_ADJUST_CFA_OFFSET -8
        ret
        CFI_ENDPROC
ENDPROC(native_load_gs_index)
       
        .section __ex_table,"a"
        .align 8
        .quad gs_change,bad_gs
        .previous
        .section .fixup,"ax"
        /* running with kernelgs */
bad_gs: 
        SWAPGS                  /* switch back to user gs */
        xorl %eax,%eax
        movl %eax,%gs
        jmp  2b
        .previous       
        
/*
 * Create a kernel thread.
 *
 * C extern interface:
 *      extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
 *
 * asm input arguments:
 *      rdi: fn, rsi: arg, rdx: flags
 */
ENTRY(kernel_thread)
        CFI_STARTPROC
        FAKE_STACK_FRAME $child_rip
        SAVE_ALL

        # rdi: flags, rsi: usp, rdx: will be &pt_regs
        movq %rdx,%rdi
        orq  kernel_thread_flags(%rip),%rdi
        movq $-1, %rsi
        movq %rsp, %rdx

        xorl %r8d,%r8d
        xorl %r9d,%r9d
        
        # clone now
        call do_fork
        movq %rax,RAX(%rsp)
        xorl %edi,%edi

        /*
         * It isn't worth to check for reschedule here,
         * so internally to the x86_64 port you can rely on kernel_thread()
         * not to reschedule the child before returning, this avoids the need
         * of hacks for example to fork off the per-CPU idle tasks.
         * [Hopefully no generic code relies on the reschedule -AK]     
         */
        RESTORE_ALL
        UNFAKE_STACK_FRAME
        ret
        CFI_ENDPROC
ENDPROC(kernel_thread)
        
child_rip:
        pushq $0                # fake return address
        CFI_STARTPROC
        /*
         * Here we are in the child and the registers are set as they were
         * at kernel_thread() invocation in the parent.
         */
        movq %rdi, %rax
        movq %rsi, %rdi
        call *%rax
        # exit
        mov %eax, %edi
        call do_exit
        CFI_ENDPROC
ENDPROC(child_rip)

/*
 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
 *
 * C extern interface:
 *       extern long execve(char *name, char **argv, char **envp)
 *
 * asm input arguments:
 *      rdi: name, rsi: argv, rdx: envp
 *
 * We want to fallback into:
 *      extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
 *
 * do_sys_execve asm fallback arguments:
 *      rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
 */
ENTRY(kernel_execve)
        CFI_STARTPROC
        FAKE_STACK_FRAME $0
        SAVE_ALL        
        movq %rsp,%rcx
        call sys_execve
        movq %rax, RAX(%rsp)    
        RESTORE_REST
        testq %rax,%rax
        je int_ret_from_sys_call
        RESTORE_ARGS
        UNFAKE_STACK_FRAME
        ret
        CFI_ENDPROC
ENDPROC(kernel_execve)

KPROBE_ENTRY(page_fault)
        errorentry do_page_fault
KPROBE_END(page_fault)

ENTRY(coprocessor_error)
        zeroentry do_coprocessor_error
END(coprocessor_error)

ENTRY(simd_coprocessor_error)
        zeroentry do_simd_coprocessor_error     
END(simd_coprocessor_error)

ENTRY(device_not_available)
        zeroentry do_device_not_available
END(device_not_available)

        /* runs on exception stack */
KPROBE_ENTRY(debug)
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        pushq $0
        CFI_ADJUST_CFA_OFFSET 8         
        paranoidentry do_debug, DEBUG_STACK
        paranoidexit
KPROBE_END(debug)

        /* runs on exception stack */   
KPROBE_ENTRY(nmi)
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        pushq $-1
        CFI_ADJUST_CFA_OFFSET 8
        paranoidentry do_nmi, 0, 0
#ifdef CONFIG_TRACE_IRQFLAGS
        paranoidexit 0
#else
        jmp paranoid_exit1
        CFI_ENDPROC
#endif
KPROBE_END(nmi)

KPROBE_ENTRY(int3)
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        pushq $0
        CFI_ADJUST_CFA_OFFSET 8
        paranoidentry do_int3, DEBUG_STACK
        jmp paranoid_exit1
        CFI_ENDPROC
KPROBE_END(int3)

ENTRY(overflow)
        zeroentry do_overflow
END(overflow)

ENTRY(bounds)
        zeroentry do_bounds
END(bounds)

ENTRY(invalid_op)
        zeroentry do_invalid_op 
END(invalid_op)

ENTRY(coprocessor_segment_overrun)
        zeroentry do_coprocessor_segment_overrun
END(coprocessor_segment_overrun)

        /* runs on exception stack */
ENTRY(double_fault)
        XCPT_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        paranoidentry do_double_fault
        jmp paranoid_exit1
        CFI_ENDPROC
END(double_fault)

ENTRY(invalid_TSS)
        errorentry do_invalid_TSS
END(invalid_TSS)

ENTRY(segment_not_present)
        errorentry do_segment_not_present
END(segment_not_present)

        /* runs on exception stack */
ENTRY(stack_segment)
        XCPT_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        paranoidentry do_stack_segment
        jmp paranoid_exit1
        CFI_ENDPROC
END(stack_segment)

KPROBE_ENTRY(general_protection)
        errorentry do_general_protection
KPROBE_END(general_protection)

ENTRY(alignment_check)
        errorentry do_alignment_check
END(alignment_check)

ENTRY(divide_error)
        zeroentry do_divide_error
END(divide_error)

ENTRY(spurious_interrupt_bug)
        zeroentry do_spurious_interrupt_bug
END(spurious_interrupt_bug)

#ifdef CONFIG_X86_MCE
        /* runs on exception stack */
ENTRY(machine_check)
        INTR_FRAME
        PARAVIRT_ADJUST_EXCEPTION_FRAME
        pushq $0
        CFI_ADJUST_CFA_OFFSET 8 
        paranoidentry do_machine_check
        jmp paranoid_exit1
        CFI_ENDPROC
END(machine_check)
#endif

/* Call softirq on interrupt stack. Interrupts are off. */
ENTRY(call_softirq)
        CFI_STARTPROC
        push %rbp
        CFI_ADJUST_CFA_OFFSET   8
        CFI_REL_OFFSET rbp,0
        mov  %rsp,%rbp
        CFI_DEF_CFA_REGISTER rbp
        incl %gs:pda_irqcount
        cmove %gs:pda_irqstackptr,%rsp
        push  %rbp                      # backlink for old unwinder
        call __do_softirq
        leaveq
        CFI_DEF_CFA_REGISTER    rsp
        CFI_ADJUST_CFA_OFFSET   -8
        decl %gs:pda_irqcount
        ret
        CFI_ENDPROC
ENDPROC(call_softirq)

KPROBE_ENTRY(ignore_sysret)
        CFI_STARTPROC
        mov $-ENOSYS,%eax
        sysret
        CFI_ENDPROC
ENDPROC(ignore_sysret)

#ifdef CONFIG_XEN
ENTRY(xen_hypervisor_callback)
        zeroentry xen_do_hypervisor_callback
END(xen_hypervisor_callback)

/*
# A note on the "critical region" in our callback handler.
# We want to avoid stacking callback handlers due to events occurring
# during handling of the last event. To do this, we keep events disabled
# until we've done all processing. HOWEVER, we must enable events before
# popping the stack frame (can't be done atomically) and so it would still
# be possible to get enough handler activations to overflow the stack.
# Although unlikely, bugs of that kind are hard to track down, so we'd
# like to avoid the possibility.
# So, on entry to the handler we detect whether we interrupted an
# existing activation in its critical region -- if so, we pop the current
# activation and restart the handler using the previous one.
*/
ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
        CFI_STARTPROC
/* Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
   see the correct pointer to the pt_regs */
        movq %rdi, %rsp            # we don't return, adjust the stack frame
        CFI_ENDPROC
        CFI_DEFAULT_STACK
11:     incl %gs:pda_irqcount
        movq %rsp,%rbp
        CFI_DEF_CFA_REGISTER rbp
        cmovzq %gs:pda_irqstackptr,%rsp
        pushq %rbp                      # backlink for old unwinder
        call xen_evtchn_do_upcall
        popq %rsp
        CFI_DEF_CFA_REGISTER rsp
        decl %gs:pda_irqcount
        jmp  error_exit
        CFI_ENDPROC
END(do_hypervisor_callback)

/*
# Hypervisor uses this for application faults while it executes.
# We get here for two reasons:
#  1. Fault while reloading DS, ES, FS or GS
#  2. Fault while executing IRET
# Category 1 we do not need to fix up as Xen has already reloaded all segment
# registers that could be reloaded and zeroed the others.
# Category 2 we fix up by killing the current process. We cannot use the
# normal Linux return path in this case because if we use the IRET hypercall
# to pop the stack frame we end up in an infinite loop of failsafe callbacks.
# We distinguish between categories by comparing each saved segment register
# with its current contents: any discrepancy means we in category 1.
*/
ENTRY(xen_failsafe_callback)
        framesz = (RIP-0x30)    /* workaround buggy gas */
        _frame framesz
        CFI_REL_OFFSET rcx, 0
        CFI_REL_OFFSET r11, 8
        movw %ds,%cx
        cmpw %cx,0x10(%rsp)
        CFI_REMEMBER_STATE
        jne 1f
        movw %es,%cx
        cmpw %cx,0x18(%rsp)
        jne 1f
        movw %fs,%cx
        cmpw %cx,0x20(%rsp)
        jne 1f
        movw %gs,%cx
        cmpw %cx,0x28(%rsp)
        jne 1f
        /* All segments match their saved values => Category 2 (Bad IRET). */
        movq (%rsp),%rcx
        CFI_RESTORE rcx
        movq 8(%rsp),%r11
        CFI_RESTORE r11
        addq $0x30,%rsp
        CFI_ADJUST_CFA_OFFSET -0x30
        pushq $0
        CFI_ADJUST_CFA_OFFSET 8
        pushq %r11
        CFI_ADJUST_CFA_OFFSET 8
        pushq %rcx
        CFI_ADJUST_CFA_OFFSET 8
        jmp general_protection
        CFI_RESTORE_STATE
1:      /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
        movq (%rsp),%rcx
        CFI_RESTORE rcx
        movq 8(%rsp),%r11
        CFI_RESTORE r11
        addq $0x30,%rsp
        CFI_ADJUST_CFA_OFFSET -0x30
        pushq $0
        CFI_ADJUST_CFA_OFFSET 8
        SAVE_ALL
        jmp error_exit
        CFI_ENDPROC
END(xen_failsafe_callback)

#endif /* CONFIG_XEN */