arch/x86/kernel/entry_32.S

   1 /*
   2  *
   3  *  Copyright (C) 1991, 1992  Linus Torvalds
   4  */
   5
   6 /*
   7  * entry.S contains the system-call and fault low-level handling routines.
   8  * This also contains the timer-interrupt handler, as well as all interrupts
   9  * and faults that can result in a task-switch.
  10  *
  11  * NOTE: This code handles signal-recognition, which happens every time
  12  * after a timer-interrupt and after each system call.
  13  *
  14  * I changed all the .align's to 4 (16 byte alignment), as that's faster
  15  * on a 486.
  16  *
  17  * Stack layout in 'syscall_exit':
  18  *      ptrace needs to have all regs on the stack.
  19  *      if the order here is changed, it needs to be
  20  *      updated in fork.c:copy_process, signal.c:do_signal,
  21  *      ptrace.c and ptrace.h
  22  *
  23  *       0(%esp) - %ebx
  24  *       4(%esp) - %ecx
  25  *       8(%esp) - %edx
  26  *       C(%esp) - %esi
  27  *      10(%esp) - %edi
  28  *      14(%esp) - %ebp
  29  *      18(%esp) - %eax
  30  *      1C(%esp) - %ds
  31  *      20(%esp) - %es
  32  *      24(%esp) - %fs
  33  *      28(%esp) - orig_eax
  34  *      2C(%esp) - %eip
  35  *      30(%esp) - %cs
  36  *      34(%esp) - %eflags
  37  *      38(%esp) - %oldesp
  38  *      3C(%esp) - %oldss
  39  *
  40  * "current" is in register %ebx during any slow entries.
  41  */
  42
  43 #include <linux/linkage.h>
  44 #include <asm/thread_info.h>
  45 #include <asm/irqflags.h>
  46 #include <asm/errno.h>
  47 #include <asm/segment.h>
  48 #include <asm/smp.h>
  49 #include <asm/page.h>
  50 #include <asm/desc.h>
  51 #include <asm/percpu.h>
  52 #include <asm/dwarf2.h>
  53 #include <asm/processor-flags.h>
  54 #include <asm/ftrace.h>
  55 #include <asm/irq_vectors.h>
  56
  57 /*
  58  * We use macros for low-level operations which need to be overridden
  59  * for paravirtualization.  The following will never clobber any registers:
  60  *   INTERRUPT_RETURN (aka. "iret")
  61  *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
  62  *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
  63  *
  64  * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
  65  * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
  66  * Allowing a register to be clobbered can shrink the paravirt replacement
  67  * enough to patch inline, increasing performance.
  68  */
  69
  70 #define nr_syscalls ((syscall_table_size)/4)
  71
  72 #ifdef CONFIG_PREEMPT
  73 #define preempt_stop(clobbers)  DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
  74 #else
  75 #define preempt_stop(clobbers)
  76 #define resume_kernel           restore_nocheck
  77 #endif
  78
  79 .macro TRACE_IRQS_IRET
  80 #ifdef CONFIG_TRACE_IRQFLAGS
  81         testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)     # interrupts off?
  82         jz 1f
  83         TRACE_IRQS_ON
  84 1:
  85 #endif
  86 .endm
  87
  88 #ifdef CONFIG_VM86
  89 #define resume_userspace_sig    check_userspace
  90 #else
  91 #define resume_userspace_sig    resume_userspace
  92 #endif
  93
  94 #define SAVE_ALL \
  95         cld; \
  96         pushl %fs; \
  97         CFI_ADJUST_CFA_OFFSET 4;\
  98         /*CFI_REL_OFFSET fs, 0;*/\
  99         pushl %es; \
 100         CFI_ADJUST_CFA_OFFSET 4;\
 101         /*CFI_REL_OFFSET es, 0;*/\
 102         pushl %ds; \
 103         CFI_ADJUST_CFA_OFFSET 4;\
 104         /*CFI_REL_OFFSET ds, 0;*/\
 105         pushl %eax; \
 106         CFI_ADJUST_CFA_OFFSET 4;\
 107         CFI_REL_OFFSET eax, 0;\
 108         pushl %ebp; \
 109         CFI_ADJUST_CFA_OFFSET 4;\
 110         CFI_REL_OFFSET ebp, 0;\
 111         pushl %edi; \
 112         CFI_ADJUST_CFA_OFFSET 4;\
 113         CFI_REL_OFFSET edi, 0;\
 114         pushl %esi; \
 115         CFI_ADJUST_CFA_OFFSET 4;\
 116         CFI_REL_OFFSET esi, 0;\
 117         pushl %edx; \
 118         CFI_ADJUST_CFA_OFFSET 4;\
 119         CFI_REL_OFFSET edx, 0;\
 120         pushl %ecx; \
 121         CFI_ADJUST_CFA_OFFSET 4;\
 122         CFI_REL_OFFSET ecx, 0;\
 123         pushl %ebx; \
 124         CFI_ADJUST_CFA_OFFSET 4;\
 125         CFI_REL_OFFSET ebx, 0;\
 126         movl $(__USER_DS), %edx; \
 127         movl %edx, %ds; \
 128         movl %edx, %es; \
 129         movl $(__KERNEL_PERCPU), %edx; \
 130         movl %edx, %fs
 131
 132 #define RESTORE_INT_REGS \
 133         popl %ebx;      \
 134         CFI_ADJUST_CFA_OFFSET -4;\
 135         CFI_RESTORE ebx;\
 136         popl %ecx;      \
 137         CFI_ADJUST_CFA_OFFSET -4;\
 138         CFI_RESTORE ecx;\
 139         popl %edx;      \
 140         CFI_ADJUST_CFA_OFFSET -4;\
 141         CFI_RESTORE edx;\
 142         popl %esi;      \
 143         CFI_ADJUST_CFA_OFFSET -4;\
 144         CFI_RESTORE esi;\
 145         popl %edi;      \
 146         CFI_ADJUST_CFA_OFFSET -4;\
 147         CFI_RESTORE edi;\
 148         popl %ebp;      \
 149         CFI_ADJUST_CFA_OFFSET -4;\
 150         CFI_RESTORE ebp;\
 151         popl %eax;      \
 152         CFI_ADJUST_CFA_OFFSET -4;\
 153         CFI_RESTORE eax
 154
 155 #define RESTORE_REGS    \
 156         RESTORE_INT_REGS; \
 157 1:      popl %ds;       \
 158         CFI_ADJUST_CFA_OFFSET -4;\
 159         /*CFI_RESTORE ds;*/\
 160 2:      popl %es;       \
 161         CFI_ADJUST_CFA_OFFSET -4;\
 162         /*CFI_RESTORE es;*/\
 163 3:      popl %fs;       \
 164         CFI_ADJUST_CFA_OFFSET -4;\
 165         /*CFI_RESTORE fs;*/\
 166 .pushsection .fixup,"ax";       \
 167 4:      movl $0,(%esp); \
 168         jmp 1b;         \
 169 5:      movl $0,(%esp); \
 170         jmp 2b;         \
 171 6:      movl $0,(%esp); \
 172         jmp 3b;         \
 173 .section __ex_table,"a";\
 174         .align 4;       \
 175         .long 1b,4b;    \
 176         .long 2b,5b;    \
 177         .long 3b,6b;    \
 178 .popsection
 179
 180 #define RING0_INT_FRAME \
 181         CFI_STARTPROC simple;\
 182         CFI_SIGNAL_FRAME;\
 183         CFI_DEF_CFA esp, 3*4;\
 184         /*CFI_OFFSET cs, -2*4;*/\
 185         CFI_OFFSET eip, -3*4
 186
 187 #define RING0_EC_FRAME \
 188         CFI_STARTPROC simple;\
 189         CFI_SIGNAL_FRAME;\
 190         CFI_DEF_CFA esp, 4*4;\
 191         /*CFI_OFFSET cs, -2*4;*/\
 192         CFI_OFFSET eip, -3*4
 193
 194 #define RING0_PTREGS_FRAME \
 195         CFI_STARTPROC simple;\
 196         CFI_SIGNAL_FRAME;\
 197         CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
 198         /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
 199         CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
 200         /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
 201         /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
 202         CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
 203         CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
 204         CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
 205         CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
 206         CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
 207         CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
 208         CFI_OFFSET ebx, PT_EBX-PT_OLDESP
 209
 210 ENTRY(ret_from_fork)
 211         CFI_STARTPROC
 212         pushl %eax
 213         CFI_ADJUST_CFA_OFFSET 4
 214         call schedule_tail
 215         GET_THREAD_INFO(%ebp)
 216         popl %eax
 217         CFI_ADJUST_CFA_OFFSET -4
 218         pushl $0x0202                   # Reset kernel eflags
 219         CFI_ADJUST_CFA_OFFSET 4
 220         popfl
 221         CFI_ADJUST_CFA_OFFSET -4
 222         jmp syscall_exit
 223         CFI_ENDPROC
 224 END(ret_from_fork)
 225
 226 /*
 227  * Return to user mode is not as complex as all this looks,
 228  * but we want the default path for a system call return to
 229  * go as quickly as possible which is why some of this is
 230  * less clear than it otherwise should be.
 231  */
 232
 233         # userspace resumption stub bypassing syscall exit tracing
 234         ALIGN
 235         RING0_PTREGS_FRAME
 236 ret_from_exception:
 237         preempt_stop(CLBR_ANY)
 238 ret_from_intr:
 239         GET_THREAD_INFO(%ebp)
 240 check_userspace:
 241         movl PT_EFLAGS(%esp), %eax      # mix EFLAGS and CS
 242         movb PT_CS(%esp), %al
 243         andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
 244         cmpl $USER_RPL, %eax
 245         jb resume_kernel                # not returning to v8086 or userspace
 246
 247 ENTRY(resume_userspace)
 248         LOCKDEP_SYS_EXIT
 249         DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 250                                         # setting need_resched or sigpending
 251                                         # between sampling and the iret
 252         TRACE_IRQS_OFF
 253         movl TI_flags(%ebp), %ecx
 254         andl $_TIF_WORK_MASK, %ecx      # is there any work to be done on
 255                                         # int/exception return?
 256         jne work_pending
 257         jmp restore_all
 258 END(ret_from_exception)
 259
 260 #ifdef CONFIG_PREEMPT
 261 ENTRY(resume_kernel)
 262         DISABLE_INTERRUPTS(CLBR_ANY)
 263         cmpl $0,TI_preempt_count(%ebp)  # non-zero preempt_count ?
 264         jnz restore_nocheck
 265 need_resched:
 266         movl TI_flags(%ebp), %ecx       # need_resched set ?
 267         testb $_TIF_NEED_RESCHED, %cl
 268         jz restore_all
 269         testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)    # interrupts off (exception path) ?
 270         jz restore_all
 271         call preempt_schedule_irq
 272         jmp need_resched
 273 END(resume_kernel)
 274 #endif
 275         CFI_ENDPROC
 276
 277 /* SYSENTER_RETURN points to after the "sysenter" instruction in
 278    the vsyscall page.  See vsyscall-sysentry.S, which defines the symbol.  */
 279
 280         # sysenter call handler stub
 281 ENTRY(ia32_sysenter_target)
 282         CFI_STARTPROC simple
 283         CFI_SIGNAL_FRAME
 284         CFI_DEF_CFA esp, 0
 285         CFI_REGISTER esp, ebp
 286         movl TSS_sysenter_sp0(%esp),%esp
 287 sysenter_past_esp:
 288         /*
 289          * Interrupts are disabled here, but we can't trace it until
 290          * enough kernel state to call TRACE_IRQS_OFF can be called - but
 291          * we immediately enable interrupts at that point anyway.
 292          */
 293         pushl $(__USER_DS)
 294         CFI_ADJUST_CFA_OFFSET 4
 295         /*CFI_REL_OFFSET ss, 0*/
 296         pushl %ebp
 297         CFI_ADJUST_CFA_OFFSET 4
 298         CFI_REL_OFFSET esp, 0
 299         pushfl
 300         orl $X86_EFLAGS_IF, (%esp)
 301         CFI_ADJUST_CFA_OFFSET 4
 302         pushl $(__USER_CS)
 303         CFI_ADJUST_CFA_OFFSET 4
 304         /*CFI_REL_OFFSET cs, 0*/
 305         /*
 306          * Push current_thread_info()->sysenter_return to the stack.
 307          * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
 308          * pushed above; +8 corresponds to copy_thread's esp0 setting.
 309          */
 310         pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
 311         CFI_ADJUST_CFA_OFFSET 4
 312         CFI_REL_OFFSET eip, 0
 313
 314         pushl %eax
 315         CFI_ADJUST_CFA_OFFSET 4
 316         SAVE_ALL
 317         ENABLE_INTERRUPTS(CLBR_NONE)
 318
 319 /*
 320  * Load the potential sixth argument from user stack.
 321  * Careful about security.
 322  */
 323         cmpl $__PAGE_OFFSET-3,%ebp
 324         jae syscall_fault
 325 1:      movl (%ebp),%ebp
 326         movl %ebp,PT_EBP(%esp)
 327 .section __ex_table,"a"
 328         .align 4
 329         .long 1b,syscall_fault
 330 .previous
 331
 332         GET_THREAD_INFO(%ebp)
 333
 334         /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
 335         testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
 336         jnz syscall_trace_entry
 337         cmpl $(nr_syscalls), %eax
 338         jae syscall_badsys
 339         call *sys_call_table(,%eax,4)
 340         movl %eax,PT_EAX(%esp)
 341         LOCKDEP_SYS_EXIT
 342         DISABLE_INTERRUPTS(CLBR_ANY)
 343         TRACE_IRQS_OFF
 344         movl TI_flags(%ebp), %ecx
 345         testw $_TIF_ALLWORK_MASK, %cx
 346         jne syscall_exit_work
 347 /* if something modifies registers it must also disable sysexit */
 348         movl PT_EIP(%esp), %edx
 349         movl PT_OLDESP(%esp), %ecx
 350         xorl %ebp,%ebp
 351         TRACE_IRQS_ON
 352 1:      mov  PT_FS(%esp), %fs
 353         ENABLE_INTERRUPTS_SYSEXIT
 354         CFI_ENDPROC
 355 .pushsection .fixup,"ax"
 356 2:      movl $0,PT_FS(%esp)
 357         jmp 1b
 358 .section __ex_table,"a"
 359         .align 4
 360         .long 1b,2b
 361 .popsection
 362 ENDPROC(ia32_sysenter_target)
 363
 364         # system call handler stub
 365 ENTRY(system_call)
 366         RING0_INT_FRAME                 # can't unwind into user space anyway
 367         pushl %eax                      # save orig_eax
 368         CFI_ADJUST_CFA_OFFSET 4
 369         SAVE_ALL
 370         GET_THREAD_INFO(%ebp)
 371                                         # system call tracing in operation / emulation
 372         /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
 373         testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
 374         jnz syscall_trace_entry
 375         cmpl $(nr_syscalls), %eax
 376         jae syscall_badsys
 377 syscall_call:
 378         call *sys_call_table(,%eax,4)
 379         movl %eax,PT_EAX(%esp)          # store the return value
 380 syscall_exit:
 381         LOCKDEP_SYS_EXIT
 382         DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 383                                         # setting need_resched or sigpending
 384                                         # between sampling and the iret
 385         TRACE_IRQS_OFF
 386         testl $X86_EFLAGS_TF,PT_EFLAGS(%esp)    # If tracing set singlestep flag on exit
 387         jz no_singlestep
 388         orl $_TIF_SINGLESTEP,TI_flags(%ebp)
 389 no_singlestep:
 390         movl TI_flags(%ebp), %ecx
 391         testw $_TIF_ALLWORK_MASK, %cx   # current->work
 392         jne syscall_exit_work
 393
 394 restore_all:
 395         movl PT_EFLAGS(%esp), %eax      # mix EFLAGS, SS and CS
 396         # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
 397         # are returning to the kernel.
 398         # See comments in process.c:copy_thread() for details.
 399         movb PT_OLDSS(%esp), %ah
 400         movb PT_CS(%esp), %al
 401         andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
 402         cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
 403         CFI_REMEMBER_STATE
 404         je ldt_ss                       # returning to user-space with LDT SS
 405 restore_nocheck:
 406         TRACE_IRQS_IRET
 407 restore_nocheck_notrace:
 408         RESTORE_REGS
 409         addl $4, %esp                   # skip orig_eax/error_code
 410         CFI_ADJUST_CFA_OFFSET -4
 411 irq_return:
 412         INTERRUPT_RETURN
 413 .section .fixup,"ax"
 414 ENTRY(iret_exc)
 415         pushl $0                        # no error code
 416         pushl $do_iret_error
 417         jmp error_code
 418 .previous
 419 .section __ex_table,"a"
 420         .align 4
 421         .long irq_return,iret_exc
 422 .previous
 423
 424         CFI_RESTORE_STATE
 425 ldt_ss:
 426         larl PT_OLDSS(%esp), %eax
 427         jnz restore_nocheck
 428         testl $0x00400000, %eax         # returning to 32bit stack?
 429         jnz restore_nocheck             # allright, normal return
 430
 431 #ifdef CONFIG_PARAVIRT
 432         /*
 433          * The kernel can't run on a non-flat stack if paravirt mode
 434          * is active.  Rather than try to fixup the high bits of
 435          * ESP, bypass this code entirely.  This may break DOSemu
 436          * and/or Wine support in a paravirt VM, although the option
 437          * is still available to implement the setting of the high
 438          * 16-bits in the INTERRUPT_RETURN paravirt-op.
 439          */
 440         cmpl $0, pv_info+PARAVIRT_enabled
 441         jne restore_nocheck
 442 #endif
 443
 444         /* If returning to userspace with 16bit stack,
 445          * try to fix the higher word of ESP, as the CPU
 446          * won't restore it.
 447          * This is an "official" bug of all the x86-compatible
 448          * CPUs, which we can try to work around to make
 449          * dosemu and wine happy. */
 450         movl PT_OLDESP(%esp), %eax
 451         movl %esp, %edx
 452         call patch_espfix_desc
 453         pushl $__ESPFIX_SS
 454         CFI_ADJUST_CFA_OFFSET 4
 455         pushl %eax
 456         CFI_ADJUST_CFA_OFFSET 4
 457         DISABLE_INTERRUPTS(CLBR_EAX)
 458         TRACE_IRQS_OFF
 459         lss (%esp), %esp
 460         CFI_ADJUST_CFA_OFFSET -8
 461         jmp restore_nocheck
 462         CFI_ENDPROC
 463 ENDPROC(system_call)
 464
 465         # perform work that needs to be done immediately before resumption
 466         ALIGN
 467         RING0_PTREGS_FRAME              # can't unwind into user space anyway
 468 work_pending:
 469         testb $_TIF_NEED_RESCHED, %cl
 470         jz work_notifysig
 471 work_resched:
 472         call schedule
 473         LOCKDEP_SYS_EXIT
 474         DISABLE_INTERRUPTS(CLBR_ANY)    # make sure we don't miss an interrupt
 475                                         # setting need_resched or sigpending
 476                                         # between sampling and the iret
 477         TRACE_IRQS_OFF
 478         movl TI_flags(%ebp), %ecx
 479         andl $_TIF_WORK_MASK, %ecx      # is there any work to be done other
 480                                         # than syscall tracing?
 481         jz restore_all
 482         testb $_TIF_NEED_RESCHED, %cl
 483         jnz work_resched
 484
 485 work_notifysig:                         # deal with pending signals and
 486                                         # notify-resume requests
 487 #ifdef CONFIG_VM86
 488         testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
 489         movl %esp, %eax
 490         jne work_notifysig_v86          # returning to kernel-space or
 491                                         # vm86-space
 492         xorl %edx, %edx
 493         call do_notify_resume
 494         jmp resume_userspace_sig
 495
 496         ALIGN
 497 work_notifysig_v86:
 498         pushl %ecx                      # save ti_flags for do_notify_resume
 499         CFI_ADJUST_CFA_OFFSET 4
 500         call save_v86_state             # %eax contains pt_regs pointer
 501         popl %ecx
 502         CFI_ADJUST_CFA_OFFSET -4
 503         movl %eax, %esp
 504 #else
 505         movl %esp, %eax
 506 #endif
 507         xorl %edx, %edx
 508         call do_notify_resume
 509         jmp resume_userspace_sig
 510 END(work_pending)
 511
 512         # perform syscall exit tracing
 513         ALIGN
 514 syscall_trace_entry:
 515         movl $-ENOSYS,PT_EAX(%esp)
 516         movl %esp, %eax
 517         xorl %edx,%edx
 518         call do_syscall_trace
 519         cmpl $0, %eax
 520         jne resume_userspace            # ret != 0 -> running under PTRACE_SYSEMU,
 521                                         # so must skip actual syscall
 522         movl PT_ORIG_EAX(%esp), %eax
 523         cmpl $(nr_syscalls), %eax
 524         jnae syscall_call
 525         jmp syscall_exit
 526 END(syscall_trace_entry)
 527
 528         # perform syscall exit tracing
 529         ALIGN
 530 syscall_exit_work:
 531         testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
 532         jz work_pending
 533         TRACE_IRQS_ON
 534         ENABLE_INTERRUPTS(CLBR_ANY)     # could let do_syscall_trace() call
 535                                         # schedule() instead
 536         movl %esp, %eax
 537         movl $1, %edx
 538         call do_syscall_trace
 539         jmp resume_userspace
 540 END(syscall_exit_work)
 541         CFI_ENDPROC
 542
 543         RING0_INT_FRAME                 # can't unwind into user space anyway
 544 syscall_fault:
 545         GET_THREAD_INFO(%ebp)
 546         movl $-EFAULT,PT_EAX(%esp)
 547         jmp resume_userspace
 548 END(syscall_fault)
 549
 550 syscall_badsys:
 551         movl $-ENOSYS,PT_EAX(%esp)
 552         jmp resume_userspace
 553 END(syscall_badsys)
 554         CFI_ENDPROC
 555
 556 #define FIXUP_ESPFIX_STACK \
 557         /* since we are on a wrong stack, we cant make it a C code :( */ \
 558         PER_CPU(gdt_page, %ebx); \
 559         GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
 560         addl %esp, %eax; \
 561         pushl $__KERNEL_DS; \
 562         CFI_ADJUST_CFA_OFFSET 4; \
 563         pushl %eax; \
 564         CFI_ADJUST_CFA_OFFSET 4; \
 565         lss (%esp), %esp; \
 566         CFI_ADJUST_CFA_OFFSET -8;
 567 #define UNWIND_ESPFIX_STACK \
 568         movl %ss, %eax; \
 569         /* see if on espfix stack */ \
 570         cmpw $__ESPFIX_SS, %ax; \
 571         jne 27f; \
 572         movl $__KERNEL_DS, %eax; \
 573         movl %eax, %ds; \
 574         movl %eax, %es; \
 575         /* switch to normal stack */ \
 576         FIXUP_ESPFIX_STACK; \
 577 27:;
 578
 579 /*
 580  * Build the entry stubs and pointer table with
 581  * some assembler magic.
 582  */
 583 .section .rodata,"a"
 584 ENTRY(interrupt)
 585 .text
 586
 587 ENTRY(irq_entries_start)
 588         RING0_INT_FRAME
 589 vector=0
 590 .rept NR_IRQS
 591         ALIGN
 592  .if vector
 593         CFI_ADJUST_CFA_OFFSET -4
 594  .endif
 595 1:      pushl $~(vector)
 596         CFI_ADJUST_CFA_OFFSET 4
 597         jmp common_interrupt
 598  .previous
 599         .long 1b
 600  .text
 601 vector=vector+1
 602 .endr
 603 END(irq_entries_start)
 604
 605 .previous
 606 END(interrupt)
 607 .previous
 608
 609 /*
 610  * the CPU automatically disables interrupts when executing an IRQ vector,
 611  * so IRQ-flags tracing has to follow that:
 612  */
 613         ALIGN
 614 common_interrupt:
 615         SAVE_ALL
 616         TRACE_IRQS_OFF
 617         movl %esp,%eax
 618         call do_IRQ
 619         jmp ret_from_intr
 620 ENDPROC(common_interrupt)
 621         CFI_ENDPROC
 622
 623 #define BUILD_INTERRUPT(name, nr)       \
 624 ENTRY(name)                             \
 625         RING0_INT_FRAME;                \
 626         pushl $~(nr);                   \
 627         CFI_ADJUST_CFA_OFFSET 4;        \
 628         SAVE_ALL;                       \
 629         TRACE_IRQS_OFF                  \
 630         movl %esp,%eax;                 \
 631         call smp_##name;                \
 632         jmp ret_from_intr;              \
 633         CFI_ENDPROC;                    \
 634 ENDPROC(name)
 635
 636 /* The include is where all of the SMP etc. interrupts come from */
 637 #include "entry_arch.h"
 638
 639 KPROBE_ENTRY(page_fault)
 640         RING0_EC_FRAME
 641         pushl $do_page_fault
 642         CFI_ADJUST_CFA_OFFSET 4
 643         ALIGN
 644 error_code:
 645         /* the function address is in %fs's slot on the stack */
 646         pushl %es
 647         CFI_ADJUST_CFA_OFFSET 4
 648         /*CFI_REL_OFFSET es, 0*/
 649         pushl %ds
 650         CFI_ADJUST_CFA_OFFSET 4
 651         /*CFI_REL_OFFSET ds, 0*/
 652         pushl %eax
 653         CFI_ADJUST_CFA_OFFSET 4
 654         CFI_REL_OFFSET eax, 0
 655         pushl %ebp
 656         CFI_ADJUST_CFA_OFFSET 4
 657         CFI_REL_OFFSET ebp, 0
 658         pushl %edi
 659         CFI_ADJUST_CFA_OFFSET 4
 660         CFI_REL_OFFSET edi, 0
 661         pushl %esi
 662         CFI_ADJUST_CFA_OFFSET 4
 663         CFI_REL_OFFSET esi, 0
 664         pushl %edx
 665         CFI_ADJUST_CFA_OFFSET 4
 666         CFI_REL_OFFSET edx, 0
 667         pushl %ecx
 668         CFI_ADJUST_CFA_OFFSET 4
 669         CFI_REL_OFFSET ecx, 0
 670         pushl %ebx
 671         CFI_ADJUST_CFA_OFFSET 4
 672         CFI_REL_OFFSET ebx, 0
 673         cld
 674         pushl %fs
 675         CFI_ADJUST_CFA_OFFSET 4
 676         /*CFI_REL_OFFSET fs, 0*/
 677         movl $(__KERNEL_PERCPU), %ecx
 678         movl %ecx, %fs
 679         UNWIND_ESPFIX_STACK
 680         popl %ecx
 681         CFI_ADJUST_CFA_OFFSET -4
 682         /*CFI_REGISTER es, ecx*/
 683         movl PT_FS(%esp), %edi          # get the function address
 684         movl PT_ORIG_EAX(%esp), %edx    # get the error code
 685         movl $-1, PT_ORIG_EAX(%esp)     # no syscall to restart
 686         mov  %ecx, PT_FS(%esp)
 687         /*CFI_REL_OFFSET fs, ES*/
 688         movl $(__USER_DS), %ecx
 689         movl %ecx, %ds
 690         movl %ecx, %es
 691         movl %esp,%eax                  # pt_regs pointer
 692         call *%edi
 693         jmp ret_from_exception
 694         CFI_ENDPROC
 695 KPROBE_END(page_fault)
 696
 697 ENTRY(coprocessor_error)
 698         RING0_INT_FRAME
 699         pushl $0
 700         CFI_ADJUST_CFA_OFFSET 4
 701         pushl $do_coprocessor_error
 702         CFI_ADJUST_CFA_OFFSET 4
 703         jmp error_code
 704         CFI_ENDPROC
 705 END(coprocessor_error)
 706
 707 ENTRY(simd_coprocessor_error)
 708         RING0_INT_FRAME
 709         pushl $0
 710         CFI_ADJUST_CFA_OFFSET 4
 711         pushl $do_simd_coprocessor_error
 712         CFI_ADJUST_CFA_OFFSET 4
 713         jmp error_code
 714         CFI_ENDPROC
 715 END(simd_coprocessor_error)
 716
 717 ENTRY(device_not_available)
 718         RING0_INT_FRAME
 719         pushl $-1                       # mark this as an int
 720         CFI_ADJUST_CFA_OFFSET 4
 721         SAVE_ALL
 722         GET_CR0_INTO_EAX
 723         testl $0x4, %eax                # EM (math emulation bit)
 724         jne device_not_available_emulate
 725         preempt_stop(CLBR_ANY)
 726         call math_state_restore
 727         jmp ret_from_exception
 728 device_not_available_emulate:
 729         pushl $0                        # temporary storage for ORIG_EIP
 730         CFI_ADJUST_CFA_OFFSET 4
 731         call math_emulate
 732         addl $4, %esp
 733         CFI_ADJUST_CFA_OFFSET -4
 734         jmp ret_from_exception
 735         CFI_ENDPROC
 736 END(device_not_available)
 737
 738 /*
 739  * Debug traps and NMI can happen at the one SYSENTER instruction
 740  * that sets up the real kernel stack. Check here, since we can't
 741  * allow the wrong stack to be used.
 742  *
 743  * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
 744  * already pushed 3 words if it hits on the sysenter instruction:
 745  * eflags, cs and eip.
 746  *
 747  * We just load the right stack, and push the three (known) values
 748  * by hand onto the new stack - while updating the return eip past
 749  * the instruction that would have done it for sysenter.
 750  */
 751 #define FIX_STACK(offset, ok, label)            \
 752         cmpw $__KERNEL_CS,4(%esp);              \
 753         jne ok;                                 \
 754 label:                                          \
 755         movl TSS_sysenter_sp0+offset(%esp),%esp;        \
 756         CFI_DEF_CFA esp, 0;                     \
 757         CFI_UNDEFINED eip;                      \
 758         pushfl;                                 \
 759         CFI_ADJUST_CFA_OFFSET 4;                \
 760         pushl $__KERNEL_CS;                     \
 761         CFI_ADJUST_CFA_OFFSET 4;                \
 762         pushl $sysenter_past_esp;               \
 763         CFI_ADJUST_CFA_OFFSET 4;                \
 764         CFI_REL_OFFSET eip, 0
 765
 766 KPROBE_ENTRY(debug)
 767         RING0_INT_FRAME
 768         cmpl $ia32_sysenter_target,(%esp)
 769         jne debug_stack_correct
 770         FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
 771 debug_stack_correct:
 772         pushl $-1                       # mark this as an int
 773         CFI_ADJUST_CFA_OFFSET 4
 774         SAVE_ALL
 775         xorl %edx,%edx                  # error code 0
 776         movl %esp,%eax                  # pt_regs pointer
 777         call do_debug
 778         jmp ret_from_exception
 779         CFI_ENDPROC
 780 KPROBE_END(debug)
 781
 782 /*
 783  * NMI is doubly nasty. It can happen _while_ we're handling
 784  * a debug fault, and the debug fault hasn't yet been able to
 785  * clear up the stack. So we first check whether we got  an
 786  * NMI on the sysenter entry path, but after that we need to
 787  * check whether we got an NMI on the debug path where the debug
 788  * fault happened on the sysenter path.
 789  */
 790 KPROBE_ENTRY(nmi)
 791         RING0_INT_FRAME
 792         pushl %eax
 793         CFI_ADJUST_CFA_OFFSET 4
 794         movl %ss, %eax
 795         cmpw $__ESPFIX_SS, %ax
 796         popl %eax
 797         CFI_ADJUST_CFA_OFFSET -4
 798         je nmi_espfix_stack
 799         cmpl $ia32_sysenter_target,(%esp)
 800         je nmi_stack_fixup
 801         pushl %eax
 802         CFI_ADJUST_CFA_OFFSET 4
 803         movl %esp,%eax
 804         /* Do not access memory above the end of our stack page,
 805          * it might not exist.
 806          */
 807         andl $(THREAD_SIZE-1),%eax
 808         cmpl $(THREAD_SIZE-20),%eax
 809         popl %eax
 810         CFI_ADJUST_CFA_OFFSET -4
 811         jae nmi_stack_correct
 812         cmpl $ia32_sysenter_target,12(%esp)
 813         je nmi_debug_stack_check
 814 nmi_stack_correct:
 815         /* We have a RING0_INT_FRAME here */
 816         pushl %eax
 817         CFI_ADJUST_CFA_OFFSET 4
 818         SAVE_ALL
 819         xorl %edx,%edx          # zero error code
 820         movl %esp,%eax          # pt_regs pointer
 821         call do_nmi
 822         jmp restore_nocheck_notrace
 823         CFI_ENDPROC
 824
 825 nmi_stack_fixup:
 826         RING0_INT_FRAME
 827         FIX_STACK(12,nmi_stack_correct, 1)
 828         jmp nmi_stack_correct
 829
 830 nmi_debug_stack_check:
 831         /* We have a RING0_INT_FRAME here */
 832         cmpw $__KERNEL_CS,16(%esp)
 833         jne nmi_stack_correct
 834         cmpl $debug,(%esp)
 835         jb nmi_stack_correct
 836         cmpl $debug_esp_fix_insn,(%esp)
 837         ja nmi_stack_correct
 838         FIX_STACK(24,nmi_stack_correct, 1)
 839         jmp nmi_stack_correct
 840
 841 nmi_espfix_stack:
 842         /* We have a RING0_INT_FRAME here.
 843          *
 844          * create the pointer to lss back
 845          */
 846         pushl %ss
 847         CFI_ADJUST_CFA_OFFSET 4
 848         pushl %esp
 849         CFI_ADJUST_CFA_OFFSET 4
 850         addw $4, (%esp)
 851         /* copy the iret frame of 12 bytes */
 852         .rept 3
 853         pushl 16(%esp)
 854         CFI_ADJUST_CFA_OFFSET 4
 855         .endr
 856         pushl %eax
 857         CFI_ADJUST_CFA_OFFSET 4
 858         SAVE_ALL
 859         FIXUP_ESPFIX_STACK              # %eax == %esp
 860         xorl %edx,%edx                  # zero error code
 861         call do_nmi
 862         RESTORE_REGS
 863         lss 12+4(%esp), %esp            # back to espfix stack
 864         CFI_ADJUST_CFA_OFFSET -24
 865         jmp irq_return
 866         CFI_ENDPROC
 867 KPROBE_END(nmi)
 868
 869 #ifdef CONFIG_PARAVIRT
 870 ENTRY(native_iret)
 871         iret
 872 .section __ex_table,"a"
 873         .align 4
 874         .long native_iret, iret_exc
 875 .previous
 876 END(native_iret)
 877
 878 ENTRY(native_irq_enable_sysexit)
 879         sti
 880         sysexit
 881 END(native_irq_enable_sysexit)
 882 #endif
 883
 884 KPROBE_ENTRY(int3)
 885         RING0_INT_FRAME
 886         pushl $-1                       # mark this as an int
 887         CFI_ADJUST_CFA_OFFSET 4
 888         SAVE_ALL
 889         xorl %edx,%edx          # zero error code
 890         movl %esp,%eax          # pt_regs pointer
 891         call do_int3
 892         jmp ret_from_exception
 893         CFI_ENDPROC
 894 KPROBE_END(int3)
 895
 896 ENTRY(overflow)
 897         RING0_INT_FRAME
 898         pushl $0
 899         CFI_ADJUST_CFA_OFFSET 4
 900         pushl $do_overflow
 901         CFI_ADJUST_CFA_OFFSET 4
 902         jmp error_code
 903         CFI_ENDPROC
 904 END(overflow)
 905
 906 ENTRY(bounds)
 907         RING0_INT_FRAME
 908         pushl $0
 909         CFI_ADJUST_CFA_OFFSET 4
 910         pushl $do_bounds
 911         CFI_ADJUST_CFA_OFFSET 4
 912         jmp error_code
 913         CFI_ENDPROC
 914 END(bounds)
 915
 916 ENTRY(invalid_op)
 917         RING0_INT_FRAME
 918         pushl $0
 919         CFI_ADJUST_CFA_OFFSET 4
 920         pushl $do_invalid_op
 921         CFI_ADJUST_CFA_OFFSET 4
 922         jmp error_code
 923         CFI_ENDPROC
 924 END(invalid_op)
 925
 926 ENTRY(coprocessor_segment_overrun)
 927         RING0_INT_FRAME
 928         pushl $0
 929         CFI_ADJUST_CFA_OFFSET 4
 930         pushl $do_coprocessor_segment_overrun
 931         CFI_ADJUST_CFA_OFFSET 4
 932         jmp error_code
 933         CFI_ENDPROC
 934 END(coprocessor_segment_overrun)
 935
 936 ENTRY(invalid_TSS)
 937         RING0_EC_FRAME
 938         pushl $do_invalid_TSS
 939         CFI_ADJUST_CFA_OFFSET 4
 940         jmp error_code
 941         CFI_ENDPROC
 942 END(invalid_TSS)
 943
 944 ENTRY(segment_not_present)
 945         RING0_EC_FRAME
 946         pushl $do_segment_not_present
 947         CFI_ADJUST_CFA_OFFSET 4
 948         jmp error_code
 949         CFI_ENDPROC
 950 END(segment_not_present)
 951
 952 ENTRY(stack_segment)
 953         RING0_EC_FRAME
 954         pushl $do_stack_segment
 955         CFI_ADJUST_CFA_OFFSET 4
 956         jmp error_code
 957         CFI_ENDPROC
 958 END(stack_segment)
 959
 960 KPROBE_ENTRY(general_protection)
 961         RING0_EC_FRAME
 962         pushl $do_general_protection
 963         CFI_ADJUST_CFA_OFFSET 4
 964         jmp error_code
 965         CFI_ENDPROC
 966 KPROBE_END(general_protection)
 967
 968 ENTRY(alignment_check)
 969         RING0_EC_FRAME
 970         pushl $do_alignment_check
 971         CFI_ADJUST_CFA_OFFSET 4
 972         jmp error_code
 973         CFI_ENDPROC
 974 END(alignment_check)
 975
 976 ENTRY(divide_error)
 977         RING0_INT_FRAME
 978         pushl $0                        # no error code
 979         CFI_ADJUST_CFA_OFFSET 4
 980         pushl $do_divide_error
 981         CFI_ADJUST_CFA_OFFSET 4
 982         jmp error_code
 983         CFI_ENDPROC
 984 END(divide_error)
 985
 986 #ifdef CONFIG_X86_MCE
 987 ENTRY(machine_check)
 988         RING0_INT_FRAME
 989         pushl $0
 990         CFI_ADJUST_CFA_OFFSET 4
 991         pushl machine_check_vector
 992         CFI_ADJUST_CFA_OFFSET 4
 993         jmp error_code
 994         CFI_ENDPROC
 995 END(machine_check)
 996 #endif
 997
 998 ENTRY(spurious_interrupt_bug)
 999         RING0_INT_FRAME
1000         pushl $0
1001         CFI_ADJUST_CFA_OFFSET 4
1002         pushl $do_spurious_interrupt_bug
1003         CFI_ADJUST_CFA_OFFSET 4
1004         jmp error_code
1005         CFI_ENDPROC
1006 END(spurious_interrupt_bug)
1007
1008 ENTRY(kernel_thread_helper)
1009         pushl $0                # fake return address for unwinder
1010         CFI_STARTPROC
1011         movl %edx,%eax
1012         push %edx
1013         CFI_ADJUST_CFA_OFFSET 4
1014         call *%ebx
1015         push %eax
1016         CFI_ADJUST_CFA_OFFSET 4
1017         call do_exit
1018         CFI_ENDPROC
1019 ENDPROC(kernel_thread_helper)
1020
1021 #ifdef CONFIG_XEN
1022 /* Xen doesn't set %esp to be precisely what the normal sysenter
1023    entrypoint expects, so fix it up before using the normal path. */
1024 ENTRY(xen_sysenter_target)
1025         RING0_INT_FRAME
1026         addl $5*4, %esp         /* remove xen-provided frame */
1027         jmp sysenter_past_esp
1028         CFI_ENDPROC
1029
1030 ENTRY(xen_hypervisor_callback)
1031         CFI_STARTPROC
1032         pushl $0
1033         CFI_ADJUST_CFA_OFFSET 4
1034         SAVE_ALL
1035         TRACE_IRQS_OFF
1036
1037         /* Check to see if we got the event in the critical
1038            region in xen_iret_direct, after we've reenabled
1039            events and checked for pending events.  This simulates
1040            iret instruction's behaviour where it delivers a
1041            pending interrupt when enabling interrupts. */
1042         movl PT_EIP(%esp),%eax
1043         cmpl $xen_iret_start_crit,%eax
1044         jb   1f
1045         cmpl $xen_iret_end_crit,%eax
1046         jae  1f
1047
1048         jmp  xen_iret_crit_fixup
1049
1050 ENTRY(xen_do_upcall)
1051 1:      mov %esp, %eax
1052         call xen_evtchn_do_upcall
1053         jmp  ret_from_intr
1054         CFI_ENDPROC
1055 ENDPROC(xen_hypervisor_callback)
1056
1057 # Hypervisor uses this for application faults while it executes.
1058 # We get here for two reasons:
1059 #  1. Fault while reloading DS, ES, FS or GS
1060 #  2. Fault while executing IRET
1061 # Category 1 we fix up by reattempting the load, and zeroing the segment
1062 # register if the load fails.
1063 # Category 2 we fix up by jumping to do_iret_error. We cannot use the
1064 # normal Linux return path in this case because if we use the IRET hypercall
1065 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1066 # We distinguish between categories by maintaining a status value in EAX.
1067 ENTRY(xen_failsafe_callback)
1068         CFI_STARTPROC
1069         pushl %eax
1070         CFI_ADJUST_CFA_OFFSET 4
1071         movl $1,%eax
1072 1:      mov 4(%esp),%ds
1073 2:      mov 8(%esp),%es
1074 3:      mov 12(%esp),%fs
1075 4:      mov 16(%esp),%gs
1076         testl %eax,%eax
1077         popl %eax
1078         CFI_ADJUST_CFA_OFFSET -4
1079         lea 16(%esp),%esp
1080         CFI_ADJUST_CFA_OFFSET -16
1081         jz 5f
1082         addl $16,%esp
1083         jmp iret_exc            # EAX != 0 => Category 2 (Bad IRET)
1084 5:      pushl $0                # EAX == 0 => Category 1 (Bad segment)
1085         CFI_ADJUST_CFA_OFFSET 4
1086         SAVE_ALL
1087         jmp ret_from_exception
1088         CFI_ENDPROC
1089
1090 .section .fixup,"ax"
1091 6:      xorl %eax,%eax
1092         movl %eax,4(%esp)
1093         jmp 1b
1094 7:      xorl %eax,%eax
1095         movl %eax,8(%esp)
1096         jmp 2b
1097 8:      xorl %eax,%eax
1098         movl %eax,12(%esp)
1099         jmp 3b
1100 9:      xorl %eax,%eax
1101         movl %eax,16(%esp)
1102         jmp 4b
1103 .previous
1104 .section __ex_table,"a"
1105         .align 4
1106         .long 1b,6b
1107         .long 2b,7b
1108         .long 3b,8b
1109         .long 4b,9b
1110 .previous
1111 ENDPROC(xen_failsafe_callback)
1112
1113 #endif  /* CONFIG_XEN */
1114
1115 #ifdef CONFIG_FTRACE
1116 #ifdef CONFIG_DYNAMIC_FTRACE
1117
1118 ENTRY(mcount)
1119         pushl %eax
1120         pushl %ecx
1121         pushl %edx
1122         movl 0xc(%esp), %eax
1123         subl $MCOUNT_INSN_SIZE, %eax
1124
1125 .globl mcount_call
1126 mcount_call:
1127         call ftrace_stub
1128
1129         popl %edx
1130         popl %ecx
1131         popl %eax
1132
1133         ret
1134 END(mcount)
1135
1136 ENTRY(ftrace_caller)
1137         pushl %eax
1138         pushl %ecx
1139         pushl %edx
1140         movl 0xc(%esp), %eax
1141         movl 0x4(%ebp), %edx
1142         subl $MCOUNT_INSN_SIZE, %eax
1143
1144 .globl ftrace_call
1145 ftrace_call:
1146         call ftrace_stub
1147
1148         popl %edx
1149         popl %ecx
1150         popl %eax
1151
1152 .globl ftrace_stub
1153 ftrace_stub:
1154         ret
1155 END(ftrace_caller)
1156
1157 #else /* ! CONFIG_DYNAMIC_FTRACE */
1158
1159 ENTRY(mcount)
1160         cmpl $ftrace_stub, ftrace_trace_function
1161         jnz trace
1162 .globl ftrace_stub
1163 ftrace_stub:
1164         ret
1165
1166         /* taken from glibc */
1167 trace:
1168         pushl %eax
1169         pushl %ecx
1170         pushl %edx
1171         movl 0xc(%esp), %eax
1172         movl 0x4(%ebp), %edx
1173         subl $MCOUNT_INSN_SIZE, %eax
1174
1175         call *ftrace_trace_function
1176
1177         popl %edx
1178         popl %ecx
1179         popl %eax
1180
1181         jmp ftrace_stub
1182 END(mcount)
1183 #endif /* CONFIG_DYNAMIC_FTRACE */
1184 #endif /* CONFIG_FTRACE */
1185
1186 .section .rodata,"a"
1187 #include "syscall_table_32.S"
1188
1189 syscall_table_size=(.-sys_call_table)