arch/x86_64/kernel/entry.S

   1 /*
   2  *  linux/arch/x86_64/entry.S
   3  *
   4  *  Copyright (C) 1991, 1992  Linus Torvalds
   5  *  Copyright (C) 2000, 2001, 2002  Andi Kleen SuSE Labs
   6  *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
   7  *
   8  *  $Id$
   9  */
  10
  11 /*
  12  * entry.S contains the system-call and fault low-level handling routines.
  13  *
  14  * NOTE: This code handles signal-recognition, which happens every time
  15  * after an interrupt and after each system call.
  16  *
  17  * Normal syscalls and interrupts don't save a full stack frame, this is
  18  * only done for syscall tracing, signals or fork/exec et.al.
  19  *
  20  * A note on terminology:
  21  * - top of stack: Architecture defined interrupt frame from SS to RIP
  22  * at the top of the kernel process stack.
  23  * - partial stack frame: partially saved registers upto R11.
  24  * - full stack frame: Like partial stack frame, but all register saved.
  25  *
  26  * TODO:
  27  * - schedule it carefully for the final hardware.
  28  */
  29
  30 #define ASSEMBLY 1
  31 #include <linux/config.h>
  32 #include <linux/linkage.h>
  33 #include <asm/segment.h>
  34 #include <asm/smp.h>
  35 #include <asm/cache.h>
  36 #include <asm/errno.h>
  37 #include <asm/dwarf2.h>
  38 #include <asm/calling.h>
  39 #include <asm/asm-offsets.h>
  40 #include <asm/msr.h>
  41 #include <asm/unistd.h>
  42 #include <asm/thread_info.h>
  43 #include <asm/hw_irq.h>
  44
  45         .code64
  46
  47 #ifndef CONFIG_PREEMPT
  48 #define retint_kernel retint_restore_args
  49 #endif
  50
  51 /*
  52  * C code is not supposed to know about undefined top of stack. Every time
  53  * a C function with an pt_regs argument is called from the SYSCALL based
  54  * fast path FIXUP_TOP_OF_STACK is needed.
  55  * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
  56  * manipulation.
  57  */
  58
  59         /* %rsp:at FRAMEEND */
  60         .macro FIXUP_TOP_OF_STACK tmp
  61         movq    %gs:pda_oldrsp,\tmp
  62         movq    \tmp,RSP(%rsp)
  63         movq    $__USER_DS,SS(%rsp)
  64         movq    $__USER_CS,CS(%rsp)
  65         movq    $-1,RCX(%rsp)
  66         movq    R11(%rsp),\tmp  /* get eflags */
  67         movq    \tmp,EFLAGS(%rsp)
  68         .endm
  69
  70         .macro RESTORE_TOP_OF_STACK tmp,offset=0
  71         movq   RSP-\offset(%rsp),\tmp
  72         movq   \tmp,%gs:pda_oldrsp
  73         movq   EFLAGS-\offset(%rsp),\tmp
  74         movq   \tmp,R11-\offset(%rsp)
  75         .endm
  76
  77         .macro FAKE_STACK_FRAME child_rip
  78         /* push in order ss, rsp, eflags, cs, rip */
  79         xorl %eax, %eax
  80         pushq %rax /* ss */
  81         CFI_ADJUST_CFA_OFFSET   8
  82         /*CFI_REL_OFFSET        ss,0*/
  83         pushq %rax /* rsp */
  84         CFI_ADJUST_CFA_OFFSET   8
  85         CFI_REL_OFFSET  rsp,0
  86         pushq $(1<<9) /* eflags - interrupts on */
  87         CFI_ADJUST_CFA_OFFSET   8
  88         /*CFI_REL_OFFSET        rflags,0*/
  89         pushq $__KERNEL_CS /* cs */
  90         CFI_ADJUST_CFA_OFFSET   8
  91         /*CFI_REL_OFFSET        cs,0*/
  92         pushq \child_rip /* rip */
  93         CFI_ADJUST_CFA_OFFSET   8
  94         CFI_REL_OFFSET  rip,0
  95         pushq   %rax /* orig rax */
  96         CFI_ADJUST_CFA_OFFSET   8
  97         .endm
  98
  99         .macro UNFAKE_STACK_FRAME
 100         addq $8*6, %rsp
 101         CFI_ADJUST_CFA_OFFSET   -(6*8)
 102         .endm
 103
 104         .macro  CFI_DEFAULT_STACK start=1
 105         .if \start
 106         CFI_STARTPROC   simple
 107         CFI_DEF_CFA     rsp,SS+8
 108         .else
 109         CFI_DEF_CFA_OFFSET SS+8
 110         .endif
 111         CFI_REL_OFFSET  r15,R15
 112         CFI_REL_OFFSET  r14,R14
 113         CFI_REL_OFFSET  r13,R13
 114         CFI_REL_OFFSET  r12,R12
 115         CFI_REL_OFFSET  rbp,RBP
 116         CFI_REL_OFFSET  rbx,RBX
 117         CFI_REL_OFFSET  r11,R11
 118         CFI_REL_OFFSET  r10,R10
 119         CFI_REL_OFFSET  r9,R9
 120         CFI_REL_OFFSET  r8,R8
 121         CFI_REL_OFFSET  rax,RAX
 122         CFI_REL_OFFSET  rcx,RCX
 123         CFI_REL_OFFSET  rdx,RDX
 124         CFI_REL_OFFSET  rsi,RSI
 125         CFI_REL_OFFSET  rdi,RDI
 126         CFI_REL_OFFSET  rip,RIP
 127         /*CFI_REL_OFFSET        cs,CS*/
 128         /*CFI_REL_OFFSET        rflags,EFLAGS*/
 129         CFI_REL_OFFSET  rsp,RSP
 130         /*CFI_REL_OFFSET        ss,SS*/
 131         .endm
 132 /*
 133  * A newly forked process directly context switches into this.
 134  */
 135 /* rdi: prev */
 136 ENTRY(ret_from_fork)
 137         CFI_DEFAULT_STACK
 138         call schedule_tail
 139         GET_THREAD_INFO(%rcx)
 140         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
 141         jnz rff_trace
 142 rff_action:
 143         RESTORE_REST
 144         testl $3,CS-ARGOFFSET(%rsp)     # from kernel_thread?
 145         je   int_ret_from_sys_call
 146         testl $_TIF_IA32,threadinfo_flags(%rcx)
 147         jnz  int_ret_from_sys_call
 148         RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
 149         jmp ret_from_sys_call
 150 rff_trace:
 151         movq %rsp,%rdi
 152         call syscall_trace_leave
 153         GET_THREAD_INFO(%rcx)
 154         jmp rff_action
 155         CFI_ENDPROC
 156
 157 /*
 158  * System call entry. Upto 6 arguments in registers are supported.
 159  *
 160  * SYSCALL does not save anything on the stack and does not change the
 161  * stack pointer.
 162  */
 163
 164 /*
 165  * Register setup:
 166  * rax  system call number
 167  * rdi  arg0
 168  * rcx  return address for syscall/sysret, C arg3
 169  * rsi  arg1
 170  * rdx  arg2
 171  * r10  arg3    (--> moved to rcx for C)
 172  * r8   arg4
 173  * r9   arg5
 174  * r11  eflags for syscall/sysret, temporary for C
 175  * r12-r15,rbp,rbx saved by C code, not touched.
 176  *
 177  * Interrupts are off on entry.
 178  * Only called from user space.
 179  *
 180  * XXX  if we had a free scratch register we could save the RSP into the stack frame
 181  *      and report it properly in ps. Unfortunately we haven't.
 182  */
 183
 184 ENTRY(system_call)
 185         CFI_STARTPROC   simple
 186         CFI_DEF_CFA     rsp,0
 187         CFI_REGISTER    rip,rcx
 188         /*CFI_REGISTER  rflags,r11*/
 189         swapgs
 190         movq    %rsp,%gs:pda_oldrsp
 191         movq    %gs:pda_kernelstack,%rsp
 192         sti
 193         SAVE_ARGS 8,1
 194         movq  %rax,ORIG_RAX-ARGOFFSET(%rsp)
 195         movq  %rcx,RIP-ARGOFFSET(%rsp)
 196         CFI_REL_OFFSET rip,RIP-ARGOFFSET
 197         GET_THREAD_INFO(%rcx)
 198         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
 199         CFI_REMEMBER_STATE
 200         jnz tracesys
 201         cmpq $__NR_syscall_max,%rax
 202         ja badsys
 203         movq %r10,%rcx
 204         call *sys_call_table(,%rax,8)  # XXX:    rip relative
 205         movq %rax,RAX-ARGOFFSET(%rsp)
 206 /*
 207  * Syscall return path ending with SYSRET (fast path)
 208  * Has incomplete stack frame and undefined top of stack.
 209  */
 210         .globl ret_from_sys_call
 211 ret_from_sys_call:
 212         movl $_TIF_ALLWORK_MASK,%edi
 213         /* edi: flagmask */
 214 sysret_check:
 215         GET_THREAD_INFO(%rcx)
 216         cli
 217         movl threadinfo_flags(%rcx),%edx
 218         andl %edi,%edx
 219         CFI_REMEMBER_STATE
 220         jnz  sysret_careful
 221         movq RIP-ARGOFFSET(%rsp),%rcx
 222         CFI_REGISTER    rip,rcx
 223         RESTORE_ARGS 0,-ARG_SKIP,1
 224         /*CFI_REGISTER  rflags,r11*/
 225         movq    %gs:pda_oldrsp,%rsp
 226         swapgs
 227         sysretq
 228
 229         /* Handle reschedules */
 230         /* edx: work, edi: workmask */
 231 sysret_careful:
 232         CFI_RESTORE_STATE
 233         bt $TIF_NEED_RESCHED,%edx
 234         jnc sysret_signal
 235         sti
 236         pushq %rdi
 237         CFI_ADJUST_CFA_OFFSET 8
 238         call schedule
 239         popq  %rdi
 240         CFI_ADJUST_CFA_OFFSET -8
 241         jmp sysret_check
 242
 243         /* Handle a signal */
 244 sysret_signal:
 245         sti
 246         testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
 247         jz    1f
 248
 249         /* Really a signal */
 250         /* edx: work flags (arg3) */
 251         leaq do_notify_resume(%rip),%rax
 252         leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
 253         xorl %esi,%esi # oldset -> arg2
 254         call ptregscall_common
 255 1:      movl $_TIF_NEED_RESCHED,%edi
 256         jmp sysret_check
 257
 258 badsys:
 259         movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
 260         jmp ret_from_sys_call
 261
 262         /* Do syscall tracing */
 263 tracesys:
 264         CFI_RESTORE_STATE
 265         SAVE_REST
 266         movq $-ENOSYS,RAX(%rsp)
 267         FIXUP_TOP_OF_STACK %rdi
 268         movq %rsp,%rdi
 269         call syscall_trace_enter
 270         LOAD_ARGS ARGOFFSET  /* reload args from stack in case ptrace changed it */
 271         RESTORE_REST
 272         cmpq $__NR_syscall_max,%rax
 273         ja  1f
 274         movq %r10,%rcx  /* fixup for C */
 275         call *sys_call_table(,%rax,8)
 276         movq %rax,RAX-ARGOFFSET(%rsp)
 277 1:      SAVE_REST
 278         movq %rsp,%rdi
 279         call syscall_trace_leave
 280         RESTORE_TOP_OF_STACK %rbx
 281         RESTORE_REST
 282         jmp ret_from_sys_call
 283         CFI_ENDPROC
 284
 285 /*
 286  * Syscall return path ending with IRET.
 287  * Has correct top of stack, but partial stack frame.
 288  */
 289 ENTRY(int_ret_from_sys_call)
 290         CFI_STARTPROC   simple
 291         CFI_DEF_CFA     rsp,SS+8-ARGOFFSET
 292         /*CFI_REL_OFFSET        ss,SS-ARGOFFSET*/
 293         CFI_REL_OFFSET  rsp,RSP-ARGOFFSET
 294         /*CFI_REL_OFFSET        rflags,EFLAGS-ARGOFFSET*/
 295         /*CFI_REL_OFFSET        cs,CS-ARGOFFSET*/
 296         CFI_REL_OFFSET  rip,RIP-ARGOFFSET
 297         CFI_REL_OFFSET  rdx,RDX-ARGOFFSET
 298         CFI_REL_OFFSET  rcx,RCX-ARGOFFSET
 299         CFI_REL_OFFSET  rax,RAX-ARGOFFSET
 300         CFI_REL_OFFSET  rdi,RDI-ARGOFFSET
 301         CFI_REL_OFFSET  rsi,RSI-ARGOFFSET
 302         CFI_REL_OFFSET  r8,R8-ARGOFFSET
 303         CFI_REL_OFFSET  r9,R9-ARGOFFSET
 304         CFI_REL_OFFSET  r10,R10-ARGOFFSET
 305         CFI_REL_OFFSET  r11,R11-ARGOFFSET
 306         cli
 307         testl $3,CS-ARGOFFSET(%rsp)
 308         je retint_restore_args
 309         movl $_TIF_ALLWORK_MASK,%edi
 310         /* edi: mask to check */
 311 int_with_check:
 312         GET_THREAD_INFO(%rcx)
 313         movl threadinfo_flags(%rcx),%edx
 314         andl %edi,%edx
 315         jnz   int_careful
 316         jmp   retint_swapgs
 317
 318         /* Either reschedule or signal or syscall exit tracking needed. */
 319         /* First do a reschedule test. */
 320         /* edx: work, edi: workmask */
 321 int_careful:
 322         bt $TIF_NEED_RESCHED,%edx
 323         jnc  int_very_careful
 324         sti
 325         pushq %rdi
 326         CFI_ADJUST_CFA_OFFSET 8
 327         call schedule
 328         popq %rdi
 329         CFI_ADJUST_CFA_OFFSET -8
 330         cli
 331         jmp int_with_check
 332
 333         /* handle signals and tracing -- both require a full stack frame */
 334 int_very_careful:
 335         sti
 336         SAVE_REST
 337         /* Check for syscall exit trace */
 338         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
 339         jz int_signal
 340         pushq %rdi
 341         CFI_ADJUST_CFA_OFFSET 8
 342         leaq 8(%rsp),%rdi       # &ptregs -> arg1
 343         call syscall_trace_leave
 344         popq %rdi
 345         CFI_ADJUST_CFA_OFFSET -8
 346         andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
 347         cli
 348         jmp int_restore_rest
 349
 350 int_signal:
 351         testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
 352         jz 1f
 353         movq %rsp,%rdi          # &ptregs -> arg1
 354         xorl %esi,%esi          # oldset -> arg2
 355         call do_notify_resume
 356 1:      movl $_TIF_NEED_RESCHED,%edi
 357 int_restore_rest:
 358         RESTORE_REST
 359         cli
 360         jmp int_with_check
 361         CFI_ENDPROC
 362
 363 /*
 364  * Certain special system calls that need to save a complete full stack frame.
 365  */
 366
 367         .macro PTREGSCALL label,func,arg
 368         .globl \label
 369 \label:
 370         leaq    \func(%rip),%rax
 371         leaq    -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
 372         jmp     ptregscall_common
 373         .endm
 374
 375         CFI_STARTPROC
 376
 377         PTREGSCALL stub_clone, sys_clone, %r8
 378         PTREGSCALL stub_fork, sys_fork, %rdi
 379         PTREGSCALL stub_vfork, sys_vfork, %rdi
 380         PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
 381         PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
 382         PTREGSCALL stub_iopl, sys_iopl, %rsi
 383
 384 ENTRY(ptregscall_common)
 385         popq %r11
 386         CFI_ADJUST_CFA_OFFSET -8
 387         CFI_REGISTER rip, r11
 388         SAVE_REST
 389         movq %r11, %r15
 390         CFI_REGISTER rip, r15
 391         FIXUP_TOP_OF_STACK %r11
 392         call *%rax
 393         RESTORE_TOP_OF_STACK %r11
 394         movq %r15, %r11
 395         CFI_REGISTER rip, r11
 396         RESTORE_REST
 397         pushq %r11
 398         CFI_ADJUST_CFA_OFFSET 8
 399         CFI_REL_OFFSET rip, 0
 400         ret
 401         CFI_ENDPROC
 402
 403 ENTRY(stub_execve)
 404         CFI_STARTPROC
 405         popq %r11
 406         CFI_ADJUST_CFA_OFFSET -8
 407         CFI_REGISTER rip, r11
 408         SAVE_REST
 409         movq %r11, %r15
 410         CFI_REGISTER rip, r15
 411         FIXUP_TOP_OF_STACK %r11
 412         call sys_execve
 413         GET_THREAD_INFO(%rcx)
 414         bt $TIF_IA32,threadinfo_flags(%rcx)
 415         CFI_REMEMBER_STATE
 416         jc exec_32bit
 417         RESTORE_TOP_OF_STACK %r11
 418         movq %r15, %r11
 419         CFI_REGISTER rip, r11
 420         RESTORE_REST
 421         pushq %r11
 422         CFI_ADJUST_CFA_OFFSET 8
 423         CFI_REL_OFFSET rip, 0
 424         ret
 425
 426 exec_32bit:
 427         CFI_RESTORE_STATE
 428         movq %rax,RAX(%rsp)
 429         RESTORE_REST
 430         jmp int_ret_from_sys_call
 431         CFI_ENDPROC
 432
 433 /*
 434  * sigreturn is special because it needs to restore all registers on return.
 435  * This cannot be done with SYSRET, so use the IRET return path instead.
 436  */
 437 ENTRY(stub_rt_sigreturn)
 438         CFI_STARTPROC
 439         addq $8, %rsp
 440         CFI_ADJUST_CFA_OFFSET   -8
 441         SAVE_REST
 442         movq %rsp,%rdi
 443         FIXUP_TOP_OF_STACK %r11
 444         call sys_rt_sigreturn
 445         movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
 446         RESTORE_REST
 447         jmp int_ret_from_sys_call
 448         CFI_ENDPROC
 449
 450 /*
 451  * initial frame state for interrupts and exceptions
 452  */
 453         .macro _frame ref
 454         CFI_STARTPROC simple
 455         CFI_DEF_CFA rsp,SS+8-\ref
 456         /*CFI_REL_OFFSET ss,SS-\ref*/
 457         CFI_REL_OFFSET rsp,RSP-\ref
 458         /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
 459         /*CFI_REL_OFFSET cs,CS-\ref*/
 460         CFI_REL_OFFSET rip,RIP-\ref
 461         .endm
 462
 463 /* initial frame state for interrupts (and exceptions without error code) */
 464 #define INTR_FRAME _frame RIP
 465 /* initial frame state for exceptions with error code (and interrupts with
 466    vector already pushed) */
 467 #define XCPT_FRAME _frame ORIG_RAX
 468
 469 /*
 470  * Interrupt entry/exit.
 471  *
 472  * Interrupt entry points save only callee clobbered registers in fast path.
 473  *
 474  * Entry runs with interrupts off.
 475  */
 476
 477 /* 0(%rsp): interrupt number */
 478         .macro interrupt func
 479         cld
 480 #ifdef CONFIG_DEBUG_INFO
 481         SAVE_ALL
 482         movq %rsp,%rdi
 483         /*
 484          * Setup a stack frame pointer.  This allows gdb to trace
 485          * back to the original stack.
 486          */
 487         movq %rsp,%rbp
 488         CFI_DEF_CFA_REGISTER    rbp
 489 #else
 490         SAVE_ARGS
 491         leaq -ARGOFFSET(%rsp),%rdi      # arg1 for handler
 492 #endif
 493         testl $3,CS(%rdi)
 494         je 1f
 495         swapgs
 496 1:      incl    %gs:pda_irqcount        # RED-PEN should check preempt count
 497         movq %gs:pda_irqstackptr,%rax
 498         cmoveq %rax,%rsp /*todo This needs CFI annotation! */
 499         pushq %rdi                      # save old stack
 500         CFI_ADJUST_CFA_OFFSET   8
 501         call \func
 502         .endm
 503
 504 ENTRY(common_interrupt)
 505         XCPT_FRAME
 506         interrupt do_IRQ
 507         /* 0(%rsp): oldrsp-ARGOFFSET */
 508 ret_from_intr:
 509         popq  %rdi
 510         CFI_ADJUST_CFA_OFFSET   -8
 511         cli
 512         decl %gs:pda_irqcount
 513 #ifdef CONFIG_DEBUG_INFO
 514         movq RBP(%rdi),%rbp
 515         CFI_DEF_CFA_REGISTER    rsp
 516 #endif
 517         leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
 518 exit_intr:
 519         GET_THREAD_INFO(%rcx)
 520         testl $3,CS-ARGOFFSET(%rsp)
 521         je retint_kernel
 522
 523         /* Interrupt came from user space */
 524         /*
 525          * Has a correct top of stack, but a partial stack frame
 526          * %rcx: thread info. Interrupts off.
 527          */
 528 retint_with_reschedule:
 529         movl $_TIF_WORK_MASK,%edi
 530 retint_check:
 531         movl threadinfo_flags(%rcx),%edx
 532         andl %edi,%edx
 533         CFI_REMEMBER_STATE
 534         jnz  retint_careful
 535 retint_swapgs:
 536         swapgs
 537 retint_restore_args:
 538         cli
 539         RESTORE_ARGS 0,8,0
 540 iret_label:
 541         iretq
 542
 543         .section __ex_table,"a"
 544         .quad iret_label,bad_iret
 545         .previous
 546         .section .fixup,"ax"
 547         /* force a signal here? this matches i386 behaviour */
 548         /* running with kernel gs */
 549 bad_iret:
 550         movq $-9999,%rdi        /* better code? */
 551         jmp do_exit
 552         .previous
 553
 554         /* edi: workmask, edx: work */
 555 retint_careful:
 556         CFI_RESTORE_STATE
 557         bt    $TIF_NEED_RESCHED,%edx
 558         jnc   retint_signal
 559         sti
 560         pushq %rdi
 561         CFI_ADJUST_CFA_OFFSET   8
 562         call  schedule
 563         popq %rdi
 564         CFI_ADJUST_CFA_OFFSET   -8
 565         GET_THREAD_INFO(%rcx)
 566         cli
 567         jmp retint_check
 568
 569 retint_signal:
 570         testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
 571         jz    retint_swapgs
 572         sti
 573         SAVE_REST
 574         movq $-1,ORIG_RAX(%rsp)
 575         xorl %esi,%esi          # oldset
 576         movq %rsp,%rdi          # &pt_regs
 577         call do_notify_resume
 578         RESTORE_REST
 579         cli
 580         movl $_TIF_NEED_RESCHED,%edi
 581         GET_THREAD_INFO(%rcx)
 582         jmp retint_check
 583
 584 #ifdef CONFIG_PREEMPT
 585         /* Returning to kernel space. Check if we need preemption */
 586         /* rcx:  threadinfo. interrupts off. */
 587         .p2align
 588 retint_kernel:
 589         cmpl $0,threadinfo_preempt_count(%rcx)
 590         jnz  retint_restore_args
 591         bt  $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
 592         jnc  retint_restore_args
 593         bt   $9,EFLAGS-ARGOFFSET(%rsp)  /* interrupts off? */
 594         jnc  retint_restore_args
 595         call preempt_schedule_irq
 596         jmp exit_intr
 597 #endif
 598         CFI_ENDPROC
 599
 600 /*
 601  * APIC interrupts.
 602  */
 603         .macro apicinterrupt num,func
 604         INTR_FRAME
 605         pushq $\num-256
 606         CFI_ADJUST_CFA_OFFSET 8
 607         interrupt \func
 608         jmp ret_from_intr
 609         CFI_ENDPROC
 610         .endm
 611
 612 ENTRY(thermal_interrupt)
 613         apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
 614
 615 #ifdef CONFIG_SMP
 616 ENTRY(reschedule_interrupt)
 617         apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
 618
 619         .macro INVALIDATE_ENTRY num
 620 ENTRY(invalidate_interrupt\num)
 621         apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
 622         .endm
 623
 624         INVALIDATE_ENTRY 0
 625         INVALIDATE_ENTRY 1
 626         INVALIDATE_ENTRY 2
 627         INVALIDATE_ENTRY 3
 628         INVALIDATE_ENTRY 4
 629         INVALIDATE_ENTRY 5
 630         INVALIDATE_ENTRY 6
 631         INVALIDATE_ENTRY 7
 632
 633 ENTRY(call_function_interrupt)
 634         apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
 635 #endif
 636
 637 #ifdef CONFIG_X86_LOCAL_APIC
 638 ENTRY(apic_timer_interrupt)
 639         apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
 640
 641 ENTRY(error_interrupt)
 642         apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
 643
 644 ENTRY(spurious_interrupt)
 645         apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
 646 #endif
 647
 648 /*
 649  * Exception entry points.
 650  */
 651         .macro zeroentry sym
 652         INTR_FRAME
 653         pushq $0        /* push error code/oldrax */
 654         CFI_ADJUST_CFA_OFFSET 8
 655         pushq %rax      /* push real oldrax to the rdi slot */
 656         CFI_ADJUST_CFA_OFFSET 8
 657         leaq  \sym(%rip),%rax
 658         jmp error_entry
 659         CFI_ENDPROC
 660         .endm
 661
 662         .macro errorentry sym
 663         XCPT_FRAME
 664         pushq %rax
 665         CFI_ADJUST_CFA_OFFSET 8
 666         leaq  \sym(%rip),%rax
 667         jmp error_entry
 668         CFI_ENDPROC
 669         .endm
 670
 671         /* error code is on the stack already */
 672         /* handle NMI like exceptions that can happen everywhere */
 673         .macro paranoidentry sym
 674         SAVE_ALL
 675         cld
 676         movl $1,%ebx
 677         movl  $MSR_GS_BASE,%ecx
 678         rdmsr
 679         testl %edx,%edx
 680         js    1f
 681         swapgs
 682         xorl  %ebx,%ebx
 683 1:      movq %rsp,%rdi
 684         movq ORIG_RAX(%rsp),%rsi
 685         movq $-1,ORIG_RAX(%rsp)
 686         call \sym
 687         cli
 688         .endm
 689
 690 /*
 691  * Exception entry point. This expects an error code/orig_rax on the stack
 692  * and the exception handler in %rax.
 693  */
 694 ENTRY(error_entry)
 695         _frame RDI
 696         /* rdi slot contains rax, oldrax contains error code */
 697         cld
 698         subq  $14*8,%rsp
 699         CFI_ADJUST_CFA_OFFSET   (14*8)
 700         movq %rsi,13*8(%rsp)
 701         CFI_REL_OFFSET  rsi,RSI
 702         movq 14*8(%rsp),%rsi    /* load rax from rdi slot */
 703         movq %rdx,12*8(%rsp)
 704         CFI_REL_OFFSET  rdx,RDX
 705         movq %rcx,11*8(%rsp)
 706         CFI_REL_OFFSET  rcx,RCX
 707         movq %rsi,10*8(%rsp)    /* store rax */
 708         CFI_REL_OFFSET  rax,RAX
 709         movq %r8, 9*8(%rsp)
 710         CFI_REL_OFFSET  r8,R8
 711         movq %r9, 8*8(%rsp)
 712         CFI_REL_OFFSET  r9,R9
 713         movq %r10,7*8(%rsp)
 714         CFI_REL_OFFSET  r10,R10
 715         movq %r11,6*8(%rsp)
 716         CFI_REL_OFFSET  r11,R11
 717         movq %rbx,5*8(%rsp)
 718         CFI_REL_OFFSET  rbx,RBX
 719         movq %rbp,4*8(%rsp)
 720         CFI_REL_OFFSET  rbp,RBP
 721         movq %r12,3*8(%rsp)
 722         CFI_REL_OFFSET  r12,R12
 723         movq %r13,2*8(%rsp)
 724         CFI_REL_OFFSET  r13,R13
 725         movq %r14,1*8(%rsp)
 726         CFI_REL_OFFSET  r14,R14
 727         movq %r15,(%rsp)
 728         CFI_REL_OFFSET  r15,R15
 729         xorl %ebx,%ebx
 730         testl $3,CS(%rsp)
 731         je  error_kernelspace
 732 error_swapgs:
 733         swapgs
 734 error_sti:
 735         movq %rdi,RDI(%rsp)
 736         movq %rsp,%rdi
 737         movq ORIG_RAX(%rsp),%rsi        /* get error code */
 738         movq $-1,ORIG_RAX(%rsp)
 739         call *%rax
 740         /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
 741 error_exit:
 742         movl %ebx,%eax
 743         RESTORE_REST
 744         cli
 745         GET_THREAD_INFO(%rcx)
 746         testl %eax,%eax
 747         jne  retint_kernel
 748         movl  threadinfo_flags(%rcx),%edx
 749         movl  $_TIF_WORK_MASK,%edi
 750         andl  %edi,%edx
 751         jnz  retint_careful
 752         swapgs
 753         RESTORE_ARGS 0,8,0
 754         iretq
 755         CFI_ENDPROC
 756
 757 error_kernelspace:
 758         incl %ebx
 759        /* There are two places in the kernel that can potentially fault with
 760           usergs. Handle them here. The exception handlers after
 761            iret run with kernel gs again, so don't set the user space flag.
 762            B stepping K8s sometimes report an truncated RIP for IRET
 763            exceptions returning to compat mode. Check for these here too. */
 764         leaq iret_label(%rip),%rbp
 765         cmpq %rbp,RIP(%rsp)
 766         je   error_swapgs
 767         movl %ebp,%ebp  /* zero extend */
 768         cmpq %rbp,RIP(%rsp)
 769         je   error_swapgs
 770         cmpq $gs_change,RIP(%rsp)
 771         je   error_swapgs
 772         jmp  error_sti
 773
 774        /* Reload gs selector with exception handling */
 775        /* edi:  new selector */
 776 ENTRY(load_gs_index)
 777         CFI_STARTPROC
 778         pushf
 779         CFI_ADJUST_CFA_OFFSET 8
 780         cli
 781         swapgs
 782 gs_change:
 783         movl %edi,%gs
 784 2:      mfence          /* workaround */
 785         swapgs
 786         popf
 787         CFI_ADJUST_CFA_OFFSET -8
 788         ret
 789         CFI_ENDPROC
 790
 791         .section __ex_table,"a"
 792         .align 8
 793         .quad gs_change,bad_gs
 794         .previous
 795         .section .fixup,"ax"
 796         /* running with kernelgs */
 797 bad_gs:
 798         swapgs                  /* switch back to user gs */
 799         xorl %eax,%eax
 800         movl %eax,%gs
 801         jmp  2b
 802         .previous
 803
 804 /*
 805  * Create a kernel thread.
 806  *
 807  * C extern interface:
 808  *      extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
 809  *
 810  * asm input arguments:
 811  *      rdi: fn, rsi: arg, rdx: flags
 812  */
 813 ENTRY(kernel_thread)
 814         CFI_STARTPROC
 815         FAKE_STACK_FRAME $child_rip
 816         SAVE_ALL
 817
 818         # rdi: flags, rsi: usp, rdx: will be &pt_regs
 819         movq %rdx,%rdi
 820         orq  kernel_thread_flags(%rip),%rdi
 821         movq $-1, %rsi
 822         movq %rsp, %rdx
 823
 824         xorl %r8d,%r8d
 825         xorl %r9d,%r9d
 826
 827         # clone now
 828         call do_fork
 829         movq %rax,RAX(%rsp)
 830         xorl %edi,%edi
 831
 832         /*
 833          * It isn't worth to check for reschedule here,
 834          * so internally to the x86_64 port you can rely on kernel_thread()
 835          * not to reschedule the child before returning, this avoids the need
 836          * of hacks for example to fork off the per-CPU idle tasks.
 837          * [Hopefully no generic code relies on the reschedule -AK]
 838          */
 839         RESTORE_ALL
 840         UNFAKE_STACK_FRAME
 841         ret
 842         CFI_ENDPROC
 843
 844
 845 child_rip:
 846         /*
 847          * Here we are in the child and the registers are set as they were
 848          * at kernel_thread() invocation in the parent.
 849          */
 850         movq %rdi, %rax
 851         movq %rsi, %rdi
 852         call *%rax
 853         # exit
 854         xorl %edi, %edi
 855         call do_exit
 856
 857 /*
 858  * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
 859  *
 860  * C extern interface:
 861  *       extern long execve(char *name, char **argv, char **envp)
 862  *
 863  * asm input arguments:
 864  *      rdi: name, rsi: argv, rdx: envp
 865  *
 866  * We want to fallback into:
 867  *      extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
 868  *
 869  * do_sys_execve asm fallback arguments:
 870  *      rdi: name, rsi: argv, rdx: envp, fake frame on the stack
 871  */
 872 ENTRY(execve)
 873         CFI_STARTPROC
 874         FAKE_STACK_FRAME $0
 875         SAVE_ALL
 876         call sys_execve
 877         movq %rax, RAX(%rsp)
 878         RESTORE_REST
 879         testq %rax,%rax
 880         je int_ret_from_sys_call
 881         RESTORE_ARGS
 882         UNFAKE_STACK_FRAME
 883         ret
 884         CFI_ENDPROC
 885
 886 KPROBE_ENTRY(page_fault)
 887         errorentry do_page_fault
 888         .previous .text
 889
 890 ENTRY(coprocessor_error)
 891         zeroentry do_coprocessor_error
 892
 893 ENTRY(simd_coprocessor_error)
 894         zeroentry do_simd_coprocessor_error
 895
 896 ENTRY(device_not_available)
 897         zeroentry math_state_restore
 898
 899         /* runs on exception stack */
 900 KPROBE_ENTRY(debug)
 901         INTR_FRAME
 902         pushq $0
 903         CFI_ADJUST_CFA_OFFSET 8
 904         paranoidentry do_debug
 905         jmp paranoid_exit
 906         CFI_ENDPROC
 907         .previous .text
 908
 909         /* runs on exception stack */
 910 ENTRY(nmi)
 911         INTR_FRAME
 912         pushq $-1
 913         CFI_ADJUST_CFA_OFFSET 8
 914         paranoidentry do_nmi
 915         /*
 916          * "Paranoid" exit path from exception stack.
 917          * Paranoid because this is used by NMIs and cannot take
 918          * any kernel state for granted.
 919          * We don't do kernel preemption checks here, because only
 920          * NMI should be common and it does not enable IRQs and
 921          * cannot get reschedule ticks.
 922          */
 923         /* ebx: no swapgs flag */
 924 paranoid_exit:
 925         testl %ebx,%ebx                         /* swapgs needed? */
 926         jnz paranoid_restore
 927         testl $3,CS(%rsp)
 928         jnz   paranoid_userspace
 929 paranoid_swapgs:
 930         swapgs
 931 paranoid_restore:
 932         RESTORE_ALL 8
 933         iretq
 934 paranoid_userspace:
 935         GET_THREAD_INFO(%rcx)
 936         movl threadinfo_flags(%rcx),%ebx
 937         andl $_TIF_WORK_MASK,%ebx
 938         jz paranoid_swapgs
 939         movq %rsp,%rdi                  /* &pt_regs */
 940         call sync_regs
 941         movq %rax,%rsp                  /* switch stack for scheduling */
 942         testl $_TIF_NEED_RESCHED,%ebx
 943         jnz paranoid_schedule
 944         movl %ebx,%edx                  /* arg3: thread flags */
 945         sti
 946         xorl %esi,%esi                  /* arg2: oldset */
 947         movq %rsp,%rdi                  /* arg1: &pt_regs */
 948         call do_notify_resume
 949         cli
 950         jmp paranoid_userspace
 951 paranoid_schedule:
 952         sti
 953         call schedule
 954         cli
 955         jmp paranoid_userspace
 956         CFI_ENDPROC
 957
 958 KPROBE_ENTRY(int3)
 959         zeroentry do_int3
 960         .previous .text
 961
 962 ENTRY(overflow)
 963         zeroentry do_overflow
 964
 965 ENTRY(bounds)
 966         zeroentry do_bounds
 967
 968 ENTRY(invalid_op)
 969         zeroentry do_invalid_op
 970
 971 ENTRY(coprocessor_segment_overrun)
 972         zeroentry do_coprocessor_segment_overrun
 973
 974 ENTRY(reserved)
 975         zeroentry do_reserved
 976
 977         /* runs on exception stack */
 978 ENTRY(double_fault)
 979         XCPT_FRAME
 980         paranoidentry do_double_fault
 981         jmp paranoid_exit
 982         CFI_ENDPROC
 983
 984 ENTRY(invalid_TSS)
 985         errorentry do_invalid_TSS
 986
 987 ENTRY(segment_not_present)
 988         errorentry do_segment_not_present
 989
 990         /* runs on exception stack */
 991 ENTRY(stack_segment)
 992         XCPT_FRAME
 993         paranoidentry do_stack_segment
 994         jmp paranoid_exit
 995         CFI_ENDPROC
 996
 997 KPROBE_ENTRY(general_protection)
 998         errorentry do_general_protection
 999         .previous .text
1000
1001 ENTRY(alignment_check)
1002         errorentry do_alignment_check
1003
1004 ENTRY(divide_error)
1005         zeroentry do_divide_error
1006
1007 ENTRY(spurious_interrupt_bug)
1008         zeroentry do_spurious_interrupt_bug
1009
1010 #ifdef CONFIG_X86_MCE
1011         /* runs on exception stack */
1012 ENTRY(machine_check)
1013         INTR_FRAME
1014         pushq $0
1015         CFI_ADJUST_CFA_OFFSET 8
1016         paranoidentry do_machine_check
1017         jmp paranoid_exit
1018         CFI_ENDPROC
1019 #endif
1020
1021 ENTRY(call_debug)
1022        zeroentry do_call_debug
1023
1024 ENTRY(call_softirq)
1025         CFI_STARTPROC
1026         movq %gs:pda_irqstackptr,%rax
1027         pushq %r15
1028         CFI_ADJUST_CFA_OFFSET 8
1029         movq %rsp,%r15
1030         CFI_DEF_CFA_REGISTER    r15
1031         incl %gs:pda_irqcount
1032         cmove %rax,%rsp
1033         call __do_softirq
1034         movq %r15,%rsp
1035         CFI_DEF_CFA_REGISTER    rsp
1036         decl %gs:pda_irqcount
1037         popq %r15
1038         CFI_ADJUST_CFA_OFFSET -8
1039         ret
1040         CFI_ENDPROC